Skip to content

Commit e9702ff

Browse files
gartunggartung
authored
Add script to produce a diff of PR and IB moduleAllocMonitor circles data. (#2670)
* Add allocation monitoring script execution to profiling workflow, remove no longer executed igprof code and files * Add links for moduleAllocMonitor.circles.json too * Initial add of comparisons/moduleAllocMonitor-circles-diff.py * Add run of moduleAllocMonitor-circles-diff.py to pr profiling * More summary page updates * Black formatting * Black formatting again * Suggested updates * Black formatting * Black formatting again * Missed putting back align=center
1 parent d8d4d39 commit e9702ff

2 files changed

Lines changed: 329 additions & 0 deletions

File tree

‎comparisons/moduleAllocMonitor-circles-diff.py‎

Lines changed: 323 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,323 @@
1+
#! /usr/bin/env python3
2+
3+
import sys
4+
import json
5+
import os
6+
7+
8+
def diff_from(metrics, data, data_total, dest, dest_total, res):
9+
for metric in metrics:
10+
dmetric = dest[metric] - data[metric]
11+
dkey = "%s diff" % metric
12+
res[dkey] = dmetric
13+
pdmetric = 0.0
14+
pdmetric = dmetric
15+
pdkey = "%s pdiff" % metric
16+
res[pdkey] = pdmetric
17+
fkey = "%s frac" % metric
18+
fdest = 100 * dest[metric] / dest_total[metric] if dest_total[metric] != 0 else 0.0
19+
dest[fkey] = fdest
20+
fdata = 100 * data[metric] / data_total[metric] if data_total[metric] != 0 else 0.0
21+
data[fkey] = fdata
22+
dfmetric = fdest - fdata
23+
dfkey = "%s frac diff" % metric
24+
res[dfkey] = dfmetric
25+
pdfmetric = 0.0
26+
pdfmetric = dfmetric
27+
dkpkey = "%s frac diff" % metric
28+
res[dkpkey] = pdfmetric
29+
30+
31+
if len(sys.argv) == 1:
32+
print("""Usage: resources-diff.py IB_FILE PR_FILE
33+
Diff the content of two "resources.json" files and print the result to standard output.""")
34+
sys.exit(1)
35+
36+
with open(sys.argv[1]) as f:
37+
ibdata = json.load(f)
38+
39+
metrics = []
40+
for resource in ibdata["resources"]:
41+
if "name" in resource:
42+
metrics.append(resource["name"])
43+
else:
44+
for key in resource:
45+
metrics.append(key)
46+
47+
datamapib = {
48+
module["label"] + "|" + module["type"] + "|" + module["record"]: module
49+
for module in ibdata["modules"]
50+
}
51+
52+
datacumulsib = {}
53+
for module in ibdata["modules"]:
54+
datacumul = datacumulsib.get(module["type"])
55+
if datacumul:
56+
datacumul["count"] += 1
57+
for metric in metrics:
58+
datacumul[metric] += module[metric]
59+
else:
60+
datacumul = {}
61+
datacumul["count"] = 1
62+
for metric in metrics:
63+
datacumul[metric] = module[metric]
64+
datacumulsib[module["type"]] = datacumul
65+
66+
with open(sys.argv[2]) as f:
67+
prdata = json.load(f)
68+
if ibdata["resources"] != prdata["resources"]:
69+
print("Error: input files describe different metrics")
70+
sys.exit(1)
71+
72+
datamappr = {
73+
module["label"] + "|" + module["type"] + "|" + module["record"]: module
74+
for module in prdata["modules"]
75+
}
76+
77+
datacumulspr = {}
78+
for module in prdata["modules"]:
79+
datacumul = datacumulspr.get(module["type"])
80+
if datacumul:
81+
datacumul["count"] += 1
82+
for metric in metrics:
83+
datacumul[metric] += module[metric]
84+
else:
85+
datacumul = {}
86+
datacumul["count"] = 1
87+
for metric in metrics:
88+
datacumul[metric] = module[metric]
89+
datacumulspr[module["type"]] = datacumul
90+
# print(datacumulspr)
91+
92+
if ibdata["total"]["label"] != prdata["total"]["label"]:
93+
print("Warning: input files describe different process names")
94+
95+
results = {}
96+
results["resources"] = []
97+
for resource in prdata["resources"]:
98+
for k, v in resource.items():
99+
dkey = "%s diff" % k
100+
results["resources"].append({k: "%s" % v})
101+
results["resources"].append({dkey: "%s diff" % v})
102+
103+
results["total"] = {}
104+
results["total"]["type"] = prdata["total"]["type"]
105+
results["total"]["label"] = prdata["total"]["label"]
106+
107+
diff_from(
108+
metrics, ibdata["total"], ibdata["total"], prdata["total"], prdata["total"], results["total"]
109+
)
110+
111+
results["modules"] = []
112+
for module in prdata["modules"]:
113+
key = module["label"] + "|" + module["type"] + "|" + module["record"]
114+
result = {}
115+
result["type"] = module["type"]
116+
result["label"] = module["label"]
117+
result["record"] = module["record"]
118+
result["transitions"] = module["transitions"]
119+
if key in datamapib:
120+
diff_from(metrics, datamapib[key], ibdata["total"], module, prdata["total"], result)
121+
results["modules"].append(result)
122+
else:
123+
datamapib[key] = module
124+
diff_from(metrics, datamapib[key], ibdata["total"], module, prdata["total"], result)
125+
results["modules"].append(result)
126+
127+
datamapres = {
128+
module["label"] + "|" + module["type"] + "|" + module["record"]: module
129+
for module in results["modules"]
130+
}
131+
132+
threshold = 5000.0
133+
error_threshold = 20000.0
134+
135+
136+
summaryLines = []
137+
summaryLines += [
138+
"<html>",
139+
"<head><style>",
140+
"table, th, td {border: 1px solid black;}</style>",
141+
"<style> th, td {padding: 15px;}</style></head>",
142+
"<body><h3>ModuleAllocMonitor Resources Difference</h3><table>",
143+
'</table><table><tr><td bgcolor="orange">',
144+
"warn threshold %0.2f kB" % threshold,
145+
'</td></tr><tr><td bgcolor="red">',
146+
"error threshold %0.2f kB" % error_threshold,
147+
'</td></tr><tr><td bgcolor="green">',
148+
"warn threshold -%0.2f kB" % threshold,
149+
'</td></tr><tr><td bgcolor="cyan">',
150+
"warn threshold -%0.2f kB" % error_threshold,
151+
"</td></tr>",
152+
"<tr><td>metric:<BR>&lt;baseline&gt;<BR>&lt;pull request&gt;<BR>&lt;PR - baseline&gt; </td>",
153+
"</tr></table>",
154+
"<table>",
155+
'<tr><td align="center">Type<BR>Label</td>',
156+
'<td align="center">added construction</td>',
157+
'<td align="center">added event</td>',
158+
'<td align="center">added event setup</td>',
159+
'<td align="center">nAlloc construction</td>',
160+
'<td align="center">nAlloc event</td>',
161+
'<td align="center">nAlloc event setup</td>',
162+
"</tr>",
163+
"<td>%s<BR>%s</td>" % (prdata["total"]["type"], prdata["total"]["label"]),
164+
'<td align="right">%0.2f<br>%0.2f<br>%0.2f</td>'
165+
% (
166+
ibdata["total"]["added construction"],
167+
prdata["total"]["added construction"],
168+
results["total"]["added construction diff"],
169+
),
170+
'<td align="right">%0.2f<br>%0.2f<br>%0.2f</td>'
171+
% (
172+
ibdata["total"]["added event"],
173+
prdata["total"]["added event"],
174+
results["total"]["added event diff"],
175+
),
176+
'<td align="right">%0.2f<br>%0.2f<br>%0.2f</td>'
177+
% (
178+
ibdata["total"]["added event setup"],
179+
prdata["total"]["added event setup"],
180+
results["total"]["added event setup diff"],
181+
),
182+
'<td align="right">%0.f<br>%0.f<br>%0.f</td>'
183+
% (
184+
ibdata["total"]["nAlloc construction"],
185+
prdata["total"]["nAlloc construction"],
186+
results["total"]["nAlloc construction diff"],
187+
),
188+
'<td align="right">%0.f<br>%0.f<br>%0.f</td>'
189+
% (
190+
ibdata["total"]["nAlloc event"],
191+
prdata["total"]["nAlloc event"],
192+
results["total"]["nAlloc event diff"],
193+
),
194+
'<td align="right">%0.f<br>%0.f<br>%0.f</td>'
195+
% (
196+
ibdata["total"]["nAlloc event setup"],
197+
prdata["total"]["nAlloc event setup"],
198+
results["total"]["nAlloc event setup diff"],
199+
),
200+
"</tr></table>",
201+
'<table><tr><td align="center">Module label<BR>Module type<BR>Module record</td>',
202+
'<td align="center">added construction (kB)</td>',
203+
'<td align="center">added event (kB)</td>',
204+
'<td align="center">added event setup (kB)</td>',
205+
'<td align="center">added total (kB)</td>',
206+
'<td align="center">nAlloc construction</td>',
207+
'<td align="center">nAlloc event</td>',
208+
'<td align="center">nAlloc event setup</td>',
209+
'<td align="center">nAlloc total</td>',
210+
'<td align="center">transitions</td>',
211+
"</tr>",
212+
]
213+
214+
for item in sorted(
215+
datamapres.items(),
216+
key=lambda x: x[1]["added construction diff"]
217+
+ x[1]["added event diff"]
218+
+ x[1]["added event setup diff"],
219+
reverse=True,
220+
):
221+
key = item[1]["label"] + "|" + item[1]["type"] + "|" + item[1]["record"]
222+
if not key == "||":
223+
moduleib = datamapib[key]
224+
modulepr = datamappr[key]
225+
moduleres = datamapres[key]
226+
cellString = '<td align="right" '
227+
color = ""
228+
added_total_pr = (
229+
modulepr.get("added event setup", 0)
230+
+ modulepr.get("added event", 0)
231+
+ modulepr.get("added construction", 0)
232+
)
233+
added_total_ib = (
234+
moduleib.get("added event setup", 0)
235+
+ moduleib.get("added event", 0)
236+
+ moduleib.get("added construction", 0)
237+
)
238+
added_total_diff = (
239+
moduleres.get("added event setup diff", 0)
240+
+ moduleres.get("added event diff", 0)
241+
+ moduleres.get("added construction diff", 0)
242+
)
243+
if added_total_diff > threshold:
244+
color = 'bgcolor="orange"'
245+
if added_total_diff > error_threshold:
246+
color = 'bgcolor="red"'
247+
if added_total_diff < -1.0 * threshold:
248+
color = 'bgcolor="cyan"'
249+
if added_total_diff < -1.0 * error_threshold:
250+
color = 'bgcolor="green"'
251+
cellString += color
252+
cellString += ">"
253+
summaryLines += [
254+
"<tr>",
255+
'<td align="center">%s<BR>%s<BR> %s</td>'
256+
% (moduleres["label"], moduleres["type"], moduleres["record"]),
257+
'<td align="right"> %0.2f<br> %0.2f<br> %0.2f</td>'
258+
% (
259+
moduleib["added construction"],
260+
modulepr["added construction"],
261+
moduleres["added construction diff"],
262+
),
263+
'<td align="right"> %0.2f<br> %0.2f<br> %0.2f</td>'
264+
% (
265+
moduleib["added event"],
266+
modulepr["added event"],
267+
moduleres["added event diff"],
268+
),
269+
'<td align="right"> %0.2f<br> %0.2f<br> %0.2f</td>'
270+
% (
271+
moduleib["added event setup"],
272+
modulepr["added event setup"],
273+
moduleres["added event setup diff"],
274+
),
275+
cellString
276+
+ "%0.2f<br> %0.2f<br> %0.2f</td>"
277+
% (
278+
added_total_ib,
279+
added_total_pr,
280+
added_total_diff,
281+
),
282+
'<td align="right">%0.f<br>%0.f<br>%0.f</td>'
283+
% (
284+
moduleib["nAlloc construction"],
285+
modulepr["nAlloc construction"],
286+
moduleres["nAlloc construction diff"],
287+
),
288+
'<td align="right">%0.f<br>%0.f<br>%0.f</td>'
289+
% (moduleib["nAlloc event"], modulepr["nAlloc event"], moduleres["nAlloc event diff"]),
290+
'<td align="right">%0.f<br>%0.f<br>%0.f</td>'
291+
% (
292+
moduleib["nAlloc event setup"],
293+
modulepr["nAlloc event setup"],
294+
moduleres["nAlloc event setup diff"],
295+
),
296+
'<td align="right">%0.f<br>%0.f<br>%0.f</td>'
297+
% (
298+
moduleib["nAlloc event setup"]
299+
+ moduleib["nAlloc event"]
300+
+ moduleib["nAlloc construction"],
301+
modulepr["nAlloc event setup"]
302+
+ modulepr["nAlloc event"]
303+
+ modulepr["nAlloc construction"],
304+
moduleres["nAlloc event setup diff"]
305+
+ moduleres["nAlloc event diff"]
306+
+ moduleres["nAlloc construction diff"],
307+
),
308+
"<td>%i<br>%i<br>%i</td>"
309+
% (moduleib["transitions"], modulepr["transitions"], moduleres["transitions"]),
310+
"</tr>",
311+
]
312+
313+
summaryLines += []
314+
summaryLines += ["</body></html>"]
315+
316+
summaryFile = os.path.dirname(sys.argv[2]) + "/diff-" + os.path.basename(sys.argv[2]) + ".html"
317+
with open(summaryFile, "w") as g:
318+
for summaryLine in summaryLines:
319+
print(summaryLine, file=g)
320+
321+
dumpfile = os.path.dirname(sys.argv[2]) + "/diff-" + os.path.basename(sys.argv[2])
322+
with open(dumpfile, "w") as f:
323+
json.dump(results, f, indent=2)

‎pr_testing/run-pr-profiling.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ for PROFILING_WORKFLOW in $WORKFLOWS;do
6464
$CMS_BOT_DIR/comparisons/resources-diff.py $CMSSW_VERSION-$BASENAME $f >$f.log || true
6565
echo "<li><a href=\"${PROFILING_WORKFLOW}/diff-$BASENAME.html\">diff-$BASENAME</a></li>" >> $WORKSPACE/upload/profiling/index-$PROFILING_WORKFLOW.html || true
6666
done
67+
for f in $(find $PROFILING_WORKFLOW -type f -name 'step*_moduleAllocMonitor.circles.resources.json' ) ; do
68+
BASENAME=$(basename $f)
69+
get_jenkins_artifacts profiling/${CMSSW_VERSION}/${SCRAM_ARCH}/$f $CMSSW_VERSION-$BASENAME || true
70+
$CMS_BOT_DIR/comparisons/moduleAllocMonitor-circles-diff.py $CMSSW_VERSION-$BASENAME $f >$f.log || true
71+
echo "<li><a href=\"${PROFILING_WORKFLOW}/diff-$BASENAME.html\">diff-$BASENAME</a></li>" >> $WORKSPACE/upload/profiling/index-$PROFILING_WORKFLOW.html || true
72+
done
6773
for f in $(find $PROFILING_WORKFLOW -type f -name '*.json.gz' -o -name '*.log' -o -name '*.txt' -o -name '*.tmp' -o -name '*.heap*' -o -name '*.json' -o -name '*.html') ; do
6874
d=$(dirname $f)
6975
mkdir -p $WORKSPACE/upload/profiling/$d || true

0 commit comments

Comments
 (0)