-
Notifications
You must be signed in to change notification settings - Fork 1
Expand file tree
/
Copy pathrun-bench.sh
More file actions
executable file
·287 lines (220 loc) · 7.25 KB
/
run-bench.sh
File metadata and controls
executable file
·287 lines (220 loc) · 7.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
#!/bin/bash
shopt -s nullglob
OPT_NOWAIT=
OPT_LAZY=
while [ $# -gt 0 ]; do
case "$1" in
--nowait) OPT_NOWAIT=1;;
--lazy) OPT_LAZY=1;;
*)
cat <<EOF
Unknown option $1, options are:
--nowait don't wait for the system load to settle down before benches
--lazy only run the benches if upstream changes are detected
EOF
exit 1
esac
shift
done
export PATH=~/local/bin:$PATH
unset OPAMROOT OPAMSWITCH OCAMLPARAM OCAMLRUNPARAM
STARTTIME=$(date +%s)
DATE=$(date +%Y-%m-%d-%H%M)
DAY=${DATE%-*}
BASELOGDIR=~/logs/operf
LOGDIR=$BASELOGDIR/$DATE
OPERFDIR=~/.cache/operf/macro/
LOCK=$BASELOGDIR/lock
if [ -e $LOCK ]; then
RUNNING_PID=$(cat $LOCK)
if ps -p $RUNNING_PID >/dev/null; then
echo "Another run-bench.sh is running (pid $RUNNING_PID). Aborting." >&2
exit 1
else
echo "Removing stale lock file $LOCK." >&2
rm $LOCK
fi
fi
trap "rm $LOCK" EXIT
echo $$ >$LOCK
( cd $BASELOGDIR && git reset --hard && git checkout master; )
if [ -n "$OPT_LAZY" ]; then
LOGBRANCH=lazy-$DAY
( cd $BASELOGDIR && git checkout $LOGBRANCH || git checkout -b $LOGBRANCH; )
else
LOGBRANCH=master
fi
( cd $BASELOGDIR && git checkout -B $DATE; )
git-sync() (
cd $BASELOGDIR
git push flambda-mirror:/var/www/flambda.ocamlpro.com/bench/ +HEAD:new
git push git@gitlab.ocamlpro.com:OCamlPro/ocaml-bench-logs +HEAD:master
ssh flambda-mirror "cd /var/www/flambda.ocamlpro.com/bench/ && git reset new --hard"
)
publish() (
cd $LOGDIR
git add $*
git add -u .
git commit -m "Add logs ($DATE)"
git-sync
)
unpublish() (
cd $BASELOGDIR
if [ $# -gt 0 ] && [ "$1" = "--wipe" ]; then
git reset --hard
rm -rf $DATE
git checkout $LOGBRANCH
git branch -D $DATE || true
else
git add $DATE
rm $LOGDIR/log # otherwise the checkout fails!
git commit -m "Extra files ($DATE) -- broken build"
git checkout $LOGBRANCH
fi
git-sync
)
git-finalise() (
cd $LOGDIR
git add .
rm $LOGDIR/log # otherwise the checkout fails!
git commit -m "Extra files ($DATE)"
git checkout $LOGBRANCH
git merge $DATE^ -m "Merge logs from $DATE"
git-sync
)
trap "unpublish; exit 2" INT
mkdir -p $LOGDIR
echo "Output and log written into $LOGDIR" >&2
exec >$LOGDIR/log 2>&1
OPAMBIN=$(which opam)
opam() {
echo "+opam $*" >&2
"$OPAMBIN" "$@"
}
echo "=== SETTING UP BENCH SWITCHES AT $DATE ==="
## Initial setup:
#
# opam 2.0~alpha6 an "operf" switch with operf-macro installed (currently
# working: ocaml 4.02.3, operf pinned to git://github.com/ocamlpro/ocaml-perf,
# operf-macro pinned to git://github.com/OCamlPro/operf-macro#opam2)
#
# opam repo add benches git+https://github.com/OCamlPro/ocamlbench-repo --dont-select
OPERF_SWITCH=operf
opam update --check benches
HAS_CHANGES=$?
COMPILERS=($(opam list --no-switch --has-flag compiler --repo=benches --all-versions --short))
SWITCHES=()
INSTALLED_BENCH_SWITCHES=($(opam switch list -s |grep '+bench$'))
for C in "${COMPILERS[@]}"; do
SWITCH=${C#*.}+bench
SWITCHES+=("$SWITCH")
if [[ ! " ${INSTALLED_BENCH_SWITCHES[@]} " =~ " $SWITCH " ]]; then
opam switch create "$SWITCH" --empty --no-switch --repositories benches,default
fi
opam pin add "${C%%.*}" "${C#*.}" --switch "$SWITCH" --yes --no-action
opam switch set-base "${C%%.*}" --switch "$SWITCH" --yes
done
# Remove switches that are no longer needed
for SW in "${INSTALLED_BENCH_SWITCHES[@]}"; do
if [[ ! " ${SWITCHES[@]} " =~ " $SW " ]]; then
opam switch remove "$SW" --yes;
fi;
done
echo
echo "=== UPGRADING operf-macro at $DATE ==="
touch $LOGDIR/stamp
publish stamp
opam update --check --switch $OPERF_SWITCH
opam upgrade --check --yes operf-macro --switch $OPERF_SWITCH --json $LOGDIR/$OPERF_SWITCH.json
HAS_CHANGES=$((HAS_CHANGES * $?))
BENCHES=($(opam list --no-switch --required-by all-bench --short --column name))
ALL_BENCHES=($(opam list --no-switch --short --column name '*-bench'))
DISABLED_BENCHES=()
for B in "${ALL_BENCHES[@]}"; do
if [[ ! " ${BENCHES[@]} " =~ " $B " ]]; then
DISABLED_BENCHES+=("$B")
fi
done
for SWITCH in "${SWITCHES[@]}"; do
if opam update --check --dev --switch $SWITCH; then
CHANGED_SWITCHES+=("$SWITCH")
fi
done
if [ -n "$OPT_LAZY" ] && [ "$HAS_CHANGES" -ne 0 ]; then
if [ "${#CHANGED_SWITCHES[*]}" -eq 0 ] ; then
echo "Lazy mode, no changes: not running benches"
unpublish --wipe
exit 0
else
echo "Lazy mode, only running benches on: ${CHANGED_SWITCHES[*]}"
BENCH_SWITCHES=("${CHANGED_SWITCHES[@]}")
fi
else
BENCH_SWITCHES=("${SWITCHES[@]}")
fi
for SWITCH in "${BENCH_SWITCHES[@]}"; do
echo
echo "=== UPGRADING SWITCH $SWITCH =="
opam remove "${DISABLED_BENCHES[@]}" --yes --switch $SWITCH
COMP=($(opam list --base --short --switch $SWITCH))
opam upgrade --all "${BENCHES[@]}" --best-effort --yes --switch $SWITCH --json $LOGDIR/$SWITCH.json
done
LOGSWITCHES=("${BENCH_SWITCHES[@]/#/$LOGDIR/}")
opamjson2html ${LOGSWITCHES[@]/%/.json*} >$LOGDIR/build.html
UPGRADE_TIME=$(($(date +%s) - STARTTIME))
echo -e "\n===== OPAM UPGRADE DONE in ${UPGRADE_TIME}s =====\n"
eval $(opam config env --switch $OPERF_SWITCH)
loadavg() {
awk '{print 100*$1}' /proc/loadavg
}
if [ -z "$OPT_NOWAIT" ]; then
# let the loadavg settle down...
sleep 60
while [ $(loadavg) -gt 60 ]; do
if [ $(($(date +%s) - STARTTIME)) -gt $((3600 * 12)) ]; then
echo "COULD NOT START FOR THE PAST 12 HOURS; ABORTING RUN" >&2
unpublish --wipe
exit 10
else
echo 'System load detected, waiting to run bench (retrying in 5 minutes)'
wall "Its BENCH STARTUP TIME, but the load is too high. Please clear the way"'!'
sleep 300
fi
done
fi
for SWITCH in "${SWITCHES[@]}"; do
rm -f $OPERFDIR/*/$SWITCH.*
done
ocaml-params() {
SWITCH=$1; shift
[ $# -eq 0 ]
opam config env --switch $SWITCH | sed -n "s/\(OCAMLPARAM='[^']*'\).*$/\1/p"
}
for SWITCH in "${SWITCHES[@]}"; do
opam show ocaml-variants --switch $SWITCH --field source-hash >$LOGDIR/${SWITCH%+bench}.hash
ocaml-params $SWITCH >$LOGDIR/${SWITCH%+bench}.params
opam pin --switch $SWITCH >$LOGDIR/${SWITCH%+bench}.pinned
done
publish log build.html "*.hash" "*.params" "*.pinned"
wall " -- STARTING BENCHES -- don't put load on the machine. Thanks"
BENCH_START_TIME=$(date +%s)
echo
echo "=== BENCH START ==="
for SWITCH in "${BENCH_SWITCHES[@]}"; do
nice -n -5 opam config exec --switch $OPERF_SWITCH -- timeout 90m operf-macro run --switch $SWITCH
done
opam config exec --switch $OPERF_SWITCH -- operf-macro summarize -b csv >$LOGDIR/summary.csv
cp -r $OPERFDIR/* $LOGDIR
BENCH_TIME=$(($(date +%s) - BENCH_START_TIME))
hours() {
printf "%02d:%02d:%02d" $(($1 / 3600)) $(($1 / 60 % 60)) $(($1 % 60))
}
cat > $LOGDIR/timings <<EOF
Upgrade: $(hours $UPGRADE_TIME)
Benches: $(hours $BENCH_TIME)
Total: $(hours $((UPGRADE_TIME + BENCH_TIME)))
EOF
publish log timings summary.csv "*/*.summary"
echo "Done"
git-finalise
cd $BASELOGDIR && echo "<html><head><title>bench index</title></head><body><ul>$(ls -d 201* latest | sed 's%\(.*\)%<li><a href="\1">\1</a></li>%')</ul></body></html>" >index.html