-
Notifications
You must be signed in to change notification settings - Fork 110
352 lines (310 loc) · 13.1 KB
/
benchmarks.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
on:
push:
branches:
- master
- jgilles/fix-callgrind-again
workflow_dispatch:
inputs:
pr_number:
description: 'Pull Request Number'
required: false
default: ''
issue_comment:
types: [created]
name: Benchmarks
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
jobs:
benchmark:
name: run criterion benchmarks
runs-on: benchmarks-runner
# filter for a comment containing 'benchmarks please'
if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }}
env:
PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
steps:
- name: Clear stdb dir
if: always()
run: |
rm -fr /stdb/*
- name: Enable CPU boost
run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost
- name: Check membership
if: ${{ github.event_name == 'issue_comment' }}
env:
CONTRIB_ORG: clockworklabs
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
run: |
curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
exit 1
fi
- name: Post initial comment
run: |
if [[ $PR_NUMBER ]]; then
comment_parent=issues/$PR_NUMBER
comment_update=issues/comments
else
comment_parent=commits/$GITHUB_SHA
comment_update=comments
fi
comment_body="Benchmark in progress..."
comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV
- name: find PR branch
if: ${{ env.PR_NUMBER }}
run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"
- name: Checkout sources
uses: actions/checkout@v4
with:
ref: ${{ env.PR_REF || github.ref }}
# if we're on master we want to know what the sha of HEAD~1 is so
# that we can compare results from it to HEAD (in the "Fetch markdown
# summary PR" step). otherwise, we can use a fully shallow checkout
fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}
- name: Install Rust toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
components: clippy
toolchain: stable
target: wasm32-unknown-unknown
override: true
- name: Install .NET toolchain
uses: actions/setup-dotnet@v3
with:
dotnet-version: "8.x"
env:
DOTNET_INSTALL_DIR: ~/.dotnet
- name: Build
working-directory: crates/bench/
run: |
cargo build --release
- name: Install latest wasm-opt for module optimisations
run: |
curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1
- name: Disable CPU boost
run: echo "0" | sudo tee /sys/devices/system/cpu/cpufreq/boost
- name: Branch; run bench
run: |
if [[ $PR_NUMBER ]]; then
BASELINE_NAME=branch
RESULTS_NAME=pr-$PR_NUMBER
BENCH_FILTER='stdb_raw'
echo "Running benchmarks without sqlite"
else
BASELINE_NAME=master
RESULTS_NAME=$GITHUB_SHA
BENCH_FILTER='(stdb_raw|sqlite)'
echo "Running benchmarks with sqlite"
fi
pushd crates/bench
rm -rf .spacetime
cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" "$BENCH_FILTER"
# sticker price benchmark
cargo bench --bench generic -- --save-baseline "$BASELINE_NAME" 'stdb_module/.*/disk/update_bulk'
cargo bench --bench special -- --save-baseline "$BASELINE_NAME"
cargo run --bin summarize pack "$BASELINE_NAME"
popd
mkdir criterion-results
[[ ! $PR_NUMBER ]] && cp target/criterion/$BASELINE_NAME.json criterion-results/
cp target/criterion/$BASELINE_NAME.json criterion-results/$RESULTS_NAME.json
# this will work for both PR and master
- name: Upload criterion results to DO spaces
uses: shallwefootball/s3-upload-action@master
with:
aws_key_id: ${{ secrets.AWS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
aws_bucket: "spacetimedb-ci-benchmarks"
source_dir: criterion-results
endpoint: https://nyc3.digitaloceanspaces.com
destination_dir: benchmarks
- name: Fetch markdown summary PR
run: |
if [[ $PR_NUMBER ]]; then
OLD=master
NEW=pr-$PR_NUMBER
else
OLD=$(git rev-parse HEAD~1)
NEW=$GITHUB_SHA
fi
echo "fetching https://benchmarks.spacetimedb.com/compare/$OLD/$NEW"
curl -sS https://benchmarks.spacetimedb.com/compare/$OLD/$NEW > report.md
- name: Post comment
run: |
BODY="<details><summary>Criterion benchmark results</summary>
$(cat report.md)
</details>"
gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
- name: Post failure comment
if: ${{ failure() && env.COMMENT_UPDATE_URL }}
run: |
BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
- name: Clean up
if: always()
run: |
rm -fr /stdb/*
callgrind_benchmark:
name: run callgrind benchmarks
# DON'T run on benchmarks-runner, using docker on a self-hosted runner has
# been broken for 4 years: https://github.com/actions/runner/issues/434 .
# Fortunately, we can run on standard Github Actions infra because we don't care
# about other stuff running on the machine!
# runs-on: benchmarks-runner
runs-on: ubuntu-latest
timeout-minutes: 20 # on a successful run, runs in 8 minutes
container:
image: rust:1.78
options: --privileged
# filter for a comment containing 'benchmarks please'
if: ${{ github.event_name != 'issue_comment' || (github.event.issue.pull_request && contains(github.event.comment.body, 'benchmarks please')) }}
env:
PR_NUMBER: ${{ github.event.inputs.pr_number || github.event.issue.number || null }}
steps:
- name: Clear stdb dir
if: always()
shell: bash
run: |
rm -fr /stdb/*
- name: Install valgrind & iai-callgrind-runner
run: |
apt-get update
apt-get install -y valgrind protobuf-compiler bash sudo curl gh
cargo install --git https://github.com/clockworklabs/iai-callgrind.git --branch main iai-callgrind-runner
git config --global --add safe.directory /__w/SpacetimeDB/SpacetimeDB
# can't do this off self hosted:
# - name: Enable CPU boost
# shell: bash
# run: echo "1" | sudo tee /sys/devices/system/cpu/cpufreq/boost
- name: Check membership
if: ${{ github.event_name == 'issue_comment' }}
env:
CONTRIB_ORG: clockworklabs
COMMENT_AUTHOR: ${{ github.event.comment.user.login }}
ORG_READ_TOKEN: ${{ secrets.ORG_READ_TOKEN }}
shell: bash
run: |
curl -OL https://github.com/cli/cli/releases/download/v2.37.0/gh_2.37.0_linux_amd64.deb && sudo dpkg -i gh_2.37.0_linux_amd64.deb
if [[ $(GH_TOKEN=$ORG_READ_TOKEN gh api --paginate /orgs/{owner}/members --jq 'any(.login == env.COMMENT_AUTHOR)') != true ]]; then
gh pr comment $PR_NUMBER -b "Sorry, you don't have permission to run benchmarks."
exit 1
fi
- name: find PR branch
if: ${{ env.PR_NUMBER }}
shell: bash
run: echo "PR_REF=$(gh pr view $PR_NUMBER --json headRefName --jq .headRefName)" >>"$GITHUB_ENV"
- name: Checkout sources
uses: actions/checkout@v3
with:
ref: ${{ env.PR_REF || github.ref }}
# if we're on master we want to know what the sha of HEAD~1 is so
# that we can compare results from it to HEAD (in the "Fetch markdown
# summary PR" step). otherwise, we can use a fully shallow checkout
fetch-depth: ${{ env.PR_NUMBER && 1 || 2 }}
- name: Unbork Github Actions state
shell: bash
run: |
echo "Letting anybody touch our git repo, in order to avoid breaking other jobs"
echo "This is necessary because we are running as root inside a docker image"
chmod -R a+rw .
- name: Post initial comment
shell: bash
run: |
set -exo pipefail
if [[ $PR_NUMBER ]]; then
comment_parent=issues/$PR_NUMBER
comment_update=issues/comments
else
comment_parent=commits/$GITHUB_SHA
comment_update=comments
fi
comment_body="Callgrind benchmark in progress..."
comment_id=$(gh api "/repos/{owner}/{repo}/$comment_parent/comments" -f body="$comment_body" --jq .id)
echo "COMMENT_UPDATE_URL=/repos/{owner}/{repo}/$comment_update/$comment_id" >>$GITHUB_ENV
- name: Install stable toolchain
uses: actions-rs/toolchain@v1
with:
profile: minimal
components: clippy
toolchain: stable
target: wasm32-unknown-unknown
override: true
- name: Build
working-directory: crates/bench/
shell: bash
run: |
cargo build --release
- name: Install latest wasm-opt for module optimisations
shell: bash
run: |
curl https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz -L | sudo tar xz -C /usr/local --strip-components=1
# leave CPU boost on, doesn't affect callgrind!
- name: Branch; run bench
shell: bash
run: |
if [[ $PR_NUMBER ]]; then
BASELINE_NAME=branch
RESULTS_NAME=pr-$PR_NUMBER
BENCH_FILTER='(special|stdb_module|stdb_raw)'
echo "Running branch callgrind benchmarks"
else
BASELINE_NAME=master
RESULTS_NAME=$GITHUB_SHA
BENCH_FILTER='.*'
echo "Running master callgrind benchmarks"
fi
pushd crates/bench
rm -rf .spacetime
cargo bench --bench callgrind -- --save-summary pretty-json
cargo run --bin summarize pack-callgrind "$BASELINE_NAME"
popd
mkdir callgrind-results
[[ ! $PR_NUMBER ]] && cp target/iai/$BASELINE_NAME.json callgrind-results/
cp target/iai/$BASELINE_NAME.json callgrind-results/$RESULTS_NAME.json
# this will work for both PR and master
- name: Upload callgrind results to DO spaces
uses: shallwefootball/s3-upload-action@master
with:
aws_key_id: ${{ secrets.AWS_KEY_ID }}
aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY}}
aws_bucket: "spacetimedb-ci-benchmarks"
source_dir: callgrind-results
endpoint: https://nyc3.digitaloceanspaces.com
destination_dir: callgrind-benchmarks
- name: Fetch markdown summary PR
shell: bash
run: |
if [[ $PR_NUMBER ]]; then
OLD=master
NEW=pr-$PR_NUMBER
else
OLD=$(git rev-parse HEAD~1)
NEW=$GITHUB_SHA
fi
echo "fetching https://benchmarks.spacetimedb.com/compare_callgrind/$OLD/$NEW"
curl -sS https://benchmarks.spacetimedb.com/compare_callgrind/$OLD/$NEW > report.md
- name: Post comment
shell: bash
run: |
BODY="<details><summary>Callgrind benchmark results</summary>
$(cat report.md)
</details>"
gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
- name: Post failure comment
if: ${{ failure() && env.COMMENT_UPDATE_URL }}
shell: bash
run: |
BODY="Benchmarking failed. Please check [the workflow run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}) for details."
gh api "$COMMENT_UPDATE_URL" -X PATCH -f body="$BODY"
- name: Clean up
if: always()
shell: bash
run: |
rm -fr /stdb/*
echo "Letting anybody touch our git repo, in order to avoid breaking other jobs"
echo "This is necessary because we are running as root inside a docker image"
chmod -R a+rw .