-
Notifications
You must be signed in to change notification settings - Fork 2
173 lines (147 loc) · 6.17 KB
/
benchmark.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
name: benchmark
on:
workflow_dispatch:
pull_request:
push:
branches:
- master
paths:
- "src/*.py"
- "benchmark/*.py"
jobs:
generate_master_traces:
if: github.event_name == 'push'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Create new branch
run: git checkout -b actions/benchmark
- name: Set branch upstream
run: git push -u origin actions/benchmark
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: actions/setup-python@v4
with:
cache: 'pip'
cache-dependency-path: setup.py
- name: Start ArangoDB Instance
run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
- name: Start Jaeger Instance
run: docker run -d --name jaeger --rm -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 9411:9411 jaegertracing/all-in-one:latest
- name: Install packages
run: |
pip install torch==2.1.0
pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
pip install -e '.[dev, tracing]'
- name: Run Python Script
run: python benchmark/write.py --output_dir master
- name: Echo PyG to ArangoDB
run: cat benchmark/traces/master/pyg_to_arangodb.json | jq
- name: Echo ArangoDB to PyG
run: cat benchmark/traces/master/arangodb_to_pyg.json | jq
# - name: Make commit for auto-generated benchmark files
# uses: EndBug/add-and-commit@v9
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# with:
# add: "./benchmark/traces/master/*.json"
# new_branch: actions/benchmark
# message: "generate benchmark files for $GITHUB_SHA"
# - name: Create pull request for the auto generated benchmark
# run: |
# echo "PR_URL=$(gh pr create \
# --title "benchmark: $GITHUB_SHA" \
# --body "beep boop, i am a robot" \
# --label documentation)" >> $GITHUB_ENV
# env:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# - name: Alert developer of open PR
# run: echo "Benchmark $PR_URL is ready to be merged by developer."
compare_traces:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.10"
cache: 'pip'
cache-dependency-path: setup.py
- name: Start ArangoDB Instance
run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
- name: Start Jaeger Instance
run: docker run -d --name jaeger --rm -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 9411:9411 jaegertracing/all-in-one:latest
- name: Install packages
run: |
pip install torch==2.1.0
pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
pip install -e '.[dev, tracing]'
- name: Write PR traces
run: python benchmark/write.py --output_dir branch
- name: Compare PR traces against Master traces
id: python_script
run: |
echo "ROOT_SPAN_COMPARISON<<EOF" >> $GITHUB_ENV
python benchmark/compare.py >> $GITHUB_ENV
echo "EOF" >> $GITHUB_ENV
- run: echo "PYG_TO_ARANGODB_DIFF=$(jq -c . < benchmark/diff/pyg_to_arangodb.json)" >> $GITHUB_ENV
- run: echo "ARANGODB_TO_PYG_DIFF=$(jq -c . < benchmark/diff/arangodb_to_pyg.json)" >> $GITHUB_ENV
- name: Prepare PR Message
run: |
echo 'MESSAGE<<EOF' >> $GITHUB_ENV
echo "Benchmark (${{ github.event.pull_request.head.sha }})" >> $GITHUB_ENV
echo '' >> $GITHUB_ENV
echo 'Root Span Comparisons:' >> $GITHUB_ENV
echo '```json' >> $GITHUB_ENV
echo "$ROOT_SPAN_COMPARISON" | jq . >> $GITHUB_ENV
echo '```' >> $GITHUB_ENV
echo '' >> $GITHUB_ENV
echo '<details><summary>ArangoDB to PyG Comparison</summary>' >> $GITHUB_ENV
echo '' >> $GITHUB_ENV
echo '```json' >> $GITHUB_ENV
cat $ARANGODB_TO_PYG_DIFF >> $GITHUB_ENV
echo '```' >> $GITHUB_ENV
echo '</details>' >> $GITHUB_ENV
echo 'EOF' >> $GITHUB_ENV
# - name: Prepare PR Message
# run: |
# echo "MESSAGE<<EOF" >> $GITHUB_ENV
# echo '**Benchmark (${GITHUB_SHA})**' >> $GITHUB_ENV
# echo '' >> $GITHUB_ENV
# echo 'Root Span Comparisons:' >> $GITHUB_ENV
# echo '```json' >> $GITHUB_ENV
# cat $ROOT_SPAN_COMPARISON >> $GITHUB_ENV
# echo '```' >> $GITHUB_ENV
# echo '' >> $GITHUB_ENV
# echo '<details><summary>PyG to ArangoDB Comparison</summary>' >> $GITHUB_ENV
# echo '' >> $GITHUB_ENV
# echo '```json' >> $GITHUB_ENV
# cat $PYG_TO_ARANGODB_DIFF >> $GITHUB_ENV
# echo '```' >> $GITHUB_ENV
# echo '</details>' >> $GITHUB_ENV
# echo '' >> $GITHUB_ENV
# echo '<details><summary>ArangoDB to PyG Comparison</summary>' >> $GITHUB_ENV
# echo '' >> $GITHUB_ENV
# echo '```json' >> $GITHUB_ENV
# cat $ARANGODB_TO_PYG_DIFF >> $GITHUB_ENV
# echo '```' >> $GITHUB_ENV
# echo '</details>' >> $GITHUB_ENV
# echo "EOF" >> $GITHUB_ENV
- name: Comment on PR
uses: actions/github-script@v7
with:
github-token: ${{secrets.GITHUB_TOKEN}}
script: |
const prNumber = context.payload.pull_request.number;
const message = process.env.MESSAGE;
github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: message
});