Create benchmark.yml

.github/workflows/benchmark.yml

@@ -0,0 +1,124 @@
+name: benchmark
+
+on:
+ workflow_dispatch:
+ push:
+ branches:
+ - master
+ paths:
+ - 'src/**'
+ - "benchmark/*.py"
+ pull_request:
+ branches:
+ - master
+ paths:
+ - 'src/**'
+ - "benchmark/*.py"
+
+jobs:
+ # write_benchmark:
+ # if: github.event_name == 'push'
+ # runs-on: ubuntu-latest
+ # steps:
+ # - uses: actions/checkout@v4
+ # with:
+ # fetch-depth: 0
+
+ # - name: Create new branch
+ # run: git checkout -b actions/benchmark
+
+ # - name: Set branch upstream
+ # run: git push -u origin actions/benchmark
+ # env:
+ # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ # - name: Setup Python
+ # uses: actions/setup-python@v4
+ # with:
+ # cache: 'pip'
+ # cache-dependency-path: setup.py
+
+ # - name: Set up ArangoDB Instance via Docker
+ # run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
+
+ # - name: Start Jaeger Instance
+ # run: docker run -d --name jaeger --rm \
+ # -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
+ # -p 16686:16686 \
+ # -p 4317:4317 \
+ # -p 4318:4318 \
+ # -p 9411:9411 \
+ # jaegertracing/all-in-one:latest
+
+ # - name: Install packages
+ # run: |
+ # pip install torch==2.1.0
+ # pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
+ # pip install -e '.[dev, tracing]'
+
+ # - name: Run Python Script
+ # run: python benchmark/write.py --output_dir head
+
+ # - name: Make commit for auto-generated benchmark files
+ # uses: EndBug/add-and-commit@v9
+ # env:
+ # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # with:
+ # add: "./benchmark/main/*.json"
+ # new_branch: actions/benchmark
+ # message: "generate benchmark files for $GITHUB_SHA"
+
+ # - name: Create pull request for the auto generated changelog
+ # run: |
+ # echo "PR_URL=$(gh pr create \
+ # --title "benchmark: $GITHUB_SHA" \
+ # --body "beep boop, i am a robot ($GITHUB_SHA)" \
+ # --label documentation)" >> $GITHUB_ENV
+ # env:
+ # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ # - name: Alert developer of open PR
+ # run: echo "Benchmark $PR_URL is ready to be merged by developer."
+
+ compare_benchmarks:
+ if: github.event_name == 'pull_request'
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout code
+ uses: actions/checkout@v4
+
+ - name: Setup Python
+ uses: actions/setup-python@v4
+ with:
+ cache: 'pip'
+ cache-dependency-path: setup.py
+
+ - name: Set up ArangoDB Instance via Docker
+ run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
+
+ - name: Start Jaeger Instance
+ run: docker run -d --name jaeger --rm \
+ -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
+ -p 16686:16686 \
+ -p 4317:4317 \
+ -p 4318:4318 \
+ -p 9411:9411 \
+ jaegertracing/all-in-one:latest
+
+ - name: Install packages
+ run: |
+ pip install torch==2.1.0
+ pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
+ pip install -e '.[dev, tracing]'
+
+ - name: Run Benchmark Write for PR
+ run: python benchmark/write.py --output_dir branch
+
+ - name: Run Benchmark Comparison against main
+ run: python benchmark/compare.py
+
+ - name: Echo PyG to ArangoDB Comparison
+ run: cat benchmark/diff/pyg_to_arangodb.json | jq
+
+ - name: Echo ArangoDB to PyG Comparison
+ run: cat benchmark/diff/arangodb_to_pyg.json | jq