benchmark.yml

name: benchmark

on:
  workflow_dispatch:
  push:
      branches:
        - master
      paths:
        - 'src/**'
        - "benchmark/*.py"
  pull_request:
    branches:
      - master
    paths:
      - 'src/**'
      - "benchmark/*.py"

jobs:
  # write_benchmark:
  #   if: github.event_name == 'push'
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v4
  #       with:
  #         fetch-depth: 0

  #     - name: Create new branch
  #       run: git checkout -b actions/benchmark

  #     - name: Set branch upstream
  #       run: git push -u origin actions/benchmark
  #       env:
  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  #     - name: Setup Python
  #       uses: actions/setup-python@v4
  #       with:
  #         cache: 'pip'
  #         cache-dependency-path: setup.py

  #     - name: Set up ArangoDB Instance via Docker
  #       run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
  
  #     - name: Start Jaeger Instance
  #       run: docker run -d --name jaeger --rm \
  #         -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
  #         -p 16686:16686 \
  #         -p 4317:4317 \
  #         -p 4318:4318 \
  #         -p 9411:9411 \
  #         jaegertracing/all-in-one:latest

  #     - name: Install packages
  #       run: |
  #         pip install torch==2.1.0
  #         pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
  #         pip install -e '.[dev, tracing]'

  #     - name: Run Python Script
  #       run: python benchmark/write.py --output_dir head

  #     - name: Make commit for auto-generated benchmark files
  #       uses: EndBug/add-and-commit@v9
  #       env:
  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  #       with:
  #         add: "./benchmark/main/*.json"
  #         new_branch: actions/benchmark
  #         message: "generate benchmark files for $GITHUB_SHA"

  #     - name: Create pull request for the auto generated changelog
  #       run: |
  #         echo "PR_URL=$(gh pr create \
  #           --title "benchmark: $GITHUB_SHA" \
  #           --body "beep boop, i am a robot ($GITHUB_SHA)" \
  #           --label documentation)" >> $GITHUB_ENV
  #       env:
  #         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  
  #     - name: Alert developer of open PR
  #       run: echo "Benchmark $PR_URL is ready to be merged by developer."

  compare_benchmarks:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          cache: 'pip'
          cache-dependency-path: setup.py

      - name: Set up ArangoDB Instance via Docker
        run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
  
      - name: Start Jaeger Instance
        run: docker run -d --name jaeger --rm \
          -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 \
          -p 16686:16686 \
          -p 4317:4317 \
          -p 4318:4318 \
          -p 9411:9411 \
          jaegertracing/all-in-one:latest

      - name: Install packages
        run: |
          pip install torch==2.1.0
          pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
          pip install -e '.[dev, tracing]'

      - name: Run Benchmark Write for PR
        run: python benchmark/write.py --output_dir branch

      - name: Run Benchmark Comparison against main
        run: python benchmark/compare.py

      - name: Echo PyG to ArangoDB Comparison
        run: cat benchmark/diff/pyg_to_arangodb.json | jq

      - name: Echo ArangoDB to PyG Comparison
        run: cat benchmark/diff/arangodb_to_pyg.json | jq