benchmark.yml

name: benchmark

on:
  workflow_dispatch:
  pull_request:
  push:
      branches:
        - master
      paths:
        - "src/*.py"
        - "benchmark/*.py"

jobs:
  generate_master_traces:
    if: github.event_name == 'push'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Create new branch
        run: git checkout -b actions/benchmark

      - name: Set branch upstream
        run: git push -u origin actions/benchmark
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          cache: 'pip'
          cache-dependency-path: setup.py

      - name: Start ArangoDB Instance
        run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
    
      - name: Start Jaeger Instance
        run: docker run -d --name jaeger --rm -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 9411:9411 jaegertracing/all-in-one:latest
  
      - name: Install packages
        run: |
          pip install torch==2.1.0
          pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
          pip install -e '.[dev, tracing]'

      - name: Run Python Script
        run: python benchmark/write.py --output_dir master

      - name: Echo PyG to ArangoDB
        run: cat benchmark/traces/master/pyg_to_arangodb.json | jq .

      - name: Echo ArangoDB to PyG 
        run: cat benchmark/traces/master/arangodb_to_pyg.json | jq .

      # - name: Make commit for auto-generated benchmark files
      #   uses: EndBug/add-and-commit@v9
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     add: "./benchmark/traces/master/*.json"
      #     new_branch: actions/benchmark
      #     message: "generate benchmark files for $GITHUB_SHA"

      # - name: Create pull request for the auto generated benchmark
      #   run: |
      #     echo "PR_URL=$(gh pr create \
      #       --title "benchmark: $GITHUB_SHA" \
      #       --body "beep boop, i am a robot" \
      #       --label documentation)" >> $GITHUB_ENV
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  
      # - name: Alert developer of open PR
      #   run: echo "Benchmark $PR_URL is ready to be merged by developer."

  compare_traces:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: 'pip'
          cache-dependency-path: setup.py

      - name: Start ArangoDB Instance
        run: docker run -d --name arangodb -p 8529:8529 -e ARANGO_ROOT_PASSWORD= arangodb/arangodb
  
      - name: Start Jaeger Instance
        run: docker run -d --name jaeger --rm -e COLLECTOR_ZIPKIN_HOST_PORT=:9411 -p 16686:16686 -p 4317:4317 -p 4318:4318 -p 9411:9411 jaegertracing/all-in-one:latest

      - name: Install packages
        run: |
          pip install torch==2.1.0
          pip install torch-scatter torch-sparse -f https://data.pyg.org/whl/torch-$(python -c 'import torch; print(torch.__version__.split("+")[0])')+cpu.html
          pip install -e '.[dev, tracing]'

      - name: Write PR traces
        run: python benchmark/write.py --output_dir branch

      - name: Compare PR traces against Master traces
        run: |
          echo "ROOT_SPAN_COMPARISON<<EOF" >> $GITHUB_ENV
          python benchmark/compare.py >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Extract PyG to ArangoDB Diff
        run: |
          echo "PYG_TO_ARANGODB_DIFF<<EOF" >> $GITHUB_ENV
          cat benchmark/diff/pyg_to_arangodb.json | jq . >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Extract ArangoDB to PyG Diff
        run: |
          echo "ARANGODB_TO_PYG_DIFF<<EOF" >> $GITHUB_ENV
          cat benchmark/diff/arangodb_to_pyg.json | jq . >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV

      - name: Prepare PR Message
        run: |
          echo 'MESSAGE<<EOF' >> $GITHUB_ENV
          echo "Benchmark (${{ github.event.pull_request.head.sha }})" >> $GITHUB_ENV
          echo '' >> $GITHUB_ENV
          echo '```json' >> $GITHUB_ENV
          echo "$ROOT_SPAN_COMPARISON" | jq . >> $GITHUB_ENV
          echo '```' >> $GITHUB_ENV
          echo '' >> $GITHUB_ENV
          echo '<details><summary>PyG to ArangoDB Diff</summary>' >> $GITHUB_ENV
          echo '' >> $GITHUB_ENV
          echo '```json' >> $GITHUB_ENV
          echo "$PYG_TO_ARANGODB_DIFF" | jq . >> $GITHUB_ENV
          echo '```' >> $GITHUB_ENV
          echo '</details>' >> $GITHUB_ENV
          echo '' >> $GITHUB_ENV
          echo '<details><summary>ArangoDB to PyG Diff</summary>' >> $GITHUB_ENV
          echo '' >> $GITHUB_ENV
          echo '```json' >> $GITHUB_ENV
          echo "$ARANGODB_TO_PYG_DIFF" | jq . >> $GITHUB_ENV
          echo '```' >> $GITHUB_ENV
          echo '</details>' >> $GITHUB_ENV
          echo 'EOF' >> $GITHUB_ENV

      - name: Comment on PR
        uses: actions/github-script@v7
        with:
          github-token: ${{secrets.GITHUB_TOKEN}}
          script: |
            const prNumber = context.payload.pull_request.number;
            const message = process.env.MESSAGE;
            github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: prNumber,
              body: message
            });