Merge pull request #303 from github/storage-test

Storage testing
github · Jul 28, 2023 · 7489606 · 7489606
2 parents bb7457a + 342d987
commit 7489606
Show file tree

Hide file tree

Showing 7 changed files with 133 additions and 56 deletions.
diff --git a/.gitattributes b/.gitattributes
@@ -0,0 +1,3 @@
+data/*.tar.gz filter=lfs diff=lfs merge=lfs -text
+data/*.tgz filter=lfs diff=lfs merge=lfs -text
+data/*.zip filter=lfs diff=lfs merge=lfs -text
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,12 +6,16 @@ on:
   schedule:
     - cron: "0 0 1,15 * *"
 
+# In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 jobs:
   test-rust:
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        os: [ubuntu-latest]
         rust: [stable]
 
     steps:
@@ -85,7 +89,7 @@ jobs:
       languages: ${{ steps.language-list.outputs.languages }}
 
   test-languages:
-    needs: [list-languages, test-rust]
+    needs: [list-languages]
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -112,3 +116,61 @@ jobs:
         run: cargo build -p ${{ matrix.language }}
       - name: Test
         run: cargo test -p ${{ matrix.language }}
+
+  test-cli:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        rust: [stable]
+    env:
+      TEST_NAME: typescript_minimal_project
+      TSSG_TS: tree-sitter-stack-graphs-typescript
+
+    steps:
+      - name: Install Rust environment
+        uses: hecrj/setup-rust-action@v1
+        with:
+          rust-version: ${{ matrix.rust }}
+      - name: Cache dependencies
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cargo
+            target
+          key: ${{ runner.OS }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+          restore-keys: |
+            ${{ runner.OS }}-cargo-
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          lfs: true
+      - name: Build
+        run: cargo build --release -p ${{ env.TSSG_TS }} --features cli
+      - name: Checkout test code
+        run: unzip ${{ env.TEST_NAME }}.zip
+        working-directory: data
+      - name: Index
+        run: |
+          target/release/${{ env.TSSG_TS }} index \
+            -D data/${{ env.TEST_NAME }}.sqlite \
+            --verbose \
+            --hide-error-details \
+            data/${{ env.TEST_NAME }}
+      - name: Status
+        run: |
+          target/release/${{ env.TSSG_TS }} status \
+            -D data/${{ env.TEST_NAME }}.sqlite \
+            --all
+      - name: Query
+        run: |
+          target/release/${{ env.TSSG_TS }} query \
+            -D data/${{ env.TEST_NAME }}.sqlite \
+            definition \
+              data/${{ env.TEST_NAME }}/index.ts:3:10
+      - name: Index (forced)
+        run: |
+          target/release/${{ env.TSSG_TS }} index \
+            -D data/${{ env.TEST_NAME }}.sqlite \
+            --verbose \
+            --hide-error-details \
+            data/${{ env.TEST_NAME }}
diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml
@@ -3,6 +3,12 @@ on:
   pull_request:
     paths:
       - 'stack-graphs/**'
+
+# In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
   BASE_REPO: ${{ github.event.pull_request.base.repo.owner.login }}/${{ github.event.pull_request.base.repo.name }}
   BASE_SHA: ${{ github.event.pull_request.base.sha }}
@@ -14,10 +20,10 @@ env:
   HEAD_DIR: head
   HEAD_DB: head.sqlite
   HEAD_ARTIFACT: head-perf-results
+  TEST_NAME: typescript_benchmark
   TEST_REPO: microsoft/TypeScript
   TEST_REF: v4.9.5
-  TEST_DIR: test
-  TEST_SRC: src/compiler
+  TEST_SRC: src/compiler/utilities.ts
   MASSIF_OUT: perf.out
   MASSIF_REPORT: perf.txt
   TSSG_TS: tree-sitter-stack-graphs-typescript
@@ -40,6 +46,7 @@ jobs:
           ref: ${{ env.BASE_SHA }}
           path: ${{ env.BASE_DIR }}
           fetch-depth: 0
+          lfs: false
       - name: Find last relevant base commit
         run: |
           printf 'BASE_SHA=%s\n' "$(git rev-list -1 ${{ env.BASE_SHA }} -- stack-graphs)" >> $GITHUB_ENV
@@ -51,6 +58,7 @@ jobs:
           ref: ${{ env.HEAD_SHA }}
           path: ${{ env.HEAD_DIR }}
           fetch-depth: 0
+          lfs: false
       - name: "Find last relevant head commit"
         run: |
           printf 'HEAD_SHA=%s\n' "$(git rev-list -1 ${{ env.HEAD_SHA }} -- stack-graphs)" >> $GITHUB_ENV
@@ -60,7 +68,7 @@ jobs:
         uses: actions/cache/restore@v3
         with:
           path: done
-          key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }}
+          key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }}
           lookup-only: 'true'
 
   ##
@@ -73,9 +81,6 @@ jobs:
     env:
       BASE_SHA: ${{ needs.changes.outputs.base-sha }}
     steps:
-      #
-      # Install tools
-      #
       - name: Install Rust environment
         uses: hecrj/setup-rust-action@v1
         with:
@@ -90,58 +95,45 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y valgrind
-      #
-      # Cache results
-      #
       - name: "Cache base result"
         id: cache-base-result
         uses: actions/cache@v3
         with:
           path: |
             ${{ env.MASSIF_OUT }}
             ${{ env.MASSIF_REPORT }}
-          key: ${{ runner.os }}-perf-result-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }}
-      #
-      # Checkout code
-      #
+          key: ${{ runner.os }}-perf-result-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.TEST_NAME }}
       - name: "Checkout base code"
         if: steps.cache-base-result.outputs.cache-hit != 'true'
         uses: actions/checkout@v3
         with:
           repository: ${{ env.BASE_REPO }}
           ref: ${{ env.BASE_SHA }}
           path: ${{ env.BASE_DIR }}
-      #
-      # Build code
-      #
+          lfs: true
       - name: "Build base CLI"
         if: steps.cache-base-result.outputs.cache-hit != 'true'
         run: cargo build --package ${{ env.TSSG_TS }} --features cli --release
         working-directory: ${{ env.BASE_DIR }}
         env:
           CARGO_PROFILE_RELEASE_DEBUG: true
-      #
-      # Test performance
-      #
-      - name: Checkout test code
+      - name: Checkout benchmark code
         if: steps.cache-base-result.outputs.cache-hit != 'true'
         uses: actions/checkout@v3
         with:
           repository: ${{ env.TEST_REPO }}
           ref: ${{ env.TEST_REF }}
-          path: ${{ env.TEST_DIR }}
+          path: ${{ env.BASE_DIR }}/data/${{ env.TEST_NAME }}
       - name: Profile base memory
         if: steps.cache-base-result.outputs.cache-hit != 'true'
         run: |
           valgrind \
             --tool=massif \
             --massif-out-file=${{ env.MASSIF_OUT }} \
             ${{ env.BASE_DIR }}/target/release/${{ env.TSSG_TS }} \
-              index -D ${{ env.BASE_DB }} --max-file-time=30 --hide-error-details -- ${{ env.TEST_DIR }}/${{ env.TEST_SRC }}
+              index -D ${{ env.BASE_DB }} --max-file-time=30 --hide-error-details -- \
+                ${{ env.BASE_DIR }}/data/${{ env.TEST_NAME }}/${{ env.TEST_SRC }}
           ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }}
-      #
-      # Upload results
-      #
       - name: Upload results
         uses: actions/upload-artifact@v3
         with:
@@ -160,9 +152,6 @@ jobs:
     env:
       HEAD_SHA: ${{ needs.changes.outputs.head-sha }}
     steps:
-      #
-      # Install tools
-      #
       - name: Install Rust environment
         uses: hecrj/setup-rust-action@v1
         with:
@@ -177,58 +166,43 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y valgrind
-      #
-      # Cache results
-      #
       - name: "Cache head result"
         id: cache-head-result
         uses: actions/cache@v3
         with:
           path: |
             ${{ env.MASSIF_OUT }}
             ${{ env.MASSIF_REPORT }}
-          key: ${{ runner.os }}-perf-result-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }}
-      #
-      # Checkout code
-      #
+          key: ${{ runner.os }}-perf-result-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }}
       - name: "Checkout head code"
         if: steps.cache-head-result.outputs.cache-hit != 'true'
         uses: actions/checkout@v3
         with:
           repository: ${{ env.HEAD_REPO }}
           ref: ${{ env.HEAD_SHA }}
           path: ${{ env.HEAD_DIR }}
-      #
-      # Build code
-      #
+          lfs: true
       - name: "Build head CLI"
         if: steps.cache-head-result.outputs.cache-hit != 'true'
         run: cargo build --package ${{ env.TSSG_TS }} --features cli --release
         working-directory: ${{ env.HEAD_DIR }}
         env:
           CARGO_PROFILE_RELEASE_DEBUG: true
-      #
-      # Test performance
-      #
-      - name: Checkout test code
+      - name: Checkout benchmark code
         if: steps.cache-head-result.outputs.cache-hit != 'true'
-        uses: actions/checkout@v3
-        with:
-          repository: ${{ env.TEST_REPO }}
-          ref: ${{ env.TEST_REF }}
-          path: ${{ env.TEST_DIR }}
+        run: |
+          unzip ${{ env.TEST_NAME }}.zip
+        working-directory: ${{ env.HEAD_DIR }}/data
       - name: Profile head memory
         if: steps.cache-head-result.outputs.cache-hit != 'true'
         run: |
           valgrind \
             --tool=massif \
             --massif-out-file=${{ env.MASSIF_OUT }} \
             ${{ env.HEAD_DIR }}/target/release/${{ env.TSSG_TS }} \
-              index -D ${{ env.HEAD_DB }} --max-file-time=30 --hide-error-details -- ${{ env.TEST_DIR }}/${{ env.TEST_SRC }}
+              index -D ${{ env.HEAD_DB }} --max-file-time=30 --hide-error-details -- \
+                ${{ env.HEAD_DIR }}/data/${{ env.TEST_NAME }}
           ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }}
-      #
-      # Upload results
-      #
       - name: Upload results
         uses: actions/upload-artifact@v3
         with:
@@ -285,7 +259,7 @@ jobs:
           ${{ env.SRC_DIR }}/script/ci-perf-summary-md \
             ${{ env.BASE_ARTIFACT }}/${{ env.MASSIF_OUT }} \
             ${{ env.HEAD_ARTIFACT }}/${{ env.MASSIF_OUT }} \
-            'Comparing base ${{ env.BASE_REPO }}@${{ env.BASE_SHA }} with head ${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }} on [${{ env.TEST_REPO }}@${{ env.TEST_REF }}](${{ github.server_url }}/${{ env.TEST_REPO }}/tree/${{ env.TEST_REF }}). For details see [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) artifacts. _Note that performance is tested on the last commits with changes in `stack-graphs`, not on every commit._' \
+            'Comparing base ${{ env.BASE_REPO }}@${{ env.BASE_SHA }} with head ${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }} on ${{ env.TEST_NAME }} benchmark. For details see [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) artifacts. _Note that performance is tested on the last commits with changes in `stack-graphs`, not on every commit._' \
           | ${{ env.SRC_DIR }}/script/ci-comment-json > ${{ env.COMMENT_JSON }}
       - name: Add summary comment to PR
         run: |
@@ -302,4 +276,4 @@ jobs:
         uses: actions/cache/save@v3
         with:
           path: done
-          key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }}
+          key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }}
diff --git a/data/typescript_benchmark.md b/data/typescript_benchmark.md
@@ -0,0 +1,32 @@
+# Performance Benchmark
+
+A set of files, taken from the TypeScript compiler v4.9.5, of different sizes:
+
+```
+      15 text files.
+      15 unique files.
+       0 files ignored.
+
+github.com/AlDanial/cloc v 1.96  T=0.12 s (121.5 files/s, 250225.1 lines/s)
+---------------------------------------------------------------------------------------------
+File                                                      blank        comment           code
+---------------------------------------------------------------------------------------------
+typescript_benchmark/types.ts                              1085           1548           6642
+typescript_benchmark/nodeFactory.ts                         598            758           5544
+typescript_benchmark/es2015.ts                              477           1082           2861
+typescript_benchmark/generators.ts                          301            903           1985
+typescript_benchmark/module.ts                              173            394           1437
+typescript_benchmark/system.ts                              204            546           1195
+typescript_benchmark/resolutionCache.ts                     112            101            920
+typescript_benchmark/sourcemap.ts                            99             47            614
+typescript_benchmark/utilities.ts                            64             92            371
+typescript_benchmark/emitNode.ts                             35             87            172
+typescript_benchmark/es5.ts                                  10             40             72
+typescript_benchmark/node.ts                                 14              4             66
+typescript_benchmark/builderPublic.ts                         8            117             56
+typescript_benchmark/es2019.ts                                4              1             33
+typescript_benchmark/builderStatePublic.ts                    1              0             13
+---------------------------------------------------------------------------------------------
+SUM:                                                       3185           5720          21981
+---------------------------------------------------------------------------------------------
+```
diff --git a/data/typescript_benchmark.zip b/data/typescript_benchmark.zip
diff --git a/data/typescript_minimal_project.zip b/data/typescript_minimal_project.zip
diff --git a/stack-graphs/force-perf-test b/stack-graphs/force-perf-test