diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..041b0e4e4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +data/*.tar.gz filter=lfs diff=lfs merge=lfs -text +data/*.tgz filter=lfs diff=lfs merge=lfs -text +data/*.zip filter=lfs diff=lfs merge=lfs -text diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4188717f1..c561bd429 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,12 +6,16 @@ on: schedule: - cron: "0 0 1,15 * *" +# In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: test-rust: - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest strategy: matrix: - os: [ubuntu-latest] rust: [stable] steps: @@ -85,7 +89,7 @@ jobs: languages: ${{ steps.language-list.outputs.languages }} test-languages: - needs: [list-languages, test-rust] + needs: [list-languages] runs-on: ubuntu-latest strategy: matrix: @@ -112,3 +116,61 @@ jobs: run: cargo build -p ${{ matrix.language }} - name: Test run: cargo test -p ${{ matrix.language }} + + test-cli: + runs-on: ubuntu-latest + strategy: + matrix: + rust: [stable] + env: + TEST_NAME: typescript_minimal_project + TSSG_TS: tree-sitter-stack-graphs-typescript + + steps: + - name: Install Rust environment + uses: hecrj/setup-rust-action@v1 + with: + rust-version: ${{ matrix.rust }} + - name: Cache dependencies + uses: actions/cache@v3 + with: + path: | + ~/.cargo + target + key: ${{ runner.OS }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.OS }}-cargo- + - name: Checkout code + uses: actions/checkout@v3 + with: + lfs: true + - name: Build + run: cargo build --release -p ${{ env.TSSG_TS }} --features cli + - name: Checkout test code + run: unzip ${{ env.TEST_NAME }}.zip + working-directory: data + - name: Index + run: | + target/release/${{ env.TSSG_TS }} index \ + -D data/${{ env.TEST_NAME }}.sqlite \ + --verbose \ + --hide-error-details \ + data/${{ env.TEST_NAME }} + - name: Status + run: | + target/release/${{ env.TSSG_TS }} status \ + -D data/${{ env.TEST_NAME }}.sqlite \ + --all + - name: Query + run: | + target/release/${{ env.TSSG_TS }} query \ + -D data/${{ env.TEST_NAME }}.sqlite \ + definition \ + data/${{ env.TEST_NAME }}/index.ts:3:10 + - name: Index (forced) + run: | + target/release/${{ env.TSSG_TS }} index \ + -D data/${{ env.TEST_NAME }}.sqlite \ + --verbose \ + --hide-error-details \ + data/${{ env.TEST_NAME }} diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index eeb851197..227bda8b0 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -3,6 +3,12 @@ on: pull_request: paths: - 'stack-graphs/**' + +# In the event that there is a new push to the ref, cancel any running jobs because there are now obsolete, and wasting resources. +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + env: BASE_REPO: ${{ github.event.pull_request.base.repo.owner.login }}/${{ github.event.pull_request.base.repo.name }} BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -14,10 +20,10 @@ env: HEAD_DIR: head HEAD_DB: head.sqlite HEAD_ARTIFACT: head-perf-results + TEST_NAME: typescript_benchmark TEST_REPO: microsoft/TypeScript TEST_REF: v4.9.5 - TEST_DIR: test - TEST_SRC: src/compiler + TEST_SRC: src/compiler/utilities.ts MASSIF_OUT: perf.out MASSIF_REPORT: perf.txt TSSG_TS: tree-sitter-stack-graphs-typescript @@ -40,6 +46,7 @@ jobs: ref: ${{ env.BASE_SHA }} path: ${{ env.BASE_DIR }} fetch-depth: 0 + lfs: false - name: Find last relevant base commit run: | printf 'BASE_SHA=%s\n' "$(git rev-list -1 ${{ env.BASE_SHA }} -- stack-graphs)" >> $GITHUB_ENV @@ -51,6 +58,7 @@ jobs: ref: ${{ env.HEAD_SHA }} path: ${{ env.HEAD_DIR }} fetch-depth: 0 + lfs: false - name: "Find last relevant head commit" run: | printf 'HEAD_SHA=%s\n' "$(git rev-list -1 ${{ env.HEAD_SHA }} -- stack-graphs)" >> $GITHUB_ENV @@ -60,7 +68,7 @@ jobs: uses: actions/cache/restore@v3 with: path: done - key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }} + key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} lookup-only: 'true' ## @@ -73,9 +81,6 @@ jobs: env: BASE_SHA: ${{ needs.changes.outputs.base-sha }} steps: - # - # Install tools - # - name: Install Rust environment uses: hecrj/setup-rust-action@v1 with: @@ -90,9 +95,6 @@ jobs: run: | sudo apt-get update sudo apt-get install -y valgrind - # - # Cache results - # - name: "Cache base result" id: cache-base-result uses: actions/cache@v3 @@ -100,10 +102,7 @@ jobs: path: | ${{ env.MASSIF_OUT }} ${{ env.MASSIF_REPORT }} - key: ${{ runner.os }}-perf-result-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }} - # - # Checkout code - # + key: ${{ runner.os }}-perf-result-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.TEST_NAME }} - name: "Checkout base code" if: steps.cache-base-result.outputs.cache-hit != 'true' uses: actions/checkout@v3 @@ -111,25 +110,20 @@ jobs: repository: ${{ env.BASE_REPO }} ref: ${{ env.BASE_SHA }} path: ${{ env.BASE_DIR }} - # - # Build code - # + lfs: true - name: "Build base CLI" if: steps.cache-base-result.outputs.cache-hit != 'true' run: cargo build --package ${{ env.TSSG_TS }} --features cli --release working-directory: ${{ env.BASE_DIR }} env: CARGO_PROFILE_RELEASE_DEBUG: true - # - # Test performance - # - - name: Checkout test code + - name: Checkout benchmark code if: steps.cache-base-result.outputs.cache-hit != 'true' uses: actions/checkout@v3 with: repository: ${{ env.TEST_REPO }} ref: ${{ env.TEST_REF }} - path: ${{ env.TEST_DIR }} + path: ${{ env.BASE_DIR }}/data/${{ env.TEST_NAME }} - name: Profile base memory if: steps.cache-base-result.outputs.cache-hit != 'true' run: | @@ -137,11 +131,9 @@ jobs: --tool=massif \ --massif-out-file=${{ env.MASSIF_OUT }} \ ${{ env.BASE_DIR }}/target/release/${{ env.TSSG_TS }} \ - index -D ${{ env.BASE_DB }} --max-file-time=30 --hide-error-details -- ${{ env.TEST_DIR }}/${{ env.TEST_SRC }} + index -D ${{ env.BASE_DB }} --max-file-time=30 --hide-error-details -- \ + ${{ env.BASE_DIR }}/data/${{ env.TEST_NAME }}/${{ env.TEST_SRC }} ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }} - # - # Upload results - # - name: Upload results uses: actions/upload-artifact@v3 with: @@ -160,9 +152,6 @@ jobs: env: HEAD_SHA: ${{ needs.changes.outputs.head-sha }} steps: - # - # Install tools - # - name: Install Rust environment uses: hecrj/setup-rust-action@v1 with: @@ -177,9 +166,6 @@ jobs: run: | sudo apt-get update sudo apt-get install -y valgrind - # - # Cache results - # - name: "Cache head result" id: cache-head-result uses: actions/cache@v3 @@ -187,10 +173,7 @@ jobs: path: | ${{ env.MASSIF_OUT }} ${{ env.MASSIF_REPORT }} - key: ${{ runner.os }}-perf-result-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }} - # - # Checkout code - # + key: ${{ runner.os }}-perf-result-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} - name: "Checkout head code" if: steps.cache-head-result.outputs.cache-hit != 'true' uses: actions/checkout@v3 @@ -198,25 +181,18 @@ jobs: repository: ${{ env.HEAD_REPO }} ref: ${{ env.HEAD_SHA }} path: ${{ env.HEAD_DIR }} - # - # Build code - # + lfs: true - name: "Build head CLI" if: steps.cache-head-result.outputs.cache-hit != 'true' run: cargo build --package ${{ env.TSSG_TS }} --features cli --release working-directory: ${{ env.HEAD_DIR }} env: CARGO_PROFILE_RELEASE_DEBUG: true - # - # Test performance - # - - name: Checkout test code + - name: Checkout benchmark code if: steps.cache-head-result.outputs.cache-hit != 'true' - uses: actions/checkout@v3 - with: - repository: ${{ env.TEST_REPO }} - ref: ${{ env.TEST_REF }} - path: ${{ env.TEST_DIR }} + run: | + unzip ${{ env.TEST_NAME }}.zip + working-directory: ${{ env.HEAD_DIR }}/data - name: Profile head memory if: steps.cache-head-result.outputs.cache-hit != 'true' run: | @@ -224,11 +200,9 @@ jobs: --tool=massif \ --massif-out-file=${{ env.MASSIF_OUT }} \ ${{ env.HEAD_DIR }}/target/release/${{ env.TSSG_TS }} \ - index -D ${{ env.HEAD_DB }} --max-file-time=30 --hide-error-details -- ${{ env.TEST_DIR }}/${{ env.TEST_SRC }} + index -D ${{ env.HEAD_DB }} --max-file-time=30 --hide-error-details -- \ + ${{ env.HEAD_DIR }}/data/${{ env.TEST_NAME }} ms_print ${{ env.MASSIF_OUT }} > ${{ env.MASSIF_REPORT }} - # - # Upload results - # - name: Upload results uses: actions/upload-artifact@v3 with: @@ -285,7 +259,7 @@ jobs: ${{ env.SRC_DIR }}/script/ci-perf-summary-md \ ${{ env.BASE_ARTIFACT }}/${{ env.MASSIF_OUT }} \ ${{ env.HEAD_ARTIFACT }}/${{ env.MASSIF_OUT }} \ - 'Comparing base ${{ env.BASE_REPO }}@${{ env.BASE_SHA }} with head ${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }} on [${{ env.TEST_REPO }}@${{ env.TEST_REF }}](${{ github.server_url }}/${{ env.TEST_REPO }}/tree/${{ env.TEST_REF }}). For details see [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) artifacts. _Note that performance is tested on the last commits with changes in `stack-graphs`, not on every commit._' \ + 'Comparing base ${{ env.BASE_REPO }}@${{ env.BASE_SHA }} with head ${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }} on ${{ env.TEST_NAME }} benchmark. For details see [workflow](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}) artifacts. _Note that performance is tested on the last commits with changes in `stack-graphs`, not on every commit._' \ | ${{ env.SRC_DIR }}/script/ci-comment-json > ${{ env.COMMENT_JSON }} - name: Add summary comment to PR run: | @@ -302,4 +276,4 @@ jobs: uses: actions/cache/save@v3 with: path: done - key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_REPO }}@${{ env.TEST_REF }}/${{ env.TEST_SRC }} + key: ${{ runner.os }}-perf-tested-${{ env.BASE_REPO }}@${{ env.BASE_SHA }}-${{ env.HEAD_REPO }}@${{ env.HEAD_SHA }}-${{ env.TEST_NAME }} diff --git a/data/typescript_benchmark.md b/data/typescript_benchmark.md new file mode 100644 index 000000000..a1362c0a0 --- /dev/null +++ b/data/typescript_benchmark.md @@ -0,0 +1,32 @@ +# Performance Benchmark + +A set of files, taken from the TypeScript compiler v4.9.5, of different sizes: + +``` + 15 text files. + 15 unique files. + 0 files ignored. + +github.com/AlDanial/cloc v 1.96 T=0.12 s (121.5 files/s, 250225.1 lines/s) +--------------------------------------------------------------------------------------------- +File blank comment code +--------------------------------------------------------------------------------------------- +typescript_benchmark/types.ts 1085 1548 6642 +typescript_benchmark/nodeFactory.ts 598 758 5544 +typescript_benchmark/es2015.ts 477 1082 2861 +typescript_benchmark/generators.ts 301 903 1985 +typescript_benchmark/module.ts 173 394 1437 +typescript_benchmark/system.ts 204 546 1195 +typescript_benchmark/resolutionCache.ts 112 101 920 +typescript_benchmark/sourcemap.ts 99 47 614 +typescript_benchmark/utilities.ts 64 92 371 +typescript_benchmark/emitNode.ts 35 87 172 +typescript_benchmark/es5.ts 10 40 72 +typescript_benchmark/node.ts 14 4 66 +typescript_benchmark/builderPublic.ts 8 117 56 +typescript_benchmark/es2019.ts 4 1 33 +typescript_benchmark/builderStatePublic.ts 1 0 13 +--------------------------------------------------------------------------------------------- +SUM: 3185 5720 21981 +--------------------------------------------------------------------------------------------- +``` diff --git a/data/typescript_benchmark.zip b/data/typescript_benchmark.zip new file mode 100644 index 000000000..97f24b96d --- /dev/null +++ b/data/typescript_benchmark.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc602330764f803afe9b30a8848c053fce2a94719226590cf951a2c7fd7ea6fd +size 238818 diff --git a/data/typescript_minimal_project.zip b/data/typescript_minimal_project.zip new file mode 100644 index 000000000..ada17bfe7 --- /dev/null +++ b/data/typescript_minimal_project.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6e352dc107e4aafdf417fd2cbbb1d027f0d3bda4beb47e9a7fca26f6a743401f +size 1117 diff --git a/stack-graphs/force-perf-test b/stack-graphs/force-perf-test new file mode 100644 index 000000000..e69de29bb