Add gzip to prerelease benchmark list #331
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Copyright (c) 2022-present, Trail of Bits, Inc. | |
| # All rights reserved. | |
| # | |
| # This source code is licensed in accordance with the terms specified in | |
| # the LICENSE file found in the root directory of this source tree. | |
| # | |
| name: "Pre Release" | |
| on: | |
| workflow_run: | |
| workflows: ["Build"] | |
| types: | |
| - completed | |
| branches: | |
| - "master" | |
| pull_request: | |
| branches: | |
| - '*' | |
| permissions: write-all | |
| env: | |
| LLVM_DISABLE_SYMBOLIZATION: 1 | |
| jobs: | |
| build: | |
| strategy: | |
| matrix: | |
| llvm-version: [19] | |
| image-version: [22.04] | |
| name: "Pre Release" | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 60 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| env: | |
| CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/" | |
| TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake | |
| LLVM_EXTERNAL_LIT: "/usr/local/bin/lit" | |
| steps: | |
| - name: Clone the VAST repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| fetch-depth: '0' # Required by the tag action | |
| #- name: Bump version and push tag | |
| # id: bump_version | |
| # uses: anothrNick/github-tag-action@v1 | |
| # env: | |
| # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # WITH_V: true # Prefix v to the tag | |
| # DEFAULT_BUMP: patch | |
| - name: Configure build | |
| run: cmake --preset ci | |
| - name: Build release | |
| run: cmake --build --preset ci-release -j $(nproc) | |
| - name: Package | |
| run: cpack --preset ci | |
| - name: Upload VAST build artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: VAST | |
| path: ./builds/ci/package/* | |
| retention-days: 1 | |
| #- name: Publish Pre-Release | |
| # uses: softprops/action-gh-release@v2 | |
| # with: | |
| # name: "latest" | |
| # tag_name: ${{ steps.bump_version.outputs.new_tag }} | |
| # generate_release_notes: true | |
| # make_latest: true | |
| # files: | | |
| # ./LICENSE | |
| # ./builds/ci/package/* | |
| # | |
| # LLVM Test Suite | |
| # | |
| run_llvm_ts: | |
| name: "Run LLVM test suite" | |
| needs: build | |
| strategy: | |
| matrix: | |
| llvm-version: [19] | |
| image-version: [22.04] | |
| vast-target: ['hl', 'llvm', 'bin'] | |
| disable-unsup: ['ON','OFF'] | |
| exclude: | |
| - vast-target: 'llvm' | |
| disable-unsup: 'ON' | |
| - vast-target: 'bin' | |
| disable-unsup: 'ON' | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 360 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| steps: | |
| - name: Fetch VAST artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: VAST | |
| - name: Unpack VAST | |
| run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1 | |
| - name: Export vast binaries | |
| run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH | |
| - name: Install test suite dependencies | |
| run: | | |
| apt-get update | |
| apt-get -y install clang | |
| pip3 install lit psutil | |
| - name: Clone test suite repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-llvm-test-suite | |
| ref: main | |
| path: llvm-test-suite | |
| fetch-depth: 1 | |
| - name: Run test-suite | |
| continue-on-error: true | |
| run: | | |
| mkdir llvm-test-suite/build && cd llvm-test-suite/build/ | |
| cmake ../ -DCMAKE_BUILD_TYPE=Release \ | |
| -DCMAKE_C_COMPILER=$(readlink -f ../tools/compiler) \ | |
| -DCMAKE_CXX_COMPILER=$(readlink -f ../tools/compiler++) \ | |
| -C=../cmake/caches/O3.cmake \ | |
| -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \ | |
| -DTEST_SUITE_SUBDIRS=SingleSource \ | |
| -DVAST_TEST_SUITE_ENABLE=ON \ | |
| -DVAST_DISABLE_UNSUPPORTED=${{ matrix.disable-unsup }} \ | |
| -DVAST_TEST_SUITE_TARGET=${{ matrix.vast-target }} | |
| make -j $(nproc) -k | |
| - name: Setup result suffix | |
| run: | | |
| if [ "${{ matrix.disable-unsup }}" = "OFF" ]; then | |
| echo "VAST_RESULTS_SUFFIX=-with-unsup" >> $GITHUB_ENV | |
| fi | |
| - name: Collect run data (mlir) | |
| if: matrix.vast-target != 'bin' | |
| working-directory: ./llvm-test-suite/build | |
| run: python3 ../utils/mlir_compile_json_gen.py >> ../../results-${{ matrix.vast-target }}$VAST_RESULTS_SUFFIX.json | |
| - name: Collect run data (bin) | |
| continue-on-error: true | |
| if: matrix.vast-target == 'bin' | |
| working-directory: ./llvm-test-suite/build | |
| run: lit --timeout=360 -v -o ../../results-${{ matrix.vast-target }}$VAST_RESULTS_SUFFIX.json . | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./results-${{ matrix.vast-target }}${{ env.VAST_RESULTS_SUFFIX }}.json | |
| name: results-llvm-ts-${{ matrix.vast-target }}${{ env.VAST_RESULTS_SUFFIX }} | |
| eval_llvm_ts: | |
| name: "Eval LLVM Singlesource results" | |
| needs: run_llvm_ts | |
| strategy: | |
| matrix: | |
| image-version: [22.04] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| steps: | |
| - name: Fetch result artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-llvm-ts-* | |
| merge-multiple: true | |
| - name: Install evaluator dependencies | |
| run: pip3 install pandas scipy tabulate | |
| - name: Clone test suite repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-llvm-test-suite | |
| sparse-checkout: utils/vast_compare.py | |
| ref: main | |
| path: llvm-test-suite | |
| fetch-depth: 1 | |
| - name: Generate the results | |
| run: | | |
| python3 llvm-test-suite/utils/vast_compare.py \ | |
| --columns "HighLevel with Unsupported",HighLevel,LLVM,Binary \ | |
| --files results-hl-with-unsup.json,results-hl.json,results-llvm-with-unsup.json,results-bin-with-unsup.json \ | |
| --output single-source-results | |
| - name: Post results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: llvm-test-suite-results | |
| path: | | |
| ./single-source-results.csv | |
| ./single-source-results.md | |
| # | |
| # SV-Comp bencmharks | |
| # | |
| run_svcomp: | |
| name: "Run SV-Comp test suite" | |
| needs: build | |
| strategy: | |
| matrix: | |
| llvm-version: [19] | |
| image-version: [22.04] | |
| vast-target: ['hl'] | |
| disable-unsup: [true, false] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 360 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| steps: | |
| - name: Fetch VAST artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: VAST | |
| - name: Unpack VAST | |
| run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1 | |
| - name: Export vast binaries | |
| run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH | |
| - name: Clone test suite repository | |
| run: git clone --depth=1 https://gitlab.com/sosy-lab/benchmarking/sv-benchmarks.git | |
| - name: Clone patch repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/sv-bench-patch | |
| ref: ci-version | |
| path: sv-bench-patch | |
| fetch-depth: 1 | |
| - name: Apply patch | |
| working-directory: sv-bench-patch | |
| run: ./apply | |
| - name: Run benchmarks | |
| if: ${{ !matrix.disable-unsup }} | |
| run: sh ./sv-bench-patch/compile-all.sh -t ${{ matrix.vast-target }} -d ./sv-benchmarks | |
| - name: Run benchmarks without unsupported | |
| if: ${{ matrix.disable-unsup }} | |
| run: sh ./sv-bench-patch/compile-all.sh -t ${{ matrix.vast-target }} --disable-unsup -d ./sv-benchmarks | |
| - name: Setup result suffix | |
| run: | | |
| if [ "${{ matrix.disable-unsup }}" = "false" ]; then | |
| echo "VAST_RESULTS_SUFFIX=-with-unsup" >> $GITHUB_ENV | |
| fi | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: ./stats | |
| name: results-sv-${{ matrix.vast-target }}${{ env.VAST_RESULTS_SUFFIX }} | |
| eval_svcomp: | |
| name: "Eval SV-Comp test results" | |
| needs: run_svcomp | |
| strategy: | |
| matrix: | |
| image-version: [22.04] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| steps: | |
| - name: Fetch result artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: results-sv-* | |
| - name: Install evaluator dependencies | |
| run: pip3 install pandas scipy tabulate | |
| - name: Clone patch repository | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/sv-bench-patch | |
| sparse-checkout: make_table.py | |
| ref: ci-version | |
| path: sv-bench-patch | |
| fetch-depth: 1 | |
| - name: Generate the results | |
| run: | | |
| python3 sv-bench-patch/make_table.py \ | |
| --columns "HighLevel with Unsupported",HighLevel \ | |
| --files results-sv-hl-with-unsup/results.txt,results-sv-hl/results.txt \ | |
| --output sv-comp-results | |
| - name: Post results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sv-comp-results | |
| path: ./sv-comp-results.md | |
| # | |
| # Linux kernel benchmark | |
| # | |
| run_linux_kernel_bench: | |
| name: "Run Linux kernel test suite" | |
| needs: build | |
| strategy: | |
| matrix: | |
| llvm-version: [19] | |
| image-version: [22.04] | |
| vast-target: ['hl'] | |
| disable-unsup: [true, false] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 360 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| steps: | |
| - name: Fetch VAST artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: VAST | |
| - name: Unpack VAST | |
| run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1 | |
| - name: Export vast binaries | |
| run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH | |
| - name: Install benchmark dependencies | |
| run: | | |
| apt-get update | |
| apt-get -y install git fakeroot build-essential \ | |
| ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison \ | |
| lld-${{ matrix.llvm-version }} | |
| - name: Clone, configure, and build the Linux kernel | |
| run: | | |
| git clone https://github.com/torvalds/linux.git --depth=1 --branch=v6.11 linux | |
| cd linux | |
| make defconfig LLVM=-${{ matrix.llvm-version }} | |
| make prepare LLVM=-${{ matrix.llvm-version }} | |
| make LLVM=-${{ matrix.llvm-version }} -j $(nproc) | |
| python3 ./scripts/clang-tools/gen_compile_commands.py | |
| - name: Clone vast benchmark directory | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-benchmarks | |
| sparse-checkout: benchmarks/linux_kernel | |
| ref: main | |
| path: vast-benchmarks/ | |
| fetch-depth: 1 | |
| - name: Setup result suffix and vast arguments | |
| run: | | |
| if [ "${{ matrix.disable-unsup }}" = "false" ]; then | |
| echo "VAST_RESULTS_SUFFIX=with_unsup" >> $GITHUB_ENV | |
| else | |
| echo "VAST_RESULTS_SUFFIX=without_unsup" >> $GITHUB_ENV | |
| echo "VAST_DISABLE_UNSUPPORTED=-vast-disable-unsupported" >> $GITHUB_ENV | |
| fi | |
| - name: Run benchmarks | |
| run: > | |
| python3 ${PWD}/vast-benchmarks/benchmarks/linux_kernel/run_vast_benchmark.py | |
| vast-front | |
| ${PWD}/linux/compile_commands.json | |
| --num_processes=$(nproc) | |
| --vast_option="-xc" | |
| --vast_option="-vast-emit-mlir=${{ matrix.vast-target }}" | |
| --vast_option="${{ env.VAST_DISABLE_UNSUPPORTED }}" | |
| --print_errors | |
| > vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv | |
| path: ./vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv | |
| convert_linux_kernel_bench: | |
| name: "Convert Linux Kernel benchmark results to Markdown" | |
| needs: run_linux_kernel_bench | |
| strategy: | |
| matrix: | |
| image-version: [22.04] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| steps: | |
| - name: Fetch result artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: vast_linux_kernel_times_* | |
| merge-multiple: true | |
| - name: Install converter dependencies | |
| run: pip3 install pandas scipy tabulate | |
| - name: Clone vast benchmark directory | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-benchmarks | |
| sparse-checkout: utils/ | |
| ref: main | |
| path: vast-benchmarks/ | |
| fetch-depth: 1 | |
| - name: Generate the results | |
| run: > | |
| python3 vast-benchmarks/utils/to_markdown.py \ | |
| '{ "HighLevel with unsupported": "vast_linux_kernel_times_hl_with_unsup.tsv", "HighLevel" : "vast_linux_kernel_times_hl_without_unsup.tsv" }' | |
| --output_filepath linux_kernel_times.md | |
| - name: Post results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: linux_kernel_times.md | |
| path: ./linux_kernel_times.md | |
| # | |
| # GZIP benchmark | |
| # | |
| run_gzip_bench: | |
| name: "Run gzip translation" | |
| needs: build | |
| strategy: | |
| matrix: | |
| llvm-version: [19] | |
| image-version: [22.04] | |
| gzip-version: ['1.10', '1.13'] | |
| vast-target: ['hl', 'llvm'] | |
| disable-unsup: ['ON','OFF'] | |
| exclude: | |
| - vast-target: 'llvm' | |
| disable-unsup: 'ON' | |
| - vast-target: 'bin' | |
| disable-unsup: 'ON' | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| timeout-minutes: 360 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| steps: | |
| - name: Fetch VAST artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: VAST | |
| - name: Unpack VAST | |
| run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1 | |
| - name: Export vast binaries | |
| run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH | |
| - name: Install benchmark dependencies | |
| run: | | |
| apt-get update | |
| apt-get -y install bear | |
| - name: Clone, configure, and build the Linux kernel | |
| run: | | |
| wget https://ftp.gnu.org/gnu/gzip/gzip-${{ matrix.gzip-version }}.tar.gz | |
| tar -xvf gzip-${{ matrix.gzip-version }}.tar.gz | |
| cd gzip-${{ matrix.gzip-version }}.tar.gz | |
| ./configure | |
| bear -- make | |
| - name: Clone vast benchmark directory | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-benchmarks | |
| sparse-checkout: benchmarks/linux_kernel | |
| ref: main | |
| path: vast-benchmarks/ | |
| fetch-depth: 1 | |
| - name: Setup result suffix and vast arguments | |
| run: | | |
| if [ "${{ matrix.disable-unsup }}" = "false" ]; then | |
| echo "VAST_RESULTS_SUFFIX=with_unsup" >> $GITHUB_ENV | |
| else | |
| echo "VAST_RESULTS_SUFFIX=without_unsup" >> $GITHUB_ENV | |
| echo "VAST_DISABLE_UNSUPPORTED=-vast-disable-unsupported" >> $GITHUB_ENV | |
| fi | |
| - name: Run benchmarks | |
| run: > | |
| python3 ${PWD}/vast-benchmarks/benchmarks/linux_kernel/run_vast_benchmark.py | |
| vast-front | |
| ${PWD}/gzip-${{ matrix.gzip-version }}/compile_commands.json | |
| --num_processes=$(nproc) | |
| --vast_option="-xc" | |
| --vast_option="-vast-emit-mlir=${{ matrix.vast-target }}" | |
| --vast_option="${{ env.VAST_DISABLE_UNSUPPORTED }}" | |
| --print_errors | |
| > gzip_${{ matrix.gzip-version }}_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv | |
| - name: Upload results | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gzip_${{ matrix.gzip-version }}_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv | |
| path: ./gzip_${{ matrix.gzip-version }}_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv | |
| convert_gzip_bench: | |
| name: "Convert results to Markdown" | |
| needs: run_gzip_bench | |
| strategy: | |
| matrix: | |
| image-version: [22.04] | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| steps: | |
| - name: Fetch result artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: vast_linux_kernel_times_* | |
| merge-multiple: true | |
| - name: Install converter dependencies | |
| run: pip3 install pandas scipy tabulate | |
| - name: Clone vast benchmark directory | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: trailofbits/vast-benchmarks | |
| sparse-checkout: utils/ | |
| ref: main | |
| path: vast-benchmarks/ | |
| fetch-depth: 1 | |
| - name: Generate the results | |
| run: > | |
| python3 vast-benchmarks/utils/to_markdown.py \ | |
| '{ "HighLevel with unsupported for 1.10": "gzip_1.10_hl_with_unsup.tsv", \ | |
| "HighLevel for 1.10" : "gzip_1.10_hl_without_unsup.tsv", \ | |
| "LLVM for 1.10" : "gzip_1.10_llvm_with_unsup.tsv", \ | |
| "HighLevel with unsupported for 1.13": "gzip_1.13_hl_with_unsup.tsv", \ | |
| "HighLevel for 1.13" : "gzip_1.13_hl_without_unsup.tsv", \ | |
| "LLVM for 1.13" : "gzip_1.13_llvm_with_unsup.tsv", \ | |
| }' | |
| --output_filepath gzip_times.md | |
| - name: Post results as artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: gzip_times.md | |
| path: ./gzip_times.md | |
| # | |
| # Webpage build | |
| # | |
| build_doc: | |
| strategy: | |
| matrix: | |
| llvm-version: [19] | |
| image-version: [22.04] | |
| name: "Build VAST doc" | |
| runs-on: ubuntu-${{ matrix.image-version }} | |
| needs: [eval_llvm_ts, eval_svcomp, convert_linux_kernel_bench] | |
| timeout-minutes: 60 | |
| container: | |
| image: | |
| ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest | |
| env: | |
| CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/" | |
| TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake | |
| LLVM_EXTERNAL_LIT: "/usr/local/bin/lit" | |
| steps: | |
| - name: Clone the VAST repository | |
| uses: actions/checkout@v4 | |
| with: | |
| submodules: true | |
| fetch-depth: 1 | |
| - name: Configure build | |
| run: cmake --preset ci | |
| - name: Build VAST Doc | |
| run: cmake --build --preset ci-release --target vast-doc | |
| - name: Fetch LLVM test suite results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: llvm-test-suite-results | |
| path: llvm-test-suite-results | |
| - name: Fetch SV-Comp benchmarks results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: sv-comp-results | |
| path: sv-comp-results | |
| - name: Fetch Linux kernel benchmark results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: linux_kernel_times.md | |
| path: linux_kernel_times | |
| - name: Fetch GZIP benchmark results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: gzip_times.md | |
| path: gzip_times | |
| - name: Build Pages | |
| run: | | |
| sh ./www/setup.sh _site ./builds/ci/ | |
| - name: Deploy docs | |
| uses: mhausenblas/mkdocs-deploy-gh-pages@master | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| CONFIG_FILE: _site/mkdocs.yml |