prerelease.yml

#
# Copyright (c) 2022-present, Trail of Bits, Inc.
# All rights reserved.
#
# This source code is licensed in accordance with the terms specified in
# the LICENSE file found in the root directory of this source tree.
#

name: "Pre Release"

on:
  workflow_run:
    workflows: ["Build"]
    types:
      - completed
    branches:
      - "master"

permissions: write-all

env:
  LLVM_DISABLE_SYMBOLIZATION: 1

jobs:
  build:
    strategy:
      matrix:
        llvm-version: [19]
        image-version: [22.04]

    name: "Pre Release"
    runs-on: ubuntu-${{ matrix.image-version }}
    timeout-minutes: 60
    container:
      image:
        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest

    env:
        CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/"
        TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake
        LLVM_EXTERNAL_LIT: "/usr/local/bin/lit"

    steps:
      - name: Clone the VAST repository
        uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: '0' # Required by the tag action

      - name: Bump version and push tag
        id: bump_version
        uses: anothrNick/github-tag-action@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          WITH_V: true # Prefix v to the tag
          DEFAULT_BUMP: patch

      - name: Configure build
        run: cmake --preset ci

      - name: Build release
        run: cmake --build --preset ci-release -j $(nproc)

      - name: Package
        run: cpack --preset ci

      - name: Upload VAST build artifact
        uses: actions/upload-artifact@v4
        with:
          name: VAST
          path: ./builds/ci/package/*
          retention-days: 1

      - name: Publish Pre-Release
        uses: softprops/action-gh-release@v2
        with:
          name: "latest"
          tag_name: ${{ steps.bump_version.outputs.new_tag }}
          generate_release_notes: true
          make_latest: true
          files: |
            ./LICENSE
            ./builds/ci/package/*
#
#     LLVM Test Suite
#
  run_llvm_ts:
    name: "Run LLVM test suite"
    needs: build
    strategy:
      matrix:
        llvm-version: [19]
        image-version: [22.04]
        vast-target: ['hl', 'llvm', 'bin']
        disable-unsup: ['ON','OFF']
        exclude:
            - vast-target: 'llvm'
              disable-unsup: 'ON'
            - vast-target: 'bin'
              disable-unsup: 'ON'

    runs-on: ubuntu-${{ matrix.image-version }}
    timeout-minutes: 360
    container:
      image:
        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest

    steps:
     - name: Fetch VAST artifact
       uses: actions/download-artifact@v4
       with:
         name: VAST

     - name: Unpack VAST
       run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1

     - name: Export vast binaries
       run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH

     - name: Install test suite dependencies
       run: |
          apt-get update
          apt-get -y install clang
          pip3 install lit psutil

     - name: Clone test suite repository
       uses: actions/checkout@v4
       with:
        repository: trailofbits/vast-llvm-test-suite
        ref: main
        path: llvm-test-suite
        fetch-depth: 1

     - name: Run test-suite
       continue-on-error: true
       run: |
        mkdir llvm-test-suite/build && cd llvm-test-suite/build/
        cmake ../ -DCMAKE_BUILD_TYPE=Release \
          -DCMAKE_C_COMPILER=$(readlink -f ../tools/compiler) \
          -DCMAKE_CXX_COMPILER=$(readlink -f ../tools/compiler++) \
          -C=../cmake/caches/O3.cmake \
          -DTEST_SUITE_COLLECT_CODE_SIZE=OFF \
          -DTEST_SUITE_SUBDIRS=SingleSource \
          -DVAST_TEST_SUITE_ENABLE=ON \
          -DVAST_DISABLE_UNSUPPORTED=${{ matrix.disable-unsup }} \
          -DVAST_TEST_SUITE_TARGET=${{ matrix.vast-target }}
        make -j $(nproc) -k

     - name: Setup result suffix
       run: |
        if [ "${{ matrix.disable-unsup }}" = "OFF" ]; then
          echo "VAST_RESULTS_SUFFIX=-with-unsup" >> $GITHUB_ENV
        fi

     - name: Collect run data (mlir)
       if: matrix.vast-target != 'bin'
       working-directory: ./llvm-test-suite/build
       run: python3 ../utils/mlir_compile_json_gen.py >> ../../results-${{ matrix.vast-target }}$VAST_RESULTS_SUFFIX.json

     - name: Collect run data (bin)
       continue-on-error: true
       if: matrix.vast-target == 'bin'
       working-directory: ./llvm-test-suite/build
       run: lit --timeout=360 -v -o ../../results-${{ matrix.vast-target }}$VAST_RESULTS_SUFFIX.json .

     - name: Upload results
       uses: actions/upload-artifact@v4
       with:
        path: ./results-${{ matrix.vast-target }}${{ env.VAST_RESULTS_SUFFIX }}.json
        name: results-llvm-ts-${{ matrix.vast-target }}${{ env.VAST_RESULTS_SUFFIX }}

  eval_llvm_ts:
    name: "Eval LLVM Singlesource results"
    needs: run_llvm_ts
    strategy:
      matrix:
        image-version: [22.04]
    runs-on: ubuntu-${{ matrix.image-version }}
    steps:
      - name: Fetch result artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: results-llvm-ts-*
          merge-multiple: true

      - name: Install evaluator dependencies
        run:  pip3 install pandas scipy tabulate

      - name: Clone test suite repository
        uses: actions/checkout@v4
        with:
          repository: trailofbits/vast-llvm-test-suite
          sparse-checkout: utils/vast_compare.py
          ref: main
          path: llvm-test-suite
          fetch-depth: 1

      - name: Generate the results
        run: |
          python3 llvm-test-suite/utils/vast_compare.py \
            --columns "HighLevel with Unsupported",HighLevel,LLVM,Binary \
            --files results-hl-with-unsup.json,results-hl.json,results-llvm-with-unsup.json,results-bin-with-unsup.json \
            --output single-source-results

      - name: Post results as artifacts
        uses: actions/upload-artifact@v4
        with:
          name: llvm-test-suite-results
          path: |
            ./single-source-results.csv
            ./single-source-results.md
#
#     SV-Comp bencmharks
#
  run_svcomp:
    name: "Run SV-Comp test suite"
    needs: build
    strategy:
      matrix:
        llvm-version: [19]
        image-version: [22.04]
        vast-target: ['hl']
        disable-unsup: [true, false]

    runs-on: ubuntu-${{ matrix.image-version }}
    timeout-minutes: 360
    container:
      image:
        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest

    steps:
     - name: Fetch VAST artifact
       uses: actions/download-artifact@v4
       with:
         name: VAST

     - name: Unpack VAST
       run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1

     - name: Export vast binaries
       run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH

     - name: Clone test suite repository
       run: git clone --depth=1 https://gitlab.com/sosy-lab/benchmarking/sv-benchmarks.git

     - name: Clone patch repository
       uses: actions/checkout@v4
       with:
        repository: trailofbits/sv-bench-patch
        ref: ci-version
        path: sv-bench-patch
        fetch-depth: 1

     - name: Apply patch
       working-directory: sv-bench-patch
       run: ./apply

     - name: Run benchmarks
       if: ${{ !matrix.disable-unsup }}
       run: sh ./sv-bench-patch/compile-all.sh -t ${{ matrix.vast-target }} -d ./sv-benchmarks

     - name: Run benchmarks without unsupported
       if: ${{ matrix.disable-unsup }}
       run: sh ./sv-bench-patch/compile-all.sh -t ${{ matrix.vast-target }} --disable-unsup -d ./sv-benchmarks

     - name: Setup result suffix
       run: |
        if [ "${{ matrix.disable-unsup }}" = "false" ]; then
          echo "VAST_RESULTS_SUFFIX=-with-unsup" >> $GITHUB_ENV
        fi

     - name: Upload results
       uses: actions/upload-artifact@v4
       with:
        path: ./stats
        name: results-sv-${{ matrix.vast-target }}${{ env.VAST_RESULTS_SUFFIX }}

  eval_svcomp:
    name: "Eval SV-Comp test results"
    needs: run_svcomp
    strategy:
      matrix:
        image-version: [22.04]
    runs-on: ubuntu-${{ matrix.image-version }}
    steps:
      - name: Fetch result artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: results-sv-*

      - name: Install evaluator dependencies
        run: pip3 install pandas scipy tabulate

      - name: Clone patch repository
        uses: actions/checkout@v4
        with:
          repository: trailofbits/sv-bench-patch
          sparse-checkout: make_table.py
          ref: ci-version
          path: sv-bench-patch
          fetch-depth: 1

      - name: Generate the results
        run: |
          python3 sv-bench-patch/make_table.py \
            --columns "HighLevel with Unsupported",HighLevel \
            --files results-sv-hl-with-unsup/results.txt,results-sv-hl/results.txt \
            --output sv-comp-results

      - name: Post results as artifacts
        uses: actions/upload-artifact@v4
        with:
          name: sv-comp-results
          path: ./sv-comp-results.md

#
#     Linux kernel benchmark
#
  run_linux_kernel_bench:
    name: "Run Linux kernel test suite"
    needs: build
    strategy:
      matrix:
        llvm-version: [19]
        image-version: [22.04]
        vast-target: ['hl']
        disable-unsup: [true, false]

    runs-on: ubuntu-${{ matrix.image-version }}
    timeout-minutes: 360
    container:
      image:
        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest

    steps:
      - name: Fetch VAST artifact
        uses: actions/download-artifact@v4
        with:
          name: VAST

      - name: Unpack VAST
        run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1

      - name: Export vast binaries
        run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH

      - name: Install benchmark dependencies
        run: |
          apt-get update
          apt-get -y install git fakeroot build-essential \
            ncurses-dev xz-utils libssl-dev bc flex libelf-dev bison \
            lld-${{ matrix.llvm-version }}

      - name: Clone, configure, and build the Linux kernel
        run: |
          git clone https://github.com/torvalds/linux.git --depth=1 --branch=v6.11 linux
          cd linux
          make defconfig LLVM=-${{ matrix.llvm-version }}
          make prepare LLVM=-${{ matrix.llvm-version }}
          make LLVM=-${{ matrix.llvm-version }} -j $(nproc)
          python3 ./scripts/clang-tools/gen_compile_commands.py

      - name: Clone vast benchmark directory
        uses: actions/checkout@v4
        with:
          repository: trailofbits/vast-benchmarks
          sparse-checkout: benchmarks/linux_kernel
          ref: main
          path: vast-benchmarks/
          fetch-depth: 1

      - name: Setup result suffix and vast arguments
        run: |
          if [ "${{ matrix.disable-unsup }}" = "false" ]; then
            echo "VAST_RESULTS_SUFFIX=with_unsup" >> $GITHUB_ENV
          else
            echo "VAST_RESULTS_SUFFIX=without_unsup" >> $GITHUB_ENV
            echo "VAST_DISABLE_UNSUPPORTED=-vast-disable-unsupported" >> $GITHUB_ENV
          fi

      - name: Run benchmarks
        run: >
          python3 ${PWD}/vast-benchmarks/benchmarks/linux_kernel/run_vast_benchmark.py
          vast-front
          ${PWD}/linux/compile_commands.json
          --num_processes=$(nproc)
          --vast_option="-xc"
          --vast_option="-vast-emit-mlir=${{ matrix.vast-target }}"
          --vast_option="${{ env.VAST_DISABLE_UNSUPPORTED }}"
          --print_errors
          > vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv
          path: ./vast_linux_kernel_times_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv

  convert_linux_kernel_bench:
    name: "Convert Linux Kernel benchmark results to Markdown"
    needs: run_linux_kernel_bench
    strategy:
      matrix:
        image-version: [22.04]
    runs-on: ubuntu-${{ matrix.image-version }}
    steps:
      - name: Fetch result artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: vast_linux_kernel_times_*
          merge-multiple: true

      - name: Install converter dependencies
        run: pip3 install pandas scipy tabulate

      - name: Clone vast benchmark directory
        uses: actions/checkout@v4
        with:
          repository: trailofbits/vast-benchmarks
          sparse-checkout: utils/
          ref: main
          path: vast-benchmarks/
          fetch-depth: 1

      - name: Generate the results
        run: >
          python3 vast-benchmarks/utils/to_markdown.py \
          '{ "HighLevel with unsupported": "vast_linux_kernel_times_hl_with_unsup.tsv", "HighLevel" : "vast_linux_kernel_times_hl_without_unsup.tsv" }'
          --output_filepath linux_kernel_times.md

      - name: Post results as artifacts
        uses: actions/upload-artifact@v4
        with:
          name: linux_kernel_times.md
          path: ./linux_kernel_times.md

#
#     GZIP benchmark
#
  run_gzip_bench:
    name: "Run gzip translation"
    needs: build
    strategy:
      matrix:
        llvm-version: [19]
        image-version: [22.04]
        gzip-version: ['1.10', '1.13']
        vast-target: ['hl', 'llvm']
        disable-unsup: [true, false]
        exclude:
            - vast-target: 'llvm'
              disable-unsup: false
            - vast-target: 'bin'
              disable-unsup: false

    runs-on: ubuntu-${{ matrix.image-version }}
    timeout-minutes: 360
    container:
      image:
        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest

    steps:
      - name: Fetch VAST artifact
        uses: actions/download-artifact@v4
        with:
          name: VAST

      - name: Unpack VAST
        run: mkdir vast && tar -xf VAST-* -C vast --strip-components=1

      - name: Export vast binaries
        run: echo "${PWD}/vast/bin/" >> $GITHUB_PATH

      - name: Install benchmark dependencies
        run: |
          apt-get update
          apt-get -y install bear

      - name: Download gzip and generate compile commands
        run: |
          wget https://ftp.gnu.org/gnu/gzip/gzip-${{ matrix.gzip-version }}.tar.gz
          tar -xvf gzip-${{ matrix.gzip-version }}.tar.gz
          cd gzip-${{ matrix.gzip-version }}
          ./configure
          bear -- make

      - name: Clone vast benchmark directory
        uses: actions/checkout@v4
        with:
          repository: trailofbits/vast-benchmarks
          sparse-checkout: benchmarks/linux_kernel
          ref: main
          path: vast-benchmarks/
          fetch-depth: 1

      - name: Setup result suffix and vast arguments
        run: |
          if [ "${{ matrix.disable-unsup }}" = "false" ]; then
            echo "VAST_RESULTS_SUFFIX=with_unsup" >> $GITHUB_ENV
          else
            echo "VAST_RESULTS_SUFFIX=without_unsup" >> $GITHUB_ENV
            echo "VAST_DISABLE_UNSUPPORTED=-vast-disable-unsupported" >> $GITHUB_ENV
          fi

      - name: Run benchmarks
        run: >
          python3 ${PWD}/vast-benchmarks/benchmarks/linux_kernel/run_vast_benchmark.py
          vast-front
          ${PWD}/gzip-${{ matrix.gzip-version }}/compile_commands.json
          --num_processes=$(nproc)
          --vast_option="-xc"
          --vast_option="-vast-emit-mlir=${{ matrix.vast-target }}"
          --vast_option="${{ env.VAST_DISABLE_UNSUPPORTED }}"
          --print_errors
          > gzip_${{ matrix.gzip-version }}_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv

      - name: Upload results
        uses: actions/upload-artifact@v4
        with:
          name: gzip_${{ matrix.gzip-version }}_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv
          path: ./gzip_${{ matrix.gzip-version }}_${{ matrix.vast-target }}_${{ env.VAST_RESULTS_SUFFIX }}.tsv

  convert_gzip_bench:
    name: "Convert gzip results to Markdown"
    needs: run_gzip_bench
    strategy:
      matrix:
        image-version: [22.04]
        gzip-version: ['1.10', '1.13']
    runs-on: ubuntu-${{ matrix.image-version }}
    steps:
      - name: Fetch result artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: gzip_*
          merge-multiple: true

      - name: Install converter dependencies
        run: pip3 install pandas scipy tabulate

      - name: Clone vast benchmark directory
        uses: actions/checkout@v4
        with:
          repository: trailofbits/vast-benchmarks
          sparse-checkout: utils/
          ref: main
          path: vast-benchmarks/
          fetch-depth: 1

      - name: Generate gzip results
        run: >
          python3 vast-benchmarks/utils/to_markdown.py '{
            "HighLevel with unsupported" : "gzip_${{ matrix.gzip-version }}_hl_with_unsup.tsv",
            "HighLevel" : "gzip_${{ matrix.gzip-version }}_hl_without_unsup.tsv",
            "LLVM" : "gzip_${{ matrix.gzip-version }}_llvm_without_unsup.tsv"
          }' --output_filepath gzip_times_${{ matrix.gzip-version }}.md

      - name: Post results as artifacts
        uses: actions/upload-artifact@v4
        with:
          name: gzip_times_${{ matrix.gzip-version }}.md
          path: ./gzip_times_${{ matrix.gzip-version }}.md

#
#     Webpage build
#
  build_doc:
    strategy:
      matrix:
        llvm-version: [19]
        image-version: [22.04]
    name: "Build VAST doc"
    runs-on: ubuntu-${{ matrix.image-version }}
    needs: [eval_llvm_ts, eval_svcomp, convert_linux_kernel_bench, convert_gzip_bench]
    timeout-minutes: 60
    container:
      image:
        ghcr.io/trailofbits/vast-ubuntu-${{ matrix.image-version }}-llvm-${{ matrix.llvm-version }}-dev:latest

    env:
        CMAKE_PREFIX_PATH: "/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/mlir/;/usr/lib/llvm-${{ matrix.llvm-version }}/lib/cmake/clang/"
        TOOLCHAIN: ${{ github.workspace }}/cmake/lld.toolchain.cmake
        LLVM_EXTERNAL_LIT: "/usr/local/bin/lit"

    steps:
      - name: Clone the VAST repository
        uses: actions/checkout@v4
        with:
          submodules: true
          fetch-depth: 1

      - name: Configure build
        run: cmake --preset ci

      - name: Build VAST Doc
        run: cmake --build --preset ci-release --target vast-doc

      - name: Fetch LLVM test suite results
        uses: actions/download-artifact@v4
        with:
          name: llvm-test-suite-results
          path: llvm-test-suite-results

      - name: Fetch SV-Comp benchmarks results
        uses: actions/download-artifact@v4
        with:
          name: sv-comp-results
          path: sv-comp-results

      - name: Fetch Linux kernel benchmark results
        uses: actions/download-artifact@v4
        with:
          name: linux_kernel_times.md
          path: linux_kernel_times

      - name: Fetch GZIP 1.10 benchmark results
        uses: actions/download-artifact@v4
        with:
            name: gzip_times_1.10.md
            path: gzip_times_1.10

      - name: Fetch GZIP 1.13 benchmark results
        uses: actions/download-artifact@v4
        with:
            name: gzip_times_1.13.md
            path: gzip_times_1.13
      - name: Build Pages
        run: |
          sh ./www/setup.sh _site ./builds/ci/

      - name: Deploy docs
        uses: mhausenblas/mkdocs-deploy-gh-pages@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          CONFIG_FILE: _site/mkdocs.yml