ci.yml

name: CI

on:
  push:
    branches:
      - main
    tags:
      - "**"
  pull_request: {}

jobs:
  resolve:
    runs-on: ubuntu-latest
    outputs:
      MSRV: ${{ steps.resolve-msrv.outputs.MSRV }}
    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: resolve MSRV
        id: resolve-msrv
        run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["workspace"]["package"]["rust-version"])'` >> $GITHUB_OUTPUT

  test:
    needs: [resolve]
    name: test rust-${{ matrix.rust-version }} on ${{ matrix.runs-on }}, Python ${{ matrix.python-version }}
    strategy:
      fail-fast: false
      matrix:
        rust-version: [stable]
        runs-on: [ubuntu, macos]
        python-version: ["3.13"]
        include:
          - rust-version: ${{ needs.resolve.outputs.MSRV }}
            runs-on: ubuntu
          - rust-version: stable
            runs-on: ubuntu
            python-version: "3.13t"
          - rust-version: nightly
            runs-on: ubuntu

    runs-on: ${{ matrix.runs-on }}-latest

    env:
      RUNS_ON: ${{ matrix.runs-on }}-latest
      RUST_VERSION: ${{ matrix.rust-version }}

    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: quansight-labs/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ matrix.rust-version }}

      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@cargo-llvm-cov

      - name: install cargo-careful
        if: matrix.rust-version == 'nightly'
        uses: taiki-e/install-action@cargo-careful

      - if: matrix.rust-version == 'nightly'
        run: cargo careful t -F python

      - run: cargo llvm-cov -F python --codecov --output-path codecov.json
        env:
          RUST_BACKTRACE: 1

      - uses: codecov/codecov-action@v4
        with:
          env_vars: RUNS_ON,RUST_VERSION
          files: codecov.json
          token: ${{ secrets.CODECOV_TOKEN }}

  test-python:
    name: test jiter-python ${{ matrix.python-version }}

    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.13", "3.13t"]

    env:
      RUNS_ON: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: quansight-labs/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - uses: dtolnay/rust-toolchain@stable
      - uses: Swatinem/rust-cache@v2
      - uses: taiki-e/install-action@cargo-llvm-cov

      - name: Install python dependencies
        run: pip install -r crates/jiter-python/tests/requirements.txt

      - name: Build jiter-python
        run: |
          source <(cargo llvm-cov show-env --export-prefix)
          cargo llvm-cov clean --workspace --profraw-only
          pip install -e crates/jiter-python --config-settings=build-args='--profile dev'

      - name: Run tests
        run: |
          source <(cargo llvm-cov show-env --export-prefix)
          pytest crates/jiter-python/tests
          python crates/jiter-python/bench.py jiter jiter-cache --fast
        env:
          RUST_BACKTRACE: 1

      - name: Coverage report
        run: |
          source <(cargo llvm-cov show-env --export-prefix)
          cargo llvm-cov report --codecov --output-path codecov.json

      - uses: codecov/codecov-action@v4
        with:
          files: codecov.json
          token: ${{ secrets.CODECOV_TOKEN }}

  bench:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - uses: moonrepo/setup-rust@v1
        with:
          channel: stable
          cache-target: release
          bins: cargo-codspeed

      - run: python crates/jiter/benches/generate_big.py

      - run: cargo codspeed build -F python -p jiter

      - uses: CodSpeedHQ/action@v3
        with:
          run: cargo codspeed run
          token: ${{ secrets.CODSPEED_TOKEN }}

  fuzz:
    name: fuzz on ${{ matrix.runs-on }}
    strategy:
      fail-fast: false
      matrix:
        runs-on: [ubuntu, macos]

    runs-on: ${{ matrix.runs-on }}-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@nightly
      - id: cache-rust
        uses: Swatinem/rust-cache@v2
      - if: steps.cache-rust.outputs.cache-hit != 'true'
        run: cargo install cargo-fuzz
      - run: |
          # cargo fuzz defaults to musl targets, which is seeming incomatible with sanitizers according to CI failures
          RUST_TARGET=$(rustc -Vv | grep host | cut -d ' ' -f 2)
          cargo fuzz run --target=$RUST_TARGET --fuzz-dir crates/fuzz compare_to_serde --release -- -max_total_time=300s

  fuzz-skip:
    name: fuzz skip
    # we only run this on ubuntu since architecture should make no difference

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: dtolnay/rust-toolchain@nightly
      - id: cache-rust
        uses: Swatinem/rust-cache@v2
      - if: steps.cache-rust.outputs.cache-hit != 'true'
        run: cargo install cargo-fuzz

      - run: |
          # cargo fuzz defaults to musl targets, which is seeming incomatible with sanitizers according to CI failures
          RUST_TARGET=$(rustc -Vv | grep host | cut -d ' ' -f 2)
          cargo fuzz run --target=$RUST_TARGET --fuzz-dir crates/fuzz compare_skip --release -- -max_total_time=300s

  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - uses: dtolnay/rust-toolchain@stable
        with:
          components: rustfmt,clippy

      - uses: Swatinem/rust-cache@v2

      - uses: pre-commit/action@v3.0.0
        with:
          extra_args: --all-files --verbose
        env:
          PRE_COMMIT_COLOR: always
          SKIP: test

      - run: cargo doc -F python

  build-sdist:
    name: build sdist
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: set up python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"
      - uses: PyO3/maturin-action@v1
        with:
          command: sdist
          args: --out dist
          rust-toolchain: stable
          working-directory: crates/jiter-python
      - uses: actions/upload-artifact@v4
        with:
          name: pypi_files_sdist
          path: crates/jiter-python/dist

  build:
    name: build on ${{ matrix.os }} (${{ matrix.target }} - ${{ matrix.interpreter || 'all' }}${{ matrix.os == 'linux' && format(' - {0}', matrix.manylinux == 'auto' && 'manylinux' || matrix.manylinux) || '' }})
    # only run on push to main and on release
    if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
    strategy:
      fail-fast: false
      matrix:
        os: [linux] # windows and macos builds added manually
        target: [x86_64, aarch64]
        manylinux: [auto]
        include:
          # manylinux for various platforms
          - os: linux
            manylinux: auto
            target: i686
          - os: linux
            manylinux: auto
            target: aarch64
          - os: linux
            manylinux: auto
            target: armv7
            interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
          - os: linux
            manylinux: auto
            target: ppc64le
            interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
          - os: linux
            manylinux: auto
            target: s390x
            interpreter: 3.8 3.9 3.10 3.11 3.12 3.13

          # musllinux
          - os: linux
            manylinux: musllinux_1_1
            target: x86_64
          - os: linux
            manylinux: musllinux_1_1
            target: aarch64

          # macos;
          # all versions x86_64
          # older pythons which can't be run on the arm hardware for PGO
          - os: macos
            target: x86_64
          - os: macos
            target: aarch64
            interpreter: 3.8 3.9

          # windows;
          # aarch64 only 3.11 and up, also not PGO optimized
          - os: windows
            target: i686
            python-architecture: x86
            interpreter: 3.8 3.9 3.10 3.11 3.12 3.13
          # FIXME had link failures, needs investigation
          # - os: windows
          #   target: aarch64
          #   interpreter: 3.11 3.12
        exclude:
          # PGO optimized below
          - os: linux
            manylinux: auto
            target: x86_64

    runs-on: ${{ (matrix.os == 'linux' && 'ubuntu') || matrix.os }}-latest
    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"
          architecture: ${{ matrix.python-architecture || 'x64' }}

      - name: build wheels
        uses: PyO3/maturin-action@v1
        with:
          target: ${{ matrix.target }}
          manylinux: ${{ matrix.manylinux }}
          args: --release --out dist --interpreter ${{ matrix.interpreter || '3.8 3.9 3.10 3.11 3.12 3.13' }}
          rust-toolchain: stable
          working-directory: crates/jiter-python

      - run: ${{ (matrix.os == 'windows' && 'dir') || 'ls -lh' }} crates/jiter-python/dist/

      - run: |
          pip install -U twine
          # FIXME: restore `twine` check when metadata version 2.4 (used by `maturin`) supported by `twine`,
          # see https://github.com/pypa/twine/pull/1180
          # twine check --strict crates/jiter-python/dist/*

      - uses: actions/upload-artifact@v4
        with:
          name: pypi_files_${{ matrix.os }}_${{ matrix.target }}_${{ matrix.interpreter || 'all' }}_${{ matrix.manylinux }}
          path: crates/jiter-python/dist

  build-pgo:
    name: build pgo-optimized on ${{ matrix.os }} / ${{ matrix.interpreter }}
    # only run on push to main and on release
    if: startsWith(github.ref, 'refs/tags/') || github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'Full Build')
    strategy:
      fail-fast: false
      matrix:
        os: [linux, windows, macos]
        interpreter: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.13t"]
        include:
          # standard runners with override for macos arm
          - os: linux
            runs-on: ubuntu-latest
          - os: windows
            ls: dir
            runs-on: windows-latest
          - os: macos
            runs-on: macos-latest
        exclude:
          # macos arm only supported from 3.10 and up
          - os: macos
            interpreter: "3.8"
          - os: macos
            interpreter: "3.9"
          # windows free-threaded build has a problematic wheel name,
          # needs something like https://github.com/PyO3/maturin/pull/2325 to fix
          - os: windows
            interpreter: "3.13t"

    runs-on: ${{ matrix.runs-on }}
    steps:
      - uses: actions/checkout@v4

      - id: setup-python
        name: set up python
        uses: quansight-labs/setup-python@v5
        with:
          python-version: ${{ matrix.interpreter }}
          allow-prereleases: true

      - name: install rust stable
        id: rust-toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          components: llvm-tools

      # linux: builds are done inside manylinux docker images, need to help maturin-action
      # find the right path
      #
      # TODO, would be nice to remove, see
      # https://github.com/PyO3/maturin-action/issues/300
      - if: ${{ matrix.os == 'linux' }}
        id: resolve-interpreter
        name: resolve interpreter
        run: |
          echo python-path=/opt/python/$(python -c "print({
            '3.8': 'cp38-cp38',
            '3.9': 'cp39-cp39',
            '3.10': 'cp310-cp310',
            '3.11': 'cp311-cp311',
            '3.12': 'cp312-cp312',
            '3.13': 'cp313-cp313',
            '3.13t': 'cp313-cp313t',
          }['${{ matrix.interpreter }}'])")/bin/python >> $GITHUB_OUTPUT

      - name: build pgo wheel
        uses: ./.github/actions/build-pgo-wheel
        with:
          interpreter: ${{ matrix.os == 'linux' && steps.resolve-interpreter.outputs.python-path || steps.setup-python.outputs.python-path }}
          rust-toolchain: ${{ steps.rust-toolchain.outputs.name }}

      - run: ${{ matrix.ls || 'ls -lh' }} crates/jiter-python/dist/

      - uses: actions/upload-artifact@v4
        with:
          name: pypi_files_${{ matrix.os }}_${{ matrix.interpreter }}
          path: crates/jiter-python/dist

  inspect-pypi-assets:
    needs: [build, build-sdist, build-pgo]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: get dist artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: pypi_files_*
          merge-multiple: true
          path: dist

      - name: list dist files
        run: |
          ls -lh dist/
          ls -l dist/
          echo "`ls dist | wc -l` files"

      - name: extract and list sdist file
        run: |
          mkdir sdist-files
          tar -xvf dist/*.tar.gz -C sdist-files
          tree -a sdist-files

      - name: extract and list wheel file
        run: |
          ls dist/*cp310-manylinux*x86_64.whl | head -n 1
          python -m zipfile --list `ls dist/*cp310-manylinux*x86_64.whl | head -n 1`

  test-builds-arch:
    name: test build on ${{ matrix.target }}-${{ matrix.distro }}
    needs: [build]
    runs-on: ubuntu-latest

    strategy:
      fail-fast: false
      matrix:
        target: [aarch64, armv7, s390x, ppc64le]
        distro: ["ubuntu22.04"]
        include:
          - target: aarch64
            distro: alpine_latest

    steps:
      - uses: actions/checkout@v4

      - name: get dist artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: pypi_files_linux_*
          merge-multiple: true
          path: crates/jiter-python/dist

      - uses: uraimo/run-on-arch-action@v2
        name: install & test
        with:
          arch: ${{ matrix.target }}
          distro: ${{ matrix.distro }}
          githubToken: ${{ github.token }}
          install: |
            set -x
            if command -v apt-get &> /dev/null; then
              echo "installing python & pip with apt-get..."
              apt-get update
              apt-get install -y --no-install-recommends python3 python3-pip python3-venv git
            else
              echo "installing python & pip with apk..."
              apk update
              apk add python3 py3-pip git
            fi
          run: |
            cd crates/jiter-python
            set -x
            # typing-extensions isn't automatically installed because of `--no-index --no-deps`
            python3 -m venv venv
            source venv/bin/activate
            python3 -m pip install -U pip -r tests/requirements.txt
            python3 -m pip install jiter --no-index --no-deps --find-links dist --force-reinstall
            python3 -m pytest
            python3 -c 'import jiter; print(jiter.__version__)'

  test-builds-os:
    name: test build on ${{ matrix.os }}
    needs: [build, build-pgo]

    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu, macos, windows]

    runs-on: ${{ matrix.os }}-latest
    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: get dist artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: pypi_files_*
          merge-multiple: true
          path: crates/jiter-python/dist

      - name: run tests
        run: |
          python3 -m pip install -U pip -r tests/requirements.txt
          python3 -m pip install jiter --no-index --no-deps --find-links dist --force-reinstall
          python3 -m pytest
          python3 -c 'import jiter; print(jiter.__version__)'
        working-directory: crates/jiter-python

  # https://github.com/marketplace/actions/alls-green#why used for branch protection checks
  check:
    if: always()
    needs: [test, test-python, bench, fuzz, fuzz-skip, lint]
    runs-on: ubuntu-latest
    steps:
      - name: Decide whether the needed jobs succeeded or failed
        uses: re-actors/alls-green@release/v1
        with:
          jobs: ${{ toJSON(needs) }}

  release:
    needs: [check]
    if: "success() && startsWith(github.ref, 'refs/tags/')"
    runs-on: ubuntu-latest
    environment: release

    steps:
      - uses: actions/checkout@v4

      - name: install rust stable
        uses: dtolnay/rust-toolchain@stable

      - uses: Swatinem/rust-cache@v2

      - run: cargo publish -p jiter
        env:
          CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}

  release-python:
    needs: [test-builds-arch, test-builds-os, build-sdist, check]
    if: "success() && startsWith(github.ref, 'refs/tags/')"
    runs-on: ubuntu-latest
    environment: release-python
    permissions:
      id-token: write

    steps:
      - uses: actions/checkout@v4

      - name: set up python
        uses: actions/setup-python@v5
        with:
          python-version: "3.13"

      - name: get dist artifacts
        uses: actions/download-artifact@v4
        with:
          pattern: pypi_files_*
          merge-multiple: true
          path: dist

      - run: pip install -U twine
      - run: ls -l dist/
      # FIXME: restore `twine` check when metadata version 2.4 (used by `maturin`) supported by `twine`
      # see https://github.com/pypa/twine/pull/1180
      # - run: twine check --strict dist/*

      - name: upload to pypi
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          packages-dir: dist/