lint.yml

name: Lint

on:
  push:
    branches:
      - master
  pull_request:

jobs:
  quick-checks:
    name: quick-checks
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Checkout PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
      - name: Clean PyTorch checkout
        run: |
          # Remove any artifacts from the previous checkouts
          git clean -fxd
      - name: Install requirements
        id: requirements
        run: pip3 install -r requirements.txt --user
      - name: Ensure consistent CircleCI YAML config
        if: ${{ always() && steps.requirements.outcome == 'success' }}
        run: cd .circleci && ./ensure-consistency.py
      - name: Lint native_functions.yaml
        if: ${{ always() && steps.requirements.outcome == 'success' }}
        run: |
          pip3 install ruamel.yaml==0.17.4 --user
          .github/scripts/lint_native_functions.py
      - name: Ensure correct trailing newlines
        if: ${{ always() && steps.requirements.outcome == 'success' }}
        run: |
          (! git --no-pager grep -Il '' -- . ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude)**.expect' ':(exclude)**.ipynb' ':(exclude)tools/clang_format_hash' | tools/linter/trailing_newlines.py || (echo "The above files do not have correct trailing newlines; please normalize them"; false))
      - name: Ensure no trailing spaces
        if: always()
        run: |
          (! git --no-pager grep -In '[[:blank:]]$' -- . ':(exclude)**/contrib/**' ':(exclude)**.diff' ':(exclude)third_party' || (echo "The above lines have trailing spaces; please remove them"; false))
      - name: Ensure no tabs
        if: always()
        run: |
          (! git --no-pager grep -In $'\t' -- . ':(exclude)*.svg' ':(exclude)**Makefile' ':(exclude)**/contrib/**' ':(exclude)third_party' ':(exclude).gitattributes' ':(exclude).gitmodules' || (echo "The above lines have tabs; please convert them to spaces"; false))
      - name: Ensure no non-breaking spaces
        if: always()
        run: |
          # NB: We use 'printf' below rather than '\u000a' since bash pre-4.2
          # does not support the '\u000a' syntax (which is relevant for local linters)
          (! git --no-pager grep -In "$(printf '\xC2\xA0')" -- . || (echo "The above lines have non-breaking spaces (U+00A0); please convert them to spaces (U+0020)"; false))
      - name: Ensure canonical include
        if: always()
        run: |
          (! git --no-pager grep -In $'#include "' -- ./c10 ./aten ./torch/csrc ':(exclude)aten/src/ATen/native/quantized/cpu/qnnpack/**' ':(exclude)torch/csrc/jit/serialization/mobile_bytecode_generated.h'|| (echo "The above lines have include with quotes; please convert them to #include <xxxx>"; false))
      - name: Ensure no versionless Python shebangs
        if: always()
        run: |
          (! git --no-pager grep -In '#!.*python$' -- . || (echo "The above lines have versionless Python shebangs; please specify either python2 or python3"; false))
      - name: Ensure no unqualified noqa
        if: always()
        run: |
          # shellcheck disable=SC2016
          (! git --no-pager grep -InP '# noqa(?!: [A-Z]+\d{3})' -- '**.py' '**.pyi' ':(exclude)caffe2' || (echo 'The above lines have unqualified `noqa`; please convert them to `noqa: XXXX`'; false))
      - name: Ensure no unqualified type ignore
        if: always()
        run: |
          # shellcheck disable=SC2016
          (! git --no-pager grep -InP '# type:\s*ignore(?!\[)' -- '**.py' '**.pyi' ':(exclude)test/test_jit.py' || (echo 'The above lines have unqualified `type: ignore`; please convert them to `type: ignore[xxxx]`'; false))
      - name: Ensure GitHub PyPi dependencies are pinned
        if: always()
        run: |
          (! git --no-pager grep --color=always -InP \
                '(pip|pip3|python -m pip|python3 -m pip|python3 -mpip|python -mpip) install ([a-z][\.a-z-0-9]*+(?!(=|.*\.whl))([[:blank:]]|))+' \
                -- .github \
                ':(exclude)**.rst' \
                ':(exclude)**.py' \
                ':(exclude)**.md' \
                ':(exclude)**.diff' \
                ':(exclude)third_party' ||
            (echo "The above lines have unpinned PyPi installs; please pin them to a specific version: e.g. 'thepackage==1.2'"; false))
      # note that this next step depends on a clean checkout;
      # if you run it locally then it will likely to complain
      # about all the generated files in torch/test
      - name: Ensure C++ source files are not executable
        if: always()
        run: |
          # shellcheck disable=SC2016
          (! find . \( -path ./third_party -o -path ./.git -o -path ./torch/bin -o -path ./build \) -prune -o -type f -executable -regextype posix-egrep -not -regex '.+(\.(bash|sh|py|so)|git-pre-commit|git-clang-format|gradlew)$' -print | grep . || (echo 'The above files have executable permission; please remove their executable permission by using `chmod -x`'; false))
      - name: C++ docs check
        if: ${{ always() && steps.requirements.outcome == 'success' }}
        run: |
          sudo apt-get install -y doxygen
          cd docs/cpp/source && ./check-doxygen.sh
      - name: CUDA kernel launch check
        if: ${{ always() && steps.requirements.outcome == 'success' }}
        run: |
          set -eux
          python torch/testing/_check_kernel_launches.py |& tee "${GITHUB_WORKSPACE}"/cuda_kernel_launch_checks.txt
      - name: Ensure no direct cub include
        if: always()
        run: |
          (! git --no-pager grep -I -no $'#include <cub/' --  ./aten  ':(exclude)aten/src/ATen/cuda/cub*.cuh' || (echo "The above files have direct cub include; please include ATen/cuda/cub.cuh instead and wrap your cub calls in at::native namespace if necessary"; false))
      - name: Ensure no raw cuda api calls
        if: always()
        run: |
          (! git --no-pager grep -I -no $'cudaStreamSynchronize' --  ./aten ./c10 ':(exclude)aten/src/ATen/test' ':(exclude)c10/cuda/CUDAFunctions.h' || (echo "The above files call raw cuda APIs directly; please use at::cuda wrappers instead"; false))
      - name: Ensure all test files have header containing ownership information
        if: always()
        run: |
          python3 -m pip install boto3==1.19.12
          .github/scripts/lint_test_ownership.py

  clang-format:
    name: clang-format
    runs-on: ubuntu-18.04
    if: ${{ github.event_name == 'pull_request' }}
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
        with:
          fetch-depth: 0 # deep clone, to allow us to use git merge-base
      - name: Run clang-format
        env:
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
        run: |
          set -eu
          # This is necessary to get the same results regardless of whether the
          # PR was opened directly or from a forked repo. See: `9f890a92` for more info.
          git remote add upstream https://github.com/pytorch/pytorch
          git fetch upstream "$GITHUB_BASE_REF"

          # only run clang-format on allowlisted files
          echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
          echo "| clang-format failures found! Run: "
          echo "|    tools/linter/clang_format_ci.sh ${BASE_SHA} "
          echo "| to fix this error. "
          echo "| For more info, see: https://github.com/pytorch/pytorch/wiki/clang-format "
          echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"

          tools/linter/clang_format_ci.sh "${BASE_SHA}"

          GIT_DIFF=$(git diff)
          if [[ -z $GIT_DIFF ]]; then
            exit 0
          fi
          echo "$GIT_DIFF"
          exit 1

  py2-setup-validate-errormsg:
    name: py2-setup-validate-errormsg
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 2.x
          architecture: x64
      - name: Checkout PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
      - name: Attempt to run setup.py
        run: |
          if ! python2 setup.py | grep -q "Python 2 has reached end-of-life and is no longer supported by PyTorch."; then
            echo 'Running setup.py with Python 2 did not give the expected error message.'
            false
          fi
      - name: Keep torch.utils.collect_env python2 compliant
        run: python2 -m py_compile torch/utils/collect_env.py

  shellcheck:
    name: shellcheck
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Checkout PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
      - name: Install requirements
        id: requirements
        run: |
          pip3 install -r requirements.txt --user
      - name: Install Jinja2
        run: |
          pip3 install Jinja2==3.0.1 --user
      - name: Checkout PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
      - name: Regenerate workflows
        id: generate_workflows
        run: .github/scripts/generate_ci_workflows.py
      - name: Assert that regenerating the workflows didn't change them
        run: |
          if ! .github/scripts/report_git_status.sh .github/workflows; then
            echo
            echo 'As shown by the above diff, the committed .github/workflows'
            echo 'are not up to date according to .github/templates.'
            echo 'Please run this command, commit, and push again to your PR:'
            echo
            echo '    .github/scripts/generate_ci_workflows.py'
            echo
            echo 'If running that command does nothing, you may need to rebase'
            echo 'onto a more recent commit from the PyTorch master branch.'
            false
          fi
      - name: Install ShellCheck
        id: install_shellcheck
        if: always()
        # https://github.com/koalaman/shellcheck/tree/v0.7.2#installing-a-pre-compiled-binary
        run: |
          set -x
          scversion="v0.7.2"
          wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${scversion?}/shellcheck-${scversion?}.linux.x86_64.tar.xz" | tar -xJv
          mkdir -p ~/.local/bin
          cp "shellcheck-${scversion}/shellcheck" ~/.local/bin/
          rm -r "shellcheck-${scversion}"
          ~/.local/bin/shellcheck --version
      - name: Extract scripts from GitHub Actions workflows
        if: ${{ always() && steps.install_shellcheck.outcome == 'success' }}
        run: |
          # For local lints, remove the .extracted_scripts folder if it was already there
          rm -rf .extracted_scripts
          tools/extract_scripts.py --out=.extracted_scripts
      - name: Run ShellCheck
        if: ${{ always() && steps.install_shellcheck.outcome == 'success' }}
        run: |
          if ! tools/linter/run_shellcheck.sh .extracted_scripts .jenkins/pytorch; then
            echo
            echo 'ShellCheck gave a nonzero exit code. Please fix the warnings'
            echo 'listed above. Note that if a path in one of the above warning'
            echo 'messages starts with .extracted_scripts/ then that means it'
            echo 'is referring to a shell script embedded within another file,'
            echo 'whose path is given by the path components immediately'
            echo 'following the .extracted_scripts/ prefix.'
            false
          fi
      - name: Check that jobs will be cancelled
        if: ${{ always() && steps.generate_workflows.outcome == 'success' }}
        run: |
          .github/scripts/ensure_actions_will_cancel.py
      - name: Run actionlint
        shell: bash
        run: |
          set -eux
          bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
          ./actionlint --color
          rm actionlint

  toc:
    name: toc
    runs-on: ubuntu-18.04
    # https://github.com/actions/virtual-environments/issues/599#issuecomment-602754687
    env:
      NPM_CONFIG_PREFIX: ~/.npm-global
    steps:
      - name: Setup Node
        uses: actions/setup-node@v2
      - name: Checkout PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
      - name: Install markdown-toc
        run: npm install -g markdown-toc
      - name: Regenerate ToCs and check that they didn't change
        run: |
          set -eu
          export PATH=~/.npm-global/bin:"$PATH"
          for FILE in $(git grep -Il '<!-- toc -->' -- '**.md'); do
            markdown-toc --bullets='-' -i "$FILE"
          done

          if ! .github/scripts/report_git_status.sh .; then
            echo
            echo 'As shown by the above diff, the table of contents in one or'
            echo 'more Markdown files is not up to date with the file contents.'
            echo 'You can either apply that Git diff directly to correct the'
            echo 'table of contents, or if you have npm installed, you can'
            echo 'install the npm package markdown-toc and run the following'
            # shellcheck disable=SC2016
            echo 'command (replacing $FILE with the filename for which you want'
            echo 'to regenerate the table of contents):'
            echo
            # shellcheck disable=SC2016
            echo "    markdown-toc --bullets='-' -i \"\$FILE\""
            false
          fi

  flake8-py3:
    name: flake8-py3
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
        with:
          fetch-depth: 2 # to allow us to use github.event.pull_request.head.sha
      - name: Prepare output dir with HEAD commit SHA
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          mkdir flake8-output
          cd flake8-output
          echo "$HEAD_SHA" > commit-sha.txt
      - name: Install dependencies
        run: |
          set -eux
          pip3 install typing-extensions==3.10 --user # for tools/linter/translate_annotations.py
          pip3 install -r requirements-flake8.txt --user
          flake8 --version
      - name: Run flake8
        run: |
          set -eux
          flake8 | tee "${GITHUB_WORKSPACE}"/flake8-output.txt
      - name: Translate annotations
        if: ${{ github.event_name == 'pull_request' }}
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          tools/linter/translate_annotations.py \
            --file="${GITHUB_WORKSPACE}"/flake8-output.txt \
            --regex='^(?P<filename>.*?):(?P<lineNumber>\d+):(?P<columnNumber>\d+): (?P<errorCode>\w+\d+) (?P<errorDesc>.*)' \
            --commit="$HEAD_SHA" \
            > flake8-output/annotations.json
      - name: Fail if there were any warnings
        run: |
          set -eu
          # Re-output flake8 status so GitHub logs show it on the step that actually failed
          cat "${GITHUB_WORKSPACE}"/flake8-output.txt
          if [ -s "${GITHUB_WORKSPACE}"/flake8-output.txt ]; then
            echo 'Please fix the above Flake8 warnings.'
            false
          fi
      - name: Add annotations
        # Don't run on forked pull requests
        if: ${{ failure() && github.event.pull_request.head.repo.full_name == github.repository }}
        uses: pytorch/add-annotations-github-action@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          check_name: 'flake8-py3'
          linter_output_path: flake8-output/annotations.json
          commit_sha: ${{ github.event.pull_request.head.sha }}
          mode: json

  clang-tidy:
    name: clang-tidy
    runs-on: [self-hosted, linux.2xlarge]
    container:
      # ubuntu20.04-cuda11.2-py3.8-tidy11
      image: ghcr.io/pytorch/cilint-clang-tidy:d8f0c777964d0dd8a147360de80aed1a13eb613a
    steps:
      - name: Clean workspace
        run: |
          rm -rf "${GITHUB_WORKSPACE}"
          mkdir "${GITHUB_WORKSPACE}"
      - name: Checkout PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
        with:
          fetch-depth: 0 # to allow tools/linter/clang_tidy.py to do its thing
      - name: Prepare output dir with HEAD commit SHA
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          cd "${GITHUB_WORKSPACE}"
          mkdir clang-tidy-output
          cd clang-tidy-output
          echo "$HEAD_SHA" > commit-sha.txt
      - name: Fetch PR diff
        if: ${{ github.event_name == 'pull_request' }}
        env:
          PR_NUMBER: ${{ github.event.pull_request.number }}
        run: |
          cd "${GITHUB_WORKSPACE}"
          wget -O pr.diff "https://patch-diff.githubusercontent.com/raw/pytorch/pytorch/pull/$PR_NUMBER.diff"
      - name: Generate build files
        run: |
          cd "${GITHUB_WORKSPACE}"
          python3 -m tools.linter.clang_tidy.generate_build_files
      - name: Run PR clang-tidy
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          cd "${GITHUB_WORKSPACE}"

          # The Docker image has our custom build, so we don't need to install it
          python3 -m tools.linter.clang_tidy \
            --clang-tidy-exe "$(which clang-tidy)" \
            --diff-file pr.diff \
            --disable-progress-bar 2>&1 | tee "${GITHUB_WORKSPACE}"/clang-tidy-output.txt

      # Run clang-tidy on a smaller subset of the codebase on master until we
      # make the repository clang-tidy clean
      - name: Run master clang-tidy
        run: |
          cd "${GITHUB_WORKSPACE}"

          python3 -m tools.linter.clang_tidy \
            --paths \
              torch/csrc/fx \
              torch/csrc/utils \
              torch/csrc/generic \
              torch/csrc/deploy \
              torch/csrc/tensor \
            --clang-tidy-exe "$(which clang-tidy)" \
            --disable-progress-bar 2>&1 | tee -a "${GITHUB_WORKSPACE}"/clang-tidy-output.txt

      - name: Annotate output
        if: ${{ github.event_name == 'pull_request' }}
        env:
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
        run: |
          cd "${GITHUB_WORKSPACE}"
          sed --in-place 's/^\.\.\///g' clang-tidy-output.txt
          tools/linter/translate_annotations.py \
            --file=clang-tidy-output.txt \
            --regex='^(?P<filename>.*?):(?P<lineNumber>\d+):(?P<columnNumber>\d+): (?P<errorDesc>.*?) \[(?P<errorCode>.*)\]' \
            --commit="$HEAD_SHA" \
            > clang-tidy-output/annotations.json
      - name: Check for warnings
        run: |
          cd "${GITHUB_WORKSPACE}"
          set -eu
          cat "${GITHUB_WORKSPACE}"/clang-tidy-output.txt
          if grep -Fq "Warnings detected!" "${GITHUB_WORKSPACE}"/clang-tidy-output.txt; then
            echo 'Please fix the above clang-tidy warnings.'
            false
          fi
      - name: Add annotations
        # Don't run on forked pull requests
        if: ${{ failure() && github.event.pull_request.head.repo.full_name == github.repository }}
        uses: pytorch/add-annotations-github-action@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          check_name: 'clang-tidy'
          linter_output_path: clang-tidy/annotations.json
          commit_sha: ${{ github.event.pull_request.head.sha }}
          mode: json

  cmakelint:
    name: cmakelint
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.x
          architecture: x64
      - name: Fetch PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
      - name: Install dependencies
        run: |
          set -eux
          pip3 install cmakelint==1.4.1 --user
          cmakelint --version
      - name: Run cmakelint
        run: |
          set -eux
          git ls-files -z -- bootstrap '*.cmake' '*.cmake.in' '*CMakeLists.txt' | \
          grep -E -z -v '^(cmake/Modules/|cmake/Modules_CUDA_fix/|cmake/Caffe2Config.cmake.in|aten/src/ATen/ATenConfig.cmake.in|cmake/Caffe2ConfigVersion.cmake.in|cmake/TorchConfig.cmake.in|cmake/TorchConfigVersion.cmake.in|cmake/cmake_uninstall.cmake.in)' | \
          xargs -0 cmakelint --config=.cmakelintrc --spaces=2 --quiet

  mypy:
    name: mypy
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
          architecture: x64
      - name: Fetch PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
      - name: Install dependencies
        run: |
          set -eux
          python3 -mpip install -r requirements.txt --user
          python3 -mpip install numpy==1.20 --user # https://github.com/pytorch/pytorch/pull/60472
          python3 -mpip install expecttest==0.1.3 mypy==0.812 --user
          # Needed to check tools/render_junit.py
          python3 -mpip install junitparser==2.1.1 rich==10.9.0 --user
      - name: Run autogen
        run: |
          set -eux
          time python3 -mtools.generate_torch_version --is_debug=false
          time python3 -mtools.codegen.gen -s aten/src/ATen -d build/aten/src/ATen
          time python3 -mtools.pyi.gen_pyi --native-functions-path aten/src/ATen/native/native_functions.yaml --deprecated-functions-path "tools/autograd/deprecated.yaml"
      - name: Run mypy
        env:
          MYPY_FORCE_COLOR: 1
          TERM: xterm-color
        run: |
          set -eux
          STATUS=
          for CONFIG in mypy*.ini; do
            if ! python3 -mmypy --config="$CONFIG"; then
              STATUS=fail
            fi
          done
          if [ -n "$STATUS" ]; then
            echo 'Please fix the above mypy warnings.'
            false
          fi

  test-tools:
    name: Test tools
    if: ${{ github.repository == 'pytorch/pytorch' }}
    runs-on: ubuntu-18.04
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
          architecture: x64
      - name: Checkout PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
        with:
          fetch-depth: 0 # deep clone, to allow us to use git log
      - name: Install dependencies
        # mypy and boto3 versions copied from
        # .circleci/docker/common/install_conda.sh
        run: |
          set -eux
          python3 -mpip install -r requirements.txt
          python3 -mpip install boto3==1.16.34
          make setup_lint
      - name: Test tools
        run: |
          python3 -m unittest discover -vs tools/test -p 'test_*.py'
          python3 -m unittest discover -vs .github/scripts -p 'test_*.py'

  test_collect_env:
    if: ${{ github.repository == 'pytorch/pytorch' }}
    name: Test collect_env
    runs-on: ubuntu-18.04
    strategy:
      matrix:
        with_torch: [with_torch, without_torch]
    steps:
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
          architecture: x64
      - name: Checkout PyTorch
        uses: zhouzhuojie/checkout@05b13c9a0d21f08f6d5e64a1d5042246d13619d9
        with:
          fetch-depth: 0 # deep clone, to allow us to use git log
      - name: Install torch
        if: matrix.with_torch == 'with_torch'
        run: |
          # Doesn't really matter what torch version, we just need ANY torch installed
          pip install 'torch==1.*'
      - name: Run collect_env.py
        run: |
          # All we need to see is that it passes
          python3 torch/utils/collect_env.py

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
  cancel-in-progress: true