diff --git a/.github/workflows/continuous-integration.yaml b/.github/workflows/continuous-integration.yaml index 71dfc91ad..d308581d4 100644 --- a/.github/workflows/continuous-integration.yaml +++ b/.github/workflows/continuous-integration.yaml @@ -92,7 +92,7 @@ env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - + FAILED_TESTS_ARE_FLAKY: "false" jobs: matrix-preparation: @@ -542,7 +542,8 @@ jobs: make determinism # Build the documentation if : - # - during weekly or release CI, as well as when the CI has been triggered manually (through + # - the current workflow takes place in a release CI with the reference build + # - the current workflow takes place in a weekly CI or it has been triggered manually (through # GitHub's Action interface) # - any documentation files has been changed # - the source code has been changed @@ -551,7 +552,8 @@ jobs: id: build-docs if: | ( - steps.changed-files-in-pr.outcome == 'skipped' + (fromJSON(env.IS_RELEASE) && fromJSON(env.IS_REF_BUILD)) + || steps.changed-files-in-pr.outcome == 'skipped' || steps.changed-files-in-pr.outputs.docs_any_changed == 'true' || steps.changed-files-in-pr.outputs.use_cases_any_changed == 'true' || steps.changed-files-in-pr.outputs.src_any_changed == 'true' @@ -562,22 +564,17 @@ jobs: && steps.determinism.outcome != 'failure' && !cancelled() run: | - make docs - - # The changelog is generated by considering all commits from the latest stable previous - # version (not a release candidate) up to the new upcoming version - # This may change in the future - # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3909 - - name: Generate release changelog - id: changelog - if: ${{ fromJSON(env.IS_RELEASE) && steps.install-deps.outcome == 'success' && !cancelled() }} + make docs_no_links + + # Do not check links during the release process in order to avoid temporary connection errors + - name: Check links + id: check_links + if: | + !fromJSON(env.IS_RELEASE) + && steps.build-docs.outcome == 'success' + && !cancelled() run: | - PROJECT_VERSION="$(poetry version --short)" - GIT_TAG="v${PROJECT_VERSION}" - CHANGELOG_FILE="CHANGELOG_${GIT_TAG}.md" - echo "changelog-file=${CHANGELOG_FILE}" >> $GITHUB_OUTPUT - poetry run python ./script/make_utils/changelog_helper.py \ - --to-ref "${{ github.sha }}" > "${CHANGELOG_FILE}" + make check_links # Make sure all necessary steps passed. For build-docs and determinism steps, we only check for # non-failures as the 'changed-files-in-pr' step might skip them @@ -589,54 +586,86 @@ jobs: ${{ steps.commit-conformance.outcome == 'success' && steps.make-pcc.outcome == 'success' - && steps.build-docs.outcome != 'failure' && steps.determinism.outcome != 'failure' + && steps.build-docs.outcome != 'failure' + && steps.check_links.outcome != 'failure' }} run: | if [[ "${CONFORMANCE_STATUS}" != "true" ]]; then echo "Conformance failed, got:" echo "Commit conformance success step: ${{ steps.commit-conformance.outcome }}" echo "Make conformance step: ${{ steps.make-pcc.outcome }}" - echo "Build docs step: ${{ steps.build-docs.outcome }}" echo "Determinism step: ${{ steps.determinism.outcome }}" + echo "Build docs step: ${{ steps.build-docs.outcome }}" + echo "Check links step: ${{ steps.check_links.outcome }}" exit 1 fi + # Tar the docs for releases with the reference build only # Taring the docs allows for much faster upload speed (from ~3min worst case to ~2s best case) - name: Tar docs artifacts id: tar-docs - if: ${{ steps.conformance.outcome == 'success' && steps.build-docs.outcome == 'success' && !cancelled() }} + if: | + fromJSON(env.IS_RELEASE) + && fromJSON(env.IS_REF_BUILD) + && steps.conformance.outcome == 'success' + && steps.build-docs.outcome == 'success' + && !cancelled() run: | cd docs/_build/html tar -cvf docs.tar ./* - # Only upload docs once from reference build - - name: Archive docs artifacts - if: ${{ fromJSON(env.IS_REF_BUILD) && steps.tar-docs.outcome == 'success' && !cancelled() }} + - name: Upload docs artifacts + if: ${{ steps.tar-docs.outcome == 'success' && !cancelled() }} uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce with: name: html-docs path: docs/_build/html/docs.tar + # Generate the changelog for releases with the reference build only + # The changelog is generated by considering all commits from the latest stable previous + # version (not a release candidate) up to the new upcoming version + # This may change in the future + # FIXME: https://github.com/zama-ai/concrete-ml-internal/issues/3909 + - name: Generate release changelog + id: changelog + if: | + fromJSON(env.IS_RELEASE) + && fromJSON(env.IS_REF_BUILD) + && steps.conformance.outcome == 'success' + && !cancelled() + run: | + PROJECT_VERSION="$(poetry version --short)" + GIT_TAG="v${PROJECT_VERSION}" + CHANGELOG_FILE="CHANGELOG_${GIT_TAG}.md" + echo "changelog-file=${CHANGELOG_FILE}" >> $GITHUB_OUTPUT + poetry run python ./script/make_utils/changelog_helper.py \ + --to-ref "${{ github.sha }}" > "${CHANGELOG_FILE}" + - name: Upload changelog artifacts - if: ${{ fromJSON(env.IS_REF_BUILD) && steps.changelog.outcome == 'success' && !cancelled() }} + if: ${{ steps.changelog.outcome == 'success' && !cancelled() }} uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce with: name: changelog path: ${{ steps.changelog.outputs.changelog-file }} + # Build the wheel for releases with the reference build only # Create packages before tests, to be able to get them if some unexpected test failure happens # Build the package only once, as we don't have binary dependency this can be used on Linux # and macOS as long as the dependencies are available - name: Build wheel id: build-wheel - if: ${{ fromJSON(env.IS_REF_BUILD) && steps.conformance.outcome == 'success' && !cancelled() }} + if: | + fromJSON(env.IS_RELEASE) + && fromJSON(env.IS_REF_BUILD) + && steps.conformance.outcome == 'success' + && !cancelled() run: | rm -rf dist poetry build -f wheel - name: Upload wheel artifacts - if: ${{ fromJSON(env.IS_REF_BUILD) && steps.build-wheel.outcome == 'success' }} + if: ${{ steps.build-wheel.outcome == 'success' && !cancelled() }} uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce with: name: py3-wheel diff --git a/Makefile b/Makefile index cf441033f..38dada693 100644 --- a/Makefile +++ b/Makefile @@ -386,8 +386,9 @@ docker_clean_volumes: .PHONY: docker_cv # Docker clean volumes docker_cv: docker_clean_volumes -.PHONY: docs # Build docs -docs: clean_docs check_docs_dollars + +.PHONY: docs_no_links # Build docs +docs_no_links: clean_docs check_docs_dollars @# Rebuild the index from README.md, to have in the home of Sphinx a copy of README.md echo " .. Warning, auto-generated by \`make docs\`, don\'t edit" > docs/index.rst echo "" >> docs/index.rst @@ -420,6 +421,9 @@ docs: clean_docs check_docs_dollars cp -r use_case_examples docs-copy/_build/ cp -r docs-copy/_build docs/ rm -rf docs-copy + +.PHONY: docs # Build docs and check links +docs: docs_no_links @# Check links "$(MAKE)" check_links