From 4d383a6458ba43642d5662434e8ee79cc4c290a8 Mon Sep 17 00:00:00 2001 From: Sebastian Wezel Date: Fri, 11 Aug 2023 11:03:36 +0000 Subject: [PATCH] tools: adding pip requirements check workflow * The workflow merges the requirements from nrf, mcuboot and zephyr * Checks if the requirements-fixed and new generated file match * Packages that where only in the current requirements-fixed are added to requirements-ci.txt * Following packages will be added to requirements-fixed.txt with this change Deprecated Markdown PyGithub PyJWT PyNaCl azure-core azure-storage-blob capstone dill editdistance exceptiongroup gitlint-core graphviz grpcio importlib-resources isodate jsonschema-specifications lark libusb-package m2r2 mistune msgpack pathspec pkgutil_resolve_name platformdirs pypng referencing rpds-py sphinx-copybutton sphinx-markdown-tables sphinx-ncs-theme sphinx-notfound-page sphinx-togglebutton sphinxcontrib-jquery sphinxcontrib-plantuml Signed-off-by: Sebastian Wezel --- .github/workflows/pip-requirements.yml | 81 +++++++ scripts/fix-requirements.sh | 36 +++ scripts/requirements-build.txt | 2 +- scripts/requirements-ci.txt | 15 ++ scripts/requirements-extra.txt | 1 - scripts/requirements-fixed.txt | 205 ++++++++++-------- scripts/requirements.txt | 2 + tests/modules/lib/zcbor/decode/CMakeLists.txt | 2 +- tests/modules/lib/zcbor/encode/CMakeLists.txt | 2 +- 9 files changed, 246 insertions(+), 100 deletions(-) create mode 100644 .github/workflows/pip-requirements.yml create mode 100755 scripts/fix-requirements.sh create mode 100644 scripts/requirements-ci.txt diff --git a/.github/workflows/pip-requirements.yml b/.github/workflows/pip-requirements.yml new file mode 100644 index 00000000000..145273cfe35 --- /dev/null +++ b/.github/workflows/pip-requirements.yml @@ -0,0 +1,81 @@ +name: Validate pip requirements-fixed.txt + +on: + pull_request: + types: [opened, synchronize, reopened] + branches: + - main + - 'v*-branch' + paths: + - 'scripts/requirements*.txt' + +jobs: + build: + runs-on: ubuntu-20.04 + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + path: ncs/nrf + fetch-depth: 0 + + - name: Get python version + id: pyv + run: | + sudo snap install yq + PYTHON_VERSION=$(yq '.python.version' ./ncs/nrf/scripts/tools-versions-linux.yml) + echo "python_version=$PYTHON_VERSION" >> $GITHUB_OUTPUT + + - name: Setup python version + uses: actions/setup-python@v4 + with: + python-version: '${{ steps.pyv.outputs.python_version }}' + + - name: Setup environment + working-directory: ncs + run: | + pip3 install --user -U setuptools wheel pip virtualenv virtualenvwrapper + pip3 install -r nrf/scripts/requirements-base.txt + west init -l nrf + west update mcuboot zephyr + + - name: Generate new requirements-fixed.txt + working-directory: ncs + run: nrf/scripts/fix-requirements.sh requirements-fixed.txt + + - name: Store requirements-fixed + uses: actions/upload-artifact@v3 + with: + name: requirements-fixed + path: ncs/requirements-fixed.txt + + - name: Compare requirements-fixed + id: diff + run: | + NEW=ncs/requirements-fixed.txt + OLD=ncs/nrf/scripts/requirements-fixed.txt + echo -e "$NEW\t\t\t\t\t"$OLD + diff -y $NEW $OLD + + - name: Post summary + if: ${{ !cancelled() }} + run: | + if [[ "failure" == "${{ steps.diff.outcome }}" ]]; then + echo -e 'New requirements-fixed.txt file differs from the current file. + Please add the newly created file to the PR.' >> $GITHUB_STEP_SUMMARY + + fi + if [[ -e ncs/requirements-fixed.txt ]]; then + echo -e ' +
+ requirements-fixed.txt + + ```' >> $GITHUB_STEP_SUMMARY + cat ncs/requirements-fixed.txt >> $GITHUB_STEP_SUMMARY + echo -e '``` + +
' >> $GITHUB_STEP_SUMMARY + else + echo -e 'Error during workflow, requirements-fixed file was not created' >> GITHUB_STEP_SUMMARY + fi diff --git a/scripts/fix-requirements.sh b/scripts/fix-requirements.sh new file mode 100755 index 00000000000..aea92be4c67 --- /dev/null +++ b/scripts/fix-requirements.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +# This script merges the py-reqs from mcuboot, zephyr, and nrf. + +OUT_FILE=$1 +[ -z "$OUT_FILE" ] && echo "Error output file not provided" && exit 1 +echo "Writing frozen requirements to: $OUT_FILE" +echo "Log python version: $(python --version)" + +TOPDIR=$(west topdir) +cd $TOPDIR + +echo "##### Generated Python Requirements #####" > $OUT_FILE +echo "##### Do not edit this file manually #####" >> $OUT_FILE +echo "###########################################################" >> $OUT_FILE +echo "##### Merged inputs: #####" >> $OUT_FILE +echo "##### bootloader/mcuboot/scripts/requirements.txt #####" >> $OUT_FILE +echo "##### zephyr/scripts/requirements.txt #####" >> $OUT_FILE +echo "##### nrf/scripts/requirements.txt #####" >> $OUT_FILE +echo "" >> $OUT_FILE + +source ~/.local/bin/virtualenvwrapper.sh +[[ $? != 0 ]] && echo "error sourcing virtualenvwrapper" && exit 1 + +rmvirtualenv pip-fixed-venv +mkvirtualenv pip-fixed-venv +workon pip-fixed-venv +pip3 install --isolated \ + -r bootloader/mcuboot/scripts/requirements.txt \ + -r zephyr/scripts/requirements.txt \ + -r nrf/scripts/requirements.txt +pip3 freeze --all | LC_ALL=C sort --ignore-case >> $OUT_FILE +#Set LC_ALL=C to have the same sorting behaviour on all systems + +deactivate +rmvirtualenv pip-fixed-venv diff --git a/scripts/requirements-build.txt b/scripts/requirements-build.txt index cf5a880b502..dba1bacfcec 100644 --- a/scripts/requirements-build.txt +++ b/scripts/requirements-build.txt @@ -6,4 +6,4 @@ imagesize>=1.2.0 intelhex protobuf pylint -zcbor>=0.7.0 +zcbor==0.6.0 diff --git a/scripts/requirements-ci.txt b/scripts/requirements-ci.txt new file mode 100644 index 00000000000..2c4110a0b1e --- /dev/null +++ b/scripts/requirements-ci.txt @@ -0,0 +1,15 @@ +aenum +bitarray +chardet +gitlint==0.18.0 +jsonschema +milksnake +py +pyparsing +python-stdnum +pytz +pyusb +qrcode +typed-ast +toml +wget diff --git a/scripts/requirements-extra.txt b/scripts/requirements-extra.txt index 5ecaec7b53f..bbf739a2f8a 100644 --- a/scripts/requirements-extra.txt +++ b/scripts/requirements-extra.txt @@ -1,4 +1,3 @@ pygit2<=1.10 editdistance>=0.5.0 PyGithub -zcbor>=0.5.1 diff --git a/scripts/requirements-fixed.txt b/scripts/requirements-fixed.txt index 07f1cc6f993..a1a3ce56524 100644 --- a/scripts/requirements-fixed.txt +++ b/scripts/requirements-fixed.txt @@ -1,117 +1,130 @@ +##### Generated Python Requirements ##### +##### Do not edit this file manually ##### ########################################################### -##### Fixed Python Requirements ##### -########################################################### +##### Merged inputs: ##### +##### bootloader/mcuboot/scripts/requirements.txt ##### +##### zephyr/scripts/requirements.txt ##### +##### nrf/scripts/requirements.txt ##### -Babel==2.9.1 -Jinja2==2.11.3 -MarkupSafe==1.1.1 -Pillow==9.4.0 -PyYAML==5.4.1 -Pygments==2.7.4 -Sphinx==3.4.3 -aenum==3.0.0 -alabaster==0.7.12 -anytree==2.8.0 +aenum==3.1.15 +anytree==2.9.0 appdirs==1.4.4 -arrow==0.17.0 -astroid==2.15.0 -attrs==20.3.0 -bitarray==2.7.3 -breathe==4.26.1 -canopen==1.2.0 -cbor2==5.4.3 +arrow==1.2.1 +astroid==2.15.6 +attrs==23.1.0 +bitarray==2.8.1 +canopen==2.1.0 +capstone==4.0.2 +cbor2==5.4.6 cbor==1.0.0 certifi==2023.7.22 -cffi==1.14.4 -chardet==4.0.0 -charset-normalizer==3.1.0 -click==7.1.2 -cmsis-pack-manager==0.2.10 +cffi==1.15.1 +chardet==5.2.0 +charset-normalizer==3.2.0 +clang-format==16.0.6 +click==8.0.3 +cmsis-pack-manager==0.5.2 colorama==0.4.6 -commonmark==0.9.1 -coverage==5.4 +coverage==7.3.0 cryptography==41.0.3 +Deprecated==1.2.14 +dill==0.3.7 docopt==0.6.2 -docutils==0.16 -ecdsa==0.16.1 +ecdsa==0.18.0 +editdistance==0.6.2 +exceptiongroup==1.1.3 future==0.18.3 -gcovr==4.2 -gitlint==0.15.0 -grpcio-tools==1.51.1 -idna==2.10 -imagesize==1.2.0 -imgtool==1.7.0 -importlib-metadata==3.4.0 -iniconfig==1.1.1 +gcovr==6.0 +gitlint-core==0.18.0 +gitlint==0.18.0 +graphviz==0.20.1 +grpcio-tools==1.57.0 +grpcio==1.57.0 +idna==3.4 +imagesize==1.4.1 +imgtool==1.10.0 +importlib-metadata==6.8.0 +importlib-resources==6.0.1 +iniconfig==2.0.0 intelhex==2.3.0 intervaltree==3.1.0 -isort==5.7.0 -jsonschema==4.4.0 -junit2html==30.0.4 -junitparser==2.8.0 -lazy-object-proxy==1.4.3 +isort==5.12.0 +Jinja2==3.1.2 +jsonschema-specifications==2023.7.1 +jsonschema==4.19.0 +junit2html==30.1.3 +junitparser==3.1.0 +lark==1.1.7 +lazy-object-proxy==1.9.0 +libusb-package==1.0.26.2 lpc-checksum==2.2.0 -lxml==4.9.1 -mccabe==0.6.1 +lxml==4.9.3 +MarkupSafe==2.1.3 +mccabe==0.7.0 milksnake==0.1.5 -mock==4.0.3 -mypy-extensions==0.4.3 -mypy==0.921 -natsort==8.3.1 -packaging==20.8 -pluggy==0.13.1 +mock==5.1.0 +msgpack==1.0.5 +mypy-extensions==1.0.0 +mypy==1.5.1 +natsort==8.4.0 +packaging==23.1 +pathspec==0.11.2 +Pillow==10.0.0 +pip==23.2.1 +pkgutil_resolve_name==1.3.10 +platformdirs==3.10.0 +pluggy==1.3.0 ply==3.11 -prettytable==2.0.0 -progress==1.5 -psutil==5.8.0 -py==1.10.0 -pycparser==2.20 -pyelftools==0.27 +prettytable==3.8.0 +progress==1.6 +protobuf==4.24.2 +psutil==5.9.5 +py==1.11.0 +pycparser==2.21 +pyelftools==0.29 pygit2==1.10.0 +PyGithub==1.59.1 +Pygments==2.16.1 +PyJWT==2.8.0 pykwalify==1.8.0 -pylink-square==0.8.1 -pylint==2.16.4 -pyocd==0.29.0 -pyparsing==2.4.7 +pylink-square==1.2.0 +pylint==2.17.5 +PyNaCl==1.5.0 +pyocd==0.35.1 +pyparsing==3.1.1 +pypng==0.20220715.0 pyserial==3.5 -pytest==6.2.2 -python-can==3.3.4 -python-dateutil==2.8.1 -python-magic==0.4.18 -python-stdnum==1.18 -pytz==2020.5 -pyusb==1.2.0 -qrcode==7.3.1 -recommonmark==0.6.0 -regex==2022.3.15 +pytest==7.4.0 +python-can==4.2.2 +python-dateutil==2.8.2 +python-magic==0.4.27 +python-stdnum==1.19 +pytz==2023.3 +pyusb==1.2.1 +PyYAML==6.0.1 +qrcode==7.4.2 +referencing==0.30.2 +regex==2023.8.8 requests==2.31.0 -ruamel.yaml.clib==0.2.6 -ruamel.yaml==0.17.21 -sh==1.14.1 -six==1.15.0 -snowballstemmer==2.1.0 -sortedcontainers==2.3.0 -sphinx-rtd-theme==0.5.1 -sphinx-tabs==2.0.0 -sphinxcontrib-applehelp==1.0.2 -sphinxcontrib-devhelp==1.0.2 -sphinxcontrib-htmlhelp==1.0.3 -sphinxcontrib-jsmath==1.0.1 -sphinxcontrib-mscgen==0.5 -sphinxcontrib-qthelp==1.0.3 -sphinxcontrib-serializinghtml==1.1.4 -sphinxcontrib-svg2pdfconverter==1.1.1 -setuptools==65.5.1 -tabulate==0.8.7 +rpds-py==0.10.0 +ruamel.yaml.clib==0.2.7 +ruamel.yaml==0.17.32 +setuptools==68.0.0 +sh==1.14.2 +six==1.16.0 +sortedcontainers==2.4.0 +tabulate==0.9.0 toml==0.10.2 tomli==2.0.1 -typed-ast==1.5.4 -typing-extensions==4.4.0 -urllib3==2.0.3 -wcwidth==0.2.5 -west==1.0.0 +tomlkit==0.12.1 +typed-ast==1.5.5 +typing_extensions==4.7.1 +urllib3==2.0.4 +wcwidth==0.2.6 +west==1.1.0 wget==3.2 -wrapt==1.12.1 -yamllint==1.30.0 -zcbor==0.5.1 -zipp==3.4.0 +wheel==0.41.1 +wrapt==1.15.0 +yamllint==1.32.0 +zcbor==0.6.0 +zipp==3.16.2 diff --git a/scripts/requirements.txt b/scripts/requirements.txt index c60572796d8..baf8bcd19c9 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -1,2 +1,4 @@ -r requirements-base.txt -r requirements-build.txt +-r requirements-ci.txt +-r requirements-extra.txt diff --git a/tests/modules/lib/zcbor/decode/CMakeLists.txt b/tests/modules/lib/zcbor/decode/CMakeLists.txt index 94e81ed92d6..d377c6e128b 100644 --- a/tests/modules/lib/zcbor/decode/CMakeLists.txt +++ b/tests/modules/lib/zcbor/decode/CMakeLists.txt @@ -12,8 +12,8 @@ project(test_zcbor_decode) find_program(ZCBOR zcbor REQUIRED) set(zcbor_args - -c ${CMAKE_CURRENT_LIST_DIR}/../cddl/test.cddl code + -c ${CMAKE_CURRENT_LIST_DIR}/../cddl/test.cddl --output-c ${PROJECT_BINARY_DIR}/src/test_decode.c --output-h ${PROJECT_BINARY_DIR}/include/test_decode.h -t Test diff --git a/tests/modules/lib/zcbor/encode/CMakeLists.txt b/tests/modules/lib/zcbor/encode/CMakeLists.txt index 4f0c7c5ea58..4af6d499f3e 100644 --- a/tests/modules/lib/zcbor/encode/CMakeLists.txt +++ b/tests/modules/lib/zcbor/encode/CMakeLists.txt @@ -12,8 +12,8 @@ project(test_zcbor_encode) find_program(ZCBOR zcbor REQUIRED) set(zcbor_args - -c ${CMAKE_CURRENT_LIST_DIR}/../cddl/test.cddl code + -c ${CMAKE_CURRENT_LIST_DIR}/../cddl/test.cddl --output-c ${PROJECT_BINARY_DIR}/src/test_encode.c --output-h ${PROJECT_BINARY_DIR}/include/test_encode.h -t Test