diff --git a/.github/actions/build-test-protocol/Dockerfile b/.github/actions/build-test-protocol/Dockerfile deleted file mode 100644 index fb4062ea0..000000000 --- a/.github/actions/build-test-protocol/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ghcr.io/aica-technology/control-libraries/development-dependencies:latest - -# Copy and set the entrypoint commands to execute when the container starts -COPY entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/build-test-protocol/action.yml b/.github/actions/build-test-protocol/action.yml deleted file mode 100644 index 134a24e6f..000000000 --- a/.github/actions/build-test-protocol/action.yml +++ /dev/null @@ -1,5 +0,0 @@ -name: 'Build and Test Protocol' -description: 'Build the protobuf message libraries and run all unit tests' -runs: - using: 'docker' - image: 'Dockerfile' \ No newline at end of file diff --git a/.github/actions/build-test-protocol/entrypoint.sh b/.github/actions/build-test-protocol/entrypoint.sh deleted file mode 100755 index 81b86e807..000000000 --- a/.github/actions/build-test-protocol/entrypoint.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -l - -echo ">>> Installing control libraries..." -bash /github/workspace/source/install.sh --auto --no-controllers --no-dynamical-systems --no-robot-model || exit 1 -ldconfig - -echo ">>> Building proto bindings..." -cd /github/workspace/protocol/protobuf && make all || exit 2 - -echo ">>> Configuring clproto_cpp cmake..." -cd /github/workspace/protocol/clproto_cpp && mkdir build && cd build \ - && cmake -DCMAKE_BUILD_TYPE=RELEASE -DBUILD_TESTING=ON .. \ - || (echo ">>> [ERROR] Configuration stage failed!" && exit 3) || exit $? - -echo ">>> Building clproto_cpp..." -make all || (echo ">>> [ERROR] Build stage failed!" && exit 4) || exit $? -echo ">>> Build stage completed successfully!" - -echo ">>> Running all test stages..." -CTEST_OUTPUT_ON_FAILURE=1 make test || (echo ">>> [ERROR] Test stage failed!" && exit 5) || exit $? -echo ">>> Test stages completed successfully!" - -exit 0 \ No newline at end of file diff --git a/.github/actions/build-test-python/Dockerfile b/.github/actions/build-test-python/Dockerfile deleted file mode 100644 index fb4062ea0..000000000 --- a/.github/actions/build-test-python/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ghcr.io/aica-technology/control-libraries/development-dependencies:latest - -# Copy and set the entrypoint commands to execute when the container starts -COPY entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/build-test-python/action.yml b/.github/actions/build-test-python/action.yml deleted file mode 100644 index e2d0133f2..000000000 --- a/.github/actions/build-test-python/action.yml +++ /dev/null @@ -1,5 +0,0 @@ -name: 'Build and Test Python' -description: 'Build the python bindings and run all unit tests' -runs: - using: 'docker' - image: 'Dockerfile' diff --git a/.github/actions/build-test-python/entrypoint.sh b/.github/actions/build-test-python/entrypoint.sh deleted file mode 100755 index 23022a820..000000000 --- a/.github/actions/build-test-python/entrypoint.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/sh -l - -echo ">>> Installing control libraries..." -bash /github/workspace/source/install.sh --auto || exit 1 -bash /github/workspace/protocol/install.sh --auto || exit 1 - -echo ">>> Building Python bindings..." - -export OSQP_INCLUDE_DIR='/usr/local/include/osqp' -pip3 install /github/workspace/python || (echo ">>> [ERROR] Build stage failed!" && exit 2) || exit $? - -echo ">>> Running all test stages..." -pytest /github/workspace/python -v || (echo ">>> [ERROR] Test stage failed!" && exit 3) || exit $? - -echo ">>> Test stages completed successfully!" - -exit 0 \ No newline at end of file diff --git a/.github/actions/build-test/Dockerfile b/.github/actions/build-test/Dockerfile deleted file mode 100644 index fb4062ea0..000000000 --- a/.github/actions/build-test/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM ghcr.io/aica-technology/control-libraries/development-dependencies:latest - -# Copy and set the entrypoint commands to execute when the container starts -COPY entrypoint.sh /entrypoint.sh -ENTRYPOINT ["/entrypoint.sh"] diff --git a/.github/actions/build-test/action.yml b/.github/actions/build-test/action.yml deleted file mode 100644 index 0ff45aa08..000000000 --- a/.github/actions/build-test/action.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: 'Build and Test' -description: 'Build the project sources and run all unit tests' -inputs: - configuration: - description: 'CMake build configuration' - required: false - default: '' -runs: - using: 'docker' - image: 'Dockerfile' - args: - - ${{ inputs.configuration }} \ No newline at end of file diff --git a/.github/actions/build-test/entrypoint.sh b/.github/actions/build-test/entrypoint.sh deleted file mode 100755 index 1c7f24aa7..000000000 --- a/.github/actions/build-test/entrypoint.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -l - -CONFIGURATION="" -if [ -n "$1" ]; then - echo ">>> Configuration: $1" - CONFIGURATION="-DCMAKE_BUILD_TYPE=$1" -fi - -cd /github/workspace/source && mkdir build && cd build || exit 1 - -echo ">>> Configuring cmake..." -cmake "${CONFIGURATION}" -DBUILD_TESTING=ON \ - -DBUILD_CONTROLLERS=ON -DBUILD_DYNAMICAL_SYSTEMS=ON -DBUILD_ROBOT_MODEL=ON .. || \ - (echo ">>> [ERROR] Configuration stage failed!" && exit 2) || exit $? - -echo ">>> Building project..." -make -j all || (echo ">>> [ERROR] Build stage failed!" && exit 3) || exit $? -echo ">>> Build stage completed successfully!" - -echo ">>> Running all test stages..." -CTEST_OUTPUT_ON_FAILURE=1 make test || (echo ">>> [ERROR] Test stage failed!" && exit 4) || exit $? -echo ">>> Test stages completed successfully!" - -exit 0 \ No newline at end of file diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 127479460..5b66b291a 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -1,6 +1,5 @@ name: Build and Test -# Run workflow on pushes to main and develop branches, on any pull request, or by manual dispatch on: push: branches: @@ -9,7 +8,6 @@ on: pull_request: workflow_dispatch: -# Define the build test jobs for each configuration jobs: check-contribution: @@ -17,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Check contributions if: ${{ github.event.pull_request.base.sha }} @@ -39,7 +37,6 @@ jobs: fi shell: bash - # check if jobs can be skipped check-skippable-changes: name: Check skippable changes runs-on: ubuntu-latest @@ -54,62 +51,30 @@ jobs: paths_ignore: '["**.md", ".**/**", "**.gitignore"]' skip_after_successful_duplicate: 'true' - build-test-debug: + build-test: needs: check-skippable-changes if: ${{ needs.check-skippable-changes.outputs.skip != 'true' }} runs-on: ubuntu-latest - name: Debug configuration test + name: Run tests steps: - # First check out the repository - name: Checkout - uses: actions/checkout@v3 - # Load the repository build-test action - - name: Build and Test - uses: ./.github/actions/build-test + uses: actions/checkout@v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Build Cache for Docker + uses: actions/cache@v3 with: - configuration: Debug - - build-test-release: - needs: check-skippable-changes - if: ${{ needs.check-skippable-changes.outputs.skip != 'true' }} - runs-on: ubuntu-latest - name: Release configuration test - steps: - # First check out the repository - - name: Checkout - uses: actions/checkout@v3 - # Load the repository build-test action + path: build-cache + key: ${{ runner.os }}-build-cache-${{ hashFiles('./') }} + - name: Inject build-cache into docker + uses: reproducible-containers/buildkit-cache-dance@v2.1.2 + with: + cache-source: build-cache + cache-target: /build - name: Build and Test - uses: ./.github/actions/build-test + uses: docker/build-push-action@v5 with: - configuration: Release - - build-test-python: - needs: - - check-skippable-changes - - build-test-release - if: ${{ needs.check-skippable-changes.outputs.skip != 'true' }} - runs-on: ubuntu-latest - name: Python bindings test - steps: - # First check out the repository - - name: Checkout - uses: actions/checkout@v3 - # Load the repository build-test-python action - - name: Build and Test Python - uses: ./.github/actions/build-test-python - - build-test-protocol: - needs: - - check-skippable-changes - - build-test-release - if: ${{ needs.check-skippable-changes.outputs.skip != 'true' }} - runs-on: ubuntu-latest - name: Protocol library test - steps: - # First check out the repository - - name: Checkout - uses: actions/checkout@v3 - # Load the repository build-test-python action - - name: Build and Test Protocol - uses: ./.github/actions/build-test-protocol + context: . + target: python-test + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/CHANGELOG.md b/CHANGELOG.md index da33030fe..411d520fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Release Versions: ## Upcoming changes (in development) +- ci: use caching from docker to run tests in CI (#429) - feat: add IO states to state representation (proto) (#172) - feat: add IO states to state representation (cpp) (#158) - build: add missing licenses (#170) diff --git a/Dockerfile b/Dockerfile index 2cc0ecb9e..8b160bd8f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,16 +76,16 @@ xargs -a /tmp/new-packages.txt dpkg-query -L \ # this root can then be copied to / to install everything globally or use LD_LIBRARY_PATH to use it locally HEREDOC -FROM base as dep-base +FROM base as base-dependencies ARG TARGETPLATFORM ARG CACHEID COPY dependencies/base_dependencies.cmake CMakeLists.txt -RUN --mount=type=cache,target=./build,id=cmake-osqp-${TARGETPLATFORM}-${CACHEID},uid=1000 \ +RUN --mount=type=cache,target=/build,id=cmake-base-deps-${TARGETPLATFORM}-${CACHEID},uid=1000 \ cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build && cmake --install build --prefix /tmp/deps -FROM base as dep-pinocchio +FROM base as pinocchio-dependencies COPY --from=apt-dependencies /tmp/apt / -COPY --from=dep-base /tmp/deps /usr +COPY --from=base-dependencies /tmp/deps /usr ARG TARGETPLATFORM ARG CACHEID ARG PINOCCHIO_TAG=v2.9.0 @@ -94,13 +94,13 @@ ARG HPP_FCL_TAG=v1.8.1 # * `pinocchio` doesn't provide an include directory we can easily plug into `target_include_directories` and thus needs to be installed first # * `pinocchio` uses hacks relying on undocumented CMake quirks which break if you use `FetchContent` # FIXME: it needs `CMAKE_INSTALL_PREFIX` and `--prefix` because it doesn't install to the right place otherwise -RUN --mount=type=cache,target=./hpp-fcl,id=cmake-hpp-fcl-src-${HPP_FCL_TAG}-${TARGETPLATFORM}-${CACHEID},uid=1000 \ - --mount=type=cache,target=./build,id=cmake-hpp-fcl-${HPP_FCL_TAG}-${TARGETPLATFORM}-${CACHEID},uid=1000 \ +RUN --mount=type=cache,target=/hpp-fcl,id=cmake-hpp-fcl-src-${HPP_FCL_TAG}-${TARGETPLATFORM}-${CACHEID},uid=1000 \ + --mount=type=cache,target=/build,id=cmake-hpp-fcl-${HPP_FCL_TAG}-${TARGETPLATFORM}-${CACHEID},uid=1000 \ if [ ! -f hpp-fcl/CMakeLists.txt ]; then rm -rf hpp-fcl/* && git clone --depth 1 -b ${HPP_FCL_TAG} --recursive https://github.com/humanoid-path-planner/hpp-fcl; fi \ && cmake -B build -S hpp-fcl -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_PYTHON_INTERFACE=OFF -DCMAKE_INSTALL_PREFIX=/tmp/deps \ && cmake --build build --target all install -RUN --mount=type=cache,target=./pinocchio,id=cmake-pinocchio-src-${PINOCCHIO_TAG}-${TARGETPLATFORM}-${CACHEID},uid=1000 \ - --mount=type=cache,target=./build,id=cmake-pinocchio-${PINOCCHIO_TAG}-${TARGETPLATFORM}-${CACHEID},uid=1000 \ +RUN --mount=type=cache,target=/pinocchio,id=cmake-pinocchio-src-${PINOCCHIO_TAG}-${TARGETPLATFORM}-${CACHEID},uid=1000 \ + --mount=type=cache,target=/build,id=cmake-pinocchio-${PINOCCHIO_TAG}-${TARGETPLATFORM}-${CACHEID},uid=1000 \ if [ ! -f pinocchio/CMakeLists.txt ]; then rm -rf pinocchio/* && git clone --depth 1 -b ${PINOCCHIO_TAG} --recursive https://github.com/stack-of-tasks/pinocchio; fi \ && cmake -B build -S pinocchio -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=Release -DBUILD_PYTHON_INTERFACE=OFF -DBUILD_WITH_COLLISION_SUPPORT=ON -DCMAKE_INSTALL_PREFIX=/tmp/deps \ && cmake --build build --target all install @@ -111,20 +111,16 @@ FROM base as dependencies ARG TARGETPLATFORM ARG CACHEID # Needed to build `osqp-eigen` -COPY --from=dep-base /tmp/deps /usr +COPY --from=base-dependencies /tmp/deps /usr COPY dependencies/dependencies.cmake CMakeLists.txt -RUN --mount=type=cache,target=./build,id=cmake-deps-${TARGETPLATFORM}-${CACHEID},uid=1000 \ - cmake -B build -DCMAKE_BUILD_TYPE=Release \ - && cmake --build build \ - && cmake --install build --prefix /tmp/deps -COPY --from=dep-base /tmp/deps /tmp/deps -COPY --from=dep-pinocchio /tmp/deps /tmp/deps +RUN --mount=type=cache,target=/build,id=cmake-deps-${TARGETPLATFORM}-${CACHEID},uid=1000 \ + cmake -B build -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release && cmake --build build && cmake --install build --prefix /tmp/deps +COPY --from=base-dependencies /tmp/deps /tmp/deps +COPY --from=pinocchio-dependencies /tmp/deps /tmp/deps FROM base as code -WORKDIR /src COPY --from=apt-dependencies /tmp/apt / COPY --from=dependencies /tmp/deps /usr -COPY . /src FROM code as development # create and configure a new user @@ -152,61 +148,60 @@ RUN ( \ COPY ./docker/sshd_entrypoint.sh /sshd_entrypoint.sh RUN chmod 744 /sshd_entrypoint.sh -RUN chown -R ${USER}:${USER} /src RUN mkdir /guidelines && cd /guidelines \ && wget https://raw.githubusercontent.com/aica-technology/.github/v0.9.0/guidelines/.clang-format USER ${USER} +WORKDIR /src +COPY --chown=${USER}:${USER} . . FROM code as build ARG TARGETPLATFORM ARG CACHEID -RUN --mount=type=cache,target=./build,id=cmake-${TARGETPLATFORM}-${CACHEID},uid=1000 \ +COPY licenses licenses +COPY protocol protocol +COPY source source +COPY CMakeLists.txt CMakeLists.txt +RUN --mount=type=cache,target=/build,id=cmake-build-${TARGETPLATFORM}-${CACHEID},uid=1000 \ cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build FROM build as cpp-test ARG TARGETPLATFORM ARG CACHEID -RUN --mount=type=cache,target=./build,id=cmake-${TARGETPLATFORM}-${CACHEID},uid=1000 \ +RUN --mount=type=cache,target=/build,id=cmake-build-${TARGETPLATFORM}-${CACHEID},uid=1000 \ cmake -B build -DBUILD_TESTING=ON && cd build && make && CTEST_OUTPUT_ON_FAILURE=1 make test FROM build as install ARG TARGETPLATFORM ARG CACHEID -RUN --mount=type=cache,target=./build,id=cmake-${TARGETPLATFORM}-${CACHEID},uid=1000 \ +RUN --mount=type=cache,target=/build,id=cmake-build-${TARGETPLATFORM}-${CACHEID},uid=1000 \ cmake --install build --prefix /tmp/cl -FROM base as python +FROM code as python ARG TARGETPLATFORM ARG CACHEID -COPY --from=apt-dependencies /tmp/apt / -COPY --from=dependencies /tmp/deps /usr COPY --from=install /tmp/cl /usr -COPY --chown=${USER}:${USER} ./python/include /python/include -COPY --chown=${USER}:${USER} ./python/source /python/source -COPY --chown=${USER}:${USER} ./python/pyproject.toml ./python/setup.py /python/ -RUN --mount=type=cache,target=${HOME}/.cache,id=pip-${TARGETPLATFORM}-${CACHEID},uid=1000 \ +COPY ./python/include /python/include +COPY ./python/source /python/source +COPY ./python/pyproject.toml ./python/setup.py /python/ +RUN --mount=type=cache,target=/.cache,id=pip-${TARGETPLATFORM}-${CACHEID},uid=1000 \ python3 -m pip install --prefix=/tmp/python /python RUN mv /tmp/python/local /tmp/python-usr FROM cpp-test as python-test -ARG TARGETPLATFORM -ARG CACHEID RUN pip install pytest COPY --from=install /tmp/cl /usr COPY --from=python /tmp/python-usr /usr -COPY --chown=${USER}:${USER} ./python/test /test +COPY ./python/test /test RUN pytest /test -FROM base as python-stubs +FROM code as python-stubs ARG TARGETPLATFORM ARG CACHEID -COPY --from=apt-dependencies /tmp/apt / -COPY --from=dependencies /tmp/deps /usr COPY --from=install /tmp/cl /usr COPY --from=python /tmp/python-usr /usr RUN pip install pybind11-stubgen -RUN --mount=type=cache,target=${HOME}/.cache,id=pip-${TARGETPLATFORM}-${CACHEID},uid=1000 \ +RUN --mount=type=cache,target=/.cache,id=pip-${TARGETPLATFORM}-${CACHEID},uid=1000 \ <