From 357e3030ac968fa32869ff751e20f2d7bbf0aeb8 Mon Sep 17 00:00:00 2001 From: Carl Delsey Date: Tue, 12 Feb 2019 17:04:46 -0800 Subject: [PATCH] Adding code coverage report to build (#553) * Script to generate coverage report. Should be run after test. * Add helper script for test suite * Modified travis to use runtestsuite.bash script * Add coverage to .travis.yml file. * Add coverage badge to main page * Run the code coverage report in the docker container. * Add lcov dependency. * Trying to add lcov tool to build. * Add coverage args to build script. * Fixing passing code coverage args. * Fixing setting env variable. * Fixing docker file coverage_enabled flag. * Fixing coverage report script. * Pass ci env through to docker * Fixing the filter rules for this workspace. * Removing unneeded dependency install line. * Add comments explaining what lcov lines do. * Rebasing on top of circle CI and common Cmake functions changes. * Fixing test suite script name. * Add codecov to CircleCI jobs * Add conditional to skip codecov if enable is false * Remove unnecessary export for codecov * Update workflow for paralel release and debug jobs * Forgot to append coverage step to job * Comment out travis codecove --- .circleci/config.yml | 54 ++++++++++++++++++++-------- .travis.yml | 41 ++++++++++----------- Dockerfile | 11 +++--- README.md | 2 ++ nav2_common/cmake/nav2_package.cmake | 7 ++++ nav2_system_tests/package.xml | 1 + tools/code_coverage_report.bash | 53 +++++++++++++++++++++++++++ tools/run_test_suite.bash | 9 +++++ 8 files changed, 140 insertions(+), 38 deletions(-) create mode 100755 tools/code_coverage_report.bash create mode 100755 tools/run_test_suite.bash diff --git a/.circleci/config.yml b/.circleci/config.yml index f455ce5d27..06526e1ba0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,7 +59,9 @@ references: else . /opt/ros/$ROS_DISTRO/setup.sh colcon build \ - --symlink-install + --symlink-install \ + --cmake-args \ + -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE fi save_upstream_cache: &save_upstream_cache save_cache: @@ -108,7 +110,10 @@ references: else . $ROS_WS/install/setup.sh colcon build \ - --symlink-install + --symlink-install \ + --cmake-args \ + -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ + -DCOVERAGE_ENABLED=$COVERAGE_ENABLED fi save_build_cache: &save_build_cache save_cache: @@ -129,15 +134,7 @@ references: name: Test Build command: | . install/setup.sh - colcon test \ - --packages-skip \ - nav2_system_tests - colcon test-result \ - --verbose - cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests - cd build/nav2_system_tests - ./ctest_retry.bash 3 - + src/navigation2/tools/run_test_suite.bash copy_test_logs: ©_test_logs run: name: Copy Test Logs @@ -146,6 +143,17 @@ references: store_test_logs: &store_test_logs store_artifacts: path: log/test + report_code_coverage: &report_code_coverage + run: + name: Report Code Coverage + command: | + if [ "$COVERAGE_ENABLED" = "True" ] + then + . install/setup.sh + src/navigation2/tools/code_coverage_report.bash codecovio + else + echo "Skipping Code Coverage Report" + fi commands: checkout_source: @@ -178,6 +186,10 @@ commands: - *test_build - *copy_test_logs - *store_test_logs + report_coverage: + description: "Report Coverage" + steps: + - *report_code_coverage executors: docker_exec: @@ -185,12 +197,25 @@ executors: - image: rosplanning/navigation2:master working_directory: /opt/nav2_ws environment: - CMAKE_BUILD_TYPE: "Release" MAKEFLAGS: "-j 1 -l 1" jobs: - build: + debug_build: + executor: docker_exec + environment: + CMAKE_BUILD_TYPE: "Debug" + COVERAGE_ENABLED: "True" + steps: + - checkout_source + - setup_upstream + - build_source + - test_source + - report_coverage + release_build: executor: docker_exec + environment: + CMAKE_BUILD_TYPE: "Release" + COVERAGE_ENABLED: "False" steps: - checkout_source - setup_upstream @@ -201,7 +226,8 @@ workflows: version: 2 build-test: jobs: - - build + - debug_build + - release_build # nightly: # triggers: # - schedule: diff --git a/.travis.yml b/.travis.yml index cac16bd303..e85c4073c4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,34 +19,35 @@ notifications: - carl.r.delsey@intel.com - matthew.k.hansen@intel.com -env: - - CMAKE_BUILD_TYPE=Release - before_install: - if [ "${TRAVIS_REPO_SLUG}" != "ros-planning/navigation2" ]; then echo "Travis CI is supported only in ros-planning/navigation2" && exit 1; fi + +matrix: + include: + - env: CMAKE_BUILD_TYPE=Release COVERAGE_ENABLED=False + after_success: + - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then + echo "Successfully built! Deploying container..." + docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD ; + docker tag navigation2:latest stevemacenski/navigation2:latest + docker push stevemacenski/navigation2:latest ; + fi + + # - env: CMAKE_BUILD_TYPE=Debug COVERAGE_ENABLED=True + # after_success: + # - ci_env=`bash <(curl -s https://codecov.io/env)` + # - docker exec --interactive --tty $ci_env nav2_bash /ros_entrypoint.sh + # src/navigation2/tools/code_coverage_report.bash codecovio + +script: - docker build --tag navigation2:latest --build-arg PULLREQ=$TRAVIS_PULL_REQUEST --build-arg CMAKE_BUILD_TYPE + --build-arg COVERAGE_ENABLED ./ - -script: - docker run --rm --detach --name nav2_bash navigation2:latest sleep infinity - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh - colcon test --packages-skip nav2_system_tests - - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh - colcon test-result --verbose - - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh - cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests - - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh - bash -c "cd build/nav2_system_tests ; ./ctest_retry.bash 3" - -after_success: - - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then - echo "Successfully built! Deploying container..." - docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD ; - docker tag navigation2:latest stevemacenski/navigation2:latest - docker push stevemacenski/navigation2:latest ; - fi + src/navigation2/tools/run_test_suite.bash diff --git a/Dockerfile b/Dockerfile index 1e586704b1..f5a8e03b55 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,7 @@ RUN . /opt/ros/$ROS_DISTRO/setup.sh && \ # build dependency package source ARG CMAKE_BUILD_TYPE=Release + RUN . /opt/ros/$ROS_DISTRO/setup.sh && \ colcon build \ --symlink-install \ @@ -67,11 +68,13 @@ RUN . $ROS_WS/install/setup.sh && \ # build navigation2 package source RUN rm $NAV2_WS/src/navigation2/nav2_system_tests/COLCON_IGNORE +ARG COVERAGE_ENABLED=False RUN . $ROS_WS/install/setup.sh && \ - colcon build \ - --symlink-install \ - --cmake-args \ - -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE + colcon build \ + --symlink-install \ + --cmake-args \ + -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \ + -DCOVERAGE_ENABLED=$COVERAGE_ENABLED # source navigation2 workspace from entrypoint RUN sed --in-place \ diff --git a/README.md b/README.md index ec094f7346..4904fab1ae 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,8 @@ ROS2 Navigation System [![Pulls](https://shields.beevelop.com/docker/pulls/stevemacenski/navigation2.svg?style=flat-square)](https://hub.docker.com/r/stevemacenski/navigation2) +[![codecov](https://codecov.io/gh/ros-planning/navigation2/branch/master/graph/badge.svg)](https://codecov.io/gh/ros-planning/navigation2) + # Overview The ROS 2 Navigation System is the control system that enables a robot to autonomously reach a goal state, such as a specific position and orientation relative to a specific map. Given a current pose, a map, and a goal, such as a destination pose, the navigation system generates a plan to reach the goal, and outputs commands to autonomously drive the robot, respecting any safety constraints and avoiding obstacles encountered along the way. diff --git a/nav2_common/cmake/nav2_package.cmake b/nav2_common/cmake/nav2_package.cmake index db9dbea89f..80ab053bbf 100644 --- a/nav2_common/cmake/nav2_package.cmake +++ b/nav2_common/cmake/nav2_package.cmake @@ -26,4 +26,11 @@ macro(nav2_package) if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang") add_compile_options(-Wall -Wextra -Wpedantic -Werror -fPIC) endif() + + option(COVERAGE_ENABLED "Enable code coverage" FALSE) + if(COVERAGE_ENABLED) + add_compile_options(--coverage) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --coverage") + endif() endmacro() diff --git a/nav2_system_tests/package.xml b/nav2_system_tests/package.xml index 741e2bb516..27b80304eb 100644 --- a/nav2_system_tests/package.xml +++ b/nav2_system_tests/package.xml @@ -42,6 +42,7 @@ launch_ros launch_testing navigation2 + lcov launch_ros launch_testing diff --git a/tools/code_coverage_report.bash b/tools/code_coverage_report.bash new file mode 100755 index 0000000000..041c075c51 --- /dev/null +++ b/tools/code_coverage_report.bash @@ -0,0 +1,53 @@ +#!/bin/bash + +if [ ! -d build ]; then + echo "Please run this script from the root of your workspace." + echo "Expected directory hierarchy is:" + echo "example_ws" + echo " - build" + echo " - - package_a" + echo " - - package_b" + exit 1 +fi + +set -e + +LCOVDIR=lcov +PWD=`pwd` + +COVERAGE_REPORT=genhtml + +for opt in "$@" ; do + case "$opt" in + clean) + rm -rf install build log $LCOVDIR + exit 0 + ;; + codecovio) + COVERAGE_REPORT=codecovio + ;; + esac +done + +mkdir -p $LCOVDIR + +# Generate initial zero-coverage data. This adds files that were otherwise not run to the report +lcov -c --initial --rc lcov_branch_coverage=1 --directory build --output-file ${LCOVDIR}/initialcoverage.info + +# Capture executed code data. +lcov -c --rc lcov_branch_coverage=1 --directory build --output-file ${LCOVDIR}/testcoverage.info + +# Combine the initial zero-coverage report with the executed lines report +lcov -a ${LCOVDIR}/initialcoverage.info -a ${LCOVDIR}/testcoverage.info --rc lcov_branch_coverage=1 --o ${LCOVDIR}/fullcoverage.info + +# Only include files that are within this workspace (eg filter out stdio.h etc) +lcov -e ${LCOVDIR}/fullcoverage.info "${PWD}/*" --rc lcov_branch_coverage=1 --output-file ${LCOVDIR}/workspacecoverage.info + +# Remove files in the build subdirectory because they are generated files (like messages, services, etc) +lcov -r ${LCOVDIR}/workspacecoverage.info "${PWD}/build/*" --rc lcov_branch_coverage=1 --output-file ${LCOVDIR}/projectcoverage.info + +if [ $COVERAGE_REPORT = codecovio ]; then + bash <(curl -s https://codecov.io/bash) -f ${LCOVDIR}/projectcoverage.info +else + genhtml ${LCOVDIR}/projectcoverage.info --output-directory ${LCOVDIR}/html --branch-coverage -p ${PWD} +fi diff --git a/tools/run_test_suite.bash b/tools/run_test_suite.bash new file mode 100755 index 0000000000..3cece7dba2 --- /dev/null +++ b/tools/run_test_suite.bash @@ -0,0 +1,9 @@ +#!/bin/bash + +set -ex + +colcon test --packages-skip nav2_system_tests +colcon test-result --verbose +cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests +cd build/nav2_system_tests +./ctest_retry.bash 3