Adding code coverage report to build (ros-navigation#553)

* Script to generate coverage report. Should be run after test.

* Add helper script for test suite

* Modified travis to use runtestsuite.bash script

* Add coverage to .travis.yml file.

* Add coverage badge to main page

* Run the code coverage report in the docker container.

* Add lcov dependency.

* Trying to add lcov tool to build.

* Add coverage args to build script.

* Fixing passing code coverage args.

* Fixing setting env variable.

* Fixing docker file coverage_enabled flag.

* Fixing coverage report script.

* Pass ci env through to docker

* Fixing the filter rules for this workspace.

* Removing unneeded dependency install line.

* Add comments explaining what lcov lines do.

* Rebasing on top of circle CI and common Cmake functions changes.

* Fixing test suite script name.

* Add codecov to CircleCI jobs

* Add conditional to skip codecov if enable is false

* Remove unnecessary export for codecov

* Update workflow for paralel release and debug jobs

* Forgot to append coverage step to job

* Comment out travis codecove

.circleci/config.yml

@@ -59,7 +59,9 @@ references:
  else
  . /opt/ros/$ROS_DISTRO/setup.sh
  colcon build \
- --symlink-install
+ --symlink-install \
+ --cmake-args \
+ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
  fi
  save_upstream_cache: &save_upstream_cache
  save_cache:
@@ -108,7 +110,10 @@ references:
  else
  . $ROS_WS/install/setup.sh
  colcon build \
- --symlink-install
+ --symlink-install \
+ --cmake-args \
+ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
+ -DCOVERAGE_ENABLED=$COVERAGE_ENABLED
  fi
  save_build_cache: &save_build_cache
  save_cache:
@@ -129,15 +134,7 @@ references:
  name: Test Build
  command: |
  . install/setup.sh
- colcon test \
- --packages-skip \
- nav2_system_tests
- colcon test-result \
- --verbose
- cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests
- cd build/nav2_system_tests
- ./ctest_retry.bash 3
-
+ src/navigation2/tools/run_test_suite.bash
  copy_test_logs: &copy_test_logs
  run:
  name: Copy Test Logs
@@ -146,6 +143,17 @@ references:
  store_test_logs: &store_test_logs
  store_artifacts:
  path: log/test
+ report_code_coverage: &report_code_coverage
+ run:
+ name: Report Code Coverage
+ command: |
+ if [ "$COVERAGE_ENABLED" = "True" ]
+ then
+ . install/setup.sh
+ src/navigation2/tools/code_coverage_report.bash codecovio
+ else
+ echo "Skipping Code Coverage Report"
+ fi
 
 commands:
  checkout_source:
@@ -178,19 +186,36 @@ commands:
  - *test_build
  - *copy_test_logs
  - *store_test_logs
+ report_coverage:
+ description: "Report Coverage"
+ steps:
+ - *report_code_coverage
 
 executors:
  docker_exec:
  docker:
  - image: rosplanning/navigation2:master
  working_directory: /opt/nav2_ws
  environment:
- CMAKE_BUILD_TYPE: "Release"
  MAKEFLAGS: "-j 1 -l 1"
 
 jobs:
- build:
+ debug_build:
+ executor: docker_exec
+ environment:
+ CMAKE_BUILD_TYPE: "Debug"
+ COVERAGE_ENABLED: "True"
+ steps:
+ - checkout_source
+ - setup_upstream
+ - build_source
+ - test_source
+ - report_coverage
+ release_build:
  executor: docker_exec
+ environment:
+ CMAKE_BUILD_TYPE: "Release"
+ COVERAGE_ENABLED: "False"
  steps:
  - checkout_source
  - setup_upstream
@@ -201,7 +226,8 @@ workflows:
  version: 2
  build-test:
  jobs:
- - build
+ - debug_build
+ - release_build
  # nightly:
  # triggers:
  # - schedule:

.travis.yml

@@ -19,34 +19,35 @@ notifications:
  - carl.r.delsey@intel.com
  - matthew.k.hansen@intel.com
 
-env:
- - CMAKE_BUILD_TYPE=Release
-
 before_install:
  - if [ "${TRAVIS_REPO_SLUG}" != "ros-planning/navigation2" ];
  then echo "Travis CI is supported only in ros-planning/navigation2" && exit 1;
  fi
+
+matrix:
+ include:
+ - env: CMAKE_BUILD_TYPE=Release COVERAGE_ENABLED=False
+ after_success:
+ - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+ echo "Successfully built! Deploying container..."
+ docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD ;
+ docker tag navigation2:latest stevemacenski/navigation2:latest
+ docker push stevemacenski/navigation2:latest ;
+ fi
+
+ # - env: CMAKE_BUILD_TYPE=Debug COVERAGE_ENABLED=True
+ # after_success:
+ # - ci_env=`bash <(curl -s https://codecov.io/env)`
+ # - docker exec --interactive --tty $ci_env nav2_bash /ros_entrypoint.sh
+ # src/navigation2/tools/code_coverage_report.bash codecovio
+
+script:
  - docker build --tag navigation2:latest
  --build-arg PULLREQ=$TRAVIS_PULL_REQUEST
  --build-arg CMAKE_BUILD_TYPE
+ --build-arg COVERAGE_ENABLED
  ./
-
-script:
  - docker run --rm --detach --name nav2_bash navigation2:latest
  sleep infinity
  - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh
- colcon test --packages-skip nav2_system_tests
- - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh
- colcon test-result --verbose
- - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh
- cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests
- - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh
- bash -c "cd build/nav2_system_tests ; ./ctest_retry.bash 3"
-
-after_success:
- - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
- echo "Successfully built! Deploying container..."
- docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD ;
- docker tag navigation2:latest stevemacenski/navigation2:latest
- docker push stevemacenski/navigation2:latest ;
- fi
+ src/navigation2/tools/run_test_suite.bash

Dockerfile

@@ -48,6 +48,7 @@ RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
 
 # build dependency package source
 ARG CMAKE_BUILD_TYPE=Release
+
 RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
  colcon build \
  --symlink-install \
@@ -67,11 +68,13 @@ RUN . $ROS_WS/install/setup.sh && \
 
 # build navigation2 package source
 RUN rm $NAV2_WS/src/navigation2/nav2_system_tests/COLCON_IGNORE
+ARG COVERAGE_ENABLED=False
 RUN . $ROS_WS/install/setup.sh && \
- colcon build \
- --symlink-install \
- --cmake-args \
- -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
+ colcon build \
+ --symlink-install \
+ --cmake-args \
+ -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
+ -DCOVERAGE_ENABLED=$COVERAGE_ENABLED
 
 # source navigation2 workspace from entrypoint
 RUN sed --in-place \

README.md

@@ -10,6 +10,8 @@ ROS2 Navigation System
 
 [![Pulls](https://shields.beevelop.com/docker/pulls/stevemacenski/navigation2.svg?style=flat-square)](https://hub.docker.com/r/stevemacenski/navigation2)
 
+[![codecov](https://codecov.io/gh/ros-planning/navigation2/branch/master/graph/badge.svg)](https://codecov.io/gh/ros-planning/navigation2)
+
 # Overview
 The ROS 2 Navigation System is the control system that enables a robot to autonomously reach a goal state, such as a specific position and orientation relative to a specific map. Given a current pose, a map, and a goal, such as a destination pose, the navigation system generates a plan to reach the goal, and outputs commands to autonomously drive the robot, respecting any safety constraints and avoiding obstacles encountered along the way.
 

nav2_common/cmake/nav2_package.cmake

@@ -26,4 +26,11 @@ macro(nav2_package)
  if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
  add_compile_options(-Wall -Wextra -Wpedantic -Werror -fPIC)
  endif()
+
+ option(COVERAGE_ENABLED "Enable code coverage" FALSE)
+ if(COVERAGE_ENABLED)
+ add_compile_options(--coverage)
+ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
+ set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --coverage")
+ endif()
 endmacro()

nav2_system_tests/package.xml

@@ -42,6 +42,7 @@
  <exec_depend>launch_ros</exec_depend>
  <exec_depend>launch_testing</exec_depend>
  <exec_depend>navigation2</exec_depend>
+ <exec_depend>lcov</exec_depend>
 
  <test_depend>launch_ros</test_depend>
  <test_depend>launch_testing</test_depend>

tools/code_coverage_report.bash

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+if [ ! -d build ]; then
+ echo "Please run this script from the root of your workspace."
+ echo "Expected directory hierarchy is:"
+ echo "example_ws"
+ echo " - build"
+ echo " - - package_a"
+ echo " - - package_b"
+ exit 1
+fi
+
+set -e
+
+LCOVDIR=lcov
+PWD=`pwd`
+
+COVERAGE_REPORT=genhtml
+
+for opt in "$@" ; do
+ case "$opt" in
+ clean)
+ rm -rf install build log $LCOVDIR
+ exit 0
+ ;;
+ codecovio)
+ COVERAGE_REPORT=codecovio
+ ;;
+ esac
+done
+
+mkdir -p $LCOVDIR
+
+# Generate initial zero-coverage data. This adds files that were otherwise not run to the report
+lcov -c --initial --rc lcov_branch_coverage=1 --directory build --output-file ${LCOVDIR}/initialcoverage.info
+
+# Capture executed code data.
+lcov -c --rc lcov_branch_coverage=1 --directory build --output-file ${LCOVDIR}/testcoverage.info
+
+# Combine the initial zero-coverage report with the executed lines report
+lcov -a ${LCOVDIR}/initialcoverage.info -a ${LCOVDIR}/testcoverage.info --rc lcov_branch_coverage=1 --o ${LCOVDIR}/fullcoverage.info
+
+# Only include files that are within this workspace (eg filter out stdio.h etc)
+lcov -e ${LCOVDIR}/fullcoverage.info "${PWD}/*" --rc lcov_branch_coverage=1 --output-file ${LCOVDIR}/workspacecoverage.info
+
+# Remove files in the build subdirectory because they are generated files (like messages, services, etc)
+lcov -r ${LCOVDIR}/workspacecoverage.info "${PWD}/build/*" --rc lcov_branch_coverage=1 --output-file ${LCOVDIR}/projectcoverage.info
+
+if [ $COVERAGE_REPORT = codecovio ]; then
+ bash <(curl -s https://codecov.io/bash) -f ${LCOVDIR}/projectcoverage.info
+else
+ genhtml ${LCOVDIR}/projectcoverage.info --output-directory ${LCOVDIR}/html --branch-coverage -p ${PWD}
+fi

tools/run_test_suite.bash

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -ex
+
+colcon test --packages-skip nav2_system_tests
+colcon test-result --verbose
+cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests
+cd build/nav2_system_tests
+./ctest_retry.bash 3