Adding code coverage report to build (#553)

* Script to generate coverage report. Should be run after test. * Add helper script for test suite * Modified travis to use runtestsuite.bash script * Add coverage to .travis.yml file. * Add coverage badge to main page * Run the code coverage report in the docker container. * Add lcov dependency. * Trying to add lcov tool to build. * Add coverage args to build script. * Fixing passing code coverage args. * Fixing setting env variable. * Fixing docker file coverage_enabled flag. * Fixing coverage report script. * Pass ci env through to docker * Fixing the filter rules for this workspace. * Removing unneeded dependency install line. * Add comments explaining what lcov lines do. * Rebasing on top of circle CI and common Cmake functions changes. * Fixing test suite script name. * Add codecov to CircleCI jobs * Add conditional to skip codecov if enable is false * Remove unnecessary export for codecov * Update workflow for paralel release and debug jobs * Forgot to append coverage step to job * Comment out travis codecove
ros-navigation · Feb 13, 2019 · 357e303 · 357e303
1 parent 0deebc1
commit 357e303
Show file tree

Hide file tree

Showing 8 changed files with 140 additions and 38 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -59,7 +59,9 @@ references:
         else
             . /opt/ros/$ROS_DISTRO/setup.sh
             colcon build \
-              --symlink-install
+              --symlink-install \
+              --cmake-args \
+                -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
         fi
   save_upstream_cache: &save_upstream_cache
     save_cache:
@@ -108,7 +110,10 @@ references:
         else
             . $ROS_WS/install/setup.sh
             colcon build \
-              --symlink-install
+              --symlink-install \
+              --cmake-args \
+                -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
+                -DCOVERAGE_ENABLED=$COVERAGE_ENABLED
         fi
   save_build_cache: &save_build_cache
     save_cache:
@@ -129,15 +134,7 @@ references:
         name: Test Build
         command: |
           . install/setup.sh
-          colcon test \
-            --packages-skip \
-              nav2_system_tests
-          colcon test-result \
-            --verbose
-          cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests
-          cd build/nav2_system_tests
-          ./ctest_retry.bash 3
-
+          src/navigation2/tools/run_test_suite.bash
   copy_test_logs: &copy_test_logs
     run:
         name: Copy Test Logs
@@ -146,6 +143,17 @@ references:
   store_test_logs: &store_test_logs
     store_artifacts:
         path: log/test
+  report_code_coverage: &report_code_coverage
+    run:
+        name: Report Code Coverage
+        command: |
+          if [ "$COVERAGE_ENABLED" = "True" ]
+          then
+            . install/setup.sh
+            src/navigation2/tools/code_coverage_report.bash codecovio
+          else
+            echo "Skipping Code Coverage Report"
+          fi
 
 commands:
   checkout_source:
@@ -178,19 +186,36 @@ commands:
       - *test_build
       - *copy_test_logs
       - *store_test_logs
+  report_coverage:
+    description: "Report Coverage"
+    steps:
+      - *report_code_coverage
 
 executors:
   docker_exec:
     docker:
       - image: rosplanning/navigation2:master
     working_directory: /opt/nav2_ws
     environment:
-      CMAKE_BUILD_TYPE: "Release"
       MAKEFLAGS: "-j 1 -l 1"
 
 jobs:
-  build:
+  debug_build:
+    executor: docker_exec
+    environment:
+      CMAKE_BUILD_TYPE: "Debug"
+      COVERAGE_ENABLED: "True"
+    steps:
+      - checkout_source
+      - setup_upstream
+      - build_source
+      - test_source
+      - report_coverage
+  release_build:
     executor: docker_exec
+    environment:
+      CMAKE_BUILD_TYPE: "Release"
+      COVERAGE_ENABLED: "False"
     steps:
       - checkout_source
       - setup_upstream
@@ -201,7 +226,8 @@ workflows:
   version: 2
   build-test:
     jobs:
-      - build
+      - debug_build
+      - release_build
   # nightly:
   #   triggers:
   #     - schedule:

diff --git a/.travis.yml b/.travis.yml
@@ -19,34 +19,35 @@ notifications:
       - carl.r.delsey@intel.com
       - matthew.k.hansen@intel.com
 
-env:
-  - CMAKE_BUILD_TYPE=Release
-
 before_install:
   - if [ "${TRAVIS_REPO_SLUG}" != "ros-planning/navigation2" ];
       then echo "Travis CI is supported only in ros-planning/navigation2" && exit 1;
       fi
+
+matrix:
+  include:
+    - env: CMAKE_BUILD_TYPE=Release COVERAGE_ENABLED=False
+      after_success:
+        - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
+            echo "Successfully built! Deploying container..."
+            docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD ;
+            docker tag navigation2:latest stevemacenski/navigation2:latest
+            docker push stevemacenski/navigation2:latest ;
+            fi
+
+    # - env: CMAKE_BUILD_TYPE=Debug COVERAGE_ENABLED=True
+    #   after_success:
+    #     - ci_env=`bash <(curl -s https://codecov.io/env)`
+    #     - docker exec --interactive --tty $ci_env nav2_bash /ros_entrypoint.sh
+    #         src/navigation2/tools/code_coverage_report.bash codecovio
+
+script:
   - docker build --tag navigation2:latest
       --build-arg PULLREQ=$TRAVIS_PULL_REQUEST
       --build-arg CMAKE_BUILD_TYPE
+      --build-arg COVERAGE_ENABLED
       ./
-
-script:
   - docker run --rm --detach --name nav2_bash navigation2:latest
       sleep infinity
   - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh
-      colcon test --packages-skip nav2_system_tests
-  - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh
-      colcon test-result --verbose
-  - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh
-      cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests
-  - docker exec --interactive --tty nav2_bash /ros_entrypoint.sh
-       bash -c "cd build/nav2_system_tests ; ./ctest_retry.bash 3"
-
-after_success:
-  - if [ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]; then
-      echo "Successfully built! Deploying container..."
-      docker login -u $DOCKER_USERNAME -p $DOCKER_PASSWORD ;
-      docker tag navigation2:latest stevemacenski/navigation2:latest
-      docker push stevemacenski/navigation2:latest ;
-      fi
+      src/navigation2/tools/run_test_suite.bash
diff --git a/Dockerfile b/Dockerfile
@@ -48,6 +48,7 @@ RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
 
 # build dependency package source
 ARG CMAKE_BUILD_TYPE=Release
+
 RUN . /opt/ros/$ROS_DISTRO/setup.sh && \
     colcon build \
       --symlink-install \
@@ -67,11 +68,13 @@ RUN . $ROS_WS/install/setup.sh && \
 
 # build navigation2 package source
 RUN rm $NAV2_WS/src/navigation2/nav2_system_tests/COLCON_IGNORE
+ARG COVERAGE_ENABLED=False
 RUN . $ROS_WS/install/setup.sh && \
-    colcon build \
-      --symlink-install \
-      --cmake-args \
-        -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE
+     colcon build \
+       --symlink-install \
+       --cmake-args \
+         -DCMAKE_BUILD_TYPE=$CMAKE_BUILD_TYPE \
+         -DCOVERAGE_ENABLED=$COVERAGE_ENABLED
 
 # source navigation2 workspace from entrypoint
 RUN sed --in-place \

diff --git a/README.md b/README.md
@@ -10,6 +10,8 @@ ROS2 Navigation System
 
 [![Pulls](https://shields.beevelop.com/docker/pulls/stevemacenski/navigation2.svg?style=flat-square)](https://hub.docker.com/r/stevemacenski/navigation2)
 
+[![codecov](https://codecov.io/gh/ros-planning/navigation2/branch/master/graph/badge.svg)](https://codecov.io/gh/ros-planning/navigation2)
+
 # Overview
 The ROS 2 Navigation System is the control system that enables a robot to autonomously reach a goal state, such as a specific position and orientation relative to a specific map. Given a current pose, a map, and a goal, such as a destination pose, the navigation system generates a plan to reach the goal, and outputs commands to autonomously drive the robot, respecting any safety constraints and avoiding obstacles encountered along the way.
 

diff --git a/nav2_common/cmake/nav2_package.cmake b/nav2_common/cmake/nav2_package.cmake
@@ -26,4 +26,11 @@ macro(nav2_package)
   if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
     add_compile_options(-Wall -Wextra -Wpedantic -Werror -fPIC)
   endif()
+
+  option(COVERAGE_ENABLED "Enable code coverage" FALSE)
+  if(COVERAGE_ENABLED)
+    add_compile_options(--coverage)
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
+    set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --coverage")
+  endif()
 endmacro()
diff --git a/nav2_system_tests/package.xml b/nav2_system_tests/package.xml
@@ -42,6 +42,7 @@
   <exec_depend>launch_ros</exec_depend>
   <exec_depend>launch_testing</exec_depend>
   <exec_depend>navigation2</exec_depend>
+  <exec_depend>lcov</exec_depend>
 
   <test_depend>launch_ros</test_depend>
   <test_depend>launch_testing</test_depend>

diff --git a/tools/code_coverage_report.bash b/tools/code_coverage_report.bash
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+if [ ! -d build ]; then
+  echo "Please run this script from the root of your workspace."
+  echo "Expected directory hierarchy is:"
+  echo "example_ws"
+  echo " - build"
+  echo " - - package_a"
+  echo " - - package_b"
+  exit 1
+fi
+
+set -e
+
+LCOVDIR=lcov
+PWD=`pwd`
+
+COVERAGE_REPORT=genhtml
+
+for opt in "$@" ; do
+  case "$opt" in
+    clean)
+      rm -rf install build log $LCOVDIR
+      exit 0
+      ;;
+    codecovio)
+      COVERAGE_REPORT=codecovio
+      ;;
+  esac
+done
+
+mkdir -p $LCOVDIR
+
+# Generate initial zero-coverage data. This adds files that were otherwise not run to the report
+lcov -c  --initial --rc lcov_branch_coverage=1 --directory build --output-file ${LCOVDIR}/initialcoverage.info
+
+# Capture executed code data.
+lcov -c --rc lcov_branch_coverage=1 --directory build --output-file ${LCOVDIR}/testcoverage.info
+
+# Combine the initial zero-coverage report with the executed lines report
+lcov -a ${LCOVDIR}/initialcoverage.info -a ${LCOVDIR}/testcoverage.info --rc lcov_branch_coverage=1 --o ${LCOVDIR}/fullcoverage.info
+
+# Only include files that are within this workspace (eg filter out stdio.h etc)
+lcov -e ${LCOVDIR}/fullcoverage.info "${PWD}/*" --rc lcov_branch_coverage=1 --output-file ${LCOVDIR}/workspacecoverage.info
+
+# Remove files in the build subdirectory because they are generated files (like messages, services, etc)
+lcov -r ${LCOVDIR}/workspacecoverage.info "${PWD}/build/*" --rc lcov_branch_coverage=1 --output-file ${LCOVDIR}/projectcoverage.info
+
+if [ $COVERAGE_REPORT = codecovio ]; then
+  bash <(curl -s https://codecov.io/bash) -f ${LCOVDIR}/projectcoverage.info
+else
+  genhtml ${LCOVDIR}/projectcoverage.info --output-directory ${LCOVDIR}/html --branch-coverage -p ${PWD}
+fi
diff --git a/tools/run_test_suite.bash b/tools/run_test_suite.bash
@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -ex
+
+colcon test --packages-skip nav2_system_tests
+colcon test-result --verbose
+cp src/navigation2/tools/ctest_retry.bash build/nav2_system_tests
+cd build/nav2_system_tests
+./ctest_retry.bash 3