Preliminary support for distro-independent CI jobs (#968)

This change adds support for CI jobs which are not associated with a ROS
distribution.

ros_buildfarm/ci_job.py

@@ -247,9 +247,12 @@ def _get_ci_job_config(
             '%s=%s' % (var, value)
             for var, value in sorted(build_file.build_environment_variables.items())]
 
-    distribution_type = index.distributions[rosdistro_name] \
-        .get('distribution_type', 'ros1')
-    assert distribution_type in ('ros1', 'ros2')
+    if rosdistro_name:
+        distribution_type = index.distributions[rosdistro_name] \
+            .get('distribution_type', 'ros1')
+        assert distribution_type in ('ros1', 'ros2')
+    else:
+        distribution_type = 'global'
     ros_version = 1 if distribution_type == 'ros1' else 2
 
     for index in range(len(underlay_source_jobs)):
@@ -261,7 +264,7 @@ def _get_ci_job_config(
         'node_label': get_node_label(
             build_file.jenkins_job_label,
             get_default_node_label('%s_%s' % (
-                rosdistro_name, 'ci'))),
+                rosdistro_name or 'global', 'ci'))),
 
         'disabled': is_disabled,
 

ros_buildfarm/common.py

@@ -190,7 +190,8 @@ def get_ci_job_name(rosdistro_name, os_name, os_code_name, arch, job_type):
 
 
 def get_ci_view_name(rosdistro_name):
-    view_name = '%sci' % rosdistro_name[0].upper()
+    # prefix with first character of rosdistro_name (if not empty)
+    view_name = '%sci' % rosdistro_name[:1].upper()
     return view_name
 
 

ros_buildfarm/config/__init__.py

@@ -76,6 +76,14 @@ def get_ci_build_files(index, dist_name):
     return build_files
 
 
+def get_global_ci_build_files(index):
+    data = _load_build_file_data(index.ci_builds)
+    build_files = {}
+    for k, (url, v) in data.items():
+        build_files[k] = CIBuildFile(k, v)
+    return build_files
+
+
 def get_release_build_files(index, dist_name):
     data = _get_build_file_data(index, dist_name, 'release_builds')
     build_files = {}

ros_buildfarm/config/index.py

@@ -92,6 +92,13 @@ def __init__(self, data, base_url):  # noqa: D107
 
         # rosdistro-independent configurations go here
 
+        self.ci_builds = {}
+        if 'ci_builds' in data and data['ci_builds']:
+            assert isinstance(data['ci_builds'], dict)
+            for k, v in data['ci_builds'].items():
+                v = _resolve_url(base_url, v)
+                self.ci_builds[k] = v
+
         self.doc_builds = {}
         if 'doc_builds' in data and data['doc_builds']:
             assert isinstance(data['doc_builds'], dict)

ros_buildfarm/scripts/ci/generate_ci_script.py

@@ -37,6 +37,7 @@
 from ros_buildfarm.ci_job import configure_ci_job
 from ros_buildfarm.common import get_ci_job_name
 from ros_buildfarm.config import get_ci_build_files
+from ros_buildfarm.config import get_global_ci_build_files
 from ros_buildfarm.config import get_index as get_config_index
 from ros_buildfarm.templates import expand_template
 
@@ -135,7 +136,10 @@ def beforeInclude(self, *_, **kwargs):
     templates.template_hooks = [hook]
 
     config = get_config_index(args.config_url)
-    build_files = get_ci_build_files(config, args.rosdistro_name)
+    if not args.rosdistro_name:
+        build_files = get_global_ci_build_files(config)
+    else:
+        build_files = get_ci_build_files(config, args.rosdistro_name)
     build_file = build_files[args.ci_build_name]
 
     underlay_source_paths = [os.path.abspath(p) for p in args.underlay_source_path or []]

ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em

@@ -58,7 +58,7 @@ COPY @repos_file /tmp/@repos_file
 ENTRYPOINT ["sh", "-c"]
 @{
 args = \
-    ' ' + rosdistro_name + \
+    ' ' + (rosdistro_name or "''") + \
     ' ' + os_name + \
     ' ' + os_code_name + \
     ' ' + arch + \

ros_buildfarm/templates/ci/ci_job.xml.em

@@ -193,7 +193,7 @@ parameters = [
         'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH',
         'if [ "$package_dependencies" = "true" ]; then package_dependencies_arg=--package-dependencies; fi',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/ci/run_ci_job.py' +
-        ' ' + rosdistro_name +
+        ' ' + (rosdistro_name or "''") +
         ' ' + os_name +
         ' ' + os_code_name +
         ' ' + arch +
@@ -220,7 +220,7 @@ parameters = [
         'echo "# BEGIN SECTION: Build Dockerfile - generating CI tasks"',
         'cd $WORKSPACE/docker_generating_dockers',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_task_generation.%s .' % (rosdistro_name),
+        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_task_generation.%s .' % (rosdistro_name or 'global'),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - generating CI tasks"',
@@ -240,7 +240,7 @@ parameters = [
         ' -v $WORKSPACE/docker_create_workspace:/tmp/docker_create_workspace' +
         ' -v $WORKSPACE/docker_build_and_install:/tmp/docker_build_and_install' +
         ' -v $WORKSPACE/docker_build_and_test:/tmp/docker_build_and_test' +
-        ' $DOCKER_IMAGE_PREFIX.ci_task_generation.%s' % (rosdistro_name),
+        ' $DOCKER_IMAGE_PREFIX.ci_task_generation.%s' % (rosdistro_name or 'global'),
         'cd -',  # restore pwd when used in scripts
         'echo "# END SECTION"',
     ]),
@@ -260,7 +260,7 @@ parameters = [
         '# build and run create_workspace Dockerfile',
         'cd $WORKSPACE/docker_create_workspace',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s .' % (rosdistro_name),
+        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s .' % (rosdistro_name or 'global'),
         'echo "# END SECTION"',
         '',
         'echo "# BEGIN SECTION: Run Dockerfile - create workspace"',
@@ -284,7 +284,7 @@ parameters = [
             for i, space in enumerate(underlay_source_paths, 1)
         ]) +
         ' -v $WORKSPACE/ws:/tmp/ws%s' % (len(underlay_source_paths) + 1 if len(underlay_source_paths) else '') +
-        ' $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s' % (rosdistro_name),
+        ' $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s' % (rosdistro_name or 'global'),
         'cd -',  # restore pwd when used in scripts
         'echo "# END SECTION"',
     ]),
@@ -314,7 +314,7 @@ parameters = [
         '# build and run build and install Dockerfile',
         'cd $WORKSPACE/docker_build_and_install',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s .' % (rosdistro_name),
+        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s .' % (rosdistro_name or 'global'),
         'echo "# END SECTION"',
         '',
     ] + ([
@@ -327,7 +327,7 @@ parameters = [
         ' --cidfile=$WORKSPACE/docker_build_and_install/docker_ccache_before.cid' +
         ' -e CCACHE_DIR=/home/buildfarm/.ccache' +
         ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
-        ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) +
+        ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name or 'global') +
         ' "ccache -s"',
         'echo "# END SECTION"',
         '',
@@ -350,7 +350,7 @@ parameters = [
             for i, space in enumerate(underlay_source_paths, 1)
         ]) +
         ' -v $WORKSPACE/ws:/tmp/ws%s' % (len(underlay_source_paths) + 1 if len(underlay_source_paths) else '') +
-        ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name),
+        ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name or 'global'),
         'cd -',  # restore pwd when used in scripts
         'echo "# END SECTION"',
     ] + ([
@@ -363,7 +363,7 @@ parameters = [
         ' --cidfile=$WORKSPACE/docker_build_and_install/docker_ccache_after.cid' +
         ' -e CCACHE_DIR=/home/buildfarm/.ccache' +
         ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
-        ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) +
+        ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name or 'global') +
         ' "ccache -s"',
         'echo "# END SECTION"',
     ] if shared_ccache else [])),
@@ -373,12 +373,12 @@ parameters = [
     script='\n'.join([
         'echo "# BEGIN SECTION: Compress install space"',
         'cd $WORKSPACE',
-        'tar -cjf ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name, os_code_name, arch) +
+        'tar -cjf ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name or 'global', os_code_name, arch) +
         ' -C ws' +
         ' --transform "s/^install_isolated/ros%d-linux/"' % (ros_version) +
         ' install_isolated',
-        'sha256sum -b ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name, os_code_name, arch) +
-        ' > ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name, os_code_name, arch),
+        'sha256sum -b ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name or 'global', os_code_name, arch) +
+        ' > ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name or 'global', os_code_name, arch),
         'cd -',
         'echo "# END SECTION"',
     ]),
@@ -398,7 +398,7 @@ parameters = [
         '# build and run build and test Dockerfile',
         'cd $WORKSPACE/docker_build_and_test',
         'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py',
-        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s .' % (rosdistro_name),
+        'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s .' % (rosdistro_name or 'global'),
         'echo "# END SECTION"',
         '',
     ] + ([
@@ -411,7 +411,7 @@ parameters = [
         ' --cidfile=$WORKSPACE/docker_build_and_test/docker_ccache_before.cid' +
         ' -e CCACHE_DIR=/home/buildfarm/.ccache' +
         ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
-        ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) +
+        ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name or 'global') +
         ' "ccache -s"',
         'echo "# END SECTION"',
         '',
@@ -436,7 +436,7 @@ parameters = [
             for i, space in enumerate(underlay_source_paths, 1)
         ]) +
         ' -v $WORKSPACE/ws:/tmp/ws%s' % (len(underlay_source_paths) + 1 if len(underlay_source_paths) else '') +
-        ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name),
+        ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name or 'global'),
         'cd -',  # restore pwd when used in scripts
         'echo "# END SECTION"',
     ] + ([
@@ -449,7 +449,7 @@ parameters = [
         ' --cidfile=$WORKSPACE/docker_build_and_test/docker_ccache_after.cid' +
         ' -e CCACHE_DIR=/home/buildfarm/.ccache' +
         ' -v $HOME/.ccache:/home/buildfarm/.ccache' +
-        ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) +
+        ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name or 'global') +
         ' "ccache -s"',
         'echo "# END SECTION"',
     ] if shared_ccache else [])),
@@ -476,8 +476,8 @@ parameters = [
 @(SNIPPET(
     'archive_artifacts',
     artifacts=[
-      'ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name, os_code_name, arch),
-      'ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name, os_code_name, arch),
+      'ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name or 'global', os_code_name, arch),
+      'ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name or 'global', os_code_name, arch),
     ] + archive_files + [
       image for images in show_images.values() for image in images
     ],
@@ -488,8 +488,8 @@ parameters = [
     config_name='ci_archives',
     remote_directory=upload_directory,
     source_files=[
-        'ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name, os_code_name, arch),
-        'ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name, os_code_name, arch),
+        'ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name or 'global', os_code_name, arch),
+        'ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name or 'global', os_code_name, arch),
     ],
     remove_prefix=None,
 ))@

ros_buildfarm/templates/ci/create_workspace.Dockerfile.em

@@ -98,7 +98,7 @@ cmds = [
 
     'PYTHONPATH=/tmp/ros_buildfarm:$PYTHONPATH python3 -u' + \
     ' /tmp/ros_buildfarm/scripts/ci/create_workspace.py' + \
-    ' ' + rosdistro_name + \
+    ' ' + (rosdistro_name or "''") + \
     ' --workspace-root ' + workspace_root[-1] + \
     ' --repos-file-urls ' + ' '.join('file:///tmp/%s' % repos_file for repos_file in repos_file_names) + \
     ' --repository-names ' + ' '.join(repository_names) + \
@@ -108,7 +108,7 @@ cmds = [
 
     'PYTHONPATH=/tmp/ros_buildfarm:$PYTHONPATH python3 -u' + \
     ' /tmp/ros_buildfarm/scripts/ci/generate_install_lists.py' + \
-    ' ' + rosdistro_name + \
+    ' ' + (rosdistro_name or "''") + \
     ' ' + os_name + \
     ' ' + os_code_name + \
     ' --package-root ' + ' '.join(base_paths) + \

ros_buildfarm/templates/devel/devel_task.Dockerfile.em

@@ -119,15 +119,15 @@ cmd = \
 if not testing:
     cmd += \
         ' /tmp/ros_buildfarm/scripts/devel/build_and_install.py' + \
-        ' --rosdistro-name ' + rosdistro_name + \
+        ' --rosdistro-name ' + (rosdistro_name or "''") + \
         ' --ros-version ' + str(ros_version) + \
         ' --clean-before'
     if run_abichecker:
         cmd += ' --run-abichecker'
 else:
     cmd += \
         ' /tmp/ros_buildfarm/scripts/devel/build_and_test.py' + \
-        ' --rosdistro-name %s' % rosdistro_name + \
+        ' --rosdistro-name %s' % (rosdistro_name or "''") + \
         ' --ros-version ' + str(ros_version)
     if require_gpu_support:
         cmd += ' --require-gpu-support'