diff --git a/ros_buildfarm/ci_job.py b/ros_buildfarm/ci_job.py index 7bd68594a..1963a4807 100644 --- a/ros_buildfarm/ci_job.py +++ b/ros_buildfarm/ci_job.py @@ -247,9 +247,12 @@ def _get_ci_job_config( '%s=%s' % (var, value) for var, value in sorted(build_file.build_environment_variables.items())] - distribution_type = index.distributions[rosdistro_name] \ - .get('distribution_type', 'ros1') - assert distribution_type in ('ros1', 'ros2') + if rosdistro_name: + distribution_type = index.distributions[rosdistro_name] \ + .get('distribution_type', 'ros1') + assert distribution_type in ('ros1', 'ros2') + else: + distribution_type = 'global' ros_version = 1 if distribution_type == 'ros1' else 2 for index in range(len(underlay_source_jobs)): @@ -261,7 +264,7 @@ def _get_ci_job_config( 'node_label': get_node_label( build_file.jenkins_job_label, get_default_node_label('%s_%s' % ( - rosdistro_name, 'ci'))), + rosdistro_name or 'global', 'ci'))), 'disabled': is_disabled, diff --git a/ros_buildfarm/common.py b/ros_buildfarm/common.py index 319d524af..30da2a12a 100644 --- a/ros_buildfarm/common.py +++ b/ros_buildfarm/common.py @@ -190,7 +190,8 @@ def get_ci_job_name(rosdistro_name, os_name, os_code_name, arch, job_type): def get_ci_view_name(rosdistro_name): - view_name = '%sci' % rosdistro_name[0].upper() + # prefix with first character of rosdistro_name (if not empty) + view_name = '%sci' % rosdistro_name[:1].upper() return view_name diff --git a/ros_buildfarm/config/__init__.py b/ros_buildfarm/config/__init__.py index 5ec534cb8..dce7a7bbb 100644 --- a/ros_buildfarm/config/__init__.py +++ b/ros_buildfarm/config/__init__.py @@ -76,6 +76,14 @@ def get_ci_build_files(index, dist_name): return build_files +def get_global_ci_build_files(index): + data = _load_build_file_data(index.ci_builds) + build_files = {} + for k, (url, v) in data.items(): + build_files[k] = CIBuildFile(k, v) + return build_files + + def get_release_build_files(index, dist_name): data = _get_build_file_data(index, dist_name, 'release_builds') build_files = {} diff --git a/ros_buildfarm/config/index.py b/ros_buildfarm/config/index.py index d0d11acb6..a5630d91e 100644 --- a/ros_buildfarm/config/index.py +++ b/ros_buildfarm/config/index.py @@ -92,6 +92,13 @@ def __init__(self, data, base_url): # noqa: D107 # rosdistro-independent configurations go here + self.ci_builds = {} + if 'ci_builds' in data and data['ci_builds']: + assert isinstance(data['ci_builds'], dict) + for k, v in data['ci_builds'].items(): + v = _resolve_url(base_url, v) + self.ci_builds[k] = v + self.doc_builds = {} if 'doc_builds' in data and data['doc_builds']: assert isinstance(data['doc_builds'], dict) diff --git a/ros_buildfarm/scripts/ci/generate_ci_script.py b/ros_buildfarm/scripts/ci/generate_ci_script.py index 5e3bef3c1..e42e510eb 100644 --- a/ros_buildfarm/scripts/ci/generate_ci_script.py +++ b/ros_buildfarm/scripts/ci/generate_ci_script.py @@ -37,6 +37,7 @@ from ros_buildfarm.ci_job import configure_ci_job from ros_buildfarm.common import get_ci_job_name from ros_buildfarm.config import get_ci_build_files +from ros_buildfarm.config import get_global_ci_build_files from ros_buildfarm.config import get_index as get_config_index from ros_buildfarm.templates import expand_template @@ -135,7 +136,10 @@ def beforeInclude(self, *_, **kwargs): templates.template_hooks = [hook] config = get_config_index(args.config_url) - build_files = get_ci_build_files(config, args.rosdistro_name) + if not args.rosdistro_name: + build_files = get_global_ci_build_files(config) + else: + build_files = get_ci_build_files(config, args.rosdistro_name) build_file = build_files[args.ci_build_name] underlay_source_paths = [os.path.abspath(p) for p in args.underlay_source_path or []] diff --git a/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em b/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em index 7a4f36be2..171fab0a9 100644 --- a/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em +++ b/ros_buildfarm/templates/ci/ci_create_tasks.Dockerfile.em @@ -58,7 +58,7 @@ COPY @repos_file /tmp/@repos_file ENTRYPOINT ["sh", "-c"] @{ args = \ - ' ' + rosdistro_name + \ + ' ' + (rosdistro_name or "''") + \ ' ' + os_name + \ ' ' + os_code_name + \ ' ' + arch + \ diff --git a/ros_buildfarm/templates/ci/ci_job.xml.em b/ros_buildfarm/templates/ci/ci_job.xml.em index 4bc993da9..6d1ed4c99 100644 --- a/ros_buildfarm/templates/ci/ci_job.xml.em +++ b/ros_buildfarm/templates/ci/ci_job.xml.em @@ -193,7 +193,7 @@ parameters = [ 'export PYTHONPATH=$WORKSPACE/ros_buildfarm:$PYTHONPATH', 'if [ "$package_dependencies" = "true" ]; then package_dependencies_arg=--package-dependencies; fi', 'python3 -u $WORKSPACE/ros_buildfarm/scripts/ci/run_ci_job.py' + - ' ' + rosdistro_name + + ' ' + (rosdistro_name or "''") + ' ' + os_name + ' ' + os_code_name + ' ' + arch + @@ -220,7 +220,7 @@ parameters = [ 'echo "# BEGIN SECTION: Build Dockerfile - generating CI tasks"', 'cd $WORKSPACE/docker_generating_dockers', 'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py', - 'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_task_generation.%s .' % (rosdistro_name), + 'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_task_generation.%s .' % (rosdistro_name or 'global'), 'echo "# END SECTION"', '', 'echo "# BEGIN SECTION: Run Dockerfile - generating CI tasks"', @@ -240,7 +240,7 @@ parameters = [ ' -v $WORKSPACE/docker_create_workspace:/tmp/docker_create_workspace' + ' -v $WORKSPACE/docker_build_and_install:/tmp/docker_build_and_install' + ' -v $WORKSPACE/docker_build_and_test:/tmp/docker_build_and_test' + - ' $DOCKER_IMAGE_PREFIX.ci_task_generation.%s' % (rosdistro_name), + ' $DOCKER_IMAGE_PREFIX.ci_task_generation.%s' % (rosdistro_name or 'global'), 'cd -', # restore pwd when used in scripts 'echo "# END SECTION"', ]), @@ -260,7 +260,7 @@ parameters = [ '# build and run create_workspace Dockerfile', 'cd $WORKSPACE/docker_create_workspace', 'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py', - 'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s .' % (rosdistro_name), + 'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s .' % (rosdistro_name or 'global'), 'echo "# END SECTION"', '', 'echo "# BEGIN SECTION: Run Dockerfile - create workspace"', @@ -284,7 +284,7 @@ parameters = [ for i, space in enumerate(underlay_source_paths, 1) ]) + ' -v $WORKSPACE/ws:/tmp/ws%s' % (len(underlay_source_paths) + 1 if len(underlay_source_paths) else '') + - ' $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s' % (rosdistro_name), + ' $DOCKER_IMAGE_PREFIX.ci_create_workspace.%s' % (rosdistro_name or 'global'), 'cd -', # restore pwd when used in scripts 'echo "# END SECTION"', ]), @@ -314,7 +314,7 @@ parameters = [ '# build and run build and install Dockerfile', 'cd $WORKSPACE/docker_build_and_install', 'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py', - 'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s .' % (rosdistro_name), + 'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s .' % (rosdistro_name or 'global'), 'echo "# END SECTION"', '', ] + ([ @@ -327,7 +327,7 @@ parameters = [ ' --cidfile=$WORKSPACE/docker_build_and_install/docker_ccache_before.cid' + ' -e CCACHE_DIR=/home/buildfarm/.ccache' + ' -v $HOME/.ccache:/home/buildfarm/.ccache' + - ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) + + ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name or 'global') + ' "ccache -s"', 'echo "# END SECTION"', '', @@ -350,7 +350,7 @@ parameters = [ for i, space in enumerate(underlay_source_paths, 1) ]) + ' -v $WORKSPACE/ws:/tmp/ws%s' % (len(underlay_source_paths) + 1 if len(underlay_source_paths) else '') + - ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name), + ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name or 'global'), 'cd -', # restore pwd when used in scripts 'echo "# END SECTION"', ] + ([ @@ -363,7 +363,7 @@ parameters = [ ' --cidfile=$WORKSPACE/docker_build_and_install/docker_ccache_after.cid' + ' -e CCACHE_DIR=/home/buildfarm/.ccache' + ' -v $HOME/.ccache:/home/buildfarm/.ccache' + - ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name) + + ' $DOCKER_IMAGE_PREFIX.ci_build_and_install.%s' % (rosdistro_name or 'global') + ' "ccache -s"', 'echo "# END SECTION"', ] if shared_ccache else [])), @@ -373,12 +373,12 @@ parameters = [ script='\n'.join([ 'echo "# BEGIN SECTION: Compress install space"', 'cd $WORKSPACE', - 'tar -cjf ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name, os_code_name, arch) + + 'tar -cjf ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name or 'global', os_code_name, arch) + ' -C ws' + ' --transform "s/^install_isolated/ros%d-linux/"' % (ros_version) + ' install_isolated', - 'sha256sum -b ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name, os_code_name, arch) + - ' > ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name, os_code_name, arch), + 'sha256sum -b ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name or 'global', os_code_name, arch) + + ' > ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name or 'global', os_code_name, arch), 'cd -', 'echo "# END SECTION"', ]), @@ -398,7 +398,7 @@ parameters = [ '# build and run build and test Dockerfile', 'cd $WORKSPACE/docker_build_and_test', 'python3 -u $WORKSPACE/ros_buildfarm/scripts/misc/docker_pull_baseimage.py', - 'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s .' % (rosdistro_name), + 'docker build --force-rm -t $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s .' % (rosdistro_name or 'global'), 'echo "# END SECTION"', '', ] + ([ @@ -411,7 +411,7 @@ parameters = [ ' --cidfile=$WORKSPACE/docker_build_and_test/docker_ccache_before.cid' + ' -e CCACHE_DIR=/home/buildfarm/.ccache' + ' -v $HOME/.ccache:/home/buildfarm/.ccache' + - ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) + + ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name or 'global') + ' "ccache -s"', 'echo "# END SECTION"', '', @@ -436,7 +436,7 @@ parameters = [ for i, space in enumerate(underlay_source_paths, 1) ]) + ' -v $WORKSPACE/ws:/tmp/ws%s' % (len(underlay_source_paths) + 1 if len(underlay_source_paths) else '') + - ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name), + ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name or 'global'), 'cd -', # restore pwd when used in scripts 'echo "# END SECTION"', ] + ([ @@ -449,7 +449,7 @@ parameters = [ ' --cidfile=$WORKSPACE/docker_build_and_test/docker_ccache_after.cid' + ' -e CCACHE_DIR=/home/buildfarm/.ccache' + ' -v $HOME/.ccache:/home/buildfarm/.ccache' + - ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name) + + ' $DOCKER_IMAGE_PREFIX.ci_build_and_test.%s' % (rosdistro_name or 'global') + ' "ccache -s"', 'echo "# END SECTION"', ] if shared_ccache else [])), @@ -476,8 +476,8 @@ parameters = [ @(SNIPPET( 'archive_artifacts', artifacts=[ - 'ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name, os_code_name, arch), - 'ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name, os_code_name, arch), + 'ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name or 'global', os_code_name, arch), + 'ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name or 'global', os_code_name, arch), ] + archive_files + [ image for images in show_images.values() for image in images ], @@ -488,8 +488,8 @@ parameters = [ config_name='ci_archives', remote_directory=upload_directory, source_files=[ - 'ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name, os_code_name, arch), - 'ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name, os_code_name, arch), + 'ros%d-%s-linux-%s-%s-ci.tar.bz2' % (ros_version, rosdistro_name or 'global', os_code_name, arch), + 'ros%d-%s-linux-%s-%s-ci-CHECKSUM' % (ros_version, rosdistro_name or 'global', os_code_name, arch), ], remove_prefix=None, ))@ diff --git a/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em b/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em index e6d6601a2..afd3d6dae 100644 --- a/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em +++ b/ros_buildfarm/templates/ci/create_workspace.Dockerfile.em @@ -98,7 +98,7 @@ cmds = [ 'PYTHONPATH=/tmp/ros_buildfarm:$PYTHONPATH python3 -u' + \ ' /tmp/ros_buildfarm/scripts/ci/create_workspace.py' + \ - ' ' + rosdistro_name + \ + ' ' + (rosdistro_name or "''") + \ ' --workspace-root ' + workspace_root[-1] + \ ' --repos-file-urls ' + ' '.join('file:///tmp/%s' % repos_file for repos_file in repos_file_names) + \ ' --repository-names ' + ' '.join(repository_names) + \ @@ -108,7 +108,7 @@ cmds = [ 'PYTHONPATH=/tmp/ros_buildfarm:$PYTHONPATH python3 -u' + \ ' /tmp/ros_buildfarm/scripts/ci/generate_install_lists.py' + \ - ' ' + rosdistro_name + \ + ' ' + (rosdistro_name or "''") + \ ' ' + os_name + \ ' ' + os_code_name + \ ' --package-root ' + ' '.join(base_paths) + \ diff --git a/ros_buildfarm/templates/devel/devel_task.Dockerfile.em b/ros_buildfarm/templates/devel/devel_task.Dockerfile.em index b25ca4722..5f9c02437 100644 --- a/ros_buildfarm/templates/devel/devel_task.Dockerfile.em +++ b/ros_buildfarm/templates/devel/devel_task.Dockerfile.em @@ -119,7 +119,7 @@ cmd = \ if not testing: cmd += \ ' /tmp/ros_buildfarm/scripts/devel/build_and_install.py' + \ - ' --rosdistro-name ' + rosdistro_name + \ + ' --rosdistro-name ' + (rosdistro_name or "''") + \ ' --ros-version ' + str(ros_version) + \ ' --clean-before' if run_abichecker: @@ -127,7 +127,7 @@ if not testing: else: cmd += \ ' /tmp/ros_buildfarm/scripts/devel/build_and_test.py' + \ - ' --rosdistro-name %s' % rosdistro_name + \ + ' --rosdistro-name %s' % (rosdistro_name or "''") + \ ' --ros-version ' + str(ros_version) if require_gpu_support: cmd += ' --require-gpu-support'