From d958b79cafa992ec0a3a13dd4680fea7e0887eb8 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Tue, 6 Feb 2024 19:02:50 +0100 Subject: [PATCH 1/4] Do not install ccache unless shared_ccache is used in binarydeb Currently the sharing of compilation artifacts through ccache is not being done unless the shared_ccache is in use. The ccache binaries are currently being installed unconditionally. The PR changes the behaviour to install the ccache only when the shared_ccache option is being used. Signed-off-by: Jose Luis Rivero --- ros_buildfarm/argument.py | 6 ++++++ .../scripts/release/create_binarydeb_task_generator.py | 3 +++ ros_buildfarm/scripts/release/run_binarydeb_job.py | 4 +++- .../release/deb/binarypkg_create_task.Dockerfile.em | 1 + ros_buildfarm/templates/release/deb/binarypkg_job.xml.em | 1 + .../templates/release/deb/binarypkg_task.Dockerfile.em | 2 ++ 6 files changed, 16 insertions(+), 1 deletion(-) diff --git a/ros_buildfarm/argument.py b/ros_buildfarm/argument.py index 20faefddf..81944aa5d 100644 --- a/ros_buildfarm/argument.py +++ b/ros_buildfarm/argument.py @@ -496,6 +496,12 @@ def add_argument_skip_tests(parser): help='Skip execution of tests as part of the package build process.') +def add_argument_install_ccache(parser): + parser.add_argument( + '--install-ccache', action='store_true', + help='Install the ccache binary packages in the system.') + + def check_len_action(minargs, maxargs): class CheckLength(argparse.Action): diff --git a/ros_buildfarm/scripts/release/create_binarydeb_task_generator.py b/ros_buildfarm/scripts/release/create_binarydeb_task_generator.py index 569801026..94390820f 100644 --- a/ros_buildfarm/scripts/release/create_binarydeb_task_generator.py +++ b/ros_buildfarm/scripts/release/create_binarydeb_task_generator.py @@ -26,6 +26,7 @@ from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir from ros_buildfarm.argument import add_argument_env_vars +from ros_buildfarm.argument import add_argument_install_ccache from ros_buildfarm.argument import add_argument_os_code_name from ros_buildfarm.argument import add_argument_os_name from ros_buildfarm.argument import add_argument_package_name @@ -56,6 +57,7 @@ def main(argv=sys.argv[1:]): add_argument_dockerfile_dir(parser) add_argument_env_vars(parser) add_argument_skip_tests(parser) + add_argument_install_ccache(parser) args = parser.parse_args(argv) debian_package_name = get_os_package_name( @@ -107,6 +109,7 @@ def main(argv=sys.argv[1:]): 'dependencies': debian_pkg_names, 'dependency_versions': debian_pkg_versions, 'skip_tests': args.skip_tests, + 'install_ccache': args.install_ccache, 'install_lists': [], 'rosdistro_name': args.rosdistro_name, diff --git a/ros_buildfarm/scripts/release/run_binarydeb_job.py b/ros_buildfarm/scripts/release/run_binarydeb_job.py index c7c4ad068..d21d3620f 100644 --- a/ros_buildfarm/scripts/release/run_binarydeb_job.py +++ b/ros_buildfarm/scripts/release/run_binarydeb_job.py @@ -16,7 +16,7 @@ import copy import sys -from ros_buildfarm.argument import add_argument_append_timestamp +from ros_buildfarm.argument import add_argument_append_timestamp, add_argument_install_ccache from ros_buildfarm.argument import add_argument_arch from ros_buildfarm.argument import add_argument_binarypkg_dir from ros_buildfarm.argument import \ @@ -55,6 +55,7 @@ def main(argv=sys.argv[1:]): add_argument_append_timestamp(parser) add_argument_env_vars(parser) add_argument_skip_tests(parser) + add_argument_install_ccache(parser) args = parser.parse_args(argv) data = copy.deepcopy(args.__dict__) @@ -69,6 +70,7 @@ def main(argv=sys.argv[1:]): 'skip_download_sourcepkg': args.skip_download_sourcepkg, 'skip_tests': args.skip_tests, + 'install_ccache': args.install_ccache, 'binarypkg_dir': '/tmp/binarydeb', 'build_environment_variables': ['%s=%s' % key_value for key_value in args.env_vars.items()], diff --git a/ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em b/ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em index 5ecc9ff29..3a36be1b5 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em @@ -105,6 +105,7 @@ cmds.append( ' --binarypkg-dir ' + binarypkg_dir + ' --env-vars ' + ' '.join(build_environment_variables) + ' --dockerfile-dir ' + dockerfile_dir + + (' --install-ccache ' if install_ccache else '')) + (' --skip-tests' if skip_tests else '')) }@ CMD ["@(' && '.join(cmds))"] diff --git a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em index 70e364ea3..eb22a3a70 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_job.xml.em @@ -112,6 +112,7 @@ but disabled since the package is blacklisted (or not whitelisted) in the config ' --dockerfile-dir $WORKSPACE/docker_generating_docker' + ' --env-vars ' + ' '.join(build_environment_variables) + (' --append-timestamp' if append_timestamp else '') + + (' --install-ccache' if shared_ccache else '') + (' --skip-tests' if skip_tests else ''), 'echo "# END SECTION"', '', diff --git a/ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em b/ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em index da24e7d8e..5928e4008 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em @@ -66,11 +66,13 @@ RUN . /etc/os-release && test "$VERSION_ID" = "20.04" && test "$(uname -m)" = "a os_code_name=os_code_name, ))@ +@[if install_ccache]@ @(TEMPLATE( 'snippet/install_ccache.Dockerfile.em', os_name=os_name, os_code_name=os_code_name, ))@ +@[endif] @(TEMPLATE( 'snippet/set_environment_variables.Dockerfile.em', From 64df76d7545f4f08e8d15523eff5ea3869ac6b50 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Mon, 12 Feb 2024 17:18:18 +0100 Subject: [PATCH 2/4] Split import one per line Signed-off-by: Jose Luis Rivero --- ros_buildfarm/scripts/release/run_binarydeb_job.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ros_buildfarm/scripts/release/run_binarydeb_job.py b/ros_buildfarm/scripts/release/run_binarydeb_job.py index d21d3620f..47fa83455 100644 --- a/ros_buildfarm/scripts/release/run_binarydeb_job.py +++ b/ros_buildfarm/scripts/release/run_binarydeb_job.py @@ -16,7 +16,7 @@ import copy import sys -from ros_buildfarm.argument import add_argument_append_timestamp, add_argument_install_ccache +from ros_buildfarm.argument import add_argument_append_timestamp from ros_buildfarm.argument import add_argument_arch from ros_buildfarm.argument import add_argument_binarypkg_dir from ros_buildfarm.argument import \ @@ -24,6 +24,7 @@ from ros_buildfarm.argument import add_argument_distribution_repository_urls from ros_buildfarm.argument import add_argument_dockerfile_dir from ros_buildfarm.argument import add_argument_env_vars +from ros_buildfarm.argument import add_argument_install_ccache from ros_buildfarm.argument import add_argument_os_code_name from ros_buildfarm.argument import add_argument_os_name from ros_buildfarm.argument import add_argument_package_name From 53dee3a979710310c3993fe250bf109ba5d828ed Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 1 Mar 2024 16:32:48 +0100 Subject: [PATCH 3/4] Found typo in format --- .../templates/release/deb/binarypkg_create_task.Dockerfile.em | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em b/ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em index 3a36be1b5..3ff2f2217 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_create_task.Dockerfile.em @@ -105,7 +105,7 @@ cmds.append( ' --binarypkg-dir ' + binarypkg_dir + ' --env-vars ' + ' '.join(build_environment_variables) + ' --dockerfile-dir ' + dockerfile_dir + - (' --install-ccache ' if install_ccache else '')) + + (' --install-ccache ' if install_ccache else '') + (' --skip-tests' if skip_tests else '')) }@ CMD ["@(' && '.join(cmds))"] From 27b8f67221b88eed2937300a759330723a78d6e2 Mon Sep 17 00:00:00 2001 From: Jose Luis Rivero Date: Fri, 1 Mar 2024 16:41:19 +0100 Subject: [PATCH 4/4] Fix empy syntax on endif --- .../templates/release/deb/binarypkg_task.Dockerfile.em | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em b/ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em index 5928e4008..61e7d4822 100644 --- a/ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em +++ b/ros_buildfarm/templates/release/deb/binarypkg_task.Dockerfile.em @@ -72,7 +72,7 @@ RUN . /etc/os-release && test "$VERSION_ID" = "20.04" && test "$(uname -m)" = "a os_name=os_name, os_code_name=os_code_name, ))@ -@[endif] +@[end if]@ @(TEMPLATE( 'snippet/set_environment_variables.Dockerfile.em',