From 0f1c7d19da389be5d37977d2e4c8b87a4c5fe1e2 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Tue, 30 Nov 2021 23:21:35 -0800 Subject: [PATCH 01/47] add an option to preserve the home directory instead of deleting it if the uid or username collide (#164) * add an option to preserve the home directory instead of deleting it if the uid or username collide Signed-off-by: Tully Foote * add tests for preserve home options Signed-off-by: Tully Foote * fix default override name string Signed-off-by: Tully Foote --- src/rocker/extensions.py | 7 ++++++- src/rocker/templates/user_snippet.Dockerfile.em | 4 ++-- test/test_extension.py | 7 ++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index 9f2a3850..1551e5ce 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -220,6 +220,7 @@ def get_snippet(self, cliargs): if 'user_override_name' in cliargs and cliargs['user_override_name']: substitutions['name'] = cliargs['user_override_name'] substitutions['dir'] = os.path.join('/home/', cliargs['user_override_name']) + substitutions['user_preserve_home'] = True if 'user_preserve_home' in cliargs and cliargs['user_preserve_home'] else False substitutions['home_extension_active'] = True if 'home' in cliargs and cliargs['home'] else False return em.expand(snippet, substitutions) @@ -231,8 +232,12 @@ def register_arguments(parser, defaults={}): help="mount the current user's id and run as that user") parser.add_argument('--user-override-name', action='store', - default=defaults.get('user-override-username', None), + default=defaults.get('user-override-name', None), help="override the current user's name") + parser.add_argument('--user-preserve-home', + action='store_true', + default=defaults.get('user-preserve-home', False), + help="Do not delete home directory if it exists when making a new user.") class Environment(RockerExtension): diff --git a/src/rocker/templates/user_snippet.Dockerfile.em b/src/rocker/templates/user_snippet.Dockerfile.em index 6effef6b..cb2d4fff 100644 --- a/src/rocker/templates/user_snippet.Dockerfile.em +++ b/src/rocker/templates/user_snippet.Dockerfile.em @@ -7,9 +7,9 @@ RUN if ! command -v sudo >/dev/null; then \ @[if name != 'root']@ RUN existing_user_by_uid=`getent passwd "@(uid)" | cut -f1 -d: || true` && \ - if [ -n "${existing_user_by_uid}" ]; then userdel -r "${existing_user_by_uid}"; fi && \ + if [ -n "${existing_user_by_uid}" ]; then userdel @('' if user_preserve_home else '-r') "${existing_user_by_uid}"; fi && \ existing_user_by_name=`getent passwd "@(name)" | cut -f1 -d: || true` && \ - if [ -n "${existing_user_by_name}" ]; then userdel -r "${existing_user_by_name}"; fi && \ + if [ -n "${existing_user_by_name}" ]; then userdel @('' if user_preserve_home else '-r') "${existing_user_by_name}"; fi && \ existing_group_by_gid=`getent group "@(gid)" | cut -f1 -d: || true` && \ if [ -z "${existing_group_by_gid}" ]; then \ groupadd -g "@(gid)" "@name"; \ diff --git a/test/test_extension.py b/test/test_extension.py index 300aa79b..1c18c480 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -248,10 +248,15 @@ def test_user_extension(self): user_override_active_cliargs = mock_cliargs user_override_active_cliargs['user_override_name'] = 'testusername' - print(p.get_snippet(user_override_active_cliargs)) snippet_result = p.get_snippet(user_override_active_cliargs) self.assertTrue('USER testusername' in snippet_result) self.assertTrue('WORKDIR /home/testusername' in snippet_result) + self.assertTrue('userdel -r' in snippet_result) + + user_override_active_cliargs['user_preserve_home'] = True + snippet_result = p.get_snippet(user_override_active_cliargs) + self.assertFalse('userdel -r' in snippet_result) + class PulseExtensionTest(unittest.TestCase): From 2e1f15403b42c6fb4f573d503e5e24e5a2ab6bbc Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Tue, 30 Nov 2021 23:23:00 -0800 Subject: [PATCH 02/47] bump to 0.2.7 Signed-off-by: Tully Foote --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a05f163b..12d6e145 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ kwargs = { 'name': 'rocker', - 'version': '0.2.6', + 'version': '0.2.7', 'packages': ['rocker'], 'package_dir': {'': 'src'}, 'package_data': {'rocker': ['templates/*.em']}, From c4c7be670fb9b2f65f0f2679ffe559a0414e97f4 Mon Sep 17 00:00:00 2001 From: Midnight Exigent <36641328+midnightexigent@users.noreply.github.com> Date: Sat, 8 Jan 2022 02:54:56 +0100 Subject: [PATCH 03/47] Update README.md (#167) --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 668feaa7..8942a6ea 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,19 @@ Rocker is available via pip you can install it via pip using `pip install rocker` +## Archlinux ([AUR](https://aur.archlinux.org/)) +Using any AUR helper, for example, with `paru` + +```bash +paru -S python-rocker +``` + +or + +```bash +paru -S python-rocker-git +``` ## Development To set things up in a virtual environment for isolation is a good way. If you don't already have it install python3's venv module. From dab596dd3c2e06b209760874b91ce8cbed117589 Mon Sep 17 00:00:00 2001 From: Marco Perin Date: Wed, 9 Feb 2022 18:41:26 +0100 Subject: [PATCH 04/47] Improve help message (#171) * Update README.md Update the "Volume mount" section, making it more readable and explicit. --- README.md | 20 ++++++++++++-------- src/rocker/volume_extension.py | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8942a6ea..fefc7697 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,11 @@ Docker installation instructions: https://docs.docker.com/install/ For the NVIDIA option this has been tested on the following systems using nvidia docker2: -| Ubuntu distribution | Linux Kernel | Nvidia drivers | -| -------------------- | ------------ | ------------------------- | -| 16.04 | 4.15 | nvidia-384 (works)
nvidia-340 (doesn't work) | -| 18.04 | | nvidia-390 (works) | -| 20.04 | 5.4.0 | nvidia-driver-460 (works) | +| Ubuntu distribution | Linux Kernel | Nvidia drivers | +| ------------------- | ------------ | ------------------------------------------------- | +| 16.04 | 4.15 | nvidia-384 (works)
nvidia-340 (doesn't work) | +| 18.04 | | nvidia-390 (works) | +| 20.04 | 5.4.0 | nvidia-driver-460 (works) | Install nvidia-docker 2: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker @@ -154,9 +154,11 @@ On Bionic ## Volume mount -For arguments with one element not colon separated. +### For arguments with one element not colon separated -`--volume` adds paths as docker volumes. The last path must be terminated with two dashes `--`. +`--volume` adds paths as docker volumes. + +**The last path must be terminated with two dashes `--`**. rocker --volume ~/.vimrc ~/.bashrc -- ubuntu:18.04 @@ -164,7 +166,9 @@ The above example of the volume option will be expanded via absolute paths for ` --volume /home//.vimrc:/home//.vimrc --volume /home//.bashrc:/home//.bashrc -For arguments with colon separation it will process the same as `docker`'s `--volume` option, `rocker --volume` takes 3 fields. +### For arguments with colon separation + +It will process the same as `docker`'s `--volume` option, `rocker --volume` takes 3 fields. - 1st field: the path to the file or directory on the host machine. - 2nd field: (optional) the path where the file or directory is mounted in the container. - If only the 1st field is supplied, same value as the 1st field will be populated as the 2nd field. diff --git a/src/rocker/volume_extension.py b/src/rocker/volume_extension.py index dd571b2e..6f2626fb 100644 --- a/src/rocker/volume_extension.py +++ b/src/rocker/volume_extension.py @@ -67,4 +67,4 @@ def register_arguments(parser): type=str, nargs='+', action='append', - help='volume volumes in container') + help='volume(s) to map into the container. The last path must be followed by two dashes "--"') From 28a8e796ed7fe9b450a80a6385b3a2f56e85e3d0 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Mon, 14 Feb 2022 14:16:12 -0800 Subject: [PATCH 05/47] If the same username exists with a different uid adopt their files so the newly created user has access. (#168) * If the same username exists with a different uid adopt their files so the newly created user has access. Signed-off-by: Tully Foote * Add unit tests for user group overrides Signed-off-by: Tully Foote --- .../templates/user_snippet.Dockerfile.em | 2 + test/test_extension.py | 47 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/src/rocker/templates/user_snippet.Dockerfile.em b/src/rocker/templates/user_snippet.Dockerfile.em index cb2d4fff..e5a09639 100644 --- a/src/rocker/templates/user_snippet.Dockerfile.em +++ b/src/rocker/templates/user_snippet.Dockerfile.em @@ -9,6 +9,8 @@ RUN if ! command -v sudo >/dev/null; then \ RUN existing_user_by_uid=`getent passwd "@(uid)" | cut -f1 -d: || true` && \ if [ -n "${existing_user_by_uid}" ]; then userdel @('' if user_preserve_home else '-r') "${existing_user_by_uid}"; fi && \ existing_user_by_name=`getent passwd "@(name)" | cut -f1 -d: || true` && \ + existing_user_uid=`getent passwd "@(name)" | cut -f3 -d: || true` && \ + if [ -n "${existing_user_by_name}" ]; then find / -uid ${existing_user_uid} -exec chown -h @(uid) {} + || true ; find / -gid ${existing_user_uid} -exec chgrp -h @(uid) {} + || true ; fi && \ if [ -n "${existing_user_by_name}" ]; then userdel @('' if user_preserve_home else '-r') "${existing_user_by_name}"; fi && \ existing_group_by_gid=`getent group "@(gid)" | cut -f1 -d: || true` && \ if [ -z "${existing_group_by_gid}" ]; then \ diff --git a/test/test_extension.py b/test/test_extension.py index 1c18c480..d9e7aff0 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -22,8 +22,11 @@ import unittest from pathlib import Path import pwd +from io import BytesIO as StringIO +from rocker.core import DockerImageGenerator +from rocker.core import docker_build from rocker.core import list_plugins from rocker.extensions import name_to_argument @@ -257,6 +260,50 @@ def test_user_extension(self): snippet_result = p.get_snippet(user_override_active_cliargs) self.assertFalse('userdel -r' in snippet_result) + def test_user_collisions(self): + plugins = list_plugins() + user_plugin = plugins['user'] + self.assertEqual(user_plugin.get_name(), 'user') + + uid = os.getuid()+1 + COLLIDING_UID_DOCKERFILE = f"""FROM ubuntu:jammy +RUN useradd test -u{uid} + +""" + iof = StringIO(COLLIDING_UID_DOCKERFILE.encode()) + image_id = docker_build( + fileobj=iof, + #output_callback=output_callback, + nocache=True, + forcerm=True, + tag="rocker:" + f"user_extension_test_uid_collision" + ) + print(f'Image id is {image_id}') + self.assertTrue(image_id, f"Image failed to build >>>{COLLIDING_UID_DOCKERFILE}<<<") + + # Test Colliding UID but not name + build_args = { + 'user': True, + 'user_override_name': 'test2', + 'user_preserve_home': True, + # 'command': 'ls -l && touch /home/test2/home_directory_access_verification', + 'command': 'touch /home/test2/testwrite', + } + dig = DockerImageGenerator([user_plugin()], build_args, image_id) + exit_code = dig.build(**build_args) + self.assertTrue(exit_code == 0, f"Build failed with exit code {exit_code}") + run_exit_code = dig.run(**build_args) + self.assertTrue(run_exit_code == 0, f"Run failed with exit code {run_exit_code}") + + + # Test colliding UID and name + build_args['user_override_name'] = 'test' + build_args['command'] = 'touch /home/test/testwrite' + dig = DockerImageGenerator([user_plugin()], build_args, image_id) + exit_code = dig.build(**build_args) + self.assertTrue(exit_code == 0, f"Build failed with exit code {exit_code}") + run_exit_code = dig.run(**build_args) + self.assertTrue(run_exit_code == 0, f"Run failed with exit code {run_exit_code}") class PulseExtensionTest(unittest.TestCase): From b2cd095c6e46da72fb2d4b504e5229dffcab08b2 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Mon, 14 Feb 2022 14:26:55 -0800 Subject: [PATCH 06/47] 0.2.8 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 12d6e145..ff2e92d7 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ kwargs = { 'name': 'rocker', - 'version': '0.2.7', + 'version': '0.2.8', 'packages': ['rocker'], 'package_dir': {'': 'src'}, 'package_data': {'rocker': ['templates/*.em']}, From 0fde53b0f6a2ff5f6e17496b52b934a09fcd0bda Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Wed, 2 Mar 2022 20:53:47 -0800 Subject: [PATCH 07/47] Update debian packaging targets (#172) Remove most EOL targests, and add a few of the newer ones Signed-off-by: Tully Foote --- stdeb.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdeb.cfg b/stdeb.cfg index 40482429..4cd9cbd4 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -3,5 +3,5 @@ Debian-Version: 100 No-Python2: Depends3: python3-docker, python3-empy, python3-pexpect, python3-packaging Conflicts3: python-rocker -Suite: xenial yakkety zesty artful bionic cosmic disco eoan focal stretch buster +Suite: bionic focal jammy stretch buster bullseye X-Python3-Version: >= 3.2 From eb0439d4d04bc88bff2042be5c992394f5f46be0 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Wed, 23 Mar 2022 15:59:24 -0700 Subject: [PATCH 08/47] 22.04 tested working (#175) Signed-off-by: Tully Foote --- README.md | 1 + src/rocker/nvidia_extension.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fefc7697..14b7c8dd 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,7 @@ For the NVIDIA option this has been tested on the following systems using nvidia | 16.04 | 4.15 | nvidia-384 (works)
nvidia-340 (doesn't work) | | 18.04 | | nvidia-390 (works) | | 20.04 | 5.4.0 | nvidia-driver-460 (works) | +| 22.04 | 5.13.0 | nvidia-driver-470 (works) | Install nvidia-docker 2: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker diff --git a/src/rocker/nvidia_extension.py b/src/rocker/nvidia_extension.py index 327ade7f..607ede94 100644 --- a/src/rocker/nvidia_extension.py +++ b/src/rocker/nvidia_extension.py @@ -85,7 +85,7 @@ def __init__(self): self._env_subs = None self.name = Nvidia.get_name() self.supported_distros = ['Ubuntu', 'Debian GNU/Linux'] - self.supported_versions = ['16.04', '18.04', '20.04', '10'] + self.supported_versions = ['16.04', '18.04', '20.04', '10', '22.04'] def get_environment_subs(self, cliargs={}): From 556596dc099f823a7364b61e68e68b9831eed4b3 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Wed, 23 Mar 2022 16:13:46 -0700 Subject: [PATCH 09/47] 0.2.9 Signed-off-by: Tully Foote --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ff2e92d7..c129e145 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ kwargs = { 'name': 'rocker', - 'version': '0.2.8', + 'version': '0.2.9', 'packages': ['rocker'], 'package_dir': {'': 'src'}, 'package_data': {'rocker': ['templates/*.em']}, From 0e528650386626470c72ef844127f6d802c57dcb Mon Sep 17 00:00:00 2001 From: Kenji Miyake <31987104+kenji-miyake@users.noreply.github.com> Date: Fri, 8 Jul 2022 03:08:10 +0900 Subject: [PATCH 10/47] Set default NVIDIA_DRIVER_CAPABILITIES if it's not set (#182) * Set default NVIDIA_DRIVER_CAPABILITIES to all if it's not set Signed-off-by: Kenji Miyake --- src/rocker/templates/nvidia_snippet.Dockerfile.em | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rocker/templates/nvidia_snippet.Dockerfile.em b/src/rocker/templates/nvidia_snippet.Dockerfile.em index 69e9a09c..2393263a 100644 --- a/src/rocker/templates/nvidia_snippet.Dockerfile.em +++ b/src/rocker/templates/nvidia_snippet.Dockerfile.em @@ -25,4 +25,4 @@ COPY --from=glvnd /usr/share/glvnd/egl_vendor.d/10_nvidia.json /usr/share/glvnd/ ENV NVIDIA_VISIBLE_DEVICES ${NVIDIA_VISIBLE_DEVICES:-all} -ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES:+$NVIDIA_DRIVER_CAPABILITIES,}graphics +ENV NVIDIA_DRIVER_CAPABILITIES ${NVIDIA_DRIVER_CAPABILITIES:-all} From e2864fb1214e19f81ca50759ef2fad979e1e056c Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Fri, 8 Jul 2022 19:09:41 -0700 Subject: [PATCH 11/47] clean up tags created for os_detector (#184) * clean up tags created for os_detector They were leaving tagged images which were eventually taking up a lot of disk space. Signed-off-by: Tully Foote --- src/rocker/os_detector.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/rocker/os_detector.py b/src/rocker/os_detector.py index 46a4295e..9559672e 100644 --- a/src/rocker/os_detector.py +++ b/src/rocker/os_detector.py @@ -17,7 +17,7 @@ from ast import literal_eval from io import BytesIO as StringIO -from .core import docker_build +from .core import docker_build, get_docker_client DETECTION_TEMPLATE=""" @@ -54,12 +54,13 @@ def detect_os(image_name, output_callback=None, nocache=False): return _detect_os_cache[image_name] iof = StringIO((DETECTION_TEMPLATE % locals()).encode()) + tag_name = "rocker:" + f"os_detect_{image_name}".replace(':', '_').replace('/', '_') image_id = docker_build( fileobj=iof, output_callback=output_callback, nocache=nocache, forcerm=True, # Remove intermediate containers from RUN commands in DETECTION_TEMPLATE - tag="rocker:" + f"os_detect_{image_name}".replace(':', '_').replace('/', '_') + tag=tag_name ) if not image_id: if output_callback: @@ -74,6 +75,11 @@ def detect_os(image_name, output_callback=None, nocache=False): if output_callback: output_callback("output: ", output) p.terminate() + + # Clean up the image + client = get_docker_client() + client.remove_image(image=tag_name) + if p.exitstatus == 0: _detect_os_cache[image_name] = literal_eval(output.strip()) return _detect_os_cache[image_name] From bf85943b6af63e721b5a75e667c9f9723e616dbc Mon Sep 17 00:00:00 2001 From: Rufus Wong Date: Thu, 14 Jul 2022 00:40:38 +0200 Subject: [PATCH 12/47] Make using user shell optional (#185) --- src/rocker/extensions.py | 9 +++++++++ src/rocker/templates/user_snippet.Dockerfile.em | 2 +- test/test_extension.py | 11 +++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index 1551e5ce..9c376124 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -222,6 +222,11 @@ def get_snippet(self, cliargs): substitutions['dir'] = os.path.join('/home/', cliargs['user_override_name']) substitutions['user_preserve_home'] = True if 'user_preserve_home' in cliargs and cliargs['user_preserve_home'] else False substitutions['home_extension_active'] = True if 'home' in cliargs and cliargs['home'] else False + if 'user_override_shell' in cliargs and cliargs['user_override_shell'] is not None: + if cliargs['user_override_shell'] == '': + substitutions['shell'] = None + else: + substitutions['shell'] = cliargs['user_override_shell'] return em.expand(snippet, substitutions) @staticmethod @@ -238,6 +243,10 @@ def register_arguments(parser, defaults={}): action='store_true', default=defaults.get('user-preserve-home', False), help="Do not delete home directory if it exists when making a new user.") + parser.add_argument('--user-override-shell', + action='store', + default=defaults.get('user-override-shell', None), + help="Override the current user's shell. Set to empty string to use container default shell") class Environment(RockerExtension): diff --git a/src/rocker/templates/user_snippet.Dockerfile.em b/src/rocker/templates/user_snippet.Dockerfile.em index e5a09639..63911409 100644 --- a/src/rocker/templates/user_snippet.Dockerfile.em +++ b/src/rocker/templates/user_snippet.Dockerfile.em @@ -16,7 +16,7 @@ RUN existing_user_by_uid=`getent passwd "@(uid)" | cut -f1 -d: || true` && \ if [ -z "${existing_group_by_gid}" ]; then \ groupadd -g "@(gid)" "@name"; \ fi && \ - useradd --no-log-init --no-create-home --uid "@(uid)" -s "@(shell)" -c "@(gecos)" -g "@(gid)" -d "@(dir)" "@(name)" && \ + useradd --no-log-init --no-create-home --uid "@(uid)" @(str('-s ' + shell) if shell else '') -c "@(gecos)" -g "@(gid)" -d "@(dir)" "@(name)" && \ echo "@(name) ALL=NOPASSWD: ALL" >> /etc/sudoers.d/rocker @[if not home_extension_active ]@ diff --git a/test/test_extension.py b/test/test_extension.py index d9e7aff0..a383500d 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -260,6 +260,17 @@ def test_user_extension(self): snippet_result = p.get_snippet(user_override_active_cliargs) self.assertFalse('userdel -r' in snippet_result) + snippet_result = p.get_snippet(user_override_active_cliargs) + self.assertTrue(('-s ' + pwd.getpwuid(os.getuid()).pw_shell) in snippet_result) + + user_override_active_cliargs['user_override_shell'] = 'testshell' + snippet_result = p.get_snippet(user_override_active_cliargs) + self.assertTrue('-s testshell' in snippet_result) + + user_override_active_cliargs['user_override_shell'] = '' + snippet_result = p.get_snippet(user_override_active_cliargs) + self.assertFalse('-s' in snippet_result) + def test_user_collisions(self): plugins = list_plugins() user_plugin = plugins['user'] From 4fa9b946ebbae98bd180fd129acd45f6f5599aad Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Fri, 29 Jul 2022 17:15:42 -0700 Subject: [PATCH 13/47] 0.2.10 Signed-off-by: Tully Foote --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c129e145..2d53d0ef 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ kwargs = { 'name': 'rocker', - 'version': '0.2.9', + 'version': '0.2.10', 'packages': ['rocker'], 'package_dir': {'': 'src'}, 'package_data': {'rocker': ['templates/*.em']}, From a75f70f71d172369bd127c9da11a97f3bc3c233b Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 3 Aug 2022 12:20:51 -0700 Subject: [PATCH 14/47] Drop executable and shebang from non-scripts in module (#191) These files are executable and contain a shebang, but do nothing when executed. --- src/rocker/cli.py | 2 -- src/rocker/core.py | 2 -- 2 files changed, 4 deletions(-) mode change 100755 => 100644 src/rocker/cli.py mode change 100755 => 100644 src/rocker/core.py diff --git a/src/rocker/cli.py b/src/rocker/cli.py old mode 100755 new mode 100644 index a1d0641d..9044cddd --- a/src/rocker/cli.py +++ b/src/rocker/cli.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright 2019 Open Source Robotics Foundation # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/src/rocker/core.py b/src/rocker/core.py old mode 100755 new mode 100644 index 9b87c400..dad8a11a --- a/src/rocker/core.py +++ b/src/rocker/core.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python3 - # Copyright 2019 Open Source Robotics Foundation # Licensed under the Apache License, Version 2.0 (the "License"); From 3227922e0601de3e55df7de727a3d7e2539c357a Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Wed, 3 Aug 2022 12:21:49 -0700 Subject: [PATCH 15/47] Persist xauth file if in a --nocleanup mode (#189) * if nocleanup argument persist xauth file * remove unused import Flagged in #188 by @tonynajjar * add simple test coverage Signed-off-by: Tully Foote --- src/rocker/nvidia_extension.py | 5 +++-- test/test_nvidia.py | 12 ++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/rocker/nvidia_extension.py b/src/rocker/nvidia_extension.py index 607ede94..07c96485 100644 --- a/src/rocker/nvidia_extension.py +++ b/src/rocker/nvidia_extension.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import grp import os import em import getpass @@ -42,9 +41,10 @@ def get_name(): def __init__(self): self.name = X11.get_name() self._env_subs = None - self._xauth = tempfile.NamedTemporaryFile(prefix='.docker', suffix='.xauth') + self._xauth = None def get_docker_args(self, cliargs): + assert self._xauth, 'xauth not initialized, get_docker_args must be called after precodition_environment' xauth = self._xauth.name return " -e DISPLAY -e TERM \ -e QT_X11_NO_MITSHM=1 \ @@ -53,6 +53,7 @@ def get_docker_args(self, cliargs): -v /etc/localtime:/etc/localtime:ro " % locals() def precondition_environment(self, cliargs): + self._xauth = tempfile.NamedTemporaryFile(prefix='.docker', suffix='.xauth', delete=not cliargs.get('nocleanup')) xauth = self._xauth.name display = os.getenv('DISPLAY') # Make sure processes in the container can connect to the x server diff --git a/test/test_nvidia.py b/test/test_nvidia.py index 01bf2d4d..69b290f7 100644 --- a/test/test_nvidia.py +++ b/test/test_nvidia.py @@ -72,6 +72,9 @@ def test_x11_extension_basic(self): p = x11_plugin() mock_cliargs = {'base_image': 'ubuntu:xenial'} + # Must be called before get_docker_args + docker_args = p.precondition_environment(mock_cliargs) + docker_args = p.get_docker_args(mock_cliargs) self.assertIn(' -e DISPLAY -e TERM', docker_args) self.assertIn(' -e QT_X11_NO_MITSHM=1', docker_args) @@ -79,6 +82,15 @@ def test_x11_extension_basic(self): self.assertIn(' -v /tmp/.X11-unix:/tmp/.X11-unix ', docker_args) self.assertIn(' -v /etc/localtime:/etc/localtime:ro ', docker_args) + def test_x11_extension_nocleanup(self): + plugins = list_plugins() + x11_plugin = plugins['x11'] + p = x11_plugin() + mock_cliargs = {'base_image': 'ubuntu:xenial', 'nocleanup': True} + docker_args = p.precondition_environment(mock_cliargs) + # TODO(tfoote) do more to check that it doesn't actually clean up. + # This is more of a smoke test + def test_no_x11_xpdyinfo(self): for tag in self.dockerfile_tags: From 74730b13c3af1f79d742415028bf07f7d0452674 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 3 Aug 2022 12:58:24 -0700 Subject: [PATCH 16/47] Run tests with pytest instead of nose (#192) The 'nose' package is no longer under active development and it is generally recommended to move to a more supported Python testing framework. --- .github/workflows/basic-ci.yaml | 4 ++-- README.md | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/basic-ci.yaml b/.github/workflows/basic-ci.yaml index 1bff4f2f..48590893 100644 --- a/.github/workflows/basic-ci.yaml +++ b/.github/workflows/basic-ci.yaml @@ -19,11 +19,11 @@ jobs: python -m pip install --upgrade pip setuptools wheel # Workaround for https://github.com/docker/docker-py/issues/2807 python -m pip install six - pip install codecov coverage nose + pip install codecov pytest pytest-cov pip install . - name: Run headless tests uses: GabrielBB/xvfb-action@v1 with: - run: nosetests -s -v --with-coverage --cover-package rocker --exclude test_nvidia_glmark2 + run: python -m pytest -s -v --cov=rocker -k "not test_nvidia_glmark2" - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/README.md b/README.md index 14b7c8dd..34120519 100644 --- a/README.md +++ b/README.md @@ -100,19 +100,19 @@ For any new terminal re activate the venv before trying to use it. ### Testing -To run tests install nose and coverage in the venv +To run tests install pytest and pytest-cov in the venv . ~/rocker_venv/bin/activate - pip install nose - pip install coverage + pip install pytest + pip install pytest-cov -Then you can run nosetests. +Then you can run pytest. - nosetests-3.4 --with-coverage --cover-package rocker + python3 -m pytest --cov=rocker Notes: -- Make sure to use the python3 instance of nosetest from inside the environment. +- Make sure to use the python3 instance of pytest from inside the environment. - The tests include an nvidia test which assumes you're using a machine with an nvidia gpu. From a548c4a96ed9fae67b90160e25b0752df69f0a84 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 3 Aug 2022 14:14:02 -0700 Subject: [PATCH 17/47] Mark tests which have external dependencies (#194) There are three classes of external requirements that I've found in these tests: * Tests which require a docker engine * Tests which require an NVIDIA GPU and drivers * Tests which require an X11 display of some kind Any of these could be unavailable for some reason, so it makes sense to add labels so that they can be easily skipped. --- .github/workflows/basic-ci.yaml | 2 +- setup.cfg | 8 ++++++++ test/test_core.py | 12 +++++++++++- test/test_extension.py | 3 +++ test/test_nvidia.py | 6 ++++++ test/test_os_detect.py | 5 +++++ 6 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 setup.cfg diff --git a/.github/workflows/basic-ci.yaml b/.github/workflows/basic-ci.yaml index 48590893..83dcfd54 100644 --- a/.github/workflows/basic-ci.yaml +++ b/.github/workflows/basic-ci.yaml @@ -24,6 +24,6 @@ jobs: - name: Run headless tests uses: GabrielBB/xvfb-action@v1 with: - run: python -m pytest -s -v --cov=rocker -k "not test_nvidia_glmark2" + run: python -m pytest -s -v --cov=rocker -m "not nvidia" - name: Upload coverage to Codecov uses: codecov/codecov-action@v1 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..e98c9901 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,8 @@ +[tool:pytest] +markers = + # Tests which require a docker engine + docker + # Tests which require an NVIDIA GPU and drivers + nvidia + # Tests which require an X11 display of some kind + x11 diff --git a/test/test_core.py b/test/test_core.py index 914d8437..f57e659b 100644 --- a/test/test_core.py +++ b/test/test_core.py @@ -17,6 +17,7 @@ import argparse import em +import pytest import unittest from itertools import chain @@ -56,18 +57,21 @@ def test_get_rocker_version(self): # Check that it can be cast to an int i = int(p) + @pytest.mark.docker def test_run_before_build(self): dig = DockerImageGenerator([], {}, 'ubuntu:bionic') self.assertEqual(dig.run('true'), 1) self.assertEqual(dig.build(), 0) self.assertEqual(dig.run('true'), 0) + @pytest.mark.docker def test_return_code_no_extensions(self): dig = DockerImageGenerator([], {}, 'ubuntu:bionic') self.assertEqual(dig.build(), 0) self.assertEqual(dig.run('true'), 0) self.assertEqual(dig.run('false'), 1) + @pytest.mark.docker def test_return_code_multiple_extensions(self): plugins = list_plugins() desired_plugins = ['home', 'user'] @@ -77,29 +81,34 @@ def test_return_code_multiple_extensions(self): self.assertEqual(dig.run('true'), 0) self.assertEqual(dig.run('false'), 1) + @pytest.mark.docker def test_noexecute(self): dig = DockerImageGenerator([], {}, 'ubuntu:bionic') self.assertEqual(dig.build(), 0) self.assertEqual(dig.run('true', noexecute=True), 0) + @pytest.mark.docker def test_dry_run(self): dig = DockerImageGenerator([], {}, 'ubuntu:bionic') self.assertEqual(dig.build(), 0) self.assertEqual(dig.run('true', mode='dry-run'), 0) self.assertEqual(dig.run('false', mode='dry-run'), 0) + @pytest.mark.docker def test_non_interactive(self): dig = DockerImageGenerator([], {}, 'ubuntu:bionic') self.assertEqual(dig.build(), 0) self.assertEqual(dig.run('true', mode='non-interactive'), 0) self.assertEqual(dig.run('false', mode='non-interactive'), 1) + @pytest.mark.docker def test_device(self): dig = DockerImageGenerator([], {}, 'ubuntu:bionic') self.assertEqual(dig.build(), 0) self.assertEqual(dig.run('true', devices=['/dev/random']), 0) self.assertEqual(dig.run('true', devices=['/dev/does_not_exist']), 0) + @pytest.mark.docker def test_network(self): dig = DockerImageGenerator([], {}, 'ubuntu:bionic') self.assertEqual(dig.build(), 0) @@ -107,6 +116,7 @@ def test_network(self): for n in networks: self.assertEqual(dig.run('true', network=n), 0) + @pytest.mark.docker def test_extension_manager(self): parser = argparse.ArgumentParser() extension_manager = RockerExtensionManager() @@ -146,4 +156,4 @@ def test_docker_cmd_nocleanup(self): self.assertIn('--rm', dig.generate_docker_cmd(mode='dry-run')) self.assertIn('--rm', dig.generate_docker_cmd(nocleanup='')) - self.assertNotIn('--rm', dig.generate_docker_cmd(nocleanup='true')) \ No newline at end of file + self.assertNotIn('--rm', dig.generate_docker_cmd(nocleanup='true')) diff --git a/test/test_extension.py b/test/test_extension.py index a383500d..34a387ff 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -22,6 +22,7 @@ import unittest from pathlib import Path import pwd +import pytest from io import BytesIO as StringIO @@ -126,6 +127,7 @@ def setUp(self): # "em.Error: interpreter stdout proxy lost" em.Interpreter._wasProxyInstalled = False + @pytest.mark.docker def test_network_extension(self): plugins = list_plugins() network_plugin = plugins['network'] @@ -271,6 +273,7 @@ def test_user_extension(self): snippet_result = p.get_snippet(user_override_active_cliargs) self.assertFalse('-s' in snippet_result) + @pytest.mark.docker def test_user_collisions(self): plugins = list_plugins() user_plugin = plugins['user'] diff --git a/test/test_nvidia.py b/test/test_nvidia.py index 69b290f7..53662331 100644 --- a/test/test_nvidia.py +++ b/test/test_nvidia.py @@ -19,6 +19,7 @@ import em import unittest import pexpect +import pytest from io import BytesIO as StringIO @@ -31,6 +32,7 @@ from test_extension import plugin_load_parser_correctly +@pytest.mark.docker class X11Test(unittest.TestCase): @classmethod def setUpClass(self): @@ -98,6 +100,7 @@ def test_no_x11_xpdyinfo(self): self.assertEqual(dig.build(), 0) self.assertNotEqual(dig.run(), 0) + @pytest.mark.x11 def test_x11_xpdyinfo(self): plugins = list_plugins() desired_plugins = ['x11'] @@ -108,6 +111,7 @@ def test_x11_xpdyinfo(self): self.assertEqual(dig.run(), 0) +@pytest.mark.docker class NvidiaTest(unittest.TestCase): @classmethod def setUpClass(self): @@ -188,6 +192,8 @@ def test_no_nvidia_glmark2(self): self.assertEqual(dig.build(), 0) self.assertNotEqual(dig.run(), 0) + @pytest.mark.nvidia + @pytest.mark.x11 def test_nvidia_glmark2(self): plugins = list_plugins() desired_plugins = ['x11', 'nvidia', 'user'] #TODO(Tfoote) encode the x11 dependency into the plugin and remove from test here diff --git a/test/test_os_detect.py b/test/test_os_detect.py index 078ba44a..dc75e5ef 100644 --- a/test/test_os_detect.py +++ b/test/test_os_detect.py @@ -16,6 +16,7 @@ # under the License. import docker +import pytest import unittest @@ -23,6 +24,7 @@ class RockerOSDetectorTest(unittest.TestCase): + @pytest.mark.docker def test_ubuntu(self): result = detect_os("ubuntu:xenial") self.assertEqual(result[0], 'Ubuntu') @@ -37,15 +39,18 @@ def test_ubuntu(self): self.assertEqual(result[0], 'Ubuntu') self.assertEqual(result[1], '18.04') + @pytest.mark.docker def test_fedora(self): result = detect_os("fedora:29") self.assertEqual(result[0], 'Fedora') self.assertEqual(result[1], '29') + @pytest.mark.docker def test_does_not_exist(self): result = detect_os("osrf/ros:does_not_exist") self.assertEqual(result, None) + @pytest.mark.docker def test_cannot_detect_os(self): # Test with output callback too get coverage of error reporting result = detect_os("scratch", output_callback=print) From 45b00b7f65f9894f06a41fa738151765247f7ab1 Mon Sep 17 00:00:00 2001 From: Scott K Logan Date: Wed, 3 Aug 2022 16:56:52 -0700 Subject: [PATCH 18/47] Declare test dependency on pytest using extras_require (#195) * Declare test dependency on pytest using extras_require This same approach to declaring test dependencies has been taken in several ros-infrastructure and colcon packages, and allows tools like colcon to invoke the appropriate testing framework. --- .github/workflows/basic-ci.yaml | 3 +-- README.md | 5 ++--- setup.py | 5 +++++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/basic-ci.yaml b/.github/workflows/basic-ci.yaml index 83dcfd54..db169448 100644 --- a/.github/workflows/basic-ci.yaml +++ b/.github/workflows/basic-ci.yaml @@ -19,8 +19,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel # Workaround for https://github.com/docker/docker-py/issues/2807 python -m pip install six - pip install codecov pytest pytest-cov - pip install . + python -m pip install -e .[test] codecov pytest-cov - name: Run headless tests uses: GabrielBB/xvfb-action@v1 with: diff --git a/README.md b/README.md index 34120519..e02b6eae 100644 --- a/README.md +++ b/README.md @@ -100,11 +100,10 @@ For any new terminal re activate the venv before trying to use it. ### Testing -To run tests install pytest and pytest-cov in the venv +To run tests install the 'test' extra and pytest-cov in the venv . ~/rocker_venv/bin/activate - pip install pytest - pip install pytest-cov + pip install -e .[test] pytest-cov Then you can run pytest. diff --git a/setup.py b/setup.py index 2d53d0ef..88aa21e3 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,11 @@ 'python_requires': '>=3.0', 'install_requires': install_requires, + 'extras_require': { + 'test': [ + 'pytest' + ] + }, 'url': 'https://github.com/osrf/rocker' } From 3a2bf74209f577f666857054095a57bbeb3c6c1d Mon Sep 17 00:00:00 2001 From: Zahi Kakish Date: Thu, 4 Aug 2022 17:22:03 -0600 Subject: [PATCH 19/47] Added note for linking Intel Xe cards with rocker (#190) * Added note for linking Intel Xe cards with rocker When running on an 11th gen Intel processor, I continually got a `(Segmentation Fault)` and `Forcing OpenGl version 0.` error when launching Rviz or any other GUI that required accelerated graphics within a container. This only happened when I linked the integrated graphics with the `--devices /dev/dri/card0` option as suggested by the documentation. Further research suggested that the `/dev/dri/renderD128` is the correct device to link for Intel chipsets with Xe graphics. Therefore, I've added an extra note on the `rocker` README for future users. Please let me know if there is anything else I should change! * Updated README to show mounting whole directory Updated method for mounting Intel integrated graphics according to official documentation. --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e02b6eae..45b345e0 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,12 @@ Note, that changing this setting will lead to a `Failed to initialize NVML: Unkn ## Intel integrated graphics support -For intel integrated graphics support you will need to mount through a specific device +For [Intel integrated graphics support](https://www.intel.com/content/www/us/en/develop/documentation/get-started-with-ai-linux/top/using-containers/using-containers-with-the-command-line.html) you will need to mount the `/dev/dri` directory as follows: ``` ---devices /dev/dri/card0 +--devices /dev/dri ``` - # Installation ## Debians (Recommended) From 675f9245979d8df1138f56b65201b052ca11320d Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Tue, 23 Aug 2022 14:35:00 -0700 Subject: [PATCH 20/47] extend README to give a better summary (#197) comparison to docker-compose List primary extensions and use cases Signed-off-by: Tully Foote --- README.md | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 45b345e0..f9612196 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,51 @@ A tool to run docker images with customized local support injected for things like nvidia support. And user id specific files for cleaner mounting file permissions. +## Difference from docker-compose + +A common question about rocker is how is it different than `docker-compose`. +`rocker` is designed to solve a similar but different problem than `docker-compose`. +The primary goal of `rocker` is to support the use of Docker in use cases where the containers will be effected by the local environment. +A primary example of this is setting up file permissions inside the container to match the users outside of the container so that mounted files inside the container have the same UID as the host. +This is done by dynamically generating overlays on the same core image after detecting the local conditions required. + +The secondary feature that `rocker` provides that docker-compose does not address is the ability to inject extra use case specific capabilities into a container before running. +A common example is the ability to use NVIDIA drivers on a standard published image. +`rocker` will take that standard published image and both inject the necessary drivers into the container which will match your host driver and simultaneously set the correct runtime flags. +This is possible to do with docker-compose or straight docker. +But the drawbacks are that you have to build and publish the combinatoric images of all possible drivers, and in addition you need to manually make sure to pass all the correct runtime arguments. +This is especially true if you want to combine multiple possible additional features, such that the number of images starts scaling in a polynomic manner and maintenance of the number of images becomes unmanagable quickly. +Whereas with `rocker` you can invoke your specific plugins and it will use multi-stage builds of docker images to customize the container for your specific use case, which lets you use official upstream docker images without requiring you to maintain a plethora of parallel built images. + + + ## Know extensions -Rocker supports extensions via entry points there are some built in but you can add your own. Here's a list of public repositories with extensions. +Rocker supports extensions via entry points there are some built in but you can add your own. + +### Integrated Extensions + +There are a number of integrated extensions here's some of the highlights. +You can get full details on the extensions from the main `rocker --help` command. + +- x11 -- Enable the use of X11 inside the container via the host X instance. +- nvidia -- Enable NVIDIA graphics cards for rendering +- cuda -- Enable NVIDIA CUDA in the container +- user -- Create a user inside the container with the same settings as the host and run commands inside the container as that user. +- home -- Mount the user's home directory into the container +- pulse -- Mount pulse audio into the container +- ssh -- Pass through ssh access to the container. + +As well as access to many of the docker arguments as well such as `device`, `env`, `volume`, `name`, `network`, and `privileged`. + +### Externally maintained extensions + +Here's a list of public repositories with extensions. - Off-your-rocker: https://github.com/sloretz/off-your-rocker + + # Prerequisites This should work on most systems using with a recent docker version available. From 5c3a73cf6d378b676e5aba3454d2ac77a3acafd0 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Thu, 25 Aug 2022 16:08:23 -0700 Subject: [PATCH 21/47] Talk about what it enables in docs (#198) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f9612196..2d1cd5fa 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ A common question about rocker is how is it different than `docker-compose`. `rocker` is designed to solve a similar but different problem than `docker-compose`. The primary goal of `rocker` is to support the use of Docker in use cases where the containers will be effected by the local environment. A primary example of this is setting up file permissions inside the container to match the users outside of the container so that mounted files inside the container have the same UID as the host. +Doing this enables quickly going in and out of different containrs while leverating the same workspace on your host for testing on different platforms etc. This is done by dynamically generating overlays on the same core image after detecting the local conditions required. The secondary feature that `rocker` provides that docker-compose does not address is the ability to inject extra use case specific capabilities into a container before running. From 27700451d0b1a0701d8fe5c80c64a52942b61ce0 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Tue, 20 Sep 2022 02:49:46 -0500 Subject: [PATCH 22/47] Add port and expose extensions with tests (#201) * Add port and expose extensions and tests * Allow multiple ports --- setup.py | 2 ++ src/rocker/extensions.py | 53 ++++++++++++++++++++++++++++++++ test/test_extension.py | 65 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+) diff --git a/setup.py b/setup.py index 88aa21e3..c60803cc 100644 --- a/setup.py +++ b/setup.py @@ -44,12 +44,14 @@ 'devices = rocker.extensions:Devices', 'dev_helpers = rocker.extensions:DevHelpers', 'env = rocker.extensions:Environment', + 'expose = rocker.extensions:Expose', 'git = rocker.git_extension:Git', 'home = rocker.extensions:HomeDir', 'volume = rocker.volume_extension:Volume', 'name = rocker.extensions:Name', 'network = rocker.extensions:Network', 'nvidia = rocker.nvidia_extension:Nvidia', + 'port = rocker.extensions:Port', 'privileged = rocker.extensions:Privileged', 'pulse = rocker.extensions:PulseAudio', 'ssh = rocker.ssh_extension:Ssh', diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index 9c376124..bdfcd4a2 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -139,6 +139,59 @@ def register_arguments(parser, defaults={}): default=defaults.get('network', None), help="What network configuration to use.") + +class Expose(RockerExtension): + @staticmethod + def get_name(): + return 'expose' + + def __init__(self): + self.name = Expose.get_name() + + def get_preamble(self, cliargs): + return '' + + def get_docker_args(self, cliargs): + args = [''] + ports = cliargs.get('expose', []) + for port in ports: + args.append(' --expose {0}'.format(port)) + return ' '.join(args) + + @staticmethod + def register_arguments(parser, defaults={}): + parser.add_argument('--expose', + default=defaults.get('expose', None), + action='append', + help="Exposes a port from the container to host machine.") + + +class Port(RockerExtension): + @staticmethod + def get_name(): + return 'port' + + def __init__(self): + self.name = Port.get_name() + + def get_preamble(self, cliargs): + return '' + + def get_docker_args(self, cliargs): + args = [''] + ports = cliargs.get('port', []) + for port in ports: + args.append(' -p {0}'.format(port)) + return ' '.join(args) + + @staticmethod + def register_arguments(parser, defaults={}): + parser.add_argument('--port', + default=defaults.get('port', None), + action='append', + help="Binds port from the container to host machine.") + + class PulseAudio(RockerExtension): @staticmethod def get_name(): diff --git a/test/test_extension.py b/test/test_extension.py index 34a387ff..0c6aa7b4 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -146,6 +146,71 @@ def test_network_extension(self): args = p.get_docker_args(mock_cliargs) self.assertTrue('--network host' in args) +class ExposeExtensionTest(unittest.TestCase): + + def setUp(self): + # Work around interference between empy Interpreter + # stdout proxy and test runner. empy installs a proxy on stdout + # to be able to capture the information. + # And the test runner creates a new stdout object for each test. + # This breaks empy as it assumes that the proxy has persistent + # between instances of the Interpreter class + # empy will error with the exception + # "em.Error: interpreter stdout proxy lost" + em.Interpreter._wasProxyInstalled = False + + @pytest.mark.docker + def test_expose_extension(self): + plugins = list_plugins() + expose_plugin = plugins['expose'] + self.assertEqual(expose_plugin.get_name(), 'expose') + + p = expose_plugin() + self.assertTrue(plugin_load_parser_correctly(expose_plugin)) + + mock_cliargs = {} + self.assertEqual(p.get_snippet(mock_cliargs), '') + self.assertEqual(p.get_preamble(mock_cliargs), '') + args = p.get_docker_args(mock_cliargs) + self.assertNotIn('--expose', args) + + mock_cliargs = {'expose': ['80', '8080']} + args = p.get_docker_args(mock_cliargs) + self.assertIn('--expose 80', args) + self.assertIn('--expose 8080', args) + +class PortExtensionTest(unittest.TestCase): + + def setUp(self): + # Work around interference between empy Interpreter + # stdout proxy and test runner. empy installs a proxy on stdout + # to be able to capture the information. + # And the test runner creates a new stdout object for each test. + # This breaks empy as it assumes that the proxy has persistent + # between instances of the Interpreter class + # empy will error with the exception + # "em.Error: interpreter stdout proxy lost" + em.Interpreter._wasProxyInstalled = False + + @pytest.mark.docker + def test_port_extension(self): + plugins = list_plugins() + port_plugin = plugins['port'] + self.assertEqual(port_plugin.get_name(), 'port') + + p = port_plugin() + self.assertTrue(plugin_load_parser_correctly(port_plugin)) + + mock_cliargs = {} + self.assertEqual(p.get_snippet(mock_cliargs), '') + self.assertEqual(p.get_preamble(mock_cliargs), '') + args = p.get_docker_args(mock_cliargs) + self.assertNotIn('-p', args) + + mock_cliargs = {'port': ['80:8080', '81:8081']} + args = p.get_docker_args(mock_cliargs) + self.assertIn('-p 80:8080', args) + self.assertIn('-p 81:8081', args) class NameExtensionTest(unittest.TestCase): From 67a414278c5940d587b9e3060eb7ab9e2e293510 Mon Sep 17 00:00:00 2001 From: George Stavrinos Date: Fri, 4 Nov 2022 10:19:00 +0200 Subject: [PATCH 23/47] Removed '-v' for rocker's version. Only --version should be available. (#205) The reasoning between this change is that it creates a conflict with the -v flag of the `docker volume` extension. --- src/rocker/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rocker/cli.py b/src/rocker/cli.py index 9044cddd..7d78641c 100644 --- a/src/rocker/cli.py +++ b/src/rocker/cli.py @@ -35,7 +35,7 @@ def main(): parser.add_argument('--nocache', action='store_true') parser.add_argument('--nocleanup', action='store_true', help='do not remove the docker container when stopped') parser.add_argument('--pull', action='store_true') - parser.add_argument('-v', '--version', action='version', + parser.add_argument('--version', action='version', version='%(prog)s ' + get_rocker_version()) try: From 721d1e3c394de3dfa5b9deab32e0a97f16375619 Mon Sep 17 00:00:00 2001 From: Ambroise Vincent Date: Mon, 24 Oct 2022 09:01:46 +0200 Subject: [PATCH 24/47] Use Go os detector Based on the distro-detect project. Directly compile a static binary without using staticx. Which gives flexibility on the runtime architecture. Issue-Id: SCM-5067 Signed-off-by: Ambroise Vincent Change-Id: I271e90e8e62f2065d0b827823ae5b3d162ad5e14 --- src/rocker/os_detector.py | 44 ++++++++++++++++++++------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/src/rocker/os_detector.py b/src/rocker/os_detector.py index 9559672e..d6b40ade 100644 --- a/src/rocker/os_detector.py +++ b/src/rocker/os_detector.py @@ -1,4 +1,4 @@ -# Copyright 2019 Open Source Robotics Foundation +# Copyright 2019-2022 Arm Ltd., Open Source Robotics Foundation # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,37 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import pexpect -from ast import literal_eval from io import BytesIO as StringIO from .core import docker_build, get_docker_client DETECTION_TEMPLATE=""" -FROM python:3-slim-stretch as detector -# Force the older version of debian for detector. -# GLIBC is forwards compatible but not necessarily backwards compatible for pyinstaller -# https://github.com/pyinstaller/pyinstaller/wiki/FAQ#gnulinux -# StaticX is supposed to take care of this but there appears to be an issue when using subprocess +FROM golang:1.19 as detector -RUN mkdir -p /tmp/distrovenv -RUN python3 -m venv /tmp/distrovenv -# patchelf needed for staticx -# binutils provides objdump needed by pyinstaller -RUN apt-get update && apt-get install -qy patchelf binutils -RUN . /tmp/distrovenv/bin/activate && pip install distro pyinstaller==4.0 staticx==0.12.3 - -RUN echo 'import distro; import sys; output = (distro.name(), distro.version(), distro.codename()); print(output) if distro.name() else sys.exit(1)' > /tmp/distrovenv/detect_os.py -RUN . /tmp/distrovenv/bin/activate && pyinstaller --onefile /tmp/distrovenv/detect_os.py - -RUN . /tmp/distrovenv/bin/activate && staticx /dist/detect_os /dist/detect_os_static && chmod go+xr /dist/detect_os_static +# For reliability, pin a distro-detect commit instead of targeting a branch. +RUN git clone -q https://github.com/dekobon/distro-detect.git && \ + cd distro-detect && \ + git checkout -q 5f5b9c724b9d9a117732d2a4292e6288905734e1 && \ + CGO_ENABLED=0 go build . FROM %(image_name)s -COPY --from=detector /dist/detect_os_static /tmp/detect_os -ENTRYPOINT [ "/tmp/detect_os" ] +COPY --from=detector /go/distro-detect/distro-detect /tmp/detect_os +ENTRYPOINT [ "/tmp/detect_os", "-format", "json-one-line" ] CMD [ "" ] """ @@ -81,7 +71,19 @@ def detect_os(image_name, output_callback=None, nocache=False): client.remove_image(image=tag_name) if p.exitstatus == 0: - _detect_os_cache[image_name] = literal_eval(output.strip()) + try: + detect_dict = json.loads(output.strip()) + except ValueError: + if output_callback: + output_callback('Failed to parse JSON') + return None + + dist = detect_dict.get('name', '') + os_release = detect_dict.get('os_release', {}) + ver = os_release.get('VERSION_ID', '') + codename = os_release.get('VERSION_CODENAME', '') + + _detect_os_cache[image_name] = (dist, ver, codename) return _detect_os_cache[image_name] else: if output_callback: From 9d5a54a776a7978eea2d9ecf05e62829f0558fe9 Mon Sep 17 00:00:00 2001 From: Woensug Choi Date: Thu, 1 Dec 2022 09:38:47 +0900 Subject: [PATCH 25/47] Re-add Cuda Devel environment (#210) * Re-add Cuda Devel environment * alphabetize the extension listings * Add pytest test flags Following work in #194 Signed-off-by: Tully Foote --- setup.py | 3 +- src/rocker/nvidia_extension.py | 60 +++++++++++++++ .../templates/cuda_snippet.Dockerfile.em | 41 +++++++++++ test/test_nvidia.py | 73 +++++++++++++++++++ 4 files changed, 176 insertions(+), 1 deletion(-) create mode 100644 src/rocker/templates/cuda_snippet.Dockerfile.em diff --git a/setup.py b/setup.py index c60803cc..a217acf4 100644 --- a/setup.py +++ b/setup.py @@ -41,13 +41,13 @@ 'detect_docker_image_os = rocker.cli:detect_image_os', ], 'rocker.extensions': [ + 'cuda = rocker.nvidia_extension:Cuda', 'devices = rocker.extensions:Devices', 'dev_helpers = rocker.extensions:DevHelpers', 'env = rocker.extensions:Environment', 'expose = rocker.extensions:Expose', 'git = rocker.git_extension:Git', 'home = rocker.extensions:HomeDir', - 'volume = rocker.volume_extension:Volume', 'name = rocker.extensions:Name', 'network = rocker.extensions:Network', 'nvidia = rocker.nvidia_extension:Nvidia', @@ -56,6 +56,7 @@ 'pulse = rocker.extensions:PulseAudio', 'ssh = rocker.ssh_extension:Ssh', 'user = rocker.extensions:User', + 'volume = rocker.volume_extension:Volume', 'x11 = rocker.nvidia_extension:X11', ] }, diff --git a/src/rocker/nvidia_extension.py b/src/rocker/nvidia_extension.py index 07c96485..1bae9fa9 100644 --- a/src/rocker/nvidia_extension.py +++ b/src/rocker/nvidia_extension.py @@ -134,4 +134,64 @@ def register_arguments(parser, defaults={}): default=defaults.get(Nvidia.get_name(), None), help="Enable nvidia") +class Cuda(RockerExtension): + @staticmethod + def get_name(): + return 'cuda' + + def __init__(self): + self._env_subs = None + self.name = Cuda.get_name() + self.supported_distros = ['Ubuntu', 'Debian GNU/Linux'] + self.supported_versions = ['20.04', '22.04', '18.04', '11'] # Debian 11 + + def get_environment_subs(self, cliargs={}): + if not self._env_subs: + self._env_subs = {} + self._env_subs['user_id'] = os.getuid() + self._env_subs['username'] = getpass.getuser() + + # non static elements test every time + detected_os = detect_os(cliargs['base_image'], print, nocache=cliargs.get('nocache', False)) + if detected_os is None: + print("WARNING unable to detect os for base image '%s', maybe the base image does not exist" % cliargs['base_image']) + sys.exit(1) + dist, ver, codename = detected_os + + self._env_subs['download_osstring'] = dist.split()[0].lower() + self._env_subs['download_verstring'] = ver.replace('.', '') + self._env_subs['download_keyid'] = '3bf863cc' + + self._env_subs['image_distro_id'] = dist + if self._env_subs['image_distro_id'] not in self.supported_distros: + print("WARNING distro id %s not supported by Cuda supported " % self._env_subs['image_distro_id'], self.supported_distros) + sys.exit(1) + self._env_subs['image_distro_version'] = ver + if self._env_subs['image_distro_version'] not in self.supported_versions: + print("WARNING distro %s version %s not in supported list by Nvidia supported versions" % (dist, ver), self.supported_versions) + sys.exit(1) + # TODO(tfoote) add a standard mechanism for checking preconditions and disabling plugins + + return self._env_subs + + def get_preamble(self, cliargs): + return '' + # preamble = pkgutil.get_data('rocker', 'templates/%s_preamble.Dockerfile.em' % self.name).decode('utf-8') + # return em.expand(preamble, self.get_environment_subs(cliargs)) + + def get_snippet(self, cliargs): + snippet = pkgutil.get_data('rocker', 'templates/%s_snippet.Dockerfile.em' % self.name).decode('utf-8') + return em.expand(snippet, self.get_environment_subs(cliargs)) + + def get_docker_args(self, cliargs): + return "" + # Runtime requires --nvidia option too + + @staticmethod + def register_arguments(parser, defaults={}): + parser.add_argument(name_to_argument(Cuda.get_name()), + action='store_true', + default=defaults.get('cuda', None), + help="Install cuda and nvidia-cuda-dev into the container") + diff --git a/src/rocker/templates/cuda_snippet.Dockerfile.em b/src/rocker/templates/cuda_snippet.Dockerfile.em new file mode 100644 index 00000000..92a12d4f --- /dev/null +++ b/src/rocker/templates/cuda_snippet.Dockerfile.em @@ -0,0 +1,41 @@ +# Installation instructions from NVIDIA: +# https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Debian&target_version=11&target_type=deb_network +# https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=22.04&target_type=deb_network + +# Keep the dockerfile non-interactive +# TODO(tfoote) make this more generic/shared across instances +ARG DEBIAN_FRONTEND=noninteractive + +# Prerequisites +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget software-properties-common gnupg2 \ + && rm -rf /var/lib/apt/lists/* + +# Enable contrib on debian to get required +# https://packages.debian.org/bullseye/glx-alternative-nvidia +# Enable non-free for nvidia-cuda-dev +# https://packages.debian.org/bullseye/nvidia-cuda-dev + +RUN \ + @[if download_osstring == 'ubuntu']@ + wget https://developer.download.nvidia.com/compute/cuda/repos/@(download_osstring)@(download_verstring)/x86_64/cuda-@(download_osstring)@(download_verstring).pin \ + && mv cuda-@(download_osstring)@(download_verstring).pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ + add-apt-repository restricted && \ + @[else]@ + add-apt-repository contrib && \ + add-apt-repository non-free && \ + @[end if]@ + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/@(download_osstring)@(download_verstring)/x86_64/@(download_keyid).pub \ + && add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/@(download_osstring)@(download_verstring)/x86_64/ /" \ + && apt-get update \ + && apt-get -y install cuda \ + && rm -rf /var/lib/apt/lists/* + +# File conflict problem with libnvidia-ml.so.1 and libcuda.so.1 +# https://github.com/NVIDIA/nvidia-docker/issues/1551 +RUN rm -rf /usr/lib/x86_64-linux-gnu/libnv* +RUN rm -rf /usr/lib/x86_64-linux-gnu/libcuda* + +# TODO(tfoote) Add documentation of why these are required +ENV PATH /usr/local/cuda/bin${PATH:+:${PATH}} +ENV LD_LIBRARY_PATH /usr/local/cuda/lib64/stubs:/usr/local/cuda/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} \ No newline at end of file diff --git a/test/test_nvidia.py b/test/test_nvidia.py index 53662331..29ae0c2c 100644 --- a/test/test_nvidia.py +++ b/test/test_nvidia.py @@ -226,3 +226,76 @@ def test_nvidia_env_subs(self): with self.assertRaises(SystemExit) as cm: p.get_environment_subs(mock_cliargs) self.assertEqual(cm.exception.code, 1) + +class CudaTest(unittest.TestCase): + @classmethod + def setUpClass(self): + client = get_docker_client() + self.dockerfile_tags = [] + for distro_version in ['focal', 'jammy']: + dockerfile = """ +FROM ubuntu:%(distro_version)s +CMD dpkg -s cuda +""" + dockerfile_tag = 'testfixture_%s_cuda' % distro_version + iof = StringIO((dockerfile % locals()).encode()) + im = client.build(fileobj = iof, tag=dockerfile_tag) + for e in im: + pass + #print(e) + self.dockerfile_tags.append(dockerfile_tag) + + def setUp(self): + # Work around interference between empy Interpreter + # stdout proxy and test runner. empy installs a proxy on stdout + # to be able to capture the information. + # And the test runner creates a new stdout object for each test. + # This breaks empy as it assumes that the proxy has persistent + # between instances of the Interpreter class + # empy will error with the exception + # "em.Error: interpreter stdout proxy lost" + em.Interpreter._wasProxyInstalled = False + + + @pytest.mark.docker + def test_no_cuda(self): + for tag in self.dockerfile_tags: + dig = DockerImageGenerator([], {}, tag) + self.assertEqual(dig.build(), 0) + self.assertNotEqual(dig.run(), 0) + + @pytest.mark.nvidia + @pytest.mark.x11 + @pytest.mark.docker + def test_cuda(self): + plugins = list_plugins() + desired_plugins = ['x11', 'nvidia', 'cuda'] #TODO(Tfoote) encode the x11 dependency into the plugin and remove from test here + active_extensions = [e() for e in plugins.values() if e.get_name() in desired_plugins] + for tag in self.dockerfile_tags: + dig = DockerImageGenerator(active_extensions, {}, tag) + self.assertEqual(dig.build(), 0) + self.assertEqual(dig.run(), 0) + + def test_cuda_env_subs(self): + plugins = list_plugins() + cuda_plugin = plugins['cuda'] + + p = cuda_plugin() + + # base image doesn't exist + mock_cliargs = {'base_image': 'ros:does-not-exist'} + with self.assertRaises(SystemExit) as cm: + p.get_environment_subs(mock_cliargs) + self.assertEqual(cm.exception.code, 1) + + # unsupported version + mock_cliargs = {'base_image': 'ubuntu:17.04'} + with self.assertRaises(SystemExit) as cm: + p.get_environment_subs(mock_cliargs) + self.assertEqual(cm.exception.code, 1) + + # unsupported os + mock_cliargs = {'base_image': 'fedora'} + with self.assertRaises(SystemExit) as cm: + p.get_environment_subs(mock_cliargs) + self.assertEqual(cm.exception.code, 1) From 49a23e70a3055898daba80fc35f1742dc01d9a09 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Tue, 14 Mar 2023 13:13:26 -0700 Subject: [PATCH 26/47] add link to mp_rocker (#213) * add link to mp_rocker * link to ghrocker --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2d1cd5fa..f851f0f6 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ As well as access to many of the docker arguments as well such as `device`, `env Here's a list of public repositories with extensions. - Off-your-rocker: https://github.com/sloretz/off-your-rocker +- mp_rocker: https://github.com/miguelprada/mp_rocker +- ghrocker: https://github.com/tfoote/ghrocker From 434d9df8dfab1acac0b3f12e20ce880bdb5f4c82 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Tue, 28 Mar 2023 17:15:35 -0700 Subject: [PATCH 27/47] catch the DockerException for the missing docker daemon (#215) This is a fix for #122 --- src/rocker/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/rocker/core.py b/src/rocker/core.py index dad8a11a..2f1a5353 100644 --- a/src/rocker/core.py +++ b/src/rocker/core.py @@ -125,11 +125,12 @@ def get_docker_client(): # Validate that the server is available docker_client.ping() return docker_client - except (docker.errors.APIError, ConnectionError) as ex: + except (docker.errors.DockerException, docker.errors.APIError, ConnectionError) as ex: raise DependencyMissing('Docker Client failed to connect to docker daemon.' ' Please verify that docker is installed and running.' ' As well as that you have permission to access the docker daemon.' - ' This is usually by being a member of the docker group.') + ' This is usually by being a member of the docker group.' + ' The underlying error was:\n"""\n%s\n"""\n' % ex) def docker_build(docker_client = None, output_callback = None, **kwargs): From 510bb2f88f90bf4bea102e396c21a9c94600fe8a Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Tue, 11 Apr 2023 02:27:47 -0700 Subject: [PATCH 28/47] Update to use 22.04 and test python 3.10 (#219) * Update to use 22.04 and test python 3.10 * fixup python version --- .github/workflows/basic-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/basic-ci.yaml b/.github/workflows/basic-ci.yaml index db169448..efacbf1f 100644 --- a/.github/workflows/basic-ci.yaml +++ b/.github/workflows/basic-ci.yaml @@ -4,10 +4,10 @@ on: [push, pull_request] jobs: basic_ci: name: Basic CI - runs-on: ubuntu-18.04 + runs-on: ubuntu-22.04 strategy: matrix: - python-version: [3.8] + python-version: [3.8, '3.10'] steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} From b16136e589ace3519e4b5c85697e770b29a151fa Mon Sep 17 00:00:00 2001 From: Miguel Prada Date: Tue, 28 Feb 2023 10:55:39 +0100 Subject: [PATCH 29/47] Add ability to preserve host user groups inside container --- src/rocker/extensions.py | 8 ++++++++ src/rocker/templates/user_snippet.Dockerfile.em | 8 ++++++++ test/test_extension.py | 4 ++++ 3 files changed, 20 insertions(+) diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index bdfcd4a2..9c94e417 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -274,6 +274,10 @@ def get_snippet(self, cliargs): substitutions['name'] = cliargs['user_override_name'] substitutions['dir'] = os.path.join('/home/', cliargs['user_override_name']) substitutions['user_preserve_home'] = True if 'user_preserve_home' in cliargs and cliargs['user_preserve_home'] else False + if 'user_preserve_groups' in cliargs and cliargs['user_preserve_groups']: + substitutions['user_groups'] = ' '.join(['{};{}'.format(g.gr_name, g.gr_gid) for g in grp.getgrall() if substitutions['name'] in g.gr_mem]) + else: + substitutions['user_groups'] = '' substitutions['home_extension_active'] = True if 'home' in cliargs and cliargs['home'] else False if 'user_override_shell' in cliargs and cliargs['user_override_shell'] is not None: if cliargs['user_override_shell'] == '': @@ -296,6 +300,10 @@ def register_arguments(parser, defaults={}): action='store_true', default=defaults.get('user-preserve-home', False), help="Do not delete home directory if it exists when making a new user.") + parser.add_argument('--user-preserve-groups', + action='store_true', + default=defaults.get('user-preserve-groups', False), + help="Assign user to same groups as he belongs in host.") parser.add_argument('--user-override-shell', action='store', default=defaults.get('user-override-shell', None), diff --git a/src/rocker/templates/user_snippet.Dockerfile.em b/src/rocker/templates/user_snippet.Dockerfile.em index 63911409..6503cb11 100644 --- a/src/rocker/templates/user_snippet.Dockerfile.em +++ b/src/rocker/templates/user_snippet.Dockerfile.em @@ -17,6 +17,14 @@ RUN existing_user_by_uid=`getent passwd "@(uid)" | cut -f1 -d: || true` && \ groupadd -g "@(gid)" "@name"; \ fi && \ useradd --no-log-init --no-create-home --uid "@(uid)" @(str('-s ' + shell) if shell else '') -c "@(gecos)" -g "@(gid)" -d "@(dir)" "@(name)" && \ +@[if user_groups != '']@ + user_groups="@(user_groups)" && \ + for groupinfo in ${user_groups}; do \ + existing_group_by_name=`getent group ${groupinfo%;*} || true`; \ + existing_group_by_gid=`getent group ${groupinfo#*;} || true`; \ + if [ -z "${existing_group_by_name}" ] && [ -z "${existing_group_by_gid}" ]; then groupadd -g "${groupinfo#*;}" "${groupinfo%;*}" && usermod -aG "${groupinfo%;*}" "@(name)"; fi \ + done && \ +@[end if]@ echo "@(name) ALL=NOPASSWD: ALL" >> /etc/sudoers.d/rocker @[if not home_extension_active ]@ diff --git a/test/test_extension.py b/test/test_extension.py index 0c6aa7b4..f2b2082c 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -317,6 +317,10 @@ def test_user_extension(self): self.assertFalse('mkhomedir_helper' in p.get_snippet(home_active_cliargs)) user_override_active_cliargs = mock_cliargs + user_override_active_cliargs['user_preserve_groups'] = True + snippet_result = p.get_snippet(user_override_active_cliargs) + self.assertTrue('usermod -aG' in snippet_result) + user_override_active_cliargs['user_override_name'] = 'testusername' snippet_result = p.get_snippet(user_override_active_cliargs) self.assertTrue('USER testusername' in snippet_result) From 44f7946653f1d58cbf52a088f8c482090f5f033c Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Mon, 17 Apr 2023 04:03:19 +0000 Subject: [PATCH 30/47] Adding a permissive option to user-preserve-groups incase there are groups on the host that aren't permissible on the target but you'd like best-effort. Signed-off-by: Tully Foote --- src/rocker/extensions.py | 6 ++++++ src/rocker/templates/user_snippet.Dockerfile.em | 2 +- test/test_extension.py | 7 +++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index 9c94e417..c9497bc8 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -278,6 +278,7 @@ def get_snippet(self, cliargs): substitutions['user_groups'] = ' '.join(['{};{}'.format(g.gr_name, g.gr_gid) for g in grp.getgrall() if substitutions['name'] in g.gr_mem]) else: substitutions['user_groups'] = '' + substitutions['user_preserve_groups_permissive'] = True if 'user_preserve_groups_permissive' in cliargs and cliargs['user_preserve_groups_permissive'] else False substitutions['home_extension_active'] = True if 'home' in cliargs and cliargs['home'] else False if 'user_override_shell' in cliargs and cliargs['user_override_shell'] is not None: if cliargs['user_override_shell'] == '': @@ -304,6 +305,11 @@ def register_arguments(parser, defaults={}): action='store_true', default=defaults.get('user-preserve-groups', False), help="Assign user to same groups as he belongs in host.") + parser.add_argument('--user-preserve-groups-permissive', + action='store_true', + default=defaults.get('user-preserve-groups-permissive', False), + help="If using user-preserve-groups allow failures in assignment." + "This is important if the host and target have different rules. https://unix.stackexchange.com/a/11481/83370" ) parser.add_argument('--user-override-shell', action='store', default=defaults.get('user-override-shell', None), diff --git a/src/rocker/templates/user_snippet.Dockerfile.em b/src/rocker/templates/user_snippet.Dockerfile.em index 6503cb11..fe37e47f 100644 --- a/src/rocker/templates/user_snippet.Dockerfile.em +++ b/src/rocker/templates/user_snippet.Dockerfile.em @@ -22,7 +22,7 @@ RUN existing_user_by_uid=`getent passwd "@(uid)" | cut -f1 -d: || true` && \ for groupinfo in ${user_groups}; do \ existing_group_by_name=`getent group ${groupinfo%;*} || true`; \ existing_group_by_gid=`getent group ${groupinfo#*;} || true`; \ - if [ -z "${existing_group_by_name}" ] && [ -z "${existing_group_by_gid}" ]; then groupadd -g "${groupinfo#*;}" "${groupinfo%;*}" && usermod -aG "${groupinfo%;*}" "@(name)"; fi \ + if [ -z "${existing_group_by_name}" ] && [ -z "${existing_group_by_gid}" ]; then groupadd -g "${groupinfo#*;}" "${groupinfo%;*}" && usermod -aG "${groupinfo%;*}" "@(name)" @(('|| (true && echo "user-preserve-group-permissive Enabled, continuing without processing group $groupinfo" )') if user_preserve_groups_permissive else ''); fi \ done && \ @[end if]@ echo "@(name) ALL=NOPASSWD: ALL" >> /etc/sudoers.d/rocker diff --git a/test/test_extension.py b/test/test_extension.py index f2b2082c..1006208a 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -321,6 +321,13 @@ def test_user_extension(self): snippet_result = p.get_snippet(user_override_active_cliargs) self.assertTrue('usermod -aG' in snippet_result) + user_override_active_cliargs = mock_cliargs + user_override_active_cliargs['user_preserve_groups'] = True + user_override_active_cliargs['user_preserve_groups_permissive'] = True + snippet_result = p.get_snippet(user_override_active_cliargs) + self.assertTrue('usermod -aG' in snippet_result) + self.assertTrue('user-preserve-group-permissive Enabled' in snippet_result) + user_override_active_cliargs['user_override_name'] = 'testusername' snippet_result = p.get_snippet(user_override_active_cliargs) self.assertTrue('USER testusername' in snippet_result) From 2e6466f4a300ecc3a65bf468cd04929bba800c39 Mon Sep 17 00:00:00 2001 From: Miguel Prada Date: Thu, 20 Apr 2023 22:45:24 +0200 Subject: [PATCH 31/47] Also preserve groups with matching name and id in host and target --- src/rocker/templates/user_snippet.Dockerfile.em | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/rocker/templates/user_snippet.Dockerfile.em b/src/rocker/templates/user_snippet.Dockerfile.em index fe37e47f..2b23b6fa 100644 --- a/src/rocker/templates/user_snippet.Dockerfile.em +++ b/src/rocker/templates/user_snippet.Dockerfile.em @@ -22,7 +22,11 @@ RUN existing_user_by_uid=`getent passwd "@(uid)" | cut -f1 -d: || true` && \ for groupinfo in ${user_groups}; do \ existing_group_by_name=`getent group ${groupinfo%;*} || true`; \ existing_group_by_gid=`getent group ${groupinfo#*;} || true`; \ - if [ -z "${existing_group_by_name}" ] && [ -z "${existing_group_by_gid}" ]; then groupadd -g "${groupinfo#*;}" "${groupinfo%;*}" && usermod -aG "${groupinfo%;*}" "@(name)" @(('|| (true && echo "user-preserve-group-permissive Enabled, continuing without processing group $groupinfo" )') if user_preserve_groups_permissive else ''); fi \ + if [ -z "${existing_group_by_name}" ] && [ -z "${existing_group_by_gid}" ]; then \ + groupadd -g "${groupinfo#*;}" "${groupinfo%;*}" && usermod -aG "${groupinfo%;*}" "@(name)" @(('|| (true && echo "user-preserve-group-permissive Enabled, continuing without processing group $groupinfo" )') if user_preserve_groups_permissive else ''); \ + elif [ "${existing_group_by_name}" = "${existing_group_by_gid}" ]; then \ + usermod -aG "${groupinfo%;*}" "@(name)"; \ + fi; \ done && \ @[end if]@ echo "@(name) ALL=NOPASSWD: ALL" >> /etc/sudoers.d/rocker From 4ce4e29041ac81830f78e1b00afbff9168194929 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Thu, 4 May 2023 01:38:52 +0000 Subject: [PATCH 32/47] add error messages for group issues --- src/rocker/templates/user_snippet.Dockerfile.em | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rocker/templates/user_snippet.Dockerfile.em b/src/rocker/templates/user_snippet.Dockerfile.em index 2b23b6fa..37857d4c 100644 --- a/src/rocker/templates/user_snippet.Dockerfile.em +++ b/src/rocker/templates/user_snippet.Dockerfile.em @@ -23,9 +23,9 @@ RUN existing_user_by_uid=`getent passwd "@(uid)" | cut -f1 -d: || true` && \ existing_group_by_name=`getent group ${groupinfo%;*} || true`; \ existing_group_by_gid=`getent group ${groupinfo#*;} || true`; \ if [ -z "${existing_group_by_name}" ] && [ -z "${existing_group_by_gid}" ]; then \ - groupadd -g "${groupinfo#*;}" "${groupinfo%;*}" && usermod -aG "${groupinfo%;*}" "@(name)" @(('|| (true && echo "user-preserve-group-permissive Enabled, continuing without processing group $groupinfo" )') if user_preserve_groups_permissive else ''); \ + groupadd -g "${groupinfo#*;}" "${groupinfo%;*}" && usermod -aG "${groupinfo%;*}" "@(name)" @(('|| (true && echo "user-preserve-group-permissive Enabled, continuing without processing group $groupinfo" )') if user_preserve_groups_permissive else '') || (echo "Failed to add group ${groupinfo%;*}, consider option --user-preserve-group-permissive" && exit 2); \ elif [ "${existing_group_by_name}" = "${existing_group_by_gid}" ]; then \ - usermod -aG "${groupinfo%;*}" "@(name)"; \ + usermod -aG "${groupinfo%;*}" "@(name)" @(('|| (true && echo "user-preserve-group-permissive Enabled, continuing without processing group $groupinfo" )') if user_preserve_groups_permissive else '') || (echo "Failed to adjust group ${groupinfo%;*}, consider option --user-preserve-group-permissive" && exit 2); \ fi; \ done && \ @[end if]@ From 514aa6fe4f58fd75cf96669f1400936787aa7dd2 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Thu, 4 May 2023 03:21:26 +0000 Subject: [PATCH 33/47] extend use group option to also support explicitly listing groups on the command line --- src/rocker/extensions.py | 18 ++++++++++++++---- test/test_extension.py | 10 ++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index c9497bc8..05c7abc5 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -274,8 +274,17 @@ def get_snippet(self, cliargs): substitutions['name'] = cliargs['user_override_name'] substitutions['dir'] = os.path.join('/home/', cliargs['user_override_name']) substitutions['user_preserve_home'] = True if 'user_preserve_home' in cliargs and cliargs['user_preserve_home'] else False - if 'user_preserve_groups' in cliargs and cliargs['user_preserve_groups']: - substitutions['user_groups'] = ' '.join(['{};{}'.format(g.gr_name, g.gr_gid) for g in grp.getgrall() if substitutions['name'] in g.gr_mem]) + if 'user_preserve_groups' in cliargs: + all_groups = grp.getgrall() + matched_groups = [g for g in all_groups if g.gr_name in cliargs['user_preserve_groups']] + matched_group_names = [g.gr_name for g in matched_groups] + unmatched_groups = [n for n in cliargs['user_preserve_groups'] if n not in matched_group_names] + if unmatched_groups: + print('Warning skipping groups %s because they do not exist on the host.' % unmatched_groups) + if cliargs['user_preserve_groups']: + substitutions['user_groups'] = ' '.join(['{};{}'.format(g.gr_name, g.gr_gid) for g in matched_groups]) + else: + substitutions['user_groups'] = ' '.join(['{};{}'.format(g.gr_name, g.gr_gid) for g in all_groups if substitutions['name'] in g.gr_mem]) else: substitutions['user_groups'] = '' substitutions['user_preserve_groups_permissive'] = True if 'user_preserve_groups_permissive' in cliargs and cliargs['user_preserve_groups_permissive'] else False @@ -302,9 +311,10 @@ def register_arguments(parser, defaults={}): default=defaults.get('user-preserve-home', False), help="Do not delete home directory if it exists when making a new user.") parser.add_argument('--user-preserve-groups', - action='store_true', + action='store', + nargs='*', default=defaults.get('user-preserve-groups', False), - help="Assign user to same groups as he belongs in host.") + help="Assign user to same groups as he belongs in host. If arguments provided they are the explicit list of groups.") parser.add_argument('--user-preserve-groups-permissive', action='store_true', default=defaults.get('user-preserve-groups-permissive', False), diff --git a/test/test_extension.py b/test/test_extension.py index 1006208a..8dbfff24 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -317,12 +317,18 @@ def test_user_extension(self): self.assertFalse('mkhomedir_helper' in p.get_snippet(home_active_cliargs)) user_override_active_cliargs = mock_cliargs - user_override_active_cliargs['user_preserve_groups'] = True + user_override_active_cliargs['user_preserve_groups'] = [] snippet_result = p.get_snippet(user_override_active_cliargs) self.assertTrue('usermod -aG' in snippet_result) user_override_active_cliargs = mock_cliargs - user_override_active_cliargs['user_preserve_groups'] = True + user_override_active_cliargs['user_preserve_groups'] = ['cdrom', 'audio'] + snippet_result = p.get_snippet(user_override_active_cliargs) + self.assertTrue('cdrom' in snippet_result) + self.assertTrue('audio' in snippet_result) + + user_override_active_cliargs = mock_cliargs + user_override_active_cliargs['user_preserve_groups'] = [] user_override_active_cliargs['user_preserve_groups_permissive'] = True snippet_result = p.get_snippet(user_override_active_cliargs) self.assertTrue('usermod -aG' in snippet_result) From a085fb73cfe907de9b3e5569627db9d6abcf20bf Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Wed, 3 May 2023 20:47:22 -0700 Subject: [PATCH 34/47] Cherry picking the group-add commits for group-add (#222) * feat: add group-add plugin Signed-off-by: Amadeusz Szymko Co-authored-by: Amadeusz Szymko --- setup.py | 1 + src/rocker/extensions.py | 27 +++++++++++++++++++++++++++ test/test_extension.py | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+) diff --git a/setup.py b/setup.py index a217acf4..b05a3031 100644 --- a/setup.py +++ b/setup.py @@ -47,6 +47,7 @@ 'env = rocker.extensions:Environment', 'expose = rocker.extensions:Expose', 'git = rocker.git_extension:Git', + 'group_add = rocker.extensions:GroupAdd', 'home = rocker.extensions:HomeDir', 'name = rocker.extensions:Name', 'network = rocker.extensions:Network', diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index 05c7abc5..72612822 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -394,3 +394,30 @@ def register_arguments(parser, defaults={}): action='store_true', default=defaults.get(Privileged.get_name(), None), help="give extended privileges to the container") + + +class GroupAdd(RockerExtension): + """Add additional groups to running container.""" + @staticmethod + def get_name(): + return 'group_add' + + def __init__(self): + self.name = GroupAdd.get_name() + + def get_preamble(self, cliargs): + return '' + + def get_docker_args(self, cliargs): + args = [''] + groups = cliargs.get('group_add', []) + for group in groups: + args.append(' --group-add {0}'.format(group)) + return ' '.join(args) + + @staticmethod + def register_arguments(parser, defaults={}): + parser.add_argument(name_to_argument(GroupAdd.get_name()), + default=defaults.get(GroupAdd.get_name(), None), + action='append', + help="Add additional groups to join.") diff --git a/test/test_extension.py b/test/test_extension.py index 8dbfff24..6e748bc3 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -517,3 +517,37 @@ def test_env_file_extension(self): self.assertEqual(p.get_snippet(mock_cliargs), '') self.assertEqual(p.get_preamble(mock_cliargs), '') self.assertEqual(p.get_docker_args(mock_cliargs), ' --env-file foo --env-file bar') + + +class GroupAddExtensionTest(unittest.TestCase): + + def setUp(self): + # Work around interference between empy Interpreter + # stdout proxy and test runner. empy installs a proxy on stdout + # to be able to capture the information. + # And the test runner creates a new stdout object for each test. + # This breaks empy as it assumes that the proxy has persistent + # between instances of the Interpreter class + # empy will error with the exception + # "em.Error: interpreter stdout proxy lost" + em.Interpreter._wasProxyInstalled = False + + @pytest.mark.docker + def test_group_add_extension(self): + plugins = list_plugins() + group_add_plugin = plugins['group_add'] + self.assertEqual(group_add_plugin.get_name(), 'group_add') + + p = group_add_plugin() + self.assertTrue(plugin_load_parser_correctly(group_add_plugin)) + + mock_cliargs = {} + self.assertEqual(p.get_snippet(mock_cliargs), '') + self.assertEqual(p.get_preamble(mock_cliargs), '') + args = p.get_docker_args(mock_cliargs) + self.assertNotIn('--group_add', args) + + mock_cliargs = {'group_add': ['sudo', 'docker']} + args = p.get_docker_args(mock_cliargs) + self.assertIn('--group-add sudo', args) + self.assertIn('--group-add docker', args) From b00c1c6e2832c6d375f8b12bcee8fe74642ef043 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Wed, 3 May 2023 20:52:02 -0700 Subject: [PATCH 35/47] adding debian bookworm support Signed-off-by: Tully Foote --- stdeb.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdeb.cfg b/stdeb.cfg index 4cd9cbd4..9e1fc94c 100644 --- a/stdeb.cfg +++ b/stdeb.cfg @@ -3,5 +3,5 @@ Debian-Version: 100 No-Python2: Depends3: python3-docker, python3-empy, python3-pexpect, python3-packaging Conflicts3: python-rocker -Suite: bionic focal jammy stretch buster bullseye +Suite: bionic focal jammy stretch buster bullseye bookworm X-Python3-Version: >= 3.2 From d5e9ecfeebf8264d86e4aba42fbdc282d84766f6 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Wed, 3 May 2023 20:53:08 -0700 Subject: [PATCH 36/47] 0.2.11 Signed-off-by: Tully Foote --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b05a3031..9c449892 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ kwargs = { 'name': 'rocker', - 'version': '0.2.10', + 'version': '0.2.11', 'packages': ['rocker'], 'package_dir': {'': 'src'}, 'package_data': {'rocker': ['templates/*.em']}, From 04cfc290351a60b7df28c7eff67dc2c27e9ab918 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Thu, 4 May 2023 11:21:43 -0700 Subject: [PATCH 37/47] Fix default logic for user-preserve-groups (#224) Fixes #223 Signed-off-by: Tully Foote --- src/rocker/extensions.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index 72612822..75030dbf 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -274,14 +274,15 @@ def get_snippet(self, cliargs): substitutions['name'] = cliargs['user_override_name'] substitutions['dir'] = os.path.join('/home/', cliargs['user_override_name']) substitutions['user_preserve_home'] = True if 'user_preserve_home' in cliargs and cliargs['user_preserve_home'] else False - if 'user_preserve_groups' in cliargs: + if 'user_preserve_groups' in cliargs and isinstance(cliargs['user_preserve_groups'], list): + query_groups = cliargs['user_preserve_groups'] all_groups = grp.getgrall() - matched_groups = [g for g in all_groups if g.gr_name in cliargs['user_preserve_groups']] - matched_group_names = [g.gr_name for g in matched_groups] - unmatched_groups = [n for n in cliargs['user_preserve_groups'] if n not in matched_group_names] - if unmatched_groups: - print('Warning skipping groups %s because they do not exist on the host.' % unmatched_groups) - if cliargs['user_preserve_groups']: + if query_groups: + matched_groups = [g for g in all_groups if g.gr_name in query_groups] + matched_group_names = [g.gr_name for g in matched_groups] + unmatched_groups = [n for n in cliargs['user_preserve_groups'] if n not in matched_group_names] + if unmatched_groups: + print('Warning skipping groups %s because they do not exist on the host.' % unmatched_groups) substitutions['user_groups'] = ' '.join(['{};{}'.format(g.gr_name, g.gr_gid) for g in matched_groups]) else: substitutions['user_groups'] = ' '.join(['{};{}'.format(g.gr_name, g.gr_gid) for g in all_groups if substitutions['name'] in g.gr_mem]) From 64e7c05f9aad804a446753cbefd4e1d439e494eb Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Thu, 4 May 2023 11:22:38 -0700 Subject: [PATCH 38/47] 0.2.12 Signed-off-by: Tully Foote --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9c449892..3ba780ef 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ kwargs = { 'name': 'rocker', - 'version': '0.2.11', + 'version': '0.2.12', 'packages': ['rocker'], 'package_dir': {'': 'src'}, 'package_data': {'rocker': ['templates/*.em']}, From 2a7e15f0fe638a7c51c7bf15a1f09e6e49b62b75 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Thu, 4 May 2023 16:04:13 -0700 Subject: [PATCH 39/47] Upgrading CI versions due to deprecated results (#228) * move to actions/checkout@v3 * move to actions/setup-python@v4 * Pin back urllib3<2 due to regression https://github.com/docker/docker-py/issues/3113 * Remove old workaround which upstream has merged a fix for. * upgrade codecov action to v3 * upgrade to coactions/setup-xvfb@v1 Signed-off-by: Tully Foote --- .github/workflows/basic-ci.yaml | 10 ++++------ setup.py | 1 + 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/basic-ci.yaml b/.github/workflows/basic-ci.yaml index efacbf1f..49db856f 100644 --- a/.github/workflows/basic-ci.yaml +++ b/.github/workflows/basic-ci.yaml @@ -9,20 +9,18 @@ jobs: matrix: python-version: [3.8, '3.10'] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install Dependencies And Self run: | python -m pip install --upgrade pip setuptools wheel - # Workaround for https://github.com/docker/docker-py/issues/2807 - python -m pip install six python -m pip install -e .[test] codecov pytest-cov - name: Run headless tests - uses: GabrielBB/xvfb-action@v1 + uses: coactions/setup-xvfb@v1 with: run: python -m pytest -s -v --cov=rocker -m "not nvidia" - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 diff --git a/setup.py b/setup.py index 3ba780ef..a12bc8d1 100644 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ 'empy', 'pexpect', 'packaging', + 'urllib3<2', # Workaround for https://github.com/docker/docker-py/issues/3113 ] # docker API used to be in a package called `docker-py` before the 2.0 release From 4716db7bc43859d7b06d5a58f35c3ecf7f4c1a6b Mon Sep 17 00:00:00 2001 From: farhan khan <86480450+BabaYaga1221@users.noreply.github.com> Date: Fri, 5 May 2023 07:02:51 +0530 Subject: [PATCH 40/47] issue #153 - Option for Hostname (#226) --- src/rocker/extensions.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/rocker/extensions.py b/src/rocker/extensions.py index 75030dbf..43bc3420 100644 --- a/src/rocker/extensions.py +++ b/src/rocker/extensions.py @@ -91,6 +91,29 @@ def register_arguments(parser, defaults={}): help="add development tools emacs and byobu to your environment") +class Hostname(RockerExtension): + @staticmethod + def get_name(): + return 'hostname' + + def __init__(self): + self.name = Hostname.get_name() + + def get_preamble(self, cliargs): + return '' + + def get_docker_args(self, cliargs): + args = '' + hostname = cliargs.get('hostname', None) + if hostname: + args += ' --hostname %s ' % hostname + return args + + @staticmethod + def register_arguments(parser, defaults={}): + parser.add_argument('--hostname', default=defaults.get('hostname', ''), + help='Hostname of the container.') + class Name(RockerExtension): @staticmethod def get_name(): From 930406667aa113e53b8845e257b5f4fd8332451a Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Thu, 4 May 2023 18:53:01 -0700 Subject: [PATCH 41/47] register Hostname plugin and add basic tests (#229) --- setup.py | 1 + test/test_extension.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/setup.py b/setup.py index a12bc8d1..d10b0d09 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ 'git = rocker.git_extension:Git', 'group_add = rocker.extensions:GroupAdd', 'home = rocker.extensions:HomeDir', + 'hostname = rocker.extensions:Hostname', 'name = rocker.extensions:Name', 'network = rocker.extensions:Network', 'nvidia = rocker.nvidia_extension:Nvidia', diff --git a/test/test_extension.py b/test/test_extension.py index 6e748bc3..a2c41a46 100644 --- a/test/test_extension.py +++ b/test/test_extension.py @@ -243,6 +243,37 @@ def test_name_extension(self): args = p.get_docker_args(mock_cliargs) self.assertTrue('--name docker_name' in args) +class HostnameExtensionTest(unittest.TestCase): + + def setUp(self): + # Work around interference between empy Interpreter + # stdout proxy and test runner. empy installs a proxy on stdout + # to be able to capture the information. + # And the test runner creates a new stdout object for each test. + # This breaks empy as it assumes that the proxy has persistent + # between instances of the Interpreter class + # empy will error with the exception + # "em.Error: interpreter stdout proxy lost" + em.Interpreter._wasProxyInstalled = False + + def test_name_extension(self): + plugins = list_plugins() + name_plugin = plugins['hostname'] + self.assertEqual(name_plugin.get_name(), 'hostname') + + p = name_plugin() + self.assertTrue(plugin_load_parser_correctly(name_plugin)) + + mock_cliargs = {'hostname': 'none'} + self.assertEqual(p.get_snippet(mock_cliargs), '') + self.assertEqual(p.get_preamble(mock_cliargs), '') + args = p.get_docker_args(mock_cliargs) + self.assertTrue('--hostname none' in args) + + mock_cliargs = {'hostname': 'docker-hostname'} + args = p.get_docker_args(mock_cliargs) + self.assertTrue('--hostname docker-hostname' in args) + class PrivilegedExtensionTest(unittest.TestCase): From 0dd2dd71c138b631ca4e0817fdd21f03d53c99ab Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Fri, 5 May 2023 14:38:42 -0700 Subject: [PATCH 42/47] Create codecov.yml (#231) Adding codecov.yml to enable comments back --- codecov.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..252003bf --- /dev/null +++ b/codecov.yml @@ -0,0 +1,3 @@ +comment: + layout: "reach, diff, flags, files" + behavior: default From e01d9cca8672e0a85b7ff42118ee456c4c414d9e Mon Sep 17 00:00:00 2001 From: Amadeusz Szymko Date: Mon, 8 May 2023 20:17:20 +0200 Subject: [PATCH 43/47] Fix nvidia runtime fail on arm64 & add --group-add plugin (#211) * Allow the override of nvidia selection Signed-off-by: Amadeusz Szymko --- src/rocker/nvidia_extension.py | 13 ++++++++++--- test/test_nvidia.py | 15 +++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/src/rocker/nvidia_extension.py b/src/rocker/nvidia_extension.py index 1bae9fa9..ec0ed5b5 100644 --- a/src/rocker/nvidia_extension.py +++ b/src/rocker/nvidia_extension.py @@ -123,6 +123,11 @@ def get_snippet(self, cliargs): return em.expand(snippet, self.get_environment_subs(cliargs)) def get_docker_args(self, cliargs): + force_flag = cliargs.get('nvidia', None) + if force_flag == 'runtime': + return " --runtime=nvidia" + if force_flag == 'gpus': + return " --gpus all" if get_docker_version() >= Version("19.03"): return " --gpus all" return " --runtime=nvidia" @@ -130,9 +135,11 @@ def get_docker_args(self, cliargs): @staticmethod def register_arguments(parser, defaults={}): parser.add_argument(name_to_argument(Nvidia.get_name()), - action='store_true', - default=defaults.get(Nvidia.get_name(), None), - help="Enable nvidia") + choices=['auto', 'runtime', 'gpus'], + nargs='?', + const='auto', + default=defaults.get(Nvidia.get_name(), 'auto'), + help="Enable nvidia. Default behavior is to pick flag based on docker version.") class Cuda(RockerExtension): @staticmethod diff --git a/test/test_nvidia.py b/test/test_nvidia.py index 29ae0c2c..d63d9794 100644 --- a/test/test_nvidia.py +++ b/test/test_nvidia.py @@ -185,6 +185,21 @@ def test_nvidia_extension_basic(self): else: self.assertIn(' --runtime=nvidia', docker_args) + mock_cliargs = {'nvidia': 'auto'} + docker_args = p.get_docker_args(mock_cliargs) + if get_docker_version() >= Version("19.03"): + self.assertIn(' --gpus all', docker_args) + else: + self.assertIn(' --runtime=nvidia', docker_args) + + mock_cliargs = {'nvidia': 'gpus'} + docker_args = p.get_docker_args(mock_cliargs) + self.assertIn(' --gpus all', docker_args) + + mock_cliargs = {'nvidia': 'runtime'} + docker_args = p.get_docker_args(mock_cliargs) + self.assertIn(' --runtime=nvidia', docker_args) + def test_no_nvidia_glmark2(self): for tag in self.dockerfile_tags: From f6d8918c9854b4fe00f09fc6e46b2c7aee330d21 Mon Sep 17 00:00:00 2001 From: Tully Foote Date: Fri, 12 May 2023 17:32:11 -0700 Subject: [PATCH 44/47] unpinning urllib3 now that the fix upstream is released (#235) Signed-off-by: Tully Foote --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d10b0d09..9bcca932 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ 'empy', 'pexpect', 'packaging', - 'urllib3<2', # Workaround for https://github.com/docker/docker-py/issues/3113 + 'urllib3', ] # docker API used to be in a package called `docker-py` before the 2.0 release From 2b8d5abb18c829d22f290679b739dc53765198ad Mon Sep 17 00:00:00 2001 From: Amadeusz Szymko Date: Mon, 15 May 2023 09:46:43 +0200 Subject: [PATCH 45/47] fix: nvidia arg (#234) Signed-off-by: Amadeusz Szymko --- src/rocker/nvidia_extension.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rocker/nvidia_extension.py b/src/rocker/nvidia_extension.py index ec0ed5b5..f731f481 100644 --- a/src/rocker/nvidia_extension.py +++ b/src/rocker/nvidia_extension.py @@ -138,7 +138,7 @@ def register_arguments(parser, defaults={}): choices=['auto', 'runtime', 'gpus'], nargs='?', const='auto', - default=defaults.get(Nvidia.get_name(), 'auto'), + default=defaults.get(Nvidia.get_name(), None), help="Enable nvidia. Default behavior is to pick flag based on docker version.") class Cuda(RockerExtension): From 52d5f28332b6fb425fcbc2d291bc3f5dd1b28d08 Mon Sep 17 00:00:00 2001 From: Sergio Portoles Diez Date: Tue, 18 Jul 2023 11:50:33 +0200 Subject: [PATCH 46/47] bump intermodalics version number --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index bb2c1dc9..0b13674c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [sdist_dsc] epoch: 1 -debian-version: 2intermodalics +debian-version: 3intermodalics [tool:pytest] markers = From d656459b5f3a181c78e60289f3f9d33dde66ccd3 Mon Sep 17 00:00:00 2001 From: Sergio Portoles Diez Date: Tue, 18 Jul 2023 19:06:47 +0200 Subject: [PATCH 47/47] fix version bump correctly after upstream version change --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0b13674c..fb52d277 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [sdist_dsc] epoch: 1 -debian-version: 3intermodalics +debian-version: 1intermodalics [tool:pytest] markers =