Merge pull request #7 from Intermodalics/merge/upstream/v0.2.6

Merge upstream v0.2.6

.github/workflows/basic-ci.yaml

@@ -17,6 +17,8 @@ jobs:
     - name: Install Dependencies And Self
       run: |
         python -m pip install --upgrade pip setuptools wheel
+        # Workaround for https://github.com/docker/docker-py/issues/2807
+        python -m pip install six
         pip install codecov coverage nose
         pip install .
     - name: Run headless tests

README.md

@@ -140,10 +140,20 @@ On Bionic
 
     rocker --nvidia --x11 osrf/ros:melodic-desktop-full gazebo
 
-## Mount volumes
+## Volume mount
 
-`--mount` adds paths as docker volumes.
-The path used inside the container is the same as the path outside.
-The last path must be terminated with two dashes `--`.
+For arguments with one element not colon separated.
 
-    rocker --mount ~/.vimrc ~/.bashrc -- ubuntu:18.04
+`--volume` adds paths as docker volumes. The last path must be terminated with two dashes `--`.
+
+    rocker --volume ~/.vimrc ~/.bashrc -- ubuntu:18.04
+
+The above example of the volume option will be expanded via absolute paths for `docker run` as follows:
+
+    --volume /home/<USERNAME>/.vimrc:/home/<USERNAME>/.vimrc --volume /home/<USERNAME>/.bashrc:/home/<USERNAME>/.bashrc  
+
+For arguments with colon separation it will process the same as `docker`'s `--volume` option, `rocker --volume` takes 3 fields.
+- 1st field: the path to the file or directory on the host machine.
+- 2nd field: (optional) the path where the file or directory is mounted in the container.
+   - If only the 1st field is supplied, same value as the 1st field will be populated as the 2nd field.
+- 3rd field: (optional) bind propagation as `ro`, `z`, and `Z`. See [docs.docker.com](https://docs.docker.com/storage/bind-mounts/) for further detail.

setup.cfg

@@ -1,3 +1,3 @@
 [sdist_dsc]
 epoch: 1
-debian-version: 4intermodalics
+debian-version: 2intermodalics

setup.py

@@ -31,7 +31,7 @@
 
 kwargs = {
     'name': 'rocker',
-    'version': '0.2.3',
+    'version': '0.2.6',
     'packages': ['rocker'],
     'package_dir': {'': 'src'},
     'package_data': {'rocker': ['templates/*.em']},
@@ -46,7 +46,7 @@
             'env = rocker.extensions:Environment',
             'git = rocker.git_extension:Git',
             'home = rocker.extensions:HomeDir',
-            'mount = rocker.mount_extension:Mount',
+            'volume = rocker.volume_extension:Volume',
             'name = rocker.extensions:Name',
             'network = rocker.extensions:Network',
             'nvidia = rocker.nvidia_extension:Nvidia',

src/rocker/core.py

@@ -103,6 +103,8 @@ def extend_cli_parser(self, parser, default_args={}):
         parser.add_argument('--mode', choices=OPERATION_MODES,
             default=OPERATIONS_INTERACTIVE,
             help="Choose mode of operation for rocker")
+        parser.add_argument('--image-name', default=None,
+            help='Tag the final image, useful with dry-run')
         parser.add_argument('--extension-blacklist', nargs='*',
             default=[],
             help='Prevent any of these extensions from being loaded.')
@@ -223,6 +225,10 @@ def build(self, **kwargs):
             arguments['rm'] = True
             arguments['nocache'] = kwargs.get('nocache', False)
             arguments['pull'] = kwargs.get('pull', False)
+            image_name = kwargs.get('image_name', None)
+            if image_name:
+                print("Running docker tag {} {}".format(self.image_id, image_name))
+                arguments['tag'] = image_name
             print("Building docker file with arguments: ", arguments)
             try:
                 self.image_id = docker_build(
@@ -260,7 +266,11 @@ def generate_docker_cmd(self, command='', **kwargs):
             for arg in kwargs['additional_docker_args']:
                 docker_args += ' ' + shlex.quote(arg)
 
-        image = self.image_id
+        image_name = kwargs.get('image_name', None)
+        if image_name:
+            image = image_name
+        else:
+            image = self.image_id
         cmd = "docker run"
         if(not kwargs.get('nocleanup')):
             # remove container only if --nocleanup is not present

src/rocker/os_detector.py

@@ -21,21 +21,23 @@
 
 
 DETECTION_TEMPLATE="""
-FROM python:3-stretch as detector
+FROM python:3-slim-stretch as detector
 # Force the older version of debian for detector.
 # GLIBC is forwards compatible but not necessarily backwards compatible for pyinstaller
 # https://github.com/pyinstaller/pyinstaller/wiki/FAQ#gnulinux
 # StaticX is supposed to take care of this but there appears to be an issue when using subprocess
 
 RUN mkdir -p /tmp/distrovenv
 RUN python3 -m venv /tmp/distrovenv
-RUN . /tmp/distrovenv/bin/activate && pip install distro pyinstaller==4.0 staticx
-RUN apt-get update && apt-get install -qy patchelf #needed for staticx
+# patchelf needed for staticx
+# binutils provides objdump needed by pyinstaller
+RUN apt-get update && apt-get install -qy patchelf binutils
+RUN . /tmp/distrovenv/bin/activate && pip install distro pyinstaller==4.0 staticx==0.12.3
 
-RUN echo 'import distro; import sys; output = distro.linux_distribution(); print(output) if output[0] else sys.exit(1)' > /tmp/distrovenv/detect_os.py
+RUN echo 'import distro; import sys; output = (distro.name(), distro.version(), distro.codename()); print(output) if distro.name() else sys.exit(1)' > /tmp/distrovenv/detect_os.py
 RUN . /tmp/distrovenv/bin/activate && pyinstaller --onefile /tmp/distrovenv/detect_os.py
 
-RUN . /tmp/distrovenv/bin/activate && staticx /dist/detect_os /dist/detect_os_static
+RUN . /tmp/distrovenv/bin/activate && staticx /dist/detect_os /dist/detect_os_static && chmod go+xr /dist/detect_os_static
 
 FROM %(image_name)s
 
@@ -57,7 +59,6 @@ def detect_os(image_name, output_callback=None, nocache=False):
         output_callback=output_callback,
         nocache=nocache,
         forcerm=True,  # Remove intermediate containers from RUN commands in DETECTION_TEMPLATE
-        tag="rocker:" + f"os_detect_{image_name}".replace(':', '_').replace('/', '_')
     )
     if not image_id:
         if output_callback:

src/rocker/volume_extension.py

@@ -0,0 +1,70 @@
+# Copyright 2019 Open Source Robotics Foundation
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from argparse import ArgumentTypeError
+import os
+from rocker.extensions import RockerExtension
+
+
+class Volume(RockerExtension):
+
+    ARG_DOCKER_VOLUME = "-v"
+    ARG_ROCKER_VOLUME = ["--volume", "--mount"]
+    name = 'volume'
+
+    @classmethod
+    def get_name(cls):
+        return cls.name
+
+    def get_docker_args(self, cli_args):
+        """
+        @param cli_args: {'volume': [[%arg%]]}
+            - 'volume' is fixed.
+            - %arg% can be:
+               - %path_host%: a path on the host. Same path will be populated in
+                   the container.
+               - %path_host%:%path_cont%
+               - %path_host%:%path_cont%:%option%
+        """
+        args = ['']
+
+        # flatten cli_args['volume']
+        volumes = [ x for sublist in cli_args[self.name] for x in sublist]
+
+        for volume in volumes:
+            elems = volume.split(':')
+            host_dir = os.path.abspath(elems[0])
+            if len(elems) == 1:
+                args.append('{0} {1}:{1}'.format(self.ARG_DOCKER_VOLUME, host_dir))
+            elif len(elems) == 2:
+                container_dir = elems[1]
+                args.append('{0} {1}:{2}'.format(self.ARG_DOCKER_VOLUME, host_dir, container_dir))
+            elif len(elems) == 3:
+                container_dir = elems[1]
+                options = elems[2]
+                args.append('{0} {1}:{2}:{3}'.format(self.ARG_DOCKER_VOLUME, host_dir, container_dir, options))
+            else:
+                raise ArgumentTypeError(
+                    '{} expects arguments in format HOST-DIR[:CONTAINER-DIR[:OPTIONS]]'.format(self.ARG_ROCKER_VOLUME))
+
+        return ' '.join(args)
+
+    @staticmethod
+    def register_arguments(parser):
+        parser.add_argument(*Volume.ARG_ROCKER_VOLUME,
+            metavar='HOST-DIR[:CONTAINER-DIR[:OPTIONS]]',
+            type=str,
+            nargs='+',
+            action='append',
+            help='volume volumes in container')

test/test_volume.py

@@ -0,0 +1,69 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import unittest
+
+from rocker.core import list_plugins
+from rocker.volume_extension import Volume
+
+
+class VolumeTest(unittest.TestCase):
+    def setUp(self):
+        self._instance = Volume()
+        self._curr_path = os.path.abspath(os.path.curdir)
+        self._virtual_path = "/path/in/container"
+
+    def _test_equals_args(self, mock_cliargs, expected):
+        """
+        @type mock_cliargs: { str: [[str]] }
+        @type expected: [[str]]
+        """
+        print("DEBUG: 'mock_cliargs' {}\n\t'expected': {}".format(mock_cliargs, expected))
+        docker_args = self._instance.get_docker_args(mock_cliargs)
+        print("DEBUG: Resulted docker_args: {}".format(docker_args))
+        for arg_expected in expected:
+            # Whitespace at the beginning is needed.
+            complete_expected = " {} {}".format(Volume.ARG_DOCKER_VOLUME, arg_expected[0])
+            self.assertTrue(complete_expected in docker_args)
+
+    def test_args_single(self):
+        """Passing source path"""
+        arg = [[self._curr_path]]
+        expected = [['{}:{}'.format(self._curr_path, self._curr_path)]]
+        mock_cliargs = {Volume.name: arg}
+        self._test_equals_args(mock_cliargs, expected)
+
+    def test_args_twopaths(self):
+        """Passing source path, dest path"""
+        arg = ["{}:{}".format(self._curr_path, self._virtual_path)]
+        mock_cliargs = {Volume.name: [arg]}
+        self._test_equals_args(mock_cliargs, arg)
+
+    def test_args_twopaths_opt(self):
+        """Passing source path, dest path, and Docker's volume option"""
+        arg = ["{}:{}:ro".format(self._curr_path, self._virtual_path)]
+        mock_cliargs = {Volume.name: [arg]}
+        self._test_equals_args(mock_cliargs, arg)
+
+    def test_args_two_volumes(self):
+        """Multiple volume points"""
+        arg_first = ["{}:{}:ro".format(self._curr_path, self._virtual_path)]
+        arg_second = ["/tmp:{}".format(os.path.join(self._virtual_path, "tmp"))]
+        args = [arg_first, arg_second]
+        mock_cliargs = {Volume.name: args}
+        self._test_equals_args(mock_cliargs, args)