Merge pull request #128 from aai-institute/126-image-pip-deps

Correctly install regular pip dependencies in declarative image builds

client/src/jobq/assembler/renderers.py

@@ -96,7 +96,8 @@ def accepts(cls, config: Config) -> bool:
     def render(self) -> str:
         result = ""
 
-        packages = cast(DependencySpec, self.config.build.dependencies).pip
+        # List will be modified, so make a copy
+        packages = cast(DependencySpec, self.config.build.dependencies).pip.copy()
         user_opts = self.config.build.user
 
         copy_options = []
@@ -114,30 +115,35 @@ def render(self) -> str:
 
         # Copy any direct Wheel dependencies to the image
         wheels = [p for p in packages if p.endswith(".whl")]
+        map(packages.remove, wheels)
         if wheels:
             result += f"COPY {' '.join(copy_options)} {' '.join(wheels)} .\n"
 
         # Copy any requirements.txt files to the image
-        reqs_files = [
-            p.split()[1] for p in packages if p.startswith(("-r", "--requirement"))
-        ]
+        reqs_packages = [p for p in packages if p.startswith(("-r", "--requirement"))]
+        reqs_files = [p.split()[1] for p in reqs_packages]
+        map(packages.remove, reqs_packages)
         if reqs_files:
             result += f"COPY {' '.join(copy_options)} {' '.join(reqs_files)} .\n"
             # ... and install those before and local projects
             result += f"RUN {' '.join(run_options)} pip install {' '.join(f'-r {r}' for r in reqs_files)}\n"
 
         # Next install local projects (built wheels or editable installs)
-        build_folders = [
-            str(folder)
+        build_packages = [
+            p
             for p in packages
             if (folder := Path(p)).is_dir() and (folder / "pyproject.toml").is_file()
         ]
+        build_folders = [str(folder) for p in build_packages]
+        map(packages.remove, build_packages)
         if build_folders:
             result += f"COPY {' '.join(copy_options)} {' '.join(build_folders)} .\n"
 
-        editable_installs = [
-            p.split()[1] for p in packages if p.startswith(("-e", "--editable"))
+        editable_packages = [
+            p for p in packages if p.startswith(("-r", "--requirement"))
         ]
+        editable_installs = [p.split()[1] for p in editable_packages]
+        map(packages.remove, editable_packages)
         for root_dir in editable_installs:
             pyproject_toml = Path(root_dir) / "pyproject.toml"
             if not pyproject_toml.exists():
@@ -150,6 +156,10 @@ def render(self) -> str:
                 f"RUN {' '.join(run_options)} pip install {' '.join(local_packages)}\n"
             )
 
+        # Finally install any remaining packages (which should be regular packages)
+        if packages:
+            result += f"RUN {' '.join(run_options)} pip install {' '.join(packages)}\n"
+
         return result
 
 

client/tests/smoke/_data/docker.yaml

@@ -1,25 +1,25 @@
 build:
-  base_image: python:3.12-slim
-  dependencies:
-    apt: [curl, git]
-    pip: [attrs, pyyaml]
-  volumes:
-    - .:.
-  user: 
-    name: no_admin
-  config:
-    env:
-      - var: secret
-    arg:
-      - build_arg: config
-    stopsignal: 1
-    shell: sh
-  meta:
-    labels: 
-      - test: test
-  workdir: /usr/src/
-  filesystem:
-    copy:
-      - . : .
-    add:
-      - . : .
+    base_image: python:3.12-slim
+    dependencies:
+        apt: [curl, git]
+        pip: [attrs, pyyaml, test.whl, marker-package, -e.]
+    volumes:
+        - .:.
+    user:
+        name: no_admin
+    config:
+        env:
+            - var: secret
+        arg:
+            - build_arg: config
+        stopsignal: 1
+        shell: sh
+    meta:
+        labels:
+            - test: test
+    workdir: /usr/src/
+    filesystem:
+        copy:
+            - .: .
+        add:
+            - .: .

client/tests/smoke/test_build_from_yaml.py

@@ -1,3 +1,4 @@
+import re
 from pathlib import Path
 
 from jobq import assembler
@@ -18,7 +19,29 @@ def test_build_image_from_yaml():
             tag="test",
         ),
     )
-    testjob._render_dockerfile()
+    dockerfile = testjob._render_dockerfile()
+
+    # Base image
+    pattern = r"FROM python:3.12-slim"
+    assert (
+        re.search(pattern, dockerfile) is not None
+    ), "Base image not found or incorrect"
+
+    # Wheel installation
+    pattern = r"RUN .* pip install.*test\.whl"
+    assert re.search(pattern, dockerfile) is not None, "Wheel installation not found"
+
+    # Editable package install
+    pattern = r"RUN .* pip install.*-e[ ]?[.]"
+    assert (
+        re.search(pattern, dockerfile) is not None
+    ), "Editable package installation not found"
+
+    # Regular package install
+    pattern = r"RUN .* pip install.*marker-package"
+    assert (
+        re.search(pattern, dockerfile) is not None
+    ), "Marker package installation not found"
 
 
 def test_image_assembler():