diff --git a/client/src/jobq/assembler/renderers.py b/client/src/jobq/assembler/renderers.py index 67a9620e..3867dafb 100644 --- a/client/src/jobq/assembler/renderers.py +++ b/client/src/jobq/assembler/renderers.py @@ -96,7 +96,8 @@ def accepts(cls, config: Config) -> bool: def render(self) -> str: result = "" - packages = cast(DependencySpec, self.config.build.dependencies).pip + # List will be modified, so make a copy + packages = cast(DependencySpec, self.config.build.dependencies).pip.copy() user_opts = self.config.build.user copy_options = [] @@ -114,30 +115,35 @@ def render(self) -> str: # Copy any direct Wheel dependencies to the image wheels = [p for p in packages if p.endswith(".whl")] + map(packages.remove, wheels) if wheels: result += f"COPY {' '.join(copy_options)} {' '.join(wheels)} .\n" # Copy any requirements.txt files to the image - reqs_files = [ - p.split()[1] for p in packages if p.startswith(("-r", "--requirement")) - ] + reqs_packages = [p for p in packages if p.startswith(("-r", "--requirement"))] + reqs_files = [p.split()[1] for p in reqs_packages] + map(packages.remove, reqs_packages) if reqs_files: result += f"COPY {' '.join(copy_options)} {' '.join(reqs_files)} .\n" # ... and install those before and local projects result += f"RUN {' '.join(run_options)} pip install {' '.join(f'-r {r}' for r in reqs_files)}\n" # Next install local projects (built wheels or editable installs) - build_folders = [ - str(folder) + build_packages = [ + p for p in packages if (folder := Path(p)).is_dir() and (folder / "pyproject.toml").is_file() ] + build_folders = [str(folder) for p in build_packages] + map(packages.remove, build_packages) if build_folders: result += f"COPY {' '.join(copy_options)} {' '.join(build_folders)} .\n" - editable_installs = [ - p.split()[1] for p in packages if p.startswith(("-e", "--editable")) + editable_packages = [ + p for p in packages if p.startswith(("-r", "--requirement")) ] + editable_installs = [p.split()[1] for p in editable_packages] + map(packages.remove, editable_packages) for root_dir in editable_installs: pyproject_toml = Path(root_dir) / "pyproject.toml" if not pyproject_toml.exists(): @@ -150,6 +156,10 @@ def render(self) -> str: f"RUN {' '.join(run_options)} pip install {' '.join(local_packages)}\n" ) + # Finally install any remaining packages (which should be regular packages) + if packages: + result += f"RUN {' '.join(run_options)} pip install {' '.join(packages)}\n" + return result diff --git a/client/tests/smoke/_data/docker.yaml b/client/tests/smoke/_data/docker.yaml index 82ff0d1e..656799ae 100644 --- a/client/tests/smoke/_data/docker.yaml +++ b/client/tests/smoke/_data/docker.yaml @@ -1,25 +1,25 @@ build: - base_image: python:3.12-slim - dependencies: - apt: [curl, git] - pip: [attrs, pyyaml] - volumes: - - .:. - user: - name: no_admin - config: - env: - - var: secret - arg: - - build_arg: config - stopsignal: 1 - shell: sh - meta: - labels: - - test: test - workdir: /usr/src/ - filesystem: - copy: - - . : . - add: - - . : . + base_image: python:3.12-slim + dependencies: + apt: [curl, git] + pip: [attrs, pyyaml, test.whl, marker-package, -e.] + volumes: + - .:. + user: + name: no_admin + config: + env: + - var: secret + arg: + - build_arg: config + stopsignal: 1 + shell: sh + meta: + labels: + - test: test + workdir: /usr/src/ + filesystem: + copy: + - .: . + add: + - .: . diff --git a/client/tests/smoke/test_build_from_yaml.py b/client/tests/smoke/test_build_from_yaml.py index 16f03d7d..44e4a69a 100644 --- a/client/tests/smoke/test_build_from_yaml.py +++ b/client/tests/smoke/test_build_from_yaml.py @@ -1,3 +1,4 @@ +import re from pathlib import Path from jobq import assembler @@ -18,7 +19,29 @@ def test_build_image_from_yaml(): tag="test", ), ) - testjob._render_dockerfile() + dockerfile = testjob._render_dockerfile() + + # Base image + pattern = r"FROM python:3.12-slim" + assert ( + re.search(pattern, dockerfile) is not None + ), "Base image not found or incorrect" + + # Wheel installation + pattern = r"RUN .* pip install.*test\.whl" + assert re.search(pattern, dockerfile) is not None, "Wheel installation not found" + + # Editable package install + pattern = r"RUN .* pip install.*-e[ ]?[.]" + assert ( + re.search(pattern, dockerfile) is not None + ), "Editable package installation not found" + + # Regular package install + pattern = r"RUN .* pip install.*marker-package" + assert ( + re.search(pattern, dockerfile) is not None + ), "Marker package installation not found" def test_image_assembler():