Fix python version parsing

For python versions like 3.10, versions were being cast from float to string, leading to "3.1" builds. - Added tests for malformed python version - Improved python validation for version strings
spillai · Oct 22, 2023 · d754fa3 · d754fa3
1 parent 863d407
commit d754fa3
Show file tree

Hide file tree

Showing 17 changed files with 111 additions and 79 deletions.
diff --git a/README.md b/README.md
@@ -58,7 +58,7 @@ Go through the [examples](./examples) and the corresponding [examples/generated]
         system:
         - wget
         - build-essential
-        python: 3.8.10
+        python: "3.8.10"
         pip:
         - loguru
         - typer
@@ -112,7 +112,7 @@ images:
     base: debian:buster-slim
     system:
         - wget
-    python: 3.8.10
+    python: "3.8.10"
     pip:
         - scikit-learn
     run:
@@ -166,7 +166,7 @@ images:
     - gnupg2
     - build-essential
     - git
-    python: 3.8.10
+    python: "3.8.10"
     pip:
     - torch==2.0.1
 ```

diff --git a/agipack/config.py b/agipack/config.py
@@ -116,7 +116,8 @@ def is_base_image(self) -> bool:
     @validator("python", pre=True)
     def validate_python_version(cls, python) -> str:
         """Validate the python version."""
-        python = str(python)
+        if not isinstance(python, str):
+            raise ValueError(f"Python version must be a string (type={type(python)})")
         if not python.startswith("3."):
             raise ValueError(f"Python version must be >= 3.6 (found {python})")
         return python

diff --git a/examples/agibuild.base-cpu.yaml b/examples/agibuild.base-cpu.yaml
@@ -3,7 +3,7 @@ images:
   base-cpu:
     system:
       - wget
-    python: 3.8
+    python: "3.8"
     conda:
       - pytorch>=2.1
       - torchvision

diff --git a/examples/agibuild.base-cu118.yaml b/examples/agibuild.base-cu118.yaml
@@ -4,7 +4,7 @@ images:
     base: nvidia/cuda:11.8.0-base-ubuntu22.04
     system:
       - wget
-    python: 3.8
+    python: "3.8"
     conda:
       - pytorch==2.1.0
       - torchvision

diff --git a/examples/agibuild.builder.yaml b/examples/agibuild.builder.yaml
@@ -2,7 +2,7 @@
 images:
   agipack-builder:
     base: debian:buster-slim
-    python: 3.8
+    python: "3.8"
     run:
       - pip install agi-pack
     command: ["agi-pack", "generate", "-f"]
diff --git a/examples/generated/Dockerfile-base-cpu b/examples/generated/Dockerfile-base-cpu
@@ -1,5 +1,5 @@
 # >>>>>>>>>>>>>>>>>>>>>>>>>>>
-# Auto-generated by agi-pack (version=0.1.14).
+# Auto-generated by agi-pack (version=0.1.15).
 FROM debian:buster-slim AS base-cpu
 
 # Setup environment variables
@@ -11,6 +11,8 @@ ENV PYTHON_VERSION 3.8
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore
+ENV PIP_CACHE_DIR /var/cache/pip
+ENV CONDA_PKGS_DIRS /var/cache/conda/pkgs
 
 # Setup conda paths
 ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}
@@ -22,21 +24,18 @@ ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV}
 # Install base system packages
 RUN apt-get -y update \
     && apt-get -y --no-install-recommends install \
-    curl bzip2 git ca-certificates \
-    && apt-get -y autoclean \
-    && apt-get -y autoremove \
-    && rm -rf /var/lib/apt/lists/*
+    curl bzip2 git ca-certificates
 
 # Install additional system packages
-RUN apt-get -y update \
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get -y update \
     && apt-get -y --no-install-recommends install \
     wget \
-    && apt-get -y autoclean \
-    && apt-get -y autoremove \
-    && rm -rf /var/lib/apt/lists/*
+    && echo "system install complete"
 
-# Install mambaforge
-RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
+# Install mambaforge, with cache mounting ${CONDA_PKGS_DIRS} for faster builds
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+  curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
   && chmod +x ~/mambaforge.sh \
   && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \
   && ${AGIPACK_PATH}/conda/bin/mamba init bash \
@@ -47,21 +46,21 @@ RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases
 # Upgrade pip
 RUN pip install --upgrade pip
 
-# Install conda packages, with cache mounting ${AGIPACK_PATH}/conda/pkgs for faster builds
+# Install conda packages, with cache mounting ${CONDA_PKGS_DIRS} for faster builds
 # Note: Cache mounts allow us to re-use the cache for conda packages
 # instead of having to re-download them every time we build.
-RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS}  \
     mamba install -yv \
     pytorch>=2.1 \
     torchvision \
     cpuonly -c pytorch \
     && echo "conda/mamba install complete"
 
-# Install pip requirements, with cache mounting ~/.cache/pip for faster builds
+# Install pip requirements, with cache mounting ${PIP_CACHE_DIR} for faster builds
 # Note: Cache mounts allow us to re-use the cache for pip packages
 # instead of having to re-download them every time we build.
 COPY requirements/requirements.txt /tmp/reqs/requirements/requirements.txt
-RUN --mount=type=cache,target=~/.cache/pip \
+RUN --mount=type=cache,target=${PIP_CACHE_DIR} \
     pip install --upgrade pip \
     && pip install -r /tmp/reqs/requirements/requirements.txt \
     && echo "pip requirements install complete"
@@ -71,21 +70,28 @@ RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.b
     && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
     && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \
     && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
-RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
-    && rm -rf ~/.cache/pip \
-    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
-    && rm -rf /tmp/reqs \
-    && echo "pip cleanup complete"
 
 # Setup working directory
 WORKDIR /app/$AGIPACK_PYENV
 
 # Run commands
 RUN echo "running commands"
-RUN python -c 'import cv2; print(cv2.__version__)'
-RUN python -c 'import torch; print(torch.__version__)'
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    python -c 'import cv2; print(cv2.__version__)'
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    python -c 'import torch; print(torch.__version__)'
 RUN echo "run commands complete"
+# Cleanup apt, mamba/conda and pip packages
+RUN apt-get -y autoclean \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/* \
+    && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
+    && rm -rf ~/.cache/pip \
+    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 
 # Setup environment variables
-ENV MY_ENV_VAR=value 
-CMD ["bash"]
+ENV MY_ENV_VAR=value 
diff --git a/examples/generated/Dockerfile-base-cu118 b/examples/generated/Dockerfile-base-cu118
@@ -1,5 +1,5 @@
 # >>>>>>>>>>>>>>>>>>>>>>>>>>>
-# Auto-generated by agi-pack (version=0.1.14).
+# Auto-generated by agi-pack (version=0.1.15).
 FROM nvidia/cuda:11.8.0-base-ubuntu22.04 AS base-gpu
 
 # Setup environment variables
@@ -11,6 +11,8 @@ ENV PYTHON_VERSION 3.8
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore
+ENV PIP_CACHE_DIR /var/cache/pip
+ENV CONDA_PKGS_DIRS /var/cache/conda/pkgs
 
 # Setup conda paths
 ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}
@@ -22,21 +24,18 @@ ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV}
 # Install base system packages
 RUN apt-get -y update \
     && apt-get -y --no-install-recommends install \
-    curl bzip2 git ca-certificates \
-    && apt-get -y autoclean \
-    && apt-get -y autoremove \
-    && rm -rf /var/lib/apt/lists/*
+    curl bzip2 git ca-certificates
 
 # Install additional system packages
-RUN apt-get -y update \
+RUN --mount=type=cache,target=/var/cache/apt \
+    apt-get -y update \
     && apt-get -y --no-install-recommends install \
     wget \
-    && apt-get -y autoclean \
-    && apt-get -y autoremove \
-    && rm -rf /var/lib/apt/lists/*
+    && echo "system install complete"
 
-# Install mambaforge
-RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
+# Install mambaforge, with cache mounting ${CONDA_PKGS_DIRS} for faster builds
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+  curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
   && chmod +x ~/mambaforge.sh \
   && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \
   && ${AGIPACK_PATH}/conda/bin/mamba init bash \
@@ -47,10 +46,10 @@ RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases
 # Upgrade pip
 RUN pip install --upgrade pip
 
-# Install conda packages, with cache mounting ${AGIPACK_PATH}/conda/pkgs for faster builds
+# Install conda packages, with cache mounting ${CONDA_PKGS_DIRS} for faster builds
 # Note: Cache mounts allow us to re-use the cache for conda packages
 # instead of having to re-download them every time we build.
-RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS}  \
     mamba install -yv \
     pytorch==2.1.0 \
     torchvision \
@@ -61,11 +60,11 @@ RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \
     -c pytorch -c nvidia \
     && echo "conda/mamba install complete"
 
-# Install pip requirements, with cache mounting ~/.cache/pip for faster builds
+# Install pip requirements, with cache mounting ${PIP_CACHE_DIR} for faster builds
 # Note: Cache mounts allow us to re-use the cache for pip packages
 # instead of having to re-download them every time we build.
 COPY requirements/requirements.txt /tmp/reqs/requirements/requirements.txt
-RUN --mount=type=cache,target=~/.cache/pip \
+RUN --mount=type=cache,target=${PIP_CACHE_DIR} \
     pip install --upgrade pip \
     && pip install -r /tmp/reqs/requirements/requirements.txt \
     && echo "pip requirements install complete"
@@ -75,23 +74,34 @@ RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.b
     && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
     && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \
     && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
-RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
-    && rm -rf ~/.cache/pip \
-    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
-    && rm -rf /tmp/reqs \
-    && echo "pip cleanup complete"
 
 # Setup working directory
 WORKDIR /app/$AGIPACK_PYENV
 
 # Run commands
 RUN echo "running commands"
-RUN echo 'pytorch: ' && python -c 'import torch; print(torch.__version__)'
-RUN echo 'cuda: ' && python -c 'import torch; print(torch.version.cuda)'
-RUN echo 'cudnn: ' && python -c 'import torch; print(torch.backends.cudnn.version())'
-RUN echo 'opencv:' && python -c 'import cv2; print(cv2.__version__)'
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    echo 'pytorch: ' && python -c 'import torch; print(torch.__version__)'
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    echo 'cuda: ' && python -c 'import torch; print(torch.version.cuda)'
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    echo 'cudnn: ' && python -c 'import torch; print(torch.backends.cudnn.version())'
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    echo 'opencv:' && python -c 'import cv2; print(cv2.__version__)'
 RUN echo "run commands complete"
+# Cleanup apt, mamba/conda and pip packages
+RUN apt-get -y autoclean \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/* \
+    && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
+    && rm -rf ~/.cache/pip \
+    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 
 # Setup environment variables
-ENV MY_ENV_VAR=value 
-CMD ["bash"]
+ENV MY_ENV_VAR=value 
diff --git a/examples/generated/Dockerfile-builder b/examples/generated/Dockerfile-builder
@@ -1,5 +1,5 @@
 # >>>>>>>>>>>>>>>>>>>>>>>>>>>
-# Auto-generated by agi-pack (version=0.1.14).
+# Auto-generated by agi-pack (version=0.1.15).
 FROM debian:buster-slim AS agipack-builder
 
 # Setup environment variables
@@ -11,6 +11,8 @@ ENV PYTHON_VERSION 3.8
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore
+ENV PIP_CACHE_DIR /var/cache/pip
+ENV CONDA_PKGS_DIRS /var/cache/conda/pkgs
 
 # Setup conda paths
 ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}
@@ -22,13 +24,11 @@ ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV}
 # Install base system packages
 RUN apt-get -y update \
     && apt-get -y --no-install-recommends install \
-    curl bzip2 git ca-certificates \
-    && apt-get -y autoclean \
-    && apt-get -y autoremove \
-    && rm -rf /var/lib/apt/lists/*
+    curl bzip2 git ca-certificates
 
-# Install mambaforge
-RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
+# Install mambaforge, with cache mounting ${CONDA_PKGS_DIRS} for faster builds
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+  curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
   && chmod +x ~/mambaforge.sh \
   && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \
   && ${AGIPACK_PATH}/conda/bin/mamba init bash \
@@ -44,17 +44,23 @@ RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.b
     && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
     && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \
     && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
-RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
-    && rm -rf ~/.cache/pip \
-    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
-    && rm -rf /tmp/reqs \
-    && echo "pip cleanup complete"
 
 # Setup working directory
 WORKDIR /app/$AGIPACK_PYENV
 
 # Run commands
 RUN echo "running commands"
-RUN pip install agi-pack
+RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \
+    --mount=type=cache,target=${PIP_CACHE_DIR} \
+    pip install agi-pack
 RUN echo "run commands complete"
+# Cleanup apt, mamba/conda and pip packages
+RUN apt-get -y autoclean \
+    && apt-get -y autoremove \
+    && rm -rf /var/lib/apt/lists/* \
+    && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
+    && rm -rf ~/.cache/pip \
+    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 CMD ["agi-pack", "generate", "-f"]
diff --git a/tests/test_config.py b/tests/test_config.py
@@ -28,6 +28,7 @@ def test_poorly_formatted_configs(test_data_dir):
     poorly_formatted_configs = [
         test_data_dir / "agibuild-no-base.yaml",
         test_data_dir / "agibuild-malformed-add.yaml",
+        test_data_dir / "agibuild-malformed-py-version.yaml",
     ]
     for filename in poorly_formatted_configs:
         with pytest.raises(ValueError):

diff --git a/tests/test_data/agibuild-different-py-versions.yaml b/tests/test_data/agibuild-different-py-versions.yaml
@@ -2,14 +2,14 @@ images:
   base-cpu:
     system:
       - wget
-    python: 3.8
+    python: "3.8"
     pip:
       - scikit-learn
     run:
       - echo "Hello, world!"
 
   dev-cpu:
     base: base-cpu
-    python: 3.9
+    python: "3.9"
     system:
       - build-essential
diff --git a/tests/test_data/agibuild-malformed-add.yaml b/tests/test_data/agibuild-malformed-add.yaml
@@ -2,7 +2,7 @@ images:
   base-cpu:
     system:
       - wget
-    python: 3.8.10
+    python: "3.8.10"
     pip:
       - scikit-learn
     add: