From d754fa37ad155df9c7b18e1fb095a9dc07b94c6a Mon Sep 17 00:00:00 2001 From: Sudeep Pillai Date: Sun, 22 Oct 2023 10:39:35 -0700 Subject: [PATCH] Fix python version parsing For python versions like 3.10, versions were being cast from float to string, leading to "3.1" builds. - Added tests for malformed python version - Improved python validation for version strings --- README.md | 6 +- agipack/config.py | 3 +- examples/agibuild.base-cpu.yaml | 2 +- examples/agibuild.base-cu118.yaml | 2 +- examples/agibuild.builder.yaml | 2 +- examples/generated/Dockerfile-base-cpu | 54 +++++++++------- examples/generated/Dockerfile-base-cu118 | 62 +++++++++++-------- examples/generated/Dockerfile-builder | 32 ++++++---- tests/test_config.py | 1 + .../agibuild-different-py-versions.yaml | 4 +- tests/test_data/agibuild-malformed-add.yaml | 2 +- .../agibuild-malformed-py-version.yaml | 5 ++ tests/test_data/agibuild-minimal.yaml | 2 +- tests/test_data/agibuild-no-deps.yaml | 2 +- tests/test_data/agibuild-no-system.yaml | 2 +- tests/test_data/agibuild-with-deps.yaml | 2 +- tests/test_examples.py | 7 ++- 17 files changed, 111 insertions(+), 79 deletions(-) create mode 100644 tests/test_data/agibuild-malformed-py-version.yaml diff --git a/README.md b/README.md index a8460f3..eebc990 100644 --- a/README.md +++ b/README.md @@ -58,7 +58,7 @@ Go through the [examples](./examples) and the corresponding [examples/generated] system: - wget - build-essential - python: 3.8.10 + python: "3.8.10" pip: - loguru - typer @@ -112,7 +112,7 @@ images: base: debian:buster-slim system: - wget - python: 3.8.10 + python: "3.8.10" pip: - scikit-learn run: @@ -166,7 +166,7 @@ images: - gnupg2 - build-essential - git - python: 3.8.10 + python: "3.8.10" pip: - torch==2.0.1 ``` diff --git a/agipack/config.py b/agipack/config.py index a2d7042..35f2838 100644 --- a/agipack/config.py +++ b/agipack/config.py @@ -116,7 +116,8 @@ def is_base_image(self) -> bool: @validator("python", pre=True) def validate_python_version(cls, python) -> str: """Validate the python version.""" - python = str(python) + if not isinstance(python, str): + raise ValueError(f"Python version must be a string (type={type(python)})") if not python.startswith("3."): raise ValueError(f"Python version must be >= 3.6 (found {python})") return python diff --git a/examples/agibuild.base-cpu.yaml b/examples/agibuild.base-cpu.yaml index 9989b83..937bc7e 100644 --- a/examples/agibuild.base-cpu.yaml +++ b/examples/agibuild.base-cpu.yaml @@ -3,7 +3,7 @@ images: base-cpu: system: - wget - python: 3.8 + python: "3.8" conda: - pytorch>=2.1 - torchvision diff --git a/examples/agibuild.base-cu118.yaml b/examples/agibuild.base-cu118.yaml index 1132c7c..f3c9957 100644 --- a/examples/agibuild.base-cu118.yaml +++ b/examples/agibuild.base-cu118.yaml @@ -4,7 +4,7 @@ images: base: nvidia/cuda:11.8.0-base-ubuntu22.04 system: - wget - python: 3.8 + python: "3.8" conda: - pytorch==2.1.0 - torchvision diff --git a/examples/agibuild.builder.yaml b/examples/agibuild.builder.yaml index 4e5970e..b2896ce 100644 --- a/examples/agibuild.builder.yaml +++ b/examples/agibuild.builder.yaml @@ -2,7 +2,7 @@ images: agipack-builder: base: debian:buster-slim - python: 3.8 + python: "3.8" run: - pip install agi-pack command: ["agi-pack", "generate", "-f"] diff --git a/examples/generated/Dockerfile-base-cpu b/examples/generated/Dockerfile-base-cpu index 50047fc..3cbbbd5 100644 --- a/examples/generated/Dockerfile-base-cpu +++ b/examples/generated/Dockerfile-base-cpu @@ -1,5 +1,5 @@ # >>>>>>>>>>>>>>>>>>>>>>>>>>> -# Auto-generated by agi-pack (version=0.1.14). +# Auto-generated by agi-pack (version=0.1.15). FROM debian:buster-slim AS base-cpu # Setup environment variables @@ -11,6 +11,8 @@ ENV PYTHON_VERSION 3.8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 ENV PYTHONWARNINGS ignore +ENV PIP_CACHE_DIR /var/cache/pip +ENV CONDA_PKGS_DIRS /var/cache/conda/pkgs # Setup conda paths ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV} @@ -22,21 +24,18 @@ ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV} # Install base system packages RUN apt-get -y update \ && apt-get -y --no-install-recommends install \ - curl bzip2 git ca-certificates \ - && apt-get -y autoclean \ - && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* + curl bzip2 git ca-certificates # Install additional system packages -RUN apt-get -y update \ +RUN --mount=type=cache,target=/var/cache/apt \ + apt-get -y update \ && apt-get -y --no-install-recommends install \ wget \ - && apt-get -y autoclean \ - && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* + && echo "system install complete" -# Install mambaforge -RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ +# Install mambaforge, with cache mounting ${CONDA_PKGS_DIRS} for faster builds +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ && chmod +x ~/mambaforge.sh \ && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \ && ${AGIPACK_PATH}/conda/bin/mamba init bash \ @@ -47,21 +46,21 @@ RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases # Upgrade pip RUN pip install --upgrade pip -# Install conda packages, with cache mounting ${AGIPACK_PATH}/conda/pkgs for faster builds +# Install conda packages, with cache mounting ${CONDA_PKGS_DIRS} for faster builds # Note: Cache mounts allow us to re-use the cache for conda packages # instead of having to re-download them every time we build. -RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \ +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ mamba install -yv \ pytorch>=2.1 \ torchvision \ cpuonly -c pytorch \ && echo "conda/mamba install complete" -# Install pip requirements, with cache mounting ~/.cache/pip for faster builds +# Install pip requirements, with cache mounting ${PIP_CACHE_DIR} for faster builds # Note: Cache mounts allow us to re-use the cache for pip packages # instead of having to re-download them every time we build. COPY requirements/requirements.txt /tmp/reqs/requirements/requirements.txt -RUN --mount=type=cache,target=~/.cache/pip \ +RUN --mount=type=cache,target=${PIP_CACHE_DIR} \ pip install --upgrade pip \ && pip install -r /tmp/reqs/requirements/requirements.txt \ && echo "pip requirements install complete" @@ -71,21 +70,28 @@ RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.b && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \ && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \ && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc -RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \ - && rm -rf ~/.cache/pip \ - && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \ - && rm -rf /tmp/reqs \ - && echo "pip cleanup complete" # Setup working directory WORKDIR /app/$AGIPACK_PYENV # Run commands RUN echo "running commands" -RUN python -c 'import cv2; print(cv2.__version__)' -RUN python -c 'import torch; print(torch.__version__)' +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + --mount=type=cache,target=${PIP_CACHE_DIR} \ + python -c 'import cv2; print(cv2.__version__)' +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + --mount=type=cache,target=${PIP_CACHE_DIR} \ + python -c 'import torch; print(torch.__version__)' RUN echo "run commands complete" +# Cleanup apt, mamba/conda and pip packages +RUN apt-get -y autoclean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* \ + && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \ + && rm -rf ~/.cache/pip \ + && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \ + && rm -rf /tmp/reqs \ + && echo "pip cleanup complete" # Setup environment variables -ENV MY_ENV_VAR=value -CMD ["bash"] \ No newline at end of file +ENV MY_ENV_VAR=value \ No newline at end of file diff --git a/examples/generated/Dockerfile-base-cu118 b/examples/generated/Dockerfile-base-cu118 index 5d1ac9f..d17d89c 100644 --- a/examples/generated/Dockerfile-base-cu118 +++ b/examples/generated/Dockerfile-base-cu118 @@ -1,5 +1,5 @@ # >>>>>>>>>>>>>>>>>>>>>>>>>>> -# Auto-generated by agi-pack (version=0.1.14). +# Auto-generated by agi-pack (version=0.1.15). FROM nvidia/cuda:11.8.0-base-ubuntu22.04 AS base-gpu # Setup environment variables @@ -11,6 +11,8 @@ ENV PYTHON_VERSION 3.8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 ENV PYTHONWARNINGS ignore +ENV PIP_CACHE_DIR /var/cache/pip +ENV CONDA_PKGS_DIRS /var/cache/conda/pkgs # Setup conda paths ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV} @@ -22,21 +24,18 @@ ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV} # Install base system packages RUN apt-get -y update \ && apt-get -y --no-install-recommends install \ - curl bzip2 git ca-certificates \ - && apt-get -y autoclean \ - && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* + curl bzip2 git ca-certificates # Install additional system packages -RUN apt-get -y update \ +RUN --mount=type=cache,target=/var/cache/apt \ + apt-get -y update \ && apt-get -y --no-install-recommends install \ wget \ - && apt-get -y autoclean \ - && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* + && echo "system install complete" -# Install mambaforge -RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ +# Install mambaforge, with cache mounting ${CONDA_PKGS_DIRS} for faster builds +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ && chmod +x ~/mambaforge.sh \ && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \ && ${AGIPACK_PATH}/conda/bin/mamba init bash \ @@ -47,10 +46,10 @@ RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases # Upgrade pip RUN pip install --upgrade pip -# Install conda packages, with cache mounting ${AGIPACK_PATH}/conda/pkgs for faster builds +# Install conda packages, with cache mounting ${CONDA_PKGS_DIRS} for faster builds # Note: Cache mounts allow us to re-use the cache for conda packages # instead of having to re-download them every time we build. -RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \ +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ mamba install -yv \ pytorch==2.1.0 \ torchvision \ @@ -61,11 +60,11 @@ RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \ -c pytorch -c nvidia \ && echo "conda/mamba install complete" -# Install pip requirements, with cache mounting ~/.cache/pip for faster builds +# Install pip requirements, with cache mounting ${PIP_CACHE_DIR} for faster builds # Note: Cache mounts allow us to re-use the cache for pip packages # instead of having to re-download them every time we build. COPY requirements/requirements.txt /tmp/reqs/requirements/requirements.txt -RUN --mount=type=cache,target=~/.cache/pip \ +RUN --mount=type=cache,target=${PIP_CACHE_DIR} \ pip install --upgrade pip \ && pip install -r /tmp/reqs/requirements/requirements.txt \ && echo "pip requirements install complete" @@ -75,23 +74,34 @@ RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.b && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \ && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \ && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc -RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \ - && rm -rf ~/.cache/pip \ - && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \ - && rm -rf /tmp/reqs \ - && echo "pip cleanup complete" # Setup working directory WORKDIR /app/$AGIPACK_PYENV # Run commands RUN echo "running commands" -RUN echo 'pytorch: ' && python -c 'import torch; print(torch.__version__)' -RUN echo 'cuda: ' && python -c 'import torch; print(torch.version.cuda)' -RUN echo 'cudnn: ' && python -c 'import torch; print(torch.backends.cudnn.version())' -RUN echo 'opencv:' && python -c 'import cv2; print(cv2.__version__)' +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + --mount=type=cache,target=${PIP_CACHE_DIR} \ + echo 'pytorch: ' && python -c 'import torch; print(torch.__version__)' +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + --mount=type=cache,target=${PIP_CACHE_DIR} \ + echo 'cuda: ' && python -c 'import torch; print(torch.version.cuda)' +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + --mount=type=cache,target=${PIP_CACHE_DIR} \ + echo 'cudnn: ' && python -c 'import torch; print(torch.backends.cudnn.version())' +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + --mount=type=cache,target=${PIP_CACHE_DIR} \ + echo 'opencv:' && python -c 'import cv2; print(cv2.__version__)' RUN echo "run commands complete" +# Cleanup apt, mamba/conda and pip packages +RUN apt-get -y autoclean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* \ + && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \ + && rm -rf ~/.cache/pip \ + && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \ + && rm -rf /tmp/reqs \ + && echo "pip cleanup complete" # Setup environment variables -ENV MY_ENV_VAR=value -CMD ["bash"] \ No newline at end of file +ENV MY_ENV_VAR=value \ No newline at end of file diff --git a/examples/generated/Dockerfile-builder b/examples/generated/Dockerfile-builder index ba8f9a9..d0e2a13 100644 --- a/examples/generated/Dockerfile-builder +++ b/examples/generated/Dockerfile-builder @@ -1,5 +1,5 @@ # >>>>>>>>>>>>>>>>>>>>>>>>>>> -# Auto-generated by agi-pack (version=0.1.14). +# Auto-generated by agi-pack (version=0.1.15). FROM debian:buster-slim AS agipack-builder # Setup environment variables @@ -11,6 +11,8 @@ ENV PYTHON_VERSION 3.8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 ENV PYTHONWARNINGS ignore +ENV PIP_CACHE_DIR /var/cache/pip +ENV CONDA_PKGS_DIRS /var/cache/conda/pkgs # Setup conda paths ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV} @@ -22,13 +24,11 @@ ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV} # Install base system packages RUN apt-get -y update \ && apt-get -y --no-install-recommends install \ - curl bzip2 git ca-certificates \ - && apt-get -y autoclean \ - && apt-get -y autoremove \ - && rm -rf /var/lib/apt/lists/* + curl bzip2 git ca-certificates -# Install mambaforge -RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ +# Install mambaforge, with cache mounting ${CONDA_PKGS_DIRS} for faster builds +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \ && chmod +x ~/mambaforge.sh \ && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \ && ${AGIPACK_PATH}/conda/bin/mamba init bash \ @@ -44,17 +44,23 @@ RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.b && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \ && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \ && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc -RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \ - && rm -rf ~/.cache/pip \ - && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \ - && rm -rf /tmp/reqs \ - && echo "pip cleanup complete" # Setup working directory WORKDIR /app/$AGIPACK_PYENV # Run commands RUN echo "running commands" -RUN pip install agi-pack +RUN --mount=type=cache,target=${CONDA_PKGS_DIRS} \ + --mount=type=cache,target=${PIP_CACHE_DIR} \ + pip install agi-pack RUN echo "run commands complete" +# Cleanup apt, mamba/conda and pip packages +RUN apt-get -y autoclean \ + && apt-get -y autoremove \ + && rm -rf /var/lib/apt/lists/* \ + && ${AGIPACK_PATH}/conda/bin/mamba clean -ya \ + && rm -rf ~/.cache/pip \ + && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \ + && rm -rf /tmp/reqs \ + && echo "pip cleanup complete" CMD ["agi-pack", "generate", "-f"] \ No newline at end of file diff --git a/tests/test_config.py b/tests/test_config.py index e410269..795f82d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -28,6 +28,7 @@ def test_poorly_formatted_configs(test_data_dir): poorly_formatted_configs = [ test_data_dir / "agibuild-no-base.yaml", test_data_dir / "agibuild-malformed-add.yaml", + test_data_dir / "agibuild-malformed-py-version.yaml", ] for filename in poorly_formatted_configs: with pytest.raises(ValueError): diff --git a/tests/test_data/agibuild-different-py-versions.yaml b/tests/test_data/agibuild-different-py-versions.yaml index aaefe4a..d7a98f4 100644 --- a/tests/test_data/agibuild-different-py-versions.yaml +++ b/tests/test_data/agibuild-different-py-versions.yaml @@ -2,7 +2,7 @@ images: base-cpu: system: - wget - python: 3.8 + python: "3.8" pip: - scikit-learn run: @@ -10,6 +10,6 @@ images: dev-cpu: base: base-cpu - python: 3.9 + python: "3.9" system: - build-essential \ No newline at end of file diff --git a/tests/test_data/agibuild-malformed-add.yaml b/tests/test_data/agibuild-malformed-add.yaml index 5de0bd4..eb933c7 100644 --- a/tests/test_data/agibuild-malformed-add.yaml +++ b/tests/test_data/agibuild-malformed-add.yaml @@ -2,7 +2,7 @@ images: base-cpu: system: - wget - python: 3.8.10 + python: "3.8.10" pip: - scikit-learn add: diff --git a/tests/test_data/agibuild-malformed-py-version.yaml b/tests/test_data/agibuild-malformed-py-version.yaml new file mode 100644 index 0000000..f22bf3a --- /dev/null +++ b/tests/test_data/agibuild-malformed-py-version.yaml @@ -0,0 +1,5 @@ +images: + base-cpu: + python: 3.10 # float casts to string ("3.1") + run: + - echo "Hello, world!" \ No newline at end of file diff --git a/tests/test_data/agibuild-minimal.yaml b/tests/test_data/agibuild-minimal.yaml index bbb1767..7d2679a 100644 --- a/tests/test_data/agibuild-minimal.yaml +++ b/tests/test_data/agibuild-minimal.yaml @@ -2,7 +2,7 @@ images: base-cpu: system: - wget - python: 3.8.10 + python: "3.8.10" pip: - scikit-learn run: diff --git a/tests/test_data/agibuild-no-deps.yaml b/tests/test_data/agibuild-no-deps.yaml index 877a98c..991f471 100644 --- a/tests/test_data/agibuild-no-deps.yaml +++ b/tests/test_data/agibuild-no-deps.yaml @@ -1,5 +1,5 @@ images: base-cpu: - python: 3.8.10 + python: "3.8.10" run: - echo "Hello, world!" \ No newline at end of file diff --git a/tests/test_data/agibuild-no-system.yaml b/tests/test_data/agibuild-no-system.yaml index 9d13214..0bd0dcd 100644 --- a/tests/test_data/agibuild-no-system.yaml +++ b/tests/test_data/agibuild-no-system.yaml @@ -1,6 +1,6 @@ images: base-cpu: - python: 3.8.10 + python: "3.8.10" pip: - scikit-learn run: diff --git a/tests/test_data/agibuild-with-deps.yaml b/tests/test_data/agibuild-with-deps.yaml index 67e919f..6aae723 100644 --- a/tests/test_data/agibuild-with-deps.yaml +++ b/tests/test_data/agibuild-with-deps.yaml @@ -2,7 +2,7 @@ images: base-cpu: system: - wget - python: 3.8.10 + python: "3.8.10" name: agipack pip: - scikit-learn diff --git a/tests/test_examples.py b/tests/test_examples.py index 6228347..42dca0b 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -15,8 +15,11 @@ @pytest.mark.parametrize("filename", EXAMPLES) def test_parse_yaml(filename): logger.info(f"Testing example {filename}") - config = AGIPackConfig.load_yaml(filename) - assert config is not None + try: + config = AGIPackConfig.load_yaml(filename) + assert config is not None + except Exception as e: + raise Exception(f"Failed to parse {filename}, e={e}") basename = filename.stem.replace("agibuild.", "") filename = EXAMPLES_DIR / "generated" / f"Dockerfile-{basename}"