Move conda builds from /opt/conda to /opt/agi-pack/conda

This avoids conflicts with other base images that reserve `/opt/conda` for their own use.
spillai · Oct 21, 2023 · b382d94 · b382d94
1 parent 84c4a20
commit b382d94
Show file tree

Hide file tree

Showing 6 changed files with 54 additions and 51 deletions.
diff --git a/agipack/config.py b/agipack/config.py
@@ -260,7 +260,10 @@ def _build_target_tree(self) -> None:
                     raise ValueError(f"First image [{target}] must be the base image")
                 self._target_tree[target] = _ImageNode(name=target, root=True)
             else:
-                assert config.base in self._target_tree
+                if config.base not in self._target_tree:
+                    raise ValueError(
+                        f"Base image for derived target `{target}` needs to be one of {list(self._target_tree.keys())}."
+                    )
                 self._target_tree[target] = _ImageNode(name=target)
                 self._target_tree[config.base].children.append(target)
         logger.debug(f"Target dependencies: {self._target_tree}")
diff --git a/agipack/templates/Dockerfile.j2 b/agipack/templates/Dockerfile.j2
@@ -21,10 +21,10 @@ ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore
 
 # Setup conda paths
-ENV CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}
+ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}
 ENV CONDA_PREFIX=${CONDA_PATH}
 ENV CONDA_EXE=${CONDA_PATH}/bin/conda
-ENV PATH=${CONDA_PATH}/bin:/opt/conda/bin:$PATH
+ENV PATH=${CONDA_PATH}/bin:${AGIPACK_PATH}/conda/bin:$PATH
 ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV}
 
 {%- endif %}
@@ -63,10 +63,10 @@ RUN apt-get -y update \
 # Install mambaforge
 RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
   && chmod +x ~/mambaforge.sh \
-  && ~/mambaforge.sh -b -p /opt/conda \
-  && /opt/conda/bin/mamba init bash \
-  && /opt/conda/bin/mamba config --add channels conda-forge \
-  && /opt/conda/bin/mamba create -n ${AGIPACK_PYENV} python=${PYTHON_VERSION} -y \
+  && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \
+  && ${AGIPACK_PATH}/conda/bin/mamba init bash \
+  && ${AGIPACK_PATH}/conda/bin/mamba config --add channels conda-forge \
+  && ${AGIPACK_PATH}/conda/bin/mamba create -n ${AGIPACK_PYENV} python=${PYTHON_VERSION} -y \
   && rm ~/mambaforge.sh
 
 # Upgrade pip
@@ -76,10 +76,10 @@ RUN pip install --upgrade pip
 
 {%- if conda|length > 0 %}
 
-# Install conda packages, with cache mounting /opt/conda/pkgs for faster builds
+# Install conda packages, with cache mounting ${AGIPACK_PATH}/conda/pkgs for faster builds
 # Note: Cache mounts allow us to re-use the cache for conda packages
 # instead of having to re-download them every time we build.
-RUN --mount=type=cache,target=/opt/conda/pkgs/ \
+RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \
     mamba install -yv \
 {%- for package in conda %}
     {{ package }} \
@@ -124,15 +124,15 @@ RUN --mount=type=cache,target=~/.cache/pip \
 {%- if is_base_image %}
 
 # Export conda environment on login
-RUN echo "export CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc \
-    && echo "export PATH=/opt/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
+RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc \
+    && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
     && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \
     && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
 
 {%- if is_prod %}
-RUN /opt/conda/bin/mamba clean -ya \
+RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
     && rm -rf ~/.cache/pip \
-    && rm -rf /opt/conda/pkgs/* \
+    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
     && rm -rf /tmp/reqs \
     && echo "pip cleanup complete"
 {%- endif %}

diff --git a/agipack/version.py b/agipack/version.py
@@ -1 +1 @@
-__version__ = "0.1.13"
+__version__ = "0.1.14"
diff --git a/examples/generated/Dockerfile-base-cpu b/examples/generated/Dockerfile-base-cpu
@@ -1,5 +1,5 @@
 # >>>>>>>>>>>>>>>>>>>>>>>>>>>
-# Auto-generated by agi-pack (version=0.1.12).
+# Auto-generated by agi-pack (version=0.1.14).
 FROM debian:buster-slim AS base-cpu
 
 # Setup environment variables
@@ -13,10 +13,10 @@ ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore
 
 # Setup conda paths
-ENV CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}
+ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}
 ENV CONDA_PREFIX=${CONDA_PATH}
 ENV CONDA_EXE=${CONDA_PATH}/bin/conda
-ENV PATH=${CONDA_PATH}/bin:/opt/conda/bin:$PATH
+ENV PATH=${CONDA_PATH}/bin:${AGIPACK_PATH}/conda/bin:$PATH
 ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV}
 
 # Install base system packages
@@ -38,19 +38,19 @@ RUN apt-get -y update \
 # Install mambaforge
 RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
   && chmod +x ~/mambaforge.sh \
-  && ~/mambaforge.sh -b -p /opt/conda \
-  && /opt/conda/bin/mamba init bash \
-  && /opt/conda/bin/mamba config --add channels conda-forge \
-  && /opt/conda/bin/mamba create -n ${AGIPACK_PYENV} python=${PYTHON_VERSION} -y \
+  && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \
+  && ${AGIPACK_PATH}/conda/bin/mamba init bash \
+  && ${AGIPACK_PATH}/conda/bin/mamba config --add channels conda-forge \
+  && ${AGIPACK_PATH}/conda/bin/mamba create -n ${AGIPACK_PYENV} python=${PYTHON_VERSION} -y \
   && rm ~/mambaforge.sh
 
 # Upgrade pip
 RUN pip install --upgrade pip
 
-# Install conda packages, with cache mounting /opt/conda/pkgs for faster builds
+# Install conda packages, with cache mounting ${AGIPACK_PATH}/conda/pkgs for faster builds
 # Note: Cache mounts allow us to re-use the cache for conda packages
 # instead of having to re-download them every time we build.
-RUN --mount=type=cache,target=/opt/conda/pkgs/ \
+RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \
     mamba install -yv \
     pytorch>=2.1 \
     torchvision \
@@ -67,13 +67,13 @@ RUN --mount=type=cache,target=~/.cache/pip \
     && echo "pip requirements install complete"
 
 # Export conda environment on login
-RUN echo "export CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc \
-    && echo "export PATH=/opt/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
+RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc \
+    && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
     && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \
     && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
-RUN /opt/conda/bin/mamba clean -ya \
+RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
     && rm -rf ~/.cache/pip \
-    && rm -rf /opt/conda/pkgs/* \
+    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
     && rm -rf /tmp/reqs \
     && echo "pip cleanup complete"
 

diff --git a/examples/generated/Dockerfile-base-cu118 b/examples/generated/Dockerfile-base-cu118
@@ -1,5 +1,5 @@
 # >>>>>>>>>>>>>>>>>>>>>>>>>>>
-# Auto-generated by agi-pack (version=0.1.12).
+# Auto-generated by agi-pack (version=0.1.14).
 FROM nvidia/cuda:11.8.0-base-ubuntu22.04 AS base-gpu
 
 # Setup environment variables
@@ -13,10 +13,10 @@ ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore
 
 # Setup conda paths
-ENV CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}
+ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}
 ENV CONDA_PREFIX=${CONDA_PATH}
 ENV CONDA_EXE=${CONDA_PATH}/bin/conda
-ENV PATH=${CONDA_PATH}/bin:/opt/conda/bin:$PATH
+ENV PATH=${CONDA_PATH}/bin:${AGIPACK_PATH}/conda/bin:$PATH
 ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV}
 
 # Install base system packages
@@ -38,19 +38,19 @@ RUN apt-get -y update \
 # Install mambaforge
 RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
   && chmod +x ~/mambaforge.sh \
-  && ~/mambaforge.sh -b -p /opt/conda \
-  && /opt/conda/bin/mamba init bash \
-  && /opt/conda/bin/mamba config --add channels conda-forge \
-  && /opt/conda/bin/mamba create -n ${AGIPACK_PYENV} python=${PYTHON_VERSION} -y \
+  && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \
+  && ${AGIPACK_PATH}/conda/bin/mamba init bash \
+  && ${AGIPACK_PATH}/conda/bin/mamba config --add channels conda-forge \
+  && ${AGIPACK_PATH}/conda/bin/mamba create -n ${AGIPACK_PYENV} python=${PYTHON_VERSION} -y \
   && rm ~/mambaforge.sh
 
 # Upgrade pip
 RUN pip install --upgrade pip
 
-# Install conda packages, with cache mounting /opt/conda/pkgs for faster builds
+# Install conda packages, with cache mounting ${AGIPACK_PATH}/conda/pkgs for faster builds
 # Note: Cache mounts allow us to re-use the cache for conda packages
 # instead of having to re-download them every time we build.
-RUN --mount=type=cache,target=/opt/conda/pkgs/ \
+RUN --mount=type=cache,target=${AGIPACK_PATH}/conda/pkgs/ \
     mamba install -yv \
     pytorch==2.1.0 \
     torchvision \
@@ -71,13 +71,13 @@ RUN --mount=type=cache,target=~/.cache/pip \
     && echo "pip requirements install complete"
 
 # Export conda environment on login
-RUN echo "export CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc \
-    && echo "export PATH=/opt/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
+RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc \
+    && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
     && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \
     && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
-RUN /opt/conda/bin/mamba clean -ya \
+RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
     && rm -rf ~/.cache/pip \
-    && rm -rf /opt/conda/pkgs/* \
+    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
     && rm -rf /tmp/reqs \
     && echo "pip cleanup complete"
 

diff --git a/examples/generated/Dockerfile-builder b/examples/generated/Dockerfile-builder
@@ -1,5 +1,5 @@
 # >>>>>>>>>>>>>>>>>>>>>>>>>>>
-# Auto-generated by agi-pack (version=0.1.12).
+# Auto-generated by agi-pack (version=0.1.14).
 FROM debian:buster-slim AS agipack-builder
 
 # Setup environment variables
@@ -13,10 +13,10 @@ ENV PYTHONUNBUFFERED 1
 ENV PYTHONWARNINGS ignore
 
 # Setup conda paths
-ENV CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}
+ENV CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}
 ENV CONDA_PREFIX=${CONDA_PATH}
 ENV CONDA_EXE=${CONDA_PATH}/bin/conda
-ENV PATH=${CONDA_PATH}/bin:/opt/conda/bin:$PATH
+ENV PATH=${CONDA_PATH}/bin:${AGIPACK_PATH}/conda/bin:$PATH
 ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV}
 
 # Install base system packages
@@ -30,23 +30,23 @@ RUN apt-get -y update \
 # Install mambaforge
 RUN curl -sLo ~/mambaforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
   && chmod +x ~/mambaforge.sh \
-  && ~/mambaforge.sh -b -p /opt/conda \
-  && /opt/conda/bin/mamba init bash \
-  && /opt/conda/bin/mamba config --add channels conda-forge \
-  && /opt/conda/bin/mamba create -n ${AGIPACK_PYENV} python=${PYTHON_VERSION} -y \
+  && ~/mambaforge.sh -b -p ${AGIPACK_PATH}/conda \
+  && ${AGIPACK_PATH}/conda/bin/mamba init bash \
+  && ${AGIPACK_PATH}/conda/bin/mamba config --add channels conda-forge \
+  && ${AGIPACK_PATH}/conda/bin/mamba create -n ${AGIPACK_PYENV} python=${PYTHON_VERSION} -y \
   && rm ~/mambaforge.sh
 
 # Upgrade pip
 RUN pip install --upgrade pip
 
 # Export conda environment on login
-RUN echo "export CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc \
-    && echo "export PATH=/opt/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
+RUN echo "export CONDA_PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc \
+    && echo "export PATH=${AGIPACK_PATH}/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc \
     && echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc \
     && echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
-RUN /opt/conda/bin/mamba clean -ya \
+RUN ${AGIPACK_PATH}/conda/bin/mamba clean -ya \
     && rm -rf ~/.cache/pip \
-    && rm -rf /opt/conda/pkgs/* \
+    && rm -rf ${AGIPACK_PATH}/conda/pkgs/* \
     && rm -rf /tmp/reqs \
     && echo "pip cleanup complete"