Added agi-pack build support with linting

agipack/builder.py

@@ -1,17 +1,19 @@
 import logging
 import os
+import subprocess
 from dataclasses import field
 from pathlib import Path
-from typing import Dict, Union
+from typing import Dict, List, Union
 
 from jinja2 import Environment, FileSystemLoader
 from pydantic.dataclasses import dataclass
+from rich import print
 
 from agipack.config import AGIPackConfig, ImageConfig
 from agipack.constants import AGIPACK_DOCKERFILE_TEMPLATE, AGIPACK_ENV, AGIPACK_TEMPLATE_DIR
 from agipack.version import __version__
 
-logging_level = os.environ.get("AGIPACK_LOGGING_LEVEL", "WARNING")
+logging_level = os.environ.get("AGIPACK_LOGGING_LEVEL", "DEBUG")
 logging.basicConfig(level=logging.getLevelName(logging_level))
 logger = logging.getLogger(__name__)
 
@@ -147,3 +149,57 @@ def render(self, **kwargs) -> Dict[str, str]:
                 pending.update(children)
 
         return dockerfiles
+
+    def build(self, filename: str, target: str, tags: List[str] = None) -> None:
+        """Builds a Docker image using the generated Dockerfile.
+
+        Args:
+            filename (str): Path to the generated Dockerfile.
+            target (str): Target image name.
+            tag (List[str[]): Tag for the Docker image.
+        """
+        logger.info(f"🚀 Building Docker image for target [{target}]")
+        image_config = self.config.images[target]
+        if tags is not None:
+            image_tags = [f"{image_config.name}:{tag}" for tag in tags]
+        else:
+            image_tags = [f"{image_config.name}:{target}"]
+        logger.debug(f"Image tags: {image_tags}")
+
+        cmd = ["docker", "build", "-f", filename, "--target", target]
+        for tag in image_tags:
+            cmd.extend(["-t", tag])
+        cmd.append(".")
+
+        logger.debug(f"Running command: {cmd}")
+        process = subprocess.Popen(
+            cmd,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+        )
+        for line in iter(process.stdout.readline, ""):
+            print(line, end="")
+        process.wait()
+
+    def lint(self, filename: str) -> bool:
+        """Lint the generated Dockerfile using hadolint.
+
+        Args:
+            filename (str): Path to the generated Dockerfile.
+        """
+        cmd = "docker pull hadolint/hadolint && "
+        cmd += f"docker run --pull=always --rm -i hadolint/hadolint < {filename}"
+        logger.info("Linting with hadolint")
+        process = subprocess.Popen(
+            cmd,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            universal_newlines=True,
+            shell=True,
+        )
+        process.wait()
+        for line in iter(process.stdout.readline, ""):
+            print(line, end="")
+        return process.returncode == 0

agipack/cli.py

@@ -55,12 +55,17 @@ def generate(
         None, "--base", "-b", help="Base image to use for the root/base target.", show_default=False
     ),
     prod: bool = typer.Option(False, "--prod", "-p", help="Generate a production Dockerfile.", show_default=False),
+    lint: bool = typer.Option(False, "--lint", "-l", help="Lint the generated Dockerfile.", show_default=False),
+    build: bool = typer.Option(False, "--build", "-b", help="Build the Docker image after generating the Dockerfile."),
 ):
     """Generate the Dockerfile with optional overrides.
 
     Usage:
         agi-pack generate -c agibuild.yaml
         agi-pack generate -c agibuild.yaml -o docker/Dockerfile
+        agi-pack generate -c agibuild.yaml -p 3.8.10
+        agi-pack generate -c agibuild.yaml -b python:3.8.10-slim
+        agi-pack generate -c agibuild.yaml --prod --lint
     """
     # Load the YAML configuration
     config = AGIPackConfig.load_yaml(config_filename)
@@ -76,12 +81,54 @@ def generate(
     builder = AGIPack(config)
     dockerfiles = builder.render(filename=filename, env="prod" if prod else "dev")
     for target, filename in dockerfiles.items():
+        image_config = config.images[target]
+
+        cmd = f"docker build -f {filename} --target {target} -t {image_config.name}:{target} ."
         tree = Tree(f"📦 [bold white]{target}[/bold white]")
         tree.add(
             f"🎉 Successfully generated Dockerfile (target=[bold white]{target}[/bold white], filename=[bold white]{filename}[/bold white])."
-        ).add(f"[green]`docker build -f {filename} --target {target} .`[/green]")
+        ).add(f"[green]`{cmd}`[/green]")
         print(tree)
 
+        # Lint the generated Dockerfile using hadolint
+        if lint:
+            print(f"🔍 Linting Dockerfile for target [{target}]")
+            builder.lint(filename=filename)
+
+        # Build the Docker image using subprocess and print all the output as it happens
+        if build:
+            print(f"🚀 Building Docker image for target [{target}]")
+            builder.build(filename=filename, target=target)
+
+
+@app.command()
+def build(
+    config_filename: str = typer.Option(
+        AGIPACK_BASENAME, "--config", "-c", help="Path to the YAML configuration file."
+    ),
+    filename: str = typer.Option(
+        "Dockerfile", "--output-filename", "-o", help="Output filename for the generated Dockerfile."
+    ),
+    python: str = typer.Option(
+        None, "--python", "-p", help="Python version to use for the base image.", show_default=False
+    ),
+    base_image: str = typer.Option(
+        None, "--base", "-b", help="Base image to use for the root/base target.", show_default=False
+    ),
+    prod: bool = typer.Option(False, "--prod", "-p", help="Generate a production Dockerfile.", show_default=False),
+    lint: bool = typer.Option(False, "--lint", "-l", help="Lint the generated Dockerfile.", show_default=False),
+):
+    """Generate the Dockerfile with optional overrides.
+
+    Usage:
+        agi-pack build -c agibuild.yaml
+        agi-pack build -c agibuild.yaml -o docker/Dockerfile
+        agi-pack build -c agibuild.yaml -p 3.8.10
+        agi-pack build -c agibuild.yaml -b python:3.8.10-slim
+        agi-pack build -c agibuild.yaml --prod --lint
+    """
+    generate(config_filename, filename, python, base_image, prod, lint, build=True)
+
 
 if __name__ == "__main__":
     app()

agipack/templates/Dockerfile.j2

@@ -42,7 +42,7 @@ ENV {{ key }}={{ value }} {% endfor %}
 
 # Install system packages
 RUN apt-get -y update \
-    && apt-get -y install \
+    && apt-get -y --no-install-recommends install \
     curl bzip2 \
 {%- for package in system %}
     {{ package }} \
@@ -77,14 +77,7 @@ RUN --mount=type=cache,target=/opt/conda/pkgs/ \
 {%- for package in pip %}
     {{ package }} \
 {%- endfor %}
-{%- if is_prod %}
-    && /opt/conda/bin/mamba clean -ya \
-    && rm -rf ~/.cache/pip \
-    && rm -rf /opt/conda/pkgs/* \
-    && echo "pip install/cleanup complete"
-{% else %}
     && echo "pip install complete"
-{%- endif %}
 
 {%- endif %}
 
@@ -102,14 +95,6 @@ RUN --mount=type=cache,target=~/.cache/pip \
 {%- for package in requirements %}
     && pip install -r /tmp/reqs/{{ package }} \
 {%- endfor %}
-{%- if is_prod %}
-    && /opt/conda/bin/mamba clean -ya \
-    && rm -rf ~/.cache/pip \
-    && rm -rf /opt/conda/pkgs/* \
-    && rm -r /tmp/reqs \
-    && echo "pip cleanup complete"
-{% else %}
-{%- endif %}
     && echo "pip install complete"
 
 {%- endif %}
@@ -123,6 +108,14 @@ RUN echo "export PATH=/opt/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc
 RUN echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc
 RUN echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
 
+{%- if is_prod %}
+RUN /opt/conda/bin/mamba clean -ya \
+    && rm -rf ~/.cache/pip \
+    && rm -rf /opt/conda/pkgs/* \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
+{%- endif %}
+
 {%- endif %}
 
 

examples/generated/Dockerfile-base-cpu

@@ -26,7 +26,7 @@ ENV MY_ENV_VAR=value
 
 # Install system packages
 RUN apt-get -y update \
-    && apt-get -y install \
+    && apt-get -y --no-install-recommends install \
     curl bzip2 \
     wget \
     && apt-get -y autoclean \
@@ -51,17 +51,18 @@ RUN --mount=type=cache,target=/opt/conda/pkgs/ \
     pytorch>=2.1 \
     torchvision \
     cpuonly -c pytorch \
-    && /opt/conda/bin/mamba clean -ya \
-    && rm -rf ~/.cache/pip \
-    && rm -rf /opt/conda/pkgs/* \
-    && echo "pip install/cleanup complete"
-
+    && echo "pip install complete"
 
 # Export conda environment on login
 RUN echo "export CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc
 RUN echo "export PATH=/opt/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc
 RUN echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc
 RUN echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
+RUN /opt/conda/bin/mamba clean -ya \
+    && rm -rf ~/.cache/pip \
+    && rm -rf /opt/conda/pkgs/* \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 
 
 # Setup working directory

examples/generated/Dockerfile-base-cu118

@@ -26,7 +26,7 @@ ENV MY_ENV_VAR=value
 
 # Install system packages
 RUN apt-get -y update \
-    && apt-get -y install \
+    && apt-get -y --no-install-recommends install \
     curl bzip2 \
     wget \
     && apt-get -y autoclean \
@@ -55,17 +55,18 @@ RUN --mount=type=cache,target=/opt/conda/pkgs/ \
     cudatoolkit=11.8 \
     cudnn=8.2.1 \
     -c pytorch -c nvidia \
-    && /opt/conda/bin/mamba clean -ya \
-    && rm -rf ~/.cache/pip \
-    && rm -rf /opt/conda/pkgs/* \
-    && echo "pip install/cleanup complete"
-
+    && echo "pip install complete"
 
 # Export conda environment on login
 RUN echo "export CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc
 RUN echo "export PATH=/opt/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc
 RUN echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc
 RUN echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
+RUN /opt/conda/bin/mamba clean -ya \
+    && rm -rf ~/.cache/pip \
+    && rm -rf /opt/conda/pkgs/* \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 
 
 # Setup working directory

examples/generated/Dockerfile-builder

@@ -23,7 +23,7 @@ ENV CONDA_DEFAULT_ENV ${AGIPACK_PYENV}
 
 # Install system packages
 RUN apt-get -y update \
-    && apt-get -y install \
+    && apt-get -y --no-install-recommends install \
     curl bzip2 \
     && apt-get -y autoclean \
     && apt-get -y autoremove \
@@ -44,6 +44,11 @@ RUN echo "export CONDA_PATH=/opt/conda/envs/${AGIPACK_PYENV}" >> ~/.bashrc
 RUN echo "export PATH=/opt/conda/envs/${AGIPACK_PYENV}/bin:$PATH" >> ~/.bashrc
 RUN echo "export CONDA_DEFAULT_ENV=${AGIPACK_PYENV}" >> ~/.bashrc
 RUN echo "mamba activate ${AGIPACK_PYENV}" > ~/.bashrc
+RUN /opt/conda/bin/mamba clean -ya \
+    && rm -rf ~/.cache/pip \
+    && rm -rf /opt/conda/pkgs/* \
+    && rm -rf /tmp/reqs \
+    && echo "pip cleanup complete"
 
 
 # Setup working directory

tests/test_builder.py

@@ -60,6 +60,7 @@ def test_builder_cls(test_data_dir):
         assert "base-cpu" in dockerfiles
         assert Path(dockerfiles["base-cpu"]).exists()
         assert Path(dockerfiles["base-cpu"]).parent == Path(tmp_dir)
+        builder.lint(filename=filename)
 
 
 def test_builder_cls_with_deps(test_data_dir):
@@ -74,3 +75,4 @@ def test_builder_cls_with_deps(test_data_dir):
         assert "dev-cpu" in dockerfiles
         assert Path(dockerfiles["base-cpu"]).exists()
         assert Path(dockerfiles["base-cpu"]).parent == Path(tmp_dir)
+        builder.lint(filename=filename)