From 28a1fca046d1c40c2ba29c1496e330ff7f4aefb5 Mon Sep 17 00:00:00 2001 From: Sudeep Pillai Date: Sun, 22 Oct 2023 22:08:35 -0700 Subject: [PATCH] Support `skip-base-builds` and `target` builds in CLI - improved pydantic config validation with forbidden extras --- README.md | 4 +-- agipack/builder.py | 8 ++++- agipack/cli.py | 44 +++++++++++++++++++++--- agipack/config.py | 35 +++++++++++++++---- agipack/templates/Dockerfile.j2 | 3 +- examples/generated/Dockerfile-base-cpu | 9 ++--- examples/generated/Dockerfile-base-cu118 | 9 ++--- examples/generated/Dockerfile-builder | 7 ++-- 8 files changed, 92 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 81296a8..9862eb2 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ 📦 **`agi-pack`** allows you to define your Docker images using a simple YAML format, and then generate them on-the-fly using [Jinja2](https://jinja.palletsprojects.com/en/3.1.x/) templates with [Pydantic](https://docs.pydantic.dev/latest/)-based validation. It's a simple tool that aims to simplify the process of building Docker images for ML. -🚨 **Disclaimer:** More than **75%** of this initial implementation was generated by GPT-4 and [Github Co-Pilot](https://github.com/features/copilot). See [attribution](#inspiration-and-attribution-🌟) section below for more details. +🚨 **Disclaimer:** More than **75%** of the original implementation was generated by GPT-4 and [Github Co-Pilot](https://github.com/features/copilot). I've spent enough time building on it that it's probably **<30%** auto-generated code now. See [attribution](#inspiration-and-attribution-🌟) section below for more details. ## Goals 🎯 @@ -151,7 +151,7 @@ Here's the corresponding [`Dockerfile`](./examples/generated/Dockerfile-multista 📦 **agi-pack** is simply a weekend project I hacked together, that started with a conversation with [ChatGPT / GPT-4](#chatgpt-prompt). -🚨 **Disclaimer:** More than **75%** of this initial implementation was generated by GPT-4 and [Github Co-Pilot](https://github.com/features/copilot). +🚨 **Disclaimer:** More than **75%** of this initial implementation was generated by GPT-4 and [Github Co-Pilot](https://github.com/features/copilot). I've spent enough time building on it that it's probably **<30%** auto-generated code now. ### ChatGPT Prompt --- diff --git a/agipack/builder.py b/agipack/builder.py index 6ccd85b..b4bb8c0 100644 --- a/agipack/builder.py +++ b/agipack/builder.py @@ -28,6 +28,9 @@ class AGIPackRenderOptions: env: str = field(default=AGIPACK_ENV) """AGIPack environment to use for the build.""" + skip_base_builds: bool = field(default=False) + """Skip building the base images.""" + def is_prod(self) -> bool: """Check if the build is for production.""" return self.env == "prod" @@ -96,7 +99,10 @@ def _render_one(self, target: str, image_config: ImageConfig, options: AGIPackRe # Render the Dockerfile template image_dict["target"] = target - image_dict["is_base_image"] = self.config.is_root(target) + if options.skip_base_builds: + image_dict["is_base_image"] = False + else: + image_dict["is_base_image"] = self.config.is_root(target) image_dict["is_prod"] = options.is_prod() image_dict["agipack_version"] = __version__ content = template.render(image_dict) diff --git a/agipack/cli.py b/agipack/cli.py index a12b882..d7f4acd 100644 --- a/agipack/cli.py +++ b/agipack/cli.py @@ -56,9 +56,13 @@ def generate( None, "--base", "-b", help="Base image to use for the root/base target.", show_default=False ), tag: str = typer.Option("{name}:{target}", "--tag", "-t", help="Image tag f-string.", show_default=True), + _target: str = typer.Option(None, "--target", help="Build specific target.", show_default=False), prod: bool = typer.Option(False, "--prod", help="Generate a production Dockerfile.", show_default=False), lint: bool = typer.Option(False, "--lint", help="Lint the generated Dockerfile.", show_default=False), build: bool = typer.Option(False, "--build", help="Build the Docker image after generating the Dockerfile."), + skip_base_builds: bool = typer.Option( + False, "--skip-base", help="Skip building the base image.", show_default=False + ), ): """Generate the Dockerfile with optional overrides. @@ -81,17 +85,25 @@ def generate( config.images[root].base = base_image # Render the Dockerfiles with the new filename and configuration + trees = [] builder = AGIPack(config) - dockerfiles = builder.render(filename=filename, env="prod" if prod else "dev") + dockerfiles = builder.render(filename=filename, env="prod" if prod else "dev", skip_base_builds=skip_base_builds) for target, filename in dockerfiles.items(): + # Skip if the target is not the one we want to build + if _target is not None and target != _target: + continue image_config = config.images[target] # Build the Dockerfile using the generated filename and target - tag_name = f"{image_config.name}:{target}" if tag is None else tag.format(name=image_config.name, target=target) + tag_name = ( + f"{image_config.name}:{target}" if tag is None else tag.format(name=image_config.name, target=target) + ) cmd = f"docker build -f {filename} --target {target} -t {tag_name} ." + + # Print the command to build the Dockerfile tree = Tree(f"📦 [bold white]{target}[/bold white]") tree.add( - f"🎉 Successfully generated Dockerfile (target=[bold white]{target}[/bold white], filename=[bold white]{filename}[/bold white])." + f"[bold green]✓[/bold green] Successfully generated Dockerfile (target=[bold white]{target}[/bold white], filename=[bold white]{filename}[/bold white])." ).add(f"[green]`{cmd}`[/green]") print(tree) @@ -105,6 +117,15 @@ def generate( print(f"🚀 Building Docker image for target [{target}]") builder.build(filename=filename, target=target, tags=[tag_name]) + tree.add( + f"[bold green]✓[/bold green] Successfully built image (target=[bold white]{target}[/bold white], image=[bold white]{tag_name}[/bold white])." + ) + trees.append(tree) + + # Re-render the tree + for tree in trees: + print(tree) + @app.command() def build( @@ -121,8 +142,12 @@ def build( None, "--base", "-b", help="Base image to use for the root/base target.", show_default=False ), tag: str = typer.Option("{name}:{target}", "--tag", "-t", help="Image tag f-string.", show_default=True), + target: str = typer.Option(None, "--target", help="Build specific target.", show_default=False), prod: bool = typer.Option(False, "--prod", help="Generate a production Dockerfile.", show_default=False), lint: bool = typer.Option(False, "--lint", help="Lint the generated Dockerfile.", show_default=False), + skip_base_builds: bool = typer.Option( + False, "--skip-base", help="Skip building the base image.", show_default=False + ), ): """Generate the Dockerfile with optional overrides. @@ -135,7 +160,18 @@ def build( agi-pack build -c agibuild.yaml -t "my-image-name:my-target" agi-pack build -c agibuild.yaml --prod --lint """ - generate(config_filename, filename, python, base_image, tag, prod, lint, build=True) + generate( + config_filename, + filename, + python, + base_image, + tag, + target, + prod, + lint, + build=True, + skip_base_builds=skip_base_builds, + ) if __name__ == "__main__": diff --git a/agipack/config.py b/agipack/config.py index 35f2838..e9a59b7 100644 --- a/agipack/config.py +++ b/agipack/config.py @@ -4,7 +4,7 @@ from typing import Dict, List, Optional, Tuple, Union import yaml -from pydantic import validator +from pydantic import Extra, validator from pydantic.dataclasses import dataclass logger = logging.getLogger(__name__) @@ -28,7 +28,13 @@ def is_leaf(self) -> bool: return not len(self.children) -@dataclass +class _ForbidExtrasConfig: + """Pydantic config to forbid extra fields.""" + + extra = Extra.forbid + + +@dataclass(config=_ForbidExtrasConfig) class ImageConfig: """AGIPack configuration for a docker target specified in `agibuild.yaml` @@ -50,16 +56,19 @@ class ImageConfig: """ - image: str = field(default="agi:latest") + image: str = field(default=None) """Name of the image repository. - Defaults to the name if not provided. + Defaults to : if not specified. """ - name: str = field(default="agi") + name: str = field(default="agipack") """Pretty-name of the project in the image and reflected in the `AGIPACK_ENV` environment variable in the image. """ + target: str = field(default=None) + """Name of the target.""" + base: str = field(default="debian:buster-slim") """Base docker image / target to use (FROM clause in the Dockerfile).""" @@ -96,6 +105,12 @@ class ImageConfig: command: Optional[Union[str, List[str]]] = field(default_factory=list) """Command to run in the image.""" + def __post_init__(self): + if self.target is None: + self.target = "latest" + if self.image is None: + self.image = f"{self.name}:{self.target}" + def additional_kwargs(self): """Additional kwargs to pass to the Jinja2 Dockerfile template.""" python_alias = f"py{''.join(self.python.split('.')[:2])}" @@ -211,9 +226,15 @@ def load_yaml(cls, filename: Union[str, Path]) -> "AGIPackConfig": Returns: AGIPackConfig: AGIPack configuration. """ + path = Path(filename) + logger.debug(f"Loading AGIPack configuration from {path}") + if not path.exists(): + raise ValueError(f"YAML file {path.name} does not exist") + if not (path.name.endswith(".yaml") or path.name.endswith(".yml")): + raise ValueError(f"YAML file {path.name} must have a .yaml or .yml extension") + # Load the YAML file - logger.debug(f"Loading AGIPack configuration from {filename}") - with open(filename, "r") as f: + with path.open("r") as f: data = yaml.safe_load(f) logger.debug(f"AGIPack configuration: {data}") diff --git a/agipack/templates/Dockerfile.j2 b/agipack/templates/Dockerfile.j2 index a8e1549..5e0205a 100644 --- a/agipack/templates/Dockerfile.j2 +++ b/agipack/templates/Dockerfile.j2 @@ -2,7 +2,6 @@ # >>>>>>>>>>>>>>>>>>>>>>>>>>> # Auto-generated by agi-pack (version={{ agipack_version }}). {%- else %} - # >>>>>>>>>>>>>>>>>>>>>>>>>>> {%- endif %} FROM {{ base }} AS {{ target }} @@ -179,7 +178,7 @@ RUN apt-get -y autoclean \ # Setup environment variables {% for key, value in env.items() -%} -ENV {{ key }}={{ value }} +ENV {{ key }}={{ value }} {% endfor %} {%- endif %} diff --git a/examples/generated/Dockerfile-base-cpu b/examples/generated/Dockerfile-base-cpu index 3cbbbd5..c2e07c3 100644 --- a/examples/generated/Dockerfile-base-cpu +++ b/examples/generated/Dockerfile-base-cpu @@ -1,12 +1,13 @@ # >>>>>>>>>>>>>>>>>>>>>>>>>>> -# Auto-generated by agi-pack (version=0.1.15). +# Auto-generated by agi-pack (version=0.1.16). FROM debian:buster-slim AS base-cpu # Setup environment variables -ENV AGIPACK_PROJECT agi -ENV AGIPACK_PYENV agi-py38 +ENV AGIPACK_PROJECT agipack +ENV AGIPACK_PYENV agipack-py38 ENV AGIPACK_PATH /opt/agi-pack +ENV DEBIAN_FRONTEND="noninteractive" ENV PYTHON_VERSION 3.8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 @@ -94,4 +95,4 @@ RUN apt-get -y autoclean \ && echo "pip cleanup complete" # Setup environment variables -ENV MY_ENV_VAR=value \ No newline at end of file +ENV MY_ENV_VAR=value diff --git a/examples/generated/Dockerfile-base-cu118 b/examples/generated/Dockerfile-base-cu118 index d17d89c..0e67a06 100644 --- a/examples/generated/Dockerfile-base-cu118 +++ b/examples/generated/Dockerfile-base-cu118 @@ -1,12 +1,13 @@ # >>>>>>>>>>>>>>>>>>>>>>>>>>> -# Auto-generated by agi-pack (version=0.1.15). +# Auto-generated by agi-pack (version=0.1.16). FROM nvidia/cuda:11.8.0-base-ubuntu22.04 AS base-gpu # Setup environment variables -ENV AGIPACK_PROJECT agi -ENV AGIPACK_PYENV agi-py38 +ENV AGIPACK_PROJECT agipack +ENV AGIPACK_PYENV agipack-py38 ENV AGIPACK_PATH /opt/agi-pack +ENV DEBIAN_FRONTEND="noninteractive" ENV PYTHON_VERSION 3.8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1 @@ -104,4 +105,4 @@ RUN apt-get -y autoclean \ && echo "pip cleanup complete" # Setup environment variables -ENV MY_ENV_VAR=value \ No newline at end of file +ENV MY_ENV_VAR=value diff --git a/examples/generated/Dockerfile-builder b/examples/generated/Dockerfile-builder index d0e2a13..2e908f6 100644 --- a/examples/generated/Dockerfile-builder +++ b/examples/generated/Dockerfile-builder @@ -1,12 +1,13 @@ # >>>>>>>>>>>>>>>>>>>>>>>>>>> -# Auto-generated by agi-pack (version=0.1.15). +# Auto-generated by agi-pack (version=0.1.16). FROM debian:buster-slim AS agipack-builder # Setup environment variables -ENV AGIPACK_PROJECT agi -ENV AGIPACK_PYENV agi-py38 +ENV AGIPACK_PROJECT agipack +ENV AGIPACK_PYENV agipack-py38 ENV AGIPACK_PATH /opt/agi-pack +ENV DEBIAN_FRONTEND="noninteractive" ENV PYTHON_VERSION 3.8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONUNBUFFERED 1