Merge pull request #6 from QGreenland-Net/submit-command-trey

Submit command

.github/CONTRIBUTING.md

@@ -10,35 +10,52 @@ description of best practices for developing scientific packages.
 You can set up a development environment by running:
 
 ```bash
-python3 -m venv .venv
+python -m venv .venv
 source ./.venv/bin/activate
 pip install -v --editable .[dev]
 ```
 
-## Testing, linting, rendering docs with `nox`
+## Testing, linting, rendering docs with Nox
 
-The fastest way to start is to use nox. If you don't have nox, you can use
+The fastest way to start is to use Nox. If you don't have Nox, you can use
 `pipx run nox` to run it without installing, or `pipx install nox`. If you don't
 have pipx, then you can install with `pip install pipx`. If you use macOS, use
 `brew install pipx nox`. To use:
 
-```
+```console
 nox
 ```
 
-This will lint and test using every installed version of Python on your system,
-skipping ones that are not installed. You can also run specific jobs:
+This will test using every installed version of Python on your system, skipping
+ones that are not installed.
+
+### Running specific tasks with Nox
+
+```console
+nox -s {job-name}
+```
+
+To view available jobs:
 
 ```console
-$ nox -s lint  # Lint only
-$ nox -s tests  # Python tests
-$ nox -s docs -- --serve  # Build and serve the docs
-$ nox -s build  # Make an SDist and wheel
+nox -l
 ```
 
 Nox handles everything for you, including setting up an temporary virtual
 environment for each run.
 
+### Re-using Nox virtual environments
+
+**By default, Nox deletes and recreates virtual environments for every run.**
+Because this is slow, you may want to skip that step with `-R` flag:
+
+```console
+nox -R  # MUCH faster!
+```
+
+Please read more in the
+[official docs](https://nox.thea.codes/en/stable/usage.html#re-using-virtualenvs)
+
 ## Automated pre-commit checks
 
 `pre-commit` can check that code passes required checks before committing:

.github/workflows/test.yml

@@ -26,7 +26,7 @@ jobs:
           fetch-depth: 0
       - uses: "actions/setup-python@v5"
         with:
-          python-version: "3.8"
+          python-version: "3.11"
 
       - name: "Install package"
         run: "python -m pip install .[test]"
@@ -40,13 +40,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.12"]
+        python-version: ["3.11", "3.12"]
         runs-on: ["ubuntu-latest", "macos-latest", "windows-latest"]
 
-        include:
-          - python-version: "pypy-3.10"
-            runs-on: "ubuntu-latest"
-
     steps:
       - uses: "actions/checkout@v4"
         with:

noxfile.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import argparse
+import os
 import shutil
 from pathlib import Path
 
@@ -11,6 +12,12 @@
 nox.needs_version = ">=2024.3.2"
 nox.options.sessions = ["typecheck", "tests"]
 nox.options.default_venv_backend = "uv|virtualenv"
+if os.environ.get("ENVIRONMENT") == "dev":
+    # Use existing venvs where possible in dev
+    nox.options.reuse_existing_virtualenvs = "yes"
+else:
+    # All other envs should have the nox venvs recreated.
+    nox.options.reuse_existing_virtualenvs = "no"
 
 
 @nox.session

pyproject.toml

@@ -8,7 +8,7 @@ authors = [
 description = "Defines OGDC recipe API(s) and submits recipes to OGDC for execution"
 readme = "README.md"
 license.file = "LICENSE"
-requires-python = ">=3.8"
+requires-python = ">=3.11"
 classifiers = [
   "Development Status :: 1 - Planning",
   "Intended Audience :: Science/Research",
@@ -18,9 +18,6 @@ classifiers = [
   "Programming Language :: Python",
   "Programming Language :: Python :: 3",
   "Programming Language :: Python :: 3 :: Only",
-  "Programming Language :: Python :: 3.8",
-  "Programming Language :: Python :: 3.9",
-  "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Topic :: Scientific/Engineering",
@@ -31,23 +28,24 @@ dependencies = [
   "click >=8",
   "pyyaml",
   "jinja2",
+  "kubernetes ~=29.0.0",
+  "pydantic ~=2.7.1",
 ]
 
 [project.optional-dependencies]
 # NOTE: "test" and "dev" are duplicated; why do we need both?
 test = [
-  "mypy",
+  "mypy ~=1.10.0",
   "pytest >=6",
   "pytest-cov >=3",
-  "types-click",
   "types-pyyaml",
 ]
 dev = [
-  "mypy",
+  "mypy ~=1.10.0",
   "pytest >=6",
   "pytest-cov >=3",
-  "types-click",
   "types-pyyaml",
+  "nox",
 ]
 docs = [
   "sphinx>=7.0",
@@ -104,7 +102,7 @@ report.exclude_also = [
 
 [tool.mypy]
 files = ["src", "tests"]
-python_version = "3.8"
+python_version = "3.11"
 warn_unused_configs = true
 strict = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
@@ -120,12 +118,14 @@ disallow_incomplete_defs = true
 [[tool.mypy.overrides]]
 module = [
   "jinja2.*",
+  "kubernetes.*",
 ]
 ignore_missing_imports = true
 
 
 [tool.ruff]
 src = ["src"]
+target-version = "py311"
 
 [tool.ruff.lint]
 extend-select = [
@@ -155,6 +155,10 @@ ignore = [
   "PLR09",    # Too many <...>
   "PLR2004",  # Magic value used in comparison
   "ISC001",   # Conflicts with formatter
+  "PIE790",   # Unnecessary placeholder (`pass` and `...`)
+  "T201",     # `print` found
+  "EM102",    # Exception must not use an f-string literal, assign to variable first
+  "RET504",   # Unnecessary assignment before return statement
 ]
 isort.required-imports = ["from __future__ import annotations"]
 # Uncomment if using a _compat.typing backport

src/ogdc_runner/__main__.py

@@ -5,6 +5,7 @@
 import click
 
 from ogdc_runner.recipe.simple import render_simple_recipe
+from ogdc_runner.submit import submit_recipe
 
 # TODO: How do we handle e.g. GitHub URL to recipe?
 recipe_path = click.argument(
@@ -25,7 +26,9 @@
 @click.group
 def cli() -> None:
     """A tool for submitting data transformation recipes to OGDC for execution."""
-    pass
+
+
+# TODO: config_cluster() subcommand?
 
 
 @cli.command
@@ -35,11 +38,11 @@ def render(recipe_path: Path) -> None:
 
     Useful for testing.
     """
-    render_simple_recipe(recipe_path)
+    print(render_simple_recipe(recipe_path))
 
 
 @cli.command
 @recipe_path
 def submit(recipe_path: Path) -> None:
     """Render and submit a recipe to OGDC for execution."""
-    raise NotImplementedError("Not yet!")
+    submit_recipe(recipe_path)

src/ogdc_runner/constants.py

@@ -4,3 +4,6 @@
 SIMPLE_RECIPE_FILENAME = "recipe.sh"
 # .yaml and .yml?
 RECIPE_CONFIG_FILENAME = "meta.yml"
+
+# Kubernetes namespace for workflows
+K8S_NAMESPACE = "qgnet"

src/ogdc_runner/jinja.py

@@ -0,0 +1,7 @@
+from __future__ import annotations
+
+from jinja2 import Environment, PackageLoader, StrictUndefined
+
+j2_environment = Environment(
+    loader=PackageLoader("ogdc_runner"), undefined=StrictUndefined
+)

src/ogdc_runner/kubernetes.py

@@ -0,0 +1,21 @@
+# classmap = {
+#     ("ConfigMap", "BatchV1"): configmap_api_instance,
+#
+#
+# }
+#
+#
+# def k8s_apply(
+#     *,
+#     yaml_file: Path,
+# ):
+#     yml = safe_load(yaml_file)
+#     kind = yml.kind
+#     api = yml.api
+#
+#     api = classmap[(kind, api)]
+#
+#     api.apply(yml)
+#
+#
+# k8s_apply("/path/to/my_yaml.yml")

src/ogdc_runner/models/recipe_config.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pydantic import AnyUrl, BaseModel, Field
+
+
+# Create a model for the recipe input
+class RecipeInput(BaseModel):
+    url: AnyUrl
+
+
+class RecipeOutput(BaseModel):
+    dataone_id: str
+
+
+# Create a model for the recipe configuration
+class RecipeConfig(BaseModel):
+    name: str
+
+    # Allow lower-case alphanumeric characters, `.`, and `,`. These are the only
+    # allowable characters in k8s object names. `id` to construct such names.
+    id: str = Field(..., pattern=r"^[a-z0-9.-]+$")
+
+    input: RecipeInput
+    output: RecipeOutput

src/ogdc_runner/recipe/__init__.py

@@ -5,9 +5,14 @@
 import yaml
 
 from ogdc_runner.constants import RECIPE_CONFIG_FILENAME
+from ogdc_runner.models.recipe_config import RecipeConfig
 
 
-def get_recipe_config(recipe_directory: Path) -> dict:
+def get_recipe_config(recipe_directory: Path) -> RecipeConfig:
     """Extract config from a recipe configuration file (meta.yml)."""
-    with open(recipe_directory / RECIPE_CONFIG_FILENAME) as config_file:
-        return yaml.safe_load(config_file)
+    with (recipe_directory / RECIPE_CONFIG_FILENAME).open() as config_file:
+        config_dict = yaml.safe_load(config_file)
+
+    config = RecipeConfig(**config_dict)
+
+    return config