Fix Runner loading and refactor code (#977)

**Pull Request Checklist** - [x] Fixes #962 - [x] Tests added - [ ] Documentation/examples added - [x] [Good commit messages](https://cbea.ms/git-commit/) and/or PR title **Description of PR** Primarily, this PR refactors the runner code. It also fixes the mapping and loading of kwargs to a RunnerInput object. Currently, the runner code is hard to follow. This PR refactors the functionality in the `runner.py` file into a `_runner` module with util submodules. The PR also makes the complex input mapping logic more testable. --------- Signed-off-by: Elliot Gunton <egunton@bloomberg.net>
argoproj-labs · Mar 4, 2024 · 799672e · 799672e
1 parent cb8417c
commit 799672e
Show file tree

Hide file tree

Showing 23 changed files with 1,669 additions and 1,220 deletions.
diff --git a/docs/examples/workflows-examples.md b/docs/examples/workflows-examples.md
@@ -107,7 +107,8 @@ Explore the examples through the side bar!
 | [template-defaults](https://github.com/argoproj/argo-workflows/blob/main/examples/template-defaults.yaml) |
 | [testvolume](https://github.com/argoproj/argo-workflows/blob/main/examples/testvolume.yaml) |
 | [timeouts-step](https://github.com/argoproj/argo-workflows/blob/main/examples/timeouts-step.yaml) |
-| [title-and-descriptin-with-markdown](https://github.com/argoproj/argo-workflows/blob/main/examples/title-and-descriptin-with-markdown.yaml) |
+| [title-and-description-with-markdown](https://github.com/argoproj/argo-workflows/blob/main/examples/title-and-description-with-markdown.yaml) |
+| [withsequence-nested-result](https://github.com/argoproj/argo-workflows/blob/main/examples/withsequence-nested-result.yaml) |
 | [work-avoidance](https://github.com/argoproj/argo-workflows/blob/main/examples/work-avoidance.yaml) |
 | [workflow-count-resourcequota](https://github.com/argoproj/argo-workflows/blob/main/examples/workflow-count-resourcequota.yaml) |
 | [workflow-event-binding/event-consumer-workfloweventbinding](https://github.com/argoproj/argo-workflows/blob/main/examples/workflow-event-binding/event-consumer-workfloweventbinding.yaml) |

diff --git a/...kflows/experimental/script_pydantic_io.md → ...orkflows/experimental/script_runner_io.md b/...kflows/experimental/script_pydantic_io.md → ...orkflows/experimental/script_runner_io.md
@@ -1,4 +1,4 @@
-# Script Pydantic Io
+# Script Runner Io
 
 
 
@@ -14,7 +14,8 @@
         from pydantic import BaseModel
 
     from hera.shared import global_config
-    from hera.workflows import Artifact, ArtifactLoader, Parameter, Workflow, script
+    from hera.workflows import Artifact, ArtifactLoader, Parameter, Steps, Workflow, script
+    from hera.workflows.archive import NoneArchiveStrategy
     from hera.workflows.io import RunnerInput, RunnerOutput
 
     try:
@@ -27,7 +28,7 @@
 
 
     class MyObject(BaseModel):
-        a_dict: dict = {}
+        a_dict: dict  # not giving a default makes the field a required input for the template
         a_str: str = "a default string"
 
 
@@ -44,15 +45,30 @@
         artifact_int: Annotated[int, Artifact(name="artifact-output")]
 
 
-    @script(constructor="runner")
+    @script(constructor="runner", image="python-image-built-with-my-package")
+    def writer() -> Annotated[int, Artifact(name="int-artifact", archive=NoneArchiveStrategy())]:
+        return 100
+
+
+    @script(constructor="runner", image="python-image-built-with-my-package")
     def pydantic_io(
         my_input: MyInput,
     ) -> MyOutput:
         return MyOutput(exit_code=1, result="Test!", param_int=42, artifact_int=my_input.param_int)
 
 
     with Workflow(generate_name="pydantic-io-") as w:
-        pydantic_io()
+        with Steps(name="use-pydantic-io"):
+            write_step = writer()
+            pydantic_io(
+                arguments=[
+                    write_step.get_artifact("int-artifact").with_name("artifact-input"),
+                    {
+                        "param_int": 101,
+                        "an_object": MyObject(a_dict={"my-new-key": "my-new-value"}),
+                    },
+                ]
+            )
     ```
 
 === "YAML"
@@ -64,6 +80,46 @@
       generateName: pydantic-io-
     spec:
       templates:
+      - name: use-pydantic-io
+        steps:
+        - - name: writer
+            template: writer
+        - - arguments:
+              artifacts:
+              - from: '{{steps.writer.outputs.artifacts.int-artifact}}'
+                name: artifact-input
+              parameters:
+              - name: param_int
+                value: '101'
+              - name: an_object
+                value: '{"a_dict": {"my-new-key": "my-new-value"}, "a_str": "a default
+                  string"}'
+            name: pydantic-io
+            template: pydantic-io
+      - name: writer
+        outputs:
+          artifacts:
+          - archive:
+              none: {}
+            name: int-artifact
+            path: /tmp/hera-outputs/artifacts/int-artifact
+        script:
+          args:
+          - -m
+          - hera.workflows.runner
+          - -e
+          - examples.workflows.experimental.script_runner_io:writer
+          command:
+          - python
+          env:
+          - name: hera__script_annotations
+            value: ''
+          - name: hera__outputs_directory
+            value: /tmp/hera-outputs
+          - name: hera__script_pydantic_io
+            value: ''
+          image: python-image-built-with-my-package
+          source: '{{inputs.parameters}}'
       - inputs:
           artifacts:
           - name: artifact-input
@@ -87,7 +143,7 @@
           - -m
           - hera.workflows.runner
           - -e
-          - examples.workflows.experimental.script_pydantic_io:pydantic_io
+          - examples.workflows.experimental.script_runner_io:pydantic_io
           command:
           - python
           env:
@@ -97,7 +153,7 @@
             value: /tmp/hera-outputs
           - name: hera__script_pydantic_io
             value: ''
-          image: python:3.8
+          image: python-image-built-with-my-package
           source: '{{inputs.parameters}}'
     ```
 
diff --git a/docs/examples/workflows/template_level_volume.md b/docs/examples/workflows/template_level_volume.md
@@ -73,35 +73,17 @@ See https://argo-workflows.readthedocs.io/en/latest/walk-through/volumes/
     spec:
       entrypoint: generate-and-use-volume
       templates:
-      - name: generate-and-use-volume
-        steps:
-        - - name: generate-volume
-            template: generate-volume
-            arguments:
-              parameters:
-                - name: pvc-size
-                  # In a real-world example, this could be generated by a previous workflow step.
-                  value: '1Gi'
-        - - name: generate
-            template: whalesay
-            arguments:
-              parameters:
-                - name: pvc-name
-                  value: '{{steps.generate-volume.outputs.parameters.pvc-name}}'
-        - - name: print
-            template: print-message
-            arguments:
-              parameters:
-                - name: pvc-name
-                  value: '{{steps.generate-volume.outputs.parameters.pvc-name}}'
-
-      - name: generate-volume
-        inputs:
+      - inputs:
+          parameters:
+          - name: pvc-size
+        name: generate-volume
+        outputs:
           parameters:
-            - name: pvc-size
+          - name: pvc-name
+            valueFrom:
+              jsonPath: '{.metadata.name}'
         resource:
           action: create
-          setOwnerReference: true
           manifest: |
             apiVersion: v1
             kind: PersistentVolumeClaim
@@ -112,42 +94,62 @@ See https://argo-workflows.readthedocs.io/en/latest/walk-through/volumes/
               resources:
                 requests:
                   storage: '{{inputs.parameters.pvc-size}}'
-        outputs:
-          parameters:
-            - name: pvc-name
-              valueFrom:
-                jsonPath: '{.metadata.name}'
-
-      - name: whalesay
-        inputs:
-          parameters:
-            - name: pvc-name
-        volumes:
-          - name: workdir
-            persistentVolumeClaim:
-              claimName: '{{inputs.parameters.pvc-name}}'
-        container:
+          setOwnerReference: true
+      - container:
+          args:
+          - echo generating message in volume; cowsay hello world | tee /mnt/vol/hello_world.txt
+          command:
+          - sh
+          - -c
           image: docker/whalesay:latest
-          command: [sh, -c]
-          args: ["echo generating message in volume; cowsay hello world | tee /mnt/vol/hello_world.txt"]
           volumeMounts:
-          - name: workdir
-            mountPath: /mnt/vol
-
-      - name: print-message
+          - mountPath: /mnt/vol
+            name: workdir
         inputs:
-            parameters:
-              - name: pvc-name
+          parameters:
+          - name: pvc-name
+        name: whalesay
         volumes:
-          - name: workdir
-            persistentVolumeClaim:
-              claimName: '{{inputs.parameters.pvc-name}}'
-        container:
+        - name: workdir
+          persistentVolumeClaim:
+            claimName: '{{inputs.parameters.pvc-name}}'
+      - container:
+          args:
+          - echo getting message from volume; find /mnt/vol; cat /mnt/vol/hello_world.txt
+          command:
+          - sh
+          - -c
           image: alpine:latest
-          command: [sh, -c]
-          args: ["echo getting message from volume; find /mnt/vol; cat /mnt/vol/hello_world.txt"]
           volumeMounts:
-          - name: workdir
-            mountPath: /mnt/vol
+          - mountPath: /mnt/vol
+            name: workdir
+        inputs:
+          parameters:
+          - name: pvc-name
+        name: print-message
+        volumes:
+        - name: workdir
+          persistentVolumeClaim:
+            claimName: '{{inputs.parameters.pvc-name}}'
+      - name: generate-and-use-volume
+        steps:
+        - - arguments:
+              parameters:
+              - name: pvc-size
+                value: 1Gi
+            name: generate-volume
+            template: generate-volume
+        - - arguments:
+              parameters:
+              - name: pvc-name
+                value: '{{steps.generate-volume.outputs.parameters.pvc-name}}'
+            name: generate
+            template: whalesay
+        - - arguments:
+              parameters:
+              - name: pvc-name
+                value: '{{steps.generate-volume.outputs.parameters.pvc-name}}'
+            name: print
+            template: print-message
     ```