Adds a dedicated command to publish operator's conda environment. (#326)

ads/jobs/builders/runtimes/python_runtime.py

@@ -119,17 +119,28 @@ def with_custom_conda(self, uri: str, region: str = None):
     def init(self, **kwargs) -> "CondaRuntime":
         """Initializes a starter specification for the runtime.
 
+        Parameters
+        ----------
+        **kwargs: Dict
+            - conda_slug: str
+                The conda environment slug.
+                If it contains '/', then the assumption that this is a custom conda environment.
+
         Returns
         -------
         CondaRuntime
             The runtime instance.
         """
         super().init(**kwargs)
+
+        conda_slug = kwargs.get("conda_slug", "")
+
+        if "/" not in conda_slug:
+            return self.with_service_conda(conda_slug)
+
         return self.with_custom_conda(
-            kwargs.get(
-                "conda_slug",
-                "{Path to the custom conda environment. Example: oci://bucket@namespace/prefix",
-            )
+            conda_slug
+            or "{Path to the custom conda environment. Example: oci://bucket@namespace/prefix}"
         )
 
 

ads/opctl/backend/ads_dataflow.py

@@ -82,12 +82,24 @@ def init(
             The YAML specification for the given resource if `uri` was not provided.
             `None` otherwise.
         """
+
+        conda_slug = kwargs.get(
+            "conda_slug", self.config["execution"].get("conda_slug", "conda_slug")
+        ).lower()
+
+        # if conda slug contains '/' then the assumption is that it is a custom conda pack
+        # the conda prefix needs to be added
+        if "/" in conda_slug:
+            conda_slug = os.path.join(
+                self.config["execution"].get(
+                    "conda_pack_os_prefix", "oci://bucket@namespace/conda_environments"
+                ),
+                conda_slug,
+            )
+
         RUNTIME_KWARGS_MAP = {
             DataFlowRuntime().type: {
-                "conda_slug": (
-                    f"{self.config['execution'].get('conda_pack_os_prefix','oci://bucket@namespace/conda_environments').rstrip('/')}"
-                    f"/{kwargs.get('conda_slug', 'conda_slug') }"
-                ),
+                "conda_slug": conda_slug,
                 "script_bucket": f"{self.config['infrastructure'].get('script_bucket','').rstrip('/')}",
             },
         }

ads/opctl/backend/ads_ml_job.py

@@ -95,25 +95,30 @@ def init(
             The YAML specification for the given resource if `uri` was not provided.
             `None` otherwise.
         """
+
+        conda_slug = kwargs.get(
+            "conda_slug", self.config["execution"].get("conda_slug", "conda_slug")
+        ).lower()
+
+        # if conda slug contains '/' then the assumption is that it is a custom conda pack
+        # the conda prefix needs to be added
+        if "/" in conda_slug:
+            conda_slug = os.path.join(
+                self.config["execution"].get(
+                    "conda_pack_os_prefix", "oci://bucket@namespace/conda_environments"
+                ),
+                conda_slug,
+            )
+
         RUNTIME_KWARGS_MAP = {
             ContainerRuntime().type: {
                 "image": (
                     f"{self.config['infrastructure'].get('docker_registry','').rstrip('/')}"
                     f"/{kwargs.get('image_name', self.config['execution'].get('image','image:latest'))}"
                 )
             },
-            ScriptRuntime().type: {
-                "conda_slug": (
-                    f"{self.config['execution'].get('conda_pack_os_prefix','oci://bucket@namespace/conda_environments').rstrip('/')}"
-                    f"/{kwargs.get('conda_slug', 'conda_slug') }"
-                )
-            },
-            PythonRuntime().type: {
-                "conda_slug": (
-                    f"{self.config['execution'].get('conda_pack_os_prefix','oci://bucket@namespace/conda_environments').rstrip('/')}"
-                    f"/{kwargs.get('conda_slug', 'conda_slug') }"
-                )
-            },
+            ScriptRuntime().type: {"conda_slug": conda_slug},
+            PythonRuntime().type: {"conda_slug": conda_slug},
             NotebookRuntime().type: {},
             GitPythonRuntime().type: {},
         }

ads/opctl/operator/cli.py

@@ -22,6 +22,7 @@
 from .cmd import info as cmd_info
 from .cmd import init as cmd_init
 from .cmd import list as cmd_list
+from .cmd import publish_conda as cmd_publish_conda
 from .cmd import publish_image as cmd_publish_image
 from .cmd import verify as cmd_verify
 
@@ -100,31 +101,14 @@ def init(debug: bool, **kwargs: Dict[str, Any]) -> None:
 @click.option(
     "--name",
     "-n",
-    help=(
-        "Name of the operator to build the image. "
-        "Only relevant for built-in service operators."
-    ),
+    help=("Name of the operator to build the image. "),
     default=None,
     required=False,
 )
-@click.option(
-    "--image",
-    "-i",
-    help="The image name. By default the operator's name will be used.",
-    default=None,
-    required=False,
-)
-@click.option(
-    "--tag",
-    "-t",
-    help="The image tag. " "By default the operator's version will be used.",
-    required=False,
-    default=None,
-)
 @click.option(
     "--rebuild-base-image",
     "-r",
-    help="Rebuild both base and operator's images.",
+    help="Rebuild operator's base image. This option is useful when developing a new operator.",
     is_flag=True,
     default=False,
 )
@@ -134,11 +118,21 @@ def build_image(debug: bool, **kwargs: Dict[str, Any]) -> None:
 
 
 @commands.command()
-@click.argument("image")
 @click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False)
 @click.help_option("--help", "-h")
 @click.option(
-    "--registry", "-r", help="Registry to publish to.", required=False, default=None
+    "--name",
+    "-n",
+    type=click.Choice(__operators__),
+    help="The name of the operator.",
+    required=True,
+)
+@click.option(
+    "--registry",
+    "-r",
+    help="Registry to publish to. By default will be taken form the ADS opctl config.",
+    required=False,
+    default=None,
 )
 @click.option(
     "--ads-config",
@@ -147,7 +141,7 @@ def build_image(debug: bool, **kwargs: Dict[str, Any]) -> None:
     default=None,
 )
 def publish_image(debug, **kwargs):
-    """Publishes operator image to the container registry."""
+    """Publishes an operator's image to the container registry."""
     suppress_traceback(debug)(cmd_publish_image)(**kwargs)
 
 
@@ -235,19 +229,16 @@ def verify(debug: bool, **kwargs: Dict[str, Any]) -> None:
 @click.option(
     "--name",
     "-n",
-    help=(
-        "Name of the operator to build the conda environment for. "
-        "Only relevant for built-in service operators."
-    ),
+    help=("Name of the operator to build the conda environment for. "),
     default=None,
     required=False,
 )
 @click.option(
     "--conda-pack-folder",
     help=(
         "The destination folder to save the conda environment. "
-        "By default will be used the path specified in the config file generated "
-        "with `ads opctl configure` command"
+        "By default will be used the path specified in the ADS config file generated "
+        "with `ads opctl configure` command."
     ),
     required=False,
     default=None,
@@ -268,3 +259,41 @@ def verify(debug: bool, **kwargs: Dict[str, Any]) -> None:
 def build_conda(debug: bool, **kwargs: Dict[str, Any]) -> None:
     """Builds a new conda environment for the particular operator."""
     suppress_traceback(debug)(cmd_build_conda)(**kwargs)
+
+
+@commands.command()
+@click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False)
+@click.help_option("--help", "-h")
+@click.option(
+    "--name",
+    "-n",
+    help=("Name of the operator to publish the conda environment for. "),
+    default=None,
+    required=False,
+)
+@click.option(
+    "--conda-pack-folder",
+    help=(
+        "The source folder to search the conda environment. "
+        "By default will be used the path specified in the ADS config file generated "
+        "with `ads opctl configure` command."
+    ),
+    required=False,
+    default=None,
+)
+@click.option(
+    "--overwrite",
+    "-o",
+    help="Overwrite conda environment if it already exists.",
+    is_flag=True,
+    default=False,
+)
+@click.option(
+    "--ads-config",
+    help="The folder where the ADS opctl config located.",
+    required=False,
+    default=None,
+)
+def publish_conda(debug: bool, **kwargs: Dict[str, Any]) -> None:
+    """Publishes an operator's conda environment to the Object Storage bucket."""
+    suppress_traceback(debug)(cmd_publish_conda)(**kwargs)