diff --git a/ads/opctl/operator/README.md b/ads/opctl/operator/README.md index d818f8b27..ed5b91e11 100644 --- a/ads/opctl/operator/README.md +++ b/ads/opctl/operator/README.md @@ -1,4 +1,4 @@ ## ML Operator ------------------------ +-------------- Welcome to ML operators. This readme will contain the instructions helping to configure and dispatch operators. diff --git a/ads/opctl/operator/cli.py b/ads/opctl/operator/cli.py index 50bdca5f3..6492f76e7 100644 --- a/ads/opctl/operator/cli.py +++ b/ads/opctl/operator/cli.py @@ -40,12 +40,12 @@ def list(debug: bool, **kwargs: Dict[str, Any]) -> None: @commands.command() -@click.option("--debug", "-d", help="Set debug mode", is_flag=True, default=False) +@click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False) @click.option( "--name", "-n", type=click.Choice(__operators__), - help="The name of the operator", + help="The name of the operator.", required=True, ) def info(debug: bool, **kwargs: Dict[str, Any]) -> None: @@ -58,10 +58,10 @@ def info(debug: bool, **kwargs: Dict[str, Any]) -> None: "--name", "-n", type=click.Choice(__operators__), - help="The name of the operator", + help="The name of the operator.", required=True, ) -@click.option("--debug", "-d", help="Set debug mode", is_flag=True, default=False) +@click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False) @click.option( "--output", help=f"The folder name to save the resulting specification templates.", @@ -71,13 +71,13 @@ def info(debug: bool, **kwargs: Dict[str, Any]) -> None: @click.option( "--overwrite", "-o", - help="Overwrite result file if it already exists", + help="Overwrite result file if it already exists.", is_flag=True, default=False, ) @click.option( "--ads-config", - help="The folder where the ADS opctl config located", + help="The folder where the ADS opctl config located.", required=False, default=None, ) @@ -87,7 +87,7 @@ def init(debug: bool, **kwargs: Dict[str, Any]) -> None: @commands.command() -@click.option("--debug", "-d", help="Set debug mode", is_flag=True, default=False) +@click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False) @click.help_option("--help", "-h") @click.option( "--gpu", @@ -110,11 +110,17 @@ def init(debug: bool, **kwargs: Dict[str, Any]) -> None: @click.option( "--image", "-i", - help="The image name. By default the operator name will be used.", + help="The image name. By default the operator's name will be used.", default=None, required=False, ) -@click.option("--tag", "-t", help="The image tag.", required=False, default=None) +@click.option( + "--tag", + "-t", + help="The image tag. " "By default the operator's version will be used.", + required=False, + default=None, +) @click.option( "--rebuild-base-image", "-r", @@ -129,14 +135,14 @@ def build_image(debug: bool, **kwargs: Dict[str, Any]) -> None: @commands.command() @click.argument("image") -@click.option("--debug", "-d", help="Set debug mode", is_flag=True, default=False) +@click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False) @click.help_option("--help", "-h") @click.option( - "--registry", "-r", help="Registry to publish to", required=False, default=None + "--registry", "-r", help="Registry to publish to.", required=False, default=None ) @click.option( "--ads-config", - help="The folder where the ADS opctl config located", + help="The folder where the ADS opctl config located.", required=False, default=None, ) @@ -146,30 +152,30 @@ def publish_image(debug, **kwargs): @commands.command(hidden=True) -@click.option("--debug", "-d", help="Set debug mode", is_flag=True, default=False) +@click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False) @click.option( "--name", "-n", type=click.Choice(__operators__), - help="The name of the operator", + help="The name of the operator.", required=True, ) @click.option( "--overwrite", "-o", - help="Overwrite result file if it already exists", + help="Overwrite result file if it already exists.", is_flag=True, default=False, ) @click.option( "--ads-config", - help="The folder where the ADS opctl config located", + help="The folder where the ADS opctl config located.", required=False, default=None, ) @click.option( "--output", - help=f"The folder to save the resulting specification template YAML", + help=f"The folder to save the resulting specification template YAML.", required=False, default=None, ) @@ -180,7 +186,7 @@ def create(debug: bool, **kwargs: Dict[str, Any]) -> None: @commands.command() @click.help_option("--help", "-h") -@click.option("--debug", "-d", help="Set debug mode", is_flag=True, default=False) +@click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False) @click.option( "--file", "-f", help="The path to resource YAML file.", required=True, default=None ) @@ -224,7 +230,7 @@ def verify(debug: bool, **kwargs: Dict[str, Any]) -> None: @commands.command() -@click.option("--debug", "-d", help="Set debug mode", is_flag=True, default=False) +@click.option("--debug", "-d", help="Set debug mode.", is_flag=True, default=False) @click.help_option("--help", "-h") @click.option( "--name", @@ -249,13 +255,13 @@ def verify(debug: bool, **kwargs: Dict[str, Any]) -> None: @click.option( "--overwrite", "-o", - help="Overwrite conda environment if it already exists", + help="Overwrite conda environment if it already exists.", is_flag=True, default=False, ) @click.option( "--ads-config", - help="The folder where the ADS opctl config located", + help="The folder where the ADS opctl config located.", required=False, default=None, ) diff --git a/ads/opctl/operator/cmd.py b/ads/opctl/operator/cmd.py index b720c631c..b70c124eb 100644 --- a/ads/opctl/operator/cmd.py +++ b/ads/opctl/operator/cmd.py @@ -53,7 +53,7 @@ def list() -> None: """Prints the list of the registered service operators.""" - logger.info( + print( tabulate( ( { diff --git a/ads/opctl/operator/lowcode/forecast/operator.py b/ads/opctl/operator/lowcode/forecast/operator.py index 9f205385c..c9c8f8d9e 100644 --- a/ads/opctl/operator/lowcode/forecast/operator.py +++ b/ads/opctl/operator/lowcode/forecast/operator.py @@ -11,6 +11,7 @@ from ads.opctl import logger + def operate(operator_config: ForecastOperatorConfig) -> None: """Runs the forecasting operator.""" ForecastOperatorModelFactory.get_model(operator_config).generate_report() @@ -22,5 +23,6 @@ def verify(spec: Dict) -> bool: msg_header = ( f"{'*' * 50} The operator config has been successfully verified {'*' * 50}" ) - logger.info(operator.to_yaml()) - logger.info("*" * len(msg_header)) + print(msg_header) + print(operator.to_yaml()) + print("*" * len(msg_header)) diff --git a/docs/source/user_guide/operators/common/explore.rst b/docs/source/user_guide/operators/common/explore.rst index 52117c765..65d03bfc1 100644 --- a/docs/source/user_guide/operators/common/explore.rst +++ b/docs/source/user_guide/operators/common/explore.rst @@ -1,20 +1,243 @@ -================= -Explore Operators -================= +============================= +Explore & Configure Operators +============================= + +After ensuring that you have all the necessary prerequisites in order, the next step is to explore and configure the operators. This guide will take you through the process, utilizing the Command Line Interface (CLI) tool to assist you in this endeavor. This step is a preliminary one that precedes the execution of operators. + +.. admonition:: Prerequisites + :class: note + + Before we start, let's ensure you have everything you need for easy starting. If you haven't already, install the accompanying CLI tool, detailed installation instructions can be found in the links below. + + - :doc:`Install ADS CLI<../../cli/quickstart>` + - :doc:`Configure Defaults<../../cli/opctl/configure>` -Once you have the prerequisites in place, it's time to explore and configure operators. CLI Overview ------------- +============ + +The ``ads opctl operator`` CLI tool is your trusty companion when working with operators. It simplifies the process, making it easy to explore the catalog of registered operators, gain insights into their specific use cases, and configure them to meet your needs. Additionally, this tool provides assistance in constructing Docker containers or setting up Conda environments to execute the operator, all while guiding you through the essential steps for running them. + + +Let's start from the very beginning by calling the following command in your terminal to see a list of supported CLI commands: + +.. code-block:: bash + + ads opctl operator --help + + +This command provides a concise overview of all available commands. + +- ``ads opctl operator list``: Retrieve a list of registered operators with this command. + +- ``ads opctl operator info``: Obtain detailed information about a specific operator using this command. It offers comprehensive instructions on how to configure and run the operator across different environments and runtimes. -Utilizing the CLI tool streamlines the process, making it straightforward to delve into the catalog of registered operators and acquire a deep understanding of the particular challenges they are designed to address. Additionally, the tool also provides assistance in configuring operators, constructing Docker containers, or setting up Conda environments to execute the operator, while also guiding you through the necessary steps for running them. +- ``ads opctl operator init``: Generate starter YAML configurations for an operator with this command. -List Operators --------------- +- ``ads opctl operator verify``: Ensure the correctness of an operator's YAML specification using this command. + +- ``ads opctl operator build-conda``: Build a new Conda environment tailored to a particular operator using this command. + +- ``ads opctl operator build-image``: Create a new image customized for the operator using this command. + +- ``ads opctl operator publish-image``: Publish the operator's image to the container registry with this command. + + +Listing Operators +----------------- Begin by browsing our operator catalog to discover the pre-packaged solutions available for various data science tasks. The catalog provides short descriptions of each operator's capabilities and use cases. -Get Details ------------ +.. code-block:: bash + + ads opctl operator list + + +.. figure:: figures/operator_list.png + :align: center + + +Getting an Operator's Details +----------------------------- Each operator is accompanied by highly detailed instructions explaining how it can be configured and executed in various environments. + +.. code-block:: bash + + ads opctl operator info --help + + +.. figure:: figures/operator_info.png + :align: center + + +.. code-block:: bash + + ads opctl operator info --name + + +.. figure:: figures/operator_info1.png + :align: center + + +Initializing an Operator's Configs +---------------------------------- + +The ``init`` command is indeed crucial as it generates the foundational configurations tailored for a specific operator. + +Before you start, make sure to complete the :doc:`Configure Defaults<../../cli/opctl/configure>` step. This step is essential as it establishes default values for different options when running the operator on OCI Data Science jobs or OCI Data Flow applications. If you have already completed this setup and are using a flexible shape, remember to adjust the ``ml_job_config.ini`` and ``dataflow_config.ini`` files with the shape configuration details and ``docker_registry`` information. + +**ml_job_config.ini** + +- ``ocpus = 1`` +- ``memory_in_gbs = 16`` +- ``docker_registry = `` + +**dataflow_config.ini** + +- ``driver_shape = VM.Standard.E4.Flex`` +- ``executor_shape = VM.Standard.E4.Flex`` +- ``driver_shape_ocpus = 1`` +- ``executor_shape_ocpus = 1`` +- ``driver_shape_memory_in_gbs = 16`` +- ``executor_shape_memory_in_gbs = 16`` + +Now let's explore the ``init`` command. + +.. code-block:: bash + + ads opctl operator init --help + +.. figure:: figures/operator_init.png + :align: center + +To create starter configuration files, execute the following command. + +.. code-block:: bash + + ads opctl operator init -n --overwrite --output ~/ + +**The essential files generated include:** + +- **.yaml**: Contains configuration related to particular operator. +- **backend_operator_local_python_config.yaml**: This file includes local backend configuration for running operator in a local environment. You must manually set up the environment before executing the operator. +- **backend_operator_local_container_config.yaml**: This file contains local backend configuration for running operator within a local container. You should build the container before running the operator, following the instructions below. +- **backend_job_container_config.yaml**: Contains Data Science job-related configuration for running operator in a container (BYOC) runtime. The container must be built and published before executing the operator, as detailed below. For comprehensive details about the supported configuration options, including the schema and available settings, please refer to the :doc:`OCI Data Science Jobs<../../jobs/yaml_schema>` documentation. +- **backend_job_python_config.yaml**: Contains Data Science job-related configuration to run operator in a Data Science job within a conda runtime. The conda environment should be built and published before running the operator. For comprehensive details about the supported configuration options, including the schema and available settings, please refer to the :doc:`OCI Data Science Jobs YAML Schema<../../jobs/yaml_schema>` documentation. +- **backend_dataflow_dataflow_config.yaml**: Contains Data Flow application-related configuration to run operator in a Data Flow application. The conda environment should be built and published before running the operator. For comprehensive details about the supported configuration options, including the schema and available settings, please refer to the :doc:`Data Flow Application YAML Schema<../../apachespark/datafloe>` documentation. + +These generated configurations are designed to be ready for use without additional adjustments. However, they are provided as starter kit configurations that can be customized as needed. + +The operator's configuration file, named as ``.yaml``, is generated based on the operator's schema and contains the essential input attributes required to run the operator. These attributes serve as the bare minimum configuration for the operator to function. + +However, in cases where the operator requires specific input or output sources of data, you may need to adjust the configuration manually to accommodate these requirements. + +Beyond the basic input attributes, additional configurations within the YAML file are generated based on the information provided during the pre-step of configuring defaults, as detailed in the :doc:`Configure Defaults<../../cli/opctl/configure>` documentation. These configurations are populated using environment variables as well, ensuring that the operator runs with the necessary settings and parameters. + +In summary, while the core configuration is automatically generated from the operator's schema, you have the flexibility to fine-tune and customize the configuration to match your specific data science needs. + + +Verifying an Operator's Config +------------------------------ + +Before proceeding to run an operator, it's essential to verify the operator's configuration. Running an operator can be resource-intensive, so it's a valuable step to ensure that the configuration is correct before initiating the operation. Once you have obtained the operator's configuration (specification) in YAML format, you can use the following command to verify it: + +.. code-block:: bash + + ads opctl operator verify --help + +.. figure:: figures/operator_config_verify.png + :align: center + +The configuration file can be located in an Object Storage bucket or in a local repository. To specify the authentication type, use the ``--auth`` attribute. + +Verification helps you catch any errors or inconsistencies in the operator's configuration, ensuring that it will run smoothly and produce the desired results when executed. + +.. code-block:: bash + + ads opctl operator verify -f ~//config/.yaml + +.. figure:: figures/operator_config_verify_result.png + :align: center + +Building an Operator's Image +---------------------------- + +In order to run an operator within a local container or utilize it with the OCI Data Science Jobs service's BYOC (Bring Your Own Container) runtime, you must first create a dedicated container image for the specific operator. This process is straightforward and only requires that you have the Docker CLI installed on your local machine. Below, we outline the steps to build your custom container. + +.. code-block:: bash + + ads opctl operator build-image --help + +.. figure:: figures/build_operator_image.png + :align: center + +The fundamental attribute you need to provide is ``--name``, which represents the name of the operator. If you do not specify ``--image`` and ``--tag``, the operator's name and version will be automatically used as the image name and tag. + +An interesting point to note is that the operator's container can be built to accommodate both CPU and GPU architectures, although this capability depends on the specific operator's requirements. + +Once the operator's image is successfully built, you have the option to publish it to the `Oracle Container Registry `_. This publishing step becomes necessary when you intend to run the operator on the OCI Data Science Jobs service within the BYOC runtime. + + +Publishing an Operator's Image +------------------------------- + +After successfully building the operator's image, the next step is to publish it to the `Oracle Container Registry `_. As mentioned previously, publishing the image is specifically required when you intend to run the operator on the OCI Data Science Jobs service within the BYOC runtime. Fortunately, this step is quite straightforward. Below, you'll find instructions on how to publish the image to the Container Registry. + +.. code-block:: bash + + ads opctl operator publish-image --help + +.. figure:: figures/publish_operator_image.png + :align: center + +The only mandatory parameter for this command is the image name that you wish to publish. + +.. code-block:: bash + + ads opctl operator publish-image :v1 + +While the image name is the only required parameter, you also have the option to provide the ``registry`` parameter if needed. By default, the information about the registry where the container should be published is retrieved from the ADS config generated during the :doc:`Configure Defaults<../../cli/opctl/configure>` step. + +For more detailed information on publishing containers, you can refer to the `Oracle Container Registry `_ documentation. + + +Building an Operator's Conda Environment +---------------------------------------- + +Another option for running an operator on OCI resources is to utilize a Conda environment. Building an operator's Conda environment is necessary if you intend to run the operator on the OCI Data Science Jobs service within the Conda runtime or on the Data Flow service. Additionally, the Conda environment can be employed within a Data Science Notebook Session to execute a specific operator. + +To build the operator's Conda environment, follow these steps: + +.. code-block:: bash + + ads opctl operator build-conda --help + +.. figure:: figures/build_operator_conda.png + :align: center + +The only mandatory parameter for this command is the ``--name`` of the operator. However, you also have the option to specify the destination folder for the Conda environment. By default, the information about the destination folder where the Conda environment should be created is retrieved from the ADS config generated during the :doc:`Configure Defaults<../../cli/opctl/configure>` step. + +.. code-block:: bash + + ads opctl operator build-conda --name + +Once you have successfully built the Conda environment, you will need to publish it to OCI Object Storage. This step allows the OCI Data Science Jobs and Data Flow services to utilize the Conda environment seamlessly. + + +Publishing an Operator's Conda Environment +------------------------------------------ + +To make a locally built Conda environment available in the OCI Object Storage bucket, follow these simple steps: + +.. code-block:: bash + + ads opctl conda publish _ + +For instance, if you have constructed a Conda environment for the "forecast" operator, the command would appear as follows: + +.. code-block:: bash + + ads opctl conda publish forecast_v1 + +Publishing the Conda environment to OCI Object Storage enables the OCI Data Science Jobs and Data Flow services to access and utilize this environment efficiently. This step is essential to ensure that your operators run seamlessly within the OCI ecosystem. diff --git a/docs/source/user_guide/operators/common/figures/build_operator_conda.png b/docs/source/user_guide/operators/common/figures/build_operator_conda.png new file mode 100644 index 000000000..ffee1b478 Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/build_operator_conda.png differ diff --git a/docs/source/user_guide/operators/common/figures/build_operator_image.png b/docs/source/user_guide/operators/common/figures/build_operator_image.png new file mode 100644 index 000000000..860682da9 Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/build_operator_image.png differ diff --git a/docs/source/user_guide/operators/common/figures/operator_config_verify.png b/docs/source/user_guide/operators/common/figures/operator_config_verify.png new file mode 100644 index 000000000..5dc070104 Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/operator_config_verify.png differ diff --git a/docs/source/user_guide/operators/common/figures/operator_config_verify_result.png b/docs/source/user_guide/operators/common/figures/operator_config_verify_result.png new file mode 100644 index 000000000..8b371c5f7 Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/operator_config_verify_result.png differ diff --git a/docs/source/user_guide/operators/common/figures/operator_info.png b/docs/source/user_guide/operators/common/figures/operator_info.png new file mode 100644 index 000000000..9349002d6 Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/operator_info.png differ diff --git a/docs/source/user_guide/operators/common/figures/operator_info1.png b/docs/source/user_guide/operators/common/figures/operator_info1.png new file mode 100644 index 000000000..c22e94f77 Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/operator_info1.png differ diff --git a/docs/source/user_guide/operators/common/figures/operator_init.png b/docs/source/user_guide/operators/common/figures/operator_init.png new file mode 100644 index 000000000..55b23e8a7 Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/operator_init.png differ diff --git a/docs/source/user_guide/operators/common/figures/operator_list.png b/docs/source/user_guide/operators/common/figures/operator_list.png new file mode 100644 index 000000000..c0c574182 Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/operator_list.png differ diff --git a/docs/source/user_guide/operators/common/figures/publish_operator_image.png b/docs/source/user_guide/operators/common/figures/publish_operator_image.png new file mode 100644 index 000000000..180da348c Binary files /dev/null and b/docs/source/user_guide/operators/common/figures/publish_operator_image.png differ diff --git a/docs/source/user_guide/operators/common/run.rst b/docs/source/user_guide/operators/common/run.rst index 3d979fb1b..efec53aea 100644 --- a/docs/source/user_guide/operators/common/run.rst +++ b/docs/source/user_guide/operators/common/run.rst @@ -2,39 +2,208 @@ How To Run ========== -Now that you've explored and configured operators, it's time to run them in your chosen backend. +It's time to run operators in your chosen backend. -Configure ---------- +.. admonition:: Prerequisites + :class: note -The initial step involves generating starter kit configurations that facilitate the execution of the operator across various backends. This can be accomplished effortlessly by employing the command ``ads opctl operator init --name ``. The entire process is completely automated and remarkably straightforward. Please adhere to the instructions provided by the CLI for guidance. + Before we start, let's ensure you have everything you need for easy starting. If you haven't already, install the accompanying CLI tool, detailed installation instructions can be found in the links below. -Local Environment ------------------ + - :doc:`Install ADS CLI<../../cli/quickstart>` + - :doc:`Configure Defaults<../../cli/opctl/configure>` + - :doc:`Explore & Configure Operators<./explore>` -- Run Within Local Conda Environment -- Run with local container - - Build Container - - Run Operator -Data Science Job ----------------- +The first step is to generate starter kit configurations that simplify the execution of the operator across different backends. This can be done easily using the following command: -- Run With Conda Environment - - Policies - - Build Conda Environment - - Publish Conda Environment - - Run Operator +.. code-block:: bash -- Run With BYOC (Bring Your Own Container) - - Policies - - Build Container - - Publish Container - - Run Operator + ads opctl operator init -n + + +Run Operator Locally +-------------------- + +There are several ways to run the operator in your local environment. The first option is to run it in the environment you've prepared on your own, assuming you've already installed all the necessary operator packages. The second option is to run the operator within a Docker container, which requires building a Docker image for the operator. + +Within Local Environment +~~~~~~~~~~~~~~~~~~~~~~~~ + +To run the operator locally, follow these steps: + +1. Create and activate a new conda environment named ````. +2. Install all the required libraries listed in the ``environment.yaml`` file generated by the ``ads opctl operator init --name `` command. +3. Review the ``.yaml`` file generated by the ``init`` command and make any necessary adjustments to input and output file locations. +4. Verify the operator's configuration using the following command: + +.. code-block:: bash + + ads opctl operator verify -f .yaml + +5. To run the operator within the ```` conda environment, use this command: + +.. code-block:: bash + + ads opctl apply -f .yaml -b local + +The operator will be run in your local environment without requiring additional modifications. + +Within Container +~~~~~~~~~~~~~~~~ + +To run the operator within a local container, follow these steps: + +1. Build the operator's container using the following command: + +.. code-block:: bash + + ads opctl operator build-image -n + +This command creates a new ``:`` image with ``/etc/operator`` as the working directory within the container. + +2. Check the ``backend_operator_local_container_config.yaml`` configuration file. It should have a ``volume`` section with the ``.oci`` configs folder mounted, as shown below: + +.. code-block:: yaml + + volume: + - "/Users//.oci:/root/.oci" + +Mounting the OCI configs folder is necessary if you intend to use an OCI Object Storage bucket to store input and output data. You can also mount ``input/output`` folders to the container as needed. + +3. Run the operator within the container using this command:: + +.. code-block:: bash + + ads opctl apply -f .yaml -b backend_operator_local_container_config.yaml + + +Run Operator In Data Science Job +-------------------------------- + +There are several options for running the operator on the OCI Data Science Jobs service, such as using the conda environment runtime or the Bring Your Own Container (BYOC) approach. + +Run With BYOC (Bring Your Own Container) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To execute the operator within a Data Science job using container runtime, follow these steps: + +1. Build the container using the following command (you can skip this if you've already done it for running the operator within a local container): + +.. code-block:: bash + + ads opctl operator build-image -n + +This creates a new ``:`` image with ``/etc/operator`` as the working directory within the container. + +2. Publish the ``:`` container to the `Oracle Container Registry (OCR) `_. + +To publish ``:`` to OCR, use this command: + +.. code-block:: bash + + ads opctl operator publish-image : --registry + +After publishing the container to OCR, you can use it within Data Science jobs service. Check the ``backend_job_container_config.yaml`` configuration file built during initializing the starter configs for the operator. It should contain pre-populated infrastructure and runtime sections. The runtime section should have an image property, like ``image: iad.ocir.io//:``. + +3. Adjust the ``.yaml`` configuration with the proper input/output folders. When running operator in a Data Science job, it won't have access to local folders, so input data and output folders should be placed in the Object Storage bucket. Open the ``.yaml`` and adjust the data path fields. + +4. Run the operator on the Data Science jobs using this command: + +.. code-block:: bash + + ads opctl apply -f .yaml -b backend_job_container_config.yaml + +You can run the operator within the ``--dry-run`` attribute to check the final configs that will be used to run the operator on the service. + +Running the operator will return a command to help you monitor the job's logs: + +.. code-block:: bash + + ads opctl watch + +Run With Conda Environment +~~~~~~~~~~~~~~~~~~~~~~~~~~ + +To execute the operator within a Data Science job using the conda runtime, follow these steps: + +1. Build the operator's conda environment using this command:: + +.. code-block:: bash + + ads opctl operator build-conda -n + +This creates a new ``_`` conda environment and places it in the folder specified within the ``ads opctl configure`` command. + +2. Publish the ``_`` conda environment to the Object Storage bucket using this command:: + +.. code-block:: bash + + ads opctl conda publish _ + +For more details on configuring the CLI, refer to the :doc:`Explore & Configure Operators<./explore>` documentation. + +3. After publishing the conda environment to Object Storage, you can use it within the Data Science Jobs service. Check the ``backend_job_python_config.yaml`` configuration file, which should contain pre-populated infrastructure and runtime sections. The runtime section should include a ``conda`` section like this:: + +.. code-block:: yaml + + conda: + type: published + uri: oci://bucket@namespace/conda_environments/cpu///_ + +4. Adjust the ``.yaml`` configuration with the proper input/output folders. When running the operator in a Data Science job, it won't have access to local folders, so input data and output folders should be placed in the Object Storage bucket. + +5. Run the operator on the Data Science Jobs service using this command:: + +.. code-block:: bash + + ads opctl apply -f .yaml --backend-config backend_job_python_config.yaml + +6. Monitor the logs using the ``ads opctl watch`` command:: + +.. code-block:: bash + + ads opctl watch Data Flow Application --------------------- - - Policies - - Build Conda Environment - - Publish Conda Environment - - Run Operator + +To execute the operator within a Data Flow application follow these steps: + +1. Build the operator's conda environment using this command:: + +.. code-block:: bash + + ads opctl operator build-conda -n + +This creates a new ``_`` conda environment and places it in the folder specified within the ``ads opctl configure`` command. + +2. Publish the ``_`` conda environment to the Object Storage bucket using this command:: + +.. code-block:: bash + + ads opctl conda publish _ + +For more details on configuring the CLI, refer to the :doc:`Explore & Configure Operators<./explore>` documentation. + +After publishing the conda environment to Object Storage, you can use it within the Data Flow service. Check the ``backend_dataflow_dataflow_config.yaml`` configuration file, which should contain pre-populated infrastructure and runtime sections. The runtime section should include a ``conda`` section like this: + +.. code-block:: yaml + + conda: + type: published + uri: oci://bucket@namespace/conda_environments/cpu///_ + + +3. Adjust the ``.yaml`` configuration with the proper input/output folders. When running the operator in a Data Flow application, it won't have access to local folders, so input data and output folders should be placed in the Object Storage bucket. + +4. Run the operator on the Data Flow service using this command:: + +.. code-block:: bash + + ads opctl apply -f .yaml --backend-config backend_dataflow_dataflow_config.yaml + +5. Monitor the logs using the ``ads opctl watch`` command:: + +.. code-block:: bash + + ads opctl watch diff --git a/docs/source/user_guide/operators/forecasting_operator/index.rst b/docs/source/user_guide/operators/forecasting_operator/index.rst index 0ebf8d1b2..a4b64e28a 100644 --- a/docs/source/user_guide/operators/forecasting_operator/index.rst +++ b/docs/source/user_guide/operators/forecasting_operator/index.rst @@ -18,6 +18,7 @@ Overview :maxdepth: 1 ./getting_started + ./yaml_schema ./examples ./advanced_use_cases ./faq diff --git a/docs/source/user_guide/operators/forecasting_operator/yaml_schema.rst b/docs/source/user_guide/operators/forecasting_operator/yaml_schema.rst new file mode 100644 index 000000000..ee676332b --- /dev/null +++ b/docs/source/user_guide/operators/forecasting_operator/yaml_schema.rst @@ -0,0 +1,5 @@ +=========== +YAML Schema +=========== + +TBD