From faac87a648b971ef932b3f2ee2b6db6ae8f3bf4a Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Mon, 27 Sep 2021 08:09:10 +0100 Subject: [PATCH 1/3] Fix metaflow local/remote decision and update to 0.5.1 release --- docs/examples/metaflow/README.ipynb | 12 ++---------- docs/examples/metaflow/README.md | 9 --------- docs/examples/metaflow/src/irisflow.py | 10 ++++++---- tempo/metaflow/utils.py | 24 ++++++++++++++++++++++-- tempo/version.py | 2 +- 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/docs/examples/metaflow/README.ipynb b/docs/examples/metaflow/README.ipynb index de3be25..e6489c6 100644 --- a/docs/examples/metaflow/README.ipynb +++ b/docs/examples/metaflow/README.ipynb @@ -53,13 +53,7 @@ "id": "b2a246ca", "metadata": {}, "source": [ - "## Run Flow locally to deploy to Docker\n", - "\n", - "To run the workflow with a local Docker deployment use the flag:\n", - "\n", - "```\n", - "--tempo-on-docker true\n", - "```\n" + "## Run Flow locally to deploy to Docker" ] }, { @@ -105,8 +99,6 @@ "\n", "### Setup AWS Metaflow Support\n", "\n", - "Note at present this is required even for a local run as artifacts are stored on S3.\n", - "\n", "[Install Metaflow with remote AWS support](https://docs.metaflow.org/metaflow-on-aws/metaflow-on-aws).\n", "\n", "### Seldon Requirements\n", @@ -313,7 +305,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.10" + "version": "3.7.9" } }, "nbformat": 4, diff --git a/docs/examples/metaflow/README.md b/docs/examples/metaflow/README.md index e24446e..ef62ad1 100644 --- a/docs/examples/metaflow/README.md +++ b/docs/examples/metaflow/README.md @@ -32,13 +32,6 @@ conda config --add channels conda-forge ## Run Flow locally to deploy to Docker -To run the workflow with a local Docker deployment use the flag: - -``` ---tempo-on-docker true -``` - - ```python !python src/irisflow.py --environment=conda run @@ -61,8 +54,6 @@ We will now run our flow on AWS Batch and will launch Tempo artifacts onto a rem ### Setup AWS Metaflow Support -Note at present this is required even for a local run as artifacts are stored on S3. - [Install Metaflow with remote AWS support](https://docs.metaflow.org/metaflow-on-aws/metaflow-on-aws). ### Seldon Requirements diff --git a/docs/examples/metaflow/src/irisflow.py b/docs/examples/metaflow/src/irisflow.py index d8c4d78..b7bf413 100644 --- a/docs/examples/metaflow/src/irisflow.py +++ b/docs/examples/metaflow/src/irisflow.py @@ -162,7 +162,7 @@ def deploy_tempo_remote(self, classifier): print(self.client_model.predict(np.array([[1, 2, 3, 4]]))) @conda(libraries={"numpy": "1.19.5"}) - @pip(libraries={"mlops-tempo": "0.5.0", "conda_env": "2.4.2"}) + @pip(libraries={"mlops-tempo": "0.5.1", "conda_env": "2.4.2"}) @step def tempo(self): """ @@ -170,11 +170,13 @@ def tempo(self): Then either deploy locally to Docker or deploy to a remote Kubernetes cluster based on the --tempo-on-docker parameter """ - classifier, remote = self.create_tempo_artifacts() - - if remote: + from tempo.metaflow.utils import running_aws_batch + classifier, s3_active = self.create_tempo_artifacts() + if s3_active and running_aws_batch(self.tempo): + print("Deploying to remote k8s cluster") self.deploy_tempo_remote(classifier) else: + print("Deploying to local Docker") self.deploy_tempo_local(classifier) self.next(self.end) diff --git a/tempo/metaflow/utils.py b/tempo/metaflow/utils.py index 022cb38..4238996 100644 --- a/tempo/metaflow/utils.py +++ b/tempo/metaflow/utils.py @@ -1,7 +1,7 @@ from typing import Any -from metaflow import S3, FlowSpec, IncludeFile - +from metaflow import S3, FlowSpec, IncludeFile, Step +from metaflow.plugins.aws.batch.batch_decorator import BatchDecorator def save_artifact(model: Any, filename: str): """ @@ -178,3 +178,23 @@ def save_pipeline_with_conda(pipeline, folder: str, conda_env: IncludeFile): with open(conda_env_path, "w") as f: f.write(conda_env) save(pipeline) + + +def running_aws_batch(step : Step) -> bool: + """ + Test if a Step is running on AWS batch + Parameters + ---------- + step The step to test + + Returns + ------- + True if flow is running on AWS Batch + + """ + running_on_aws_batch = False + print(step.decorators) # pylint: disable=maybe-no-member + for deco in step.decorators: # pylint: disable=maybe-no-member + if isinstance(deco, BatchDecorator): + running_on_aws_batch = True + return running_on_aws_batch diff --git a/tempo/version.py b/tempo/version.py index 3d18726..dd9b22c 100644 --- a/tempo/version.py +++ b/tempo/version.py @@ -1 +1 @@ -__version__ = "0.5.0" +__version__ = "0.5.1" From 6d777433e158c5baef3c844e5dc128b481d3bd39 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Mon, 27 Sep 2021 08:11:38 +0100 Subject: [PATCH 2/3] fmt --- docs/examples/metaflow/src/irisflow.py | 1 + tempo/metaflow/utils.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/examples/metaflow/src/irisflow.py b/docs/examples/metaflow/src/irisflow.py index b7bf413..0cedf44 100644 --- a/docs/examples/metaflow/src/irisflow.py +++ b/docs/examples/metaflow/src/irisflow.py @@ -171,6 +171,7 @@ def tempo(self): --tempo-on-docker parameter """ from tempo.metaflow.utils import running_aws_batch + classifier, s3_active = self.create_tempo_artifacts() if s3_active and running_aws_batch(self.tempo): print("Deploying to remote k8s cluster") diff --git a/tempo/metaflow/utils.py b/tempo/metaflow/utils.py index 4238996..a6eae99 100644 --- a/tempo/metaflow/utils.py +++ b/tempo/metaflow/utils.py @@ -3,6 +3,7 @@ from metaflow import S3, FlowSpec, IncludeFile, Step from metaflow.plugins.aws.batch.batch_decorator import BatchDecorator + def save_artifact(model: Any, filename: str): """ @@ -180,7 +181,7 @@ def save_pipeline_with_conda(pipeline, folder: str, conda_env: IncludeFile): save(pipeline) -def running_aws_batch(step : Step) -> bool: +def running_aws_batch(step: Step) -> bool: """ Test if a Step is running on AWS batch Parameters From 238ad3ae6baf1cd5fb7e059d59050798cd0107d6 Mon Sep 17 00:00:00 2001 From: Clive Cox Date: Mon, 27 Sep 2021 16:44:48 +0100 Subject: [PATCH 3/3] remove print statement --- tempo/metaflow/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tempo/metaflow/utils.py b/tempo/metaflow/utils.py index a6eae99..4bd5901 100644 --- a/tempo/metaflow/utils.py +++ b/tempo/metaflow/utils.py @@ -194,7 +194,6 @@ def running_aws_batch(step: Step) -> bool: """ running_on_aws_batch = False - print(step.decorators) # pylint: disable=maybe-no-member for deco in step.decorators: # pylint: disable=maybe-no-member if isinstance(deco, BatchDecorator): running_on_aws_batch = True