From 02e0dc96aed223d6951361633fc2f9860d9fa24d Mon Sep 17 00:00:00 2001 From: Chun-Mao Lai <72752478+Mecoli1219@users.noreply.github.com> Date: Sat, 21 Dec 2024 11:50:34 -0800 Subject: [PATCH] Jupyter notebook basic example (#1754) * Jupyter notebook basic example Signed-off-by: Mecoli1219 * Update jupyter notebook & fix doc Signed-off-by: Mecoli1219 * Update readme Signed-off-by: Mecoli1219 * Move to basics Signed-off-by: Mecoli1219 --------- Signed-off-by: Mecoli1219 --- .../basics/basic_interactive_mode.ipynb | 289 ++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 examples/basics/basics/basic_interactive_mode.ipynb diff --git a/examples/basics/basics/basic_interactive_mode.ipynb b/examples/basics/basics/basic_interactive_mode.ipynb new file mode 100644 index 000000000..8a6d89e26 --- /dev/null +++ b/examples/basics/basics/basic_interactive_mode.ipynb @@ -0,0 +1,289 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Basic Interactive Mode Usage in Jupyter Notebook Environment" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Before you start, make sure your flytekit version is higher than 1.14.0. If not, please upgrade it by running the following command:\n", + "```bash\n", + "!pip install flytekit==1.14.0\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 01 Setup the Interface to Connect to the Flyte Backend\n", + "\n", + "Before starting to develop and run the tasks and workflows remotely via Jupiter Notebook, let's setup the interface via [flytekit.remote](https://docs.flyte.org/projects/flytekit/en/latest/design/control_plane.html) to interact with the Flyte backend. \n", + "\n", + "If you are running in Jupyter Notebook, it should automatically enable the interactive mode. However, you can also enable it explicitly by setting `interactive_mode_enabled` to `True`. " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
21:39:52.768159 WARNING  remote.py:273 - Jupyter notebook and interactive task  \n",
+       "                         support is still alpha.                                \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[2;36m21:39:52.768159\u001b[0m\u001b[2;36m \u001b[0m\u001b[33mWARNING \u001b[0m remote.py:\u001b[1;36m273\u001b[0m - Jupyter notebook and interactive task \n", + "\u001b[2;36m \u001b[0m support is still alpha. \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from flytekit.remote import FlyteRemote\n", + "from flytekit.configuration import Config\n", + "\n", + "# The `for_sandbox` method instantiates a connection to the demo cluster.\n", + "remote = FlyteRemote(\n", + " Config.for_sandbox(), \n", + " default_project=\"flytesnacks\", \n", + " default_domain=\"development\", \n", + " interactive_mode_enabled=True, # This is not necessary if you are in a notebook.\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 02 Develop the tasks & workflows" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's create a simple task and workflow to demonstrate how to develop and run the tasks and workflows remotely via Jupiter Notebook." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from flytekit import task, workflow\n", + "\n", + "@task\n", + "def hello(name: str) -> str:\n", + " return f\"Hello {name}!\"\n", + "\n", + "@task\n", + "def world(pre: str) -> str:\n", + " return f\"{pre} Welcome to the Jupyter Notebook!\"\n", + "\n", + "@workflow\n", + "def wf(name: str) -> str:\n", + " return world(pre=hello(name=name))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "When executing the task or workflow via `FlyteRemote` interface, the related entities will be automatically registered to the Flyte backend. You can navigate to the Flyte Console via the URL to see the progress and outputs of the executions, or you can use `remote.wait()` to wait for the execution to complete. This may take a few seconds to minutes depending on the size of the task and workflow." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "http://localhost:30080/console/projects/flytesnacks/domains/development/executions/ah5cm5s482w9fl5jzz7c\n", + "{'o0': 'Hello Flyte!'}\n" + ] + } + ], + "source": [ + "# Execute the task\n", + "exe = remote.execute(hello, inputs={\"name\": \"Flyte\"})\n", + "\n", + "# This will print the URL to the console\n", + "print(exe.execution_url)\n", + "\n", + "# Wait for the task to complete\n", + "exe = exe.wait(poll_interval=1)\n", + "\n", + "# Print the outputs\n", + "print(exe.outputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you are not interested in the URL to the Flyte Console and want the execution wait directly to complete, you can set `wait=True` in the `remote.execute()` method." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'o0': 'Hello world!'}\n" + ] + } + ], + "source": [ + "# Execute the workflow and wait for it to complete\n", + "exe = remote.execute(hello, inputs={\"name\": \"world\"}, wait=True)\n", + "\n", + "# Print the outputs\n", + "print(exe.outputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Similarly, we can also execute the workflows remotely and extract the outputs." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'o0': 'Hello world! Welcome to the Jupyter Notebook!'}\n" + ] + } + ], + "source": [ + "# Execute the workflow and wait for it to complete\n", + "exe = remote.execute(wf, inputs={\"name\": \"world\"})\n", + "\n", + "# Wait for the task to complete\n", + "exe = exe.wait(poll_interval=1)\n", + "\n", + "# Print the outputs\n", + "print(exe.outputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you repeat the execution, it will not re-register the entities to the backend, instead it will reuse the existing entities. However, if you re-execute the cells that define the tasks or workflows, it will register the updated entities to the backend when you execute them.\n", + "\n", + "Feel free to update the tasks and workflows definition and see how it works!" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 03 Develop the Workflow Contains `map_task`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`map_task` is also supported in the remote execution. You can define the `map_task` in the workflow and execute it remotely." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "from flytekit import task, workflow, map_task\n", + "from functools import partial\n", + "\n", + "@task\n", + "def fn(x: int, y: int) -> int:\n", + " return x + y + 2\n", + "\n", + "@workflow\n", + "def workflow_with_maptask(data: list[int], y: int) -> list[int]:\n", + " partial_fn = partial(fn, y=y)\n", + " return map_task(partial_fn)(x=data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's execute the workflow remotely and extract the outputs." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{'o0': [7, 8, 9]}\n" + ] + } + ], + "source": [ + "# Execute the workflow with map_task and wait for it to complete\n", + "out = remote.execute(workflow_with_maptask, inputs={\"data\": [1, 2, 3], \"y\": 4})\n", + "\n", + "# Wait for the task to complete\n", + "out = out.wait(poll_interval=1)\n", + "\n", + "# Print the outputs\n", + "print(out.outputs)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "flytekit3.12", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}