Skip to content

Commit

Permalink
Merge pull request #354 from v3io/development
Browse files Browse the repository at this point in the history 
Remove magic commands and fix scripts
  • Loading branch information
@gilad-shaham
gilad-shaham authored Nov 8, 2021
2 parents 3f1ff16 + 1848f5d commit d4e7795
Show file tree
Hide file tree
Showing 13 changed files with 163 additions and 412 deletions.
129 changes: 63 additions & 66 deletions data-ingestion-and-preparation/dask-cluster.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Both server & client are aligned (0.6.0rc13).\n"
"Both server & client are aligned (0.7.1).\n"
]
}
],
Expand Down Expand Up @@ -79,7 +79,7 @@
"text": [
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
" Dload Upload Total Spent Left Speed\n",
"100 84.9M 100 84.9M 0 0 8206k 0 0:00:10 0:00:10 --:--:-- 8639k\n"
"100 84.9M 100 84.9M 0 0 21.7M 0 0:00:03 0:00:03 --:--:-- 21.7M\n"
]
}
],
Expand All @@ -103,26 +103,6 @@
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"%nuclio: setting kind to 'job'\n",
"%nuclio: setting spec.image to 'mlrun/ml-models'\n"
]
}
],
"source": [
"# nuclio: start-code\n",
"%nuclio config kind = \"job\"\n",
"%nuclio config spec.image = \"mlrun/ml-models\""
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"from mlrun.execution import MLClientCtx\n",
Expand Down Expand Up @@ -157,7 +137,7 @@
},
{
"cell_type": "code",
"execution_count": 7,
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -178,7 +158,7 @@
},
{
"cell_type": "code",
"execution_count": 8,
"execution_count": 7,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -194,9 +174,17 @@
},
{
"cell_type": "code",
"execution_count": 9,
"execution_count": 8,
"metadata": {},
"outputs": [],
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"> 2021-10-12 09:43:54,745 [info] loaded project default from MLRun DB\n"
]
}
],
"source": [
"import mlrun\n",
"artifact_path = mlrun.set_environment(api_path = mlrun.mlconf.dbpath or 'http://mlrun-api:8080',\n",
Expand All @@ -220,7 +208,7 @@
},
{
"cell_type": "code",
"execution_count": 10,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -236,24 +224,16 @@
},
{
"cell_type": "code",
"execution_count": 11,
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"> 2021-02-04 19:07:34,933 [info] using in-cluster config.\n"
]
}
],
"outputs": [],
"source": [
"dsf = mlrun.new_function(\"dask_init\", kind='dask', image='mlrun/ml-models').apply(mlrun.mount_v3io())"
]
},
{
"cell_type": "code",
"execution_count": 12,
"execution_count": 11,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -266,21 +246,21 @@
},
{
"cell_type": "code",
"execution_count": 13,
"execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"> 2021-02-04 19:07:35,158 [info] trying dask client at: tcp://mlrun-dask-init-b7725876-7.default-tenant:8786\n",
"> 2021-02-04 19:07:35,234 [info] using remote dask scheduler (mlrun-dask-init-b7725876-7) at: tcp://mlrun-dask-init-b7725876-7.default-tenant:8786\n"
"> 2021-10-12 09:44:06,095 [info] trying dask client at: tcp://mlrun-dask-init-b76f44f5-0.default-tenant:8786\n",
"> 2021-10-12 09:44:06,116 [info] using remote dask scheduler (mlrun-dask-init-b76f44f5-0) at: tcp://mlrun-dask-init-b76f44f5-0.default-tenant:8786\n"
]
},
{
"data": {
"text/html": [
"<a href=\"http://default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com:30053/status\" target=\"_blank\" >dashboard link: default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com:30053</a>"
"<a href=\"http://default-tenant.app.app-lab-eks-testing.iguazio-cd1.com:30990/status\" target=\"_blank\" >dashboard link: default-tenant.app.app-lab-eks-testing.iguazio-cd1.com:30990</a>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
Expand All @@ -297,26 +277,26 @@
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Client</h3>\n",
"<ul style=\"text-align: left; list-style: none; margin: 0; padding: 0;\">\n",
" <li><b>Scheduler: </b>tcp://mlrun-dask-init-b7725876-7.default-tenant:8786</li>\n",
" <li><b>Dashboard: </b><a href='http://mlrun-dask-init-b7725876-7.default-tenant:8787/status' target='_blank'>http://mlrun-dask-init-b7725876-7.default-tenant:8787/status</a></li>\n",
" <li><b>Scheduler: </b>tcp://mlrun-dask-init-b76f44f5-0.default-tenant:8786</li>\n",
" <li><b>Dashboard: </b><a href='http://mlrun-dask-init-b76f44f5-0.default-tenant:8787/status' target='_blank'>http://mlrun-dask-init-b76f44f5-0.default-tenant:8787/status</a></li>\n",
"</ul>\n",
"</td>\n",
"<td style=\"vertical-align: top; border: 0px solid white\">\n",
"<h3 style=\"text-align: left;\">Cluster</h3>\n",
"<ul style=\"text-align: left; list-style:none; margin: 0; padding: 0;\">\n",
" <li><b>Workers: </b>2</li>\n",
" <li><b>Cores: </b>2</li>\n",
" <li><b>Memory: </b>8.31 GB</li>\n",
" <li><b>Memory: </b>8.34 GB</li>\n",
"</ul>\n",
"</td>\n",
"</tr>\n",
"</table>"
],
"text/plain": [
"<Client: 'tcp://10.200.0.59:8786' processes=2 threads=2, memory=8.31 GB>"
"<Client: 'tcp://172.31.13.112:8786' processes=2 threads=2, memory=8.34 GB>"
]
},
"execution_count": 13,
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
Expand All @@ -335,7 +315,7 @@
},
{
"cell_type": "code",
"execution_count": 14,
"execution_count": 13,
"metadata": {},
"outputs": [],
"source": [
Expand All @@ -360,16 +340,16 @@
},
{
"cell_type": "code",
"execution_count": 15,
"execution_count": 14,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"> 2021-02-04 19:07:35,288 [info] starting run test-dask-test_dask uid=4973d57c968747e491d6fcaa401968e0 DB=http://mlrun-api:8080\n",
"> 2021-02-04 19:07:35,769 [info] Job is running in the background, pod: test-dask-test-dask-4m29c\n",
"> 2021-02-04 19:07:56,270 [info] run executed, status=completed\n",
"> 2021-10-12 09:44:06,154 [info] starting run test-dask-test_dask uid=3d3ec4b558a140b5849700dc5d77c97c DB=http://mlrun-api:8080\n",
"> 2021-10-12 09:44:06,317 [info] Job is running in the background, pod: test-dask-test-dask-w28ff\n",
"> 2021-10-12 09:44:20,436 [info] run executed, status=completed\n",
"final state: completed\n"
]
},
Expand Down Expand Up @@ -544,26 +524,26 @@
" <tbody>\n",
" <tr>\n",
" <td>default</td>\n",
" <td><div title=\"4973d57c968747e491d6fcaa401968e0\"><a href=\"https://dashboard.default-tenant.app.bsmhzmwkycjf.iguazio-cd2.com/mlprojects/default/jobs/monitor/4973d57c968747e491d6fcaa401968e0/info\" target=\"_blank\" >...401968e0</a></div></td>\n",
" <td><div title=\"3d3ec4b558a140b5849700dc5d77c97c\"><a href=\"https://dashboard.default-tenant.app.app-lab-eks-testing.iguazio-cd1.com/mlprojects/default/jobs/monitor/3d3ec4b558a140b5849700dc5d77c97c/overview\" target=\"_blank\" >...5d77c97c</a></div></td>\n",
" <td>0</td>\n",
" <td>Feb 04 19:07:42</td>\n",
" <td>Oct 12 09:44:11</td>\n",
" <td>completed</td>\n",
" <td>test-dask-test_dask</td>\n",
" <td><div class=\"dictlist\">v3io_user=admin</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=admin</div><div class=\"dictlist\">host=test-dask-test-dask-4m29c</div></td>\n",
" <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"resultf45735f7\" title=\"/files/examples/ytrip.csv\">dataset</div></td>\n",
" <td><div class=\"dictlist\">dask_client=tcp://mlrun-dask-init-b7725876-7.default-tenant:8786</div></td>\n",
" <td><div class=\"dictlist\">v3io_user=dani</div><div class=\"dictlist\">kind=job</div><div class=\"dictlist\">owner=dani</div><div class=\"dictlist\">host=test-dask-test-dask-w28ff</div></td>\n",
" <td><div class=\"artifact\" onclick=\"expandPanel(this)\" paneName=\"result46fb98e5\" title=\"/files/examples/ytrip.csv\">dataset</div></td>\n",
" <td><div class=\"dictlist\">dask_client=tcp://mlrun-dask-init-b76f44f5-0.default-tenant:8786</div></td>\n",
" <td></td>\n",
" <td></td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div></div>\n",
" <div id=\"resultf45735f7-pane\" class=\"right-pane block hidden\">\n",
" <div id=\"result46fb98e5-pane\" class=\"right-pane block hidden\">\n",
" <div class=\"pane-header\">\n",
" <span id=\"resultf45735f7-title\" class=\"pane-header-title\">Title</span>\n",
" <span onclick=\"closePanel(this)\" paneName=\"resultf45735f7\" class=\"close clickable\">&times;</span>\n",
" <span id=\"result46fb98e5-title\" class=\"pane-header-title\">Title</span>\n",
" <span onclick=\"closePanel(this)\" paneName=\"result46fb98e5\" class=\"close clickable\">&times;</span>\n",
" </div>\n",
" <iframe class=\"fileview\" id=\"resultf45735f7-body\"></iframe>\n",
" <iframe class=\"fileview\" id=\"result46fb98e5-body\"></iframe>\n",
" </div>\n",
"</div>\n"
],
Expand All @@ -578,18 +558,35 @@
"name": "stdout",
"output_type": "stream",
"text": [
"to track results use .show() or .logs() or in CLI: \n",
"!mlrun get run 4973d57c968747e491d6fcaa401968e0 --project default , !mlrun logs 4973d57c968747e491d6fcaa401968e0 --project default\n",
"> 2021-02-04 19:08:05,372 [info] run executed, status=completed\n"
"\n"
]
},
{
"data": {
"text/html": [
"<b> > to track results use the .show() or .logs() methods or <a href=\"https://dashboard.default-tenant.app.app-lab-eks-testing.iguazio-cd1.com/mlprojects/default/jobs/monitor/3d3ec4b558a140b5849700dc5d77c97c/overview\" target=\"_blank\">click here</a> to open in UI</b>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"> 2021-10-12 09:44:25,557 [info] run executed, status=completed\n"
]
},
{
"data": {
"text/plain": [
"<mlrun.model.RunObject at 0x7efe58c67c50>"
"<mlrun.model.RunObject at 0x7fe5ec4a1d50>"
]
},
"execution_count": 15,
"execution_count": 14,
"metadata": {},
"output_type": "execute_result"
}
Expand Down
Loading

0 comments on commit d4e7795

Please sign in to comment.