upload_dir_worker_plugin.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "34312b35-79d6-4f74-b154-68ad7b4e9511",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: numpy==1.21.4 in /opt/conda/lib/python3.8/site-packages (1.21.4)\n",
      "Requirement already satisfied: pandas==1.3.4 in /opt/conda/lib/python3.8/site-packages (1.3.4)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /opt/conda/lib/python3.8/site-packages (from pandas==1.3.4) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in /opt/conda/lib/python3.8/site-packages (from pandas==1.3.4) (2021.3)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.8/site-packages (from python-dateutil>=2.7.3->pandas==1.3.4) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install numpy==1.21.4 pandas==1.3.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ae1e3070-5b0d-484f-9810-f878b3701816",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'tcp://172.17.0.10:39093': (['.ipynb_checkpoints', 'coffea_casa.py'],\n",
       "  ['/dask-worker-space/worker-juplvzdj',\n",
       "   '/opt/conda/bin',\n",
       "   '/opt/conda/lib/python38.zip',\n",
       "   '/opt/conda/lib/python3.8',\n",
       "   '/opt/conda/lib/python3.8/lib-dynload',\n",
       "   '/opt/conda/lib/python3.8/site-packages']),\n",
       " 'tcp://172.17.0.5:40989': (['.ipynb_checkpoints', 'coffea_casa.py'],\n",
       "  ['/dask-worker-space/worker-ry_w50vg',\n",
       "   '/opt/conda/bin',\n",
       "   '/opt/conda/lib/python38.zip',\n",
       "   '/opt/conda/lib/python3.8',\n",
       "   '/opt/conda/lib/python3.8/lib-dynload',\n",
       "   '/opt/conda/lib/python3.8/site-packages']),\n",
       " 'tcp://172.17.0.8:42699': (['.ipynb_checkpoints', 'coffea_casa.py'],\n",
       "  ['/dask-worker-space/worker-h8w1izet',\n",
       "   '/opt/conda/bin',\n",
       "   '/opt/conda/lib/python38.zip',\n",
       "   '/opt/conda/lib/python3.8',\n",
       "   '/opt/conda/lib/python3.8/lib-dynload',\n",
       "   '/opt/conda/lib/python3.8/site-packages'])}"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os, pathlib\n",
    "from distributed import Client\n",
    "from distributed.diagnostics.plugin import UploadDirectory\n",
    "\n",
    "directory = \"tmp_test\"\n",
    "\n",
    "if not os.path.exists(directory):\n",
    "    os.makedirs(directory)\n",
    "\n",
    "with open(pathlib.Path(directory) / \"foo.py\", \"w\") as f:\n",
    "    f.write(\"x = 123\")\n",
    "with open(pathlib.Path(directory) / \"bar.py\", \"w\") as f:\n",
    "    f.write(\"from foo import x\")\n",
    "\n",
    "client = Client(\"tcp://172.17.0.7:8786\")\n",
    "client.register_worker_plugin(UploadDirectory(directory, restart=False, update_path=True), nanny=True)\n",
    "\n",
    "\n",
    "def get_info():\n",
    "    import sys, os\n",
    "    return os.listdir('/dask-worker-space/example_upload_dir'), sys.path\n",
    "    import bar\n",
    "    return bar.x\n",
    "\n",
    "client.run(get_info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "221f8254-973d-4350-8895-e104bdd1b84d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}