Multiprocessing and Class.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d64323ff-3c4c-4128-ba50-c96f07fcb9cf",
   "metadata": {},
   "source": [
    "## Multiprocessing and Class"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86ab3a3e-4c3d-4601-988c-2545fac6dffa",
   "metadata": {},
   "source": [
    "$\\color{red}{\\text{why Multiprocessing not working on Windows and how to make it work}}$ "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e92aaca9-c795-4d99-87d8-70be41893758",
   "metadata": {},
   "source": [
    "### Define a class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e4bbf683-fb3e-48b0-a2a0-dcdb5ce6d85e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class Listener:\n",
    "    def __init__(self,opla_job):\n",
    "        self.opla_job=opla_job\n",
    "        self.lst=[]\n",
    "    def test(self):\n",
    "        for i in range(0,3):\n",
    "            self.lst.append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1ed63e30-4b64-4008-8c4f-75b25223d14d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ea8a7909-ce58-4857-9891-605527b5e5e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "active_lst=['listener1','listener2']\n",
    "all_active_process = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "8845553b-d80d-405e-ac64-3fc392e03efe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting to create a new process: listener1\n",
      "<__main__.Listener object at 0x000001637050F6A0>\n",
      "{'listener1': <Process name='listener1' pid=20484 parent=3364 started>}\n",
      "Starting to create a new process: listener2\n",
      "<__main__.Listener object at 0x000001637050F9A0>\n",
      "{'listener1': <Process name='listener1' pid=20484 parent=3364 stopped exitcode=1>, 'listener2': <Process name='listener2' pid=18984 parent=3364 started>}\n"
     ]
    }
   ],
   "source": [
    "for opla_job in active_lst:\n",
    "    print('Starting to create a new process:', opla_job)\n",
    "    instance = Listener(opla_job)\n",
    "    print(instance)\n",
    "    p = multiprocessing.Process(target=instance.test, name=opla_job)\n",
    "    p.start()\n",
    "    all_active_process[opla_job] = p\n",
    "    print(all_active_process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b058c9b3-01d8-439d-9421-9a5a5dd9cba5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'listener1': <Process name='listener1' pid=20484 parent=3364 stopped exitcode=1>,\n",
       " 'listener2': <Process name='listener2' pid=18984 parent=3364 stopped exitcode=1>}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "time.sleep(1)\n",
    "all_active_process"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ff0f782-f170-46dc-bb36-355f448ad3c1",
   "metadata": {},
   "source": [
    "### note exitcode = 1 above"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6de201c2-3e0c-4003-bab8-0d33aa9c95be",
   "metadata": {},
   "source": [
    "### To fix: add ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e538a521-c244-4f4c-8caf-3d437bf31bda",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting to create a new process: listener1\n",
      "<__main__.Listener object at 0x000001637050F550>\n",
      "{'listener1': <Process name='listener1' pid=12840 parent=3364 started>, 'listener2': <Process name='listener2' pid=18984 parent=3364 stopped exitcode=1>}\n",
      "Starting to create a new process: listener2\n",
      "<__main__.Listener object at 0x000001636CC5F100>\n",
      "{'listener1': <Process name='listener1' pid=12840 parent=3364 stopped exitcode=0>, 'listener2': <Process name='listener2' pid=2828 parent=3364 started>}\n"
     ]
    }
   ],
   "source": [
    "for opla_job in active_lst:\n",
    "    print('Starting to create a new process:', opla_job)\n",
    "    instance = Listener(opla_job)\n",
    "    print(instance)\n",
    "    p = multiprocessing.Process(target=instance.test(), name=opla_job)\n",
    "    p.start()\n",
    "    all_active_process[opla_job] = p\n",
    "    print(all_active_process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "47603644-c14d-49d1-89a2-765c65f889b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'listener1': <Process name='listener1' pid=12840 parent=3364 stopped exitcode=0>,\n",
       " 'listener2': <Process name='listener2' pid=2828 parent=3364 stopped exitcode=0>}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "time.sleep(1)\n",
    "all_active_process"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d0c72ca2-6eb3-409b-a335-18221322ec5a",
   "metadata": {},
   "source": [
    "### note exitcode = 0 above"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efaacdfc-47fb-44e0-b4a6-d25dc8e1f653",
   "metadata": {},
   "source": [
    "### what if use import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1bb138a0-3a2f-429b-8111-1f504fdf960a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from Listener import Listener\n",
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0b309187-573a-4217-9039-80c3e1358a06",
   "metadata": {},
   "outputs": [],
   "source": [
    "active_lst=['listener1','listener2']\n",
    "all_active_process = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "04845ffc-dc69-4781-b3c3-960fcbffe0cb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting to create a new process: listener1\n",
      "<Listener.Listener object at 0x0000024D81B99CA0>\n",
      "{'listener1': <Process name='listener1' pid=20336 parent=9184 started>}\n",
      "Starting to create a new process: listener2\n",
      "<Listener.Listener object at 0x0000024D81B99580>\n",
      "{'listener1': <Process name='listener1' pid=20336 parent=9184 stopped exitcode=0>, 'listener2': <Process name='listener2' pid=10840 parent=9184 started>}\n"
     ]
    }
   ],
   "source": [
    "for opla_job in active_lst:\n",
    "    print('Starting to create a new process:', opla_job)\n",
    "    instance = Listener(opla_job)\n",
    "    print(instance)\n",
    "    p = multiprocessing.Process(target=instance.test, name=opla_job)\n",
    "    p.start()\n",
    "    all_active_process[opla_job] = p\n",
    "    print(all_active_process)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3b38b893-cde1-4cad-9be0-c26455710fc7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'listener1': <Process name='listener1' pid=20336 parent=9184 stopped exitcode=0>,\n",
       " 'listener2': <Process name='listener2' pid=10840 parent=9184 stopped exitcode=0>}"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import time\n",
    "time.sleep(1)\n",
    "all_active_process"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a00cdec8-4a1d-4e86-bea0-bb30a019f677",
   "metadata": {},
   "source": [
    "### note exitcode = 0 above, but no need to add ()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8873279e-f318-4492-b397-63cc59ca1b3e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cb4c96b7-f057-4ab7-9eea-94e5802bc10f",
   "metadata": {},
   "outputs": [],
   "source": [
    "listener1=Listener('yy')\n",
    "listener2=Listener('qq')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7a08a20c-97ec-4ef5-bb2c-44c881e9a5d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "listener1.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d3ae691-b32e-45c0-84de-c2cc001ae4d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "listener1.lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "23d4a81c-8cfe-4cdf-9904-319029d37fb9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Listener.test of <Listener.Listener object at 0x0000026F9BF71B20>>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "listener2.test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "84a98cdd-8ab5-4d22-a7fb-1aaa0cac3795",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "listener2.lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0b8f6f4b-dfdd-4c4b-8b0e-696133498229",
   "metadata": {},
   "outputs": [],
   "source": [
    "listener2.test()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f549db46-81c6-4c2f-a45f-8d5cd538631f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0, 1, 2]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "listener2.lst"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "680ed03a-5f2f-4476-b2c4-aa65d3638522",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "04b3828f-19ff-4c38-a6e1-f3774aa035a4",
   "metadata": {},
   "source": [
    "# Root Problem I guess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "64f6c060-03b5-482a-88c6-63cf32141b20",
   "metadata": {},
   "outputs": [],
   "source": [
    "from multiprocessing import Process\n",
    "\n",
    "def f(name):\n",
    "    print('hello', name)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    p = Process(target=f, args=('bob',))\n",
    "    p.start()\n",
    "    p.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a7c52720-b94f-4e19-980e-b057fc96d941",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Process name='Process-1' pid=16000 parent=20956 stopped exitcode=1>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee4d7dd4-4d47-48e4-8e7d-10f1ca1c2d06",
   "metadata": {},
   "source": [
    "### $\\color{red}{\\text{Multiprocessing not working on Windows !!!}}$ "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6431320-bd67-448b-a1e5-d6a8c6c40353",
   "metadata": {},
   "source": [
    "**The Windows API only supports the spawn method of process creation.** In POSIX (except macOS), the default is the fork method, for which the child inherits the interactive main module of the parent. If you switch to the spawn method in Linux via **multiprocessing.set_start_method('spawn')**, you'll see the same error. \n",
    "\n",
    "**multiprocessing is one package where it's necessary in Windows to test examples using a script.** This is implied in the guidelines when it says to \"[m]ake sure that the main module can be safely imported by a new Python interpreter without causing unintended side effects (such a starting a new process)\". It's the fact the main module has to be importable by child processes that matters in this case. The behavior is noted with an example at the end of the introduction:\n",
    "\n",
    "    Note\n",
    "    Functionality within this package requires that the __main__ module\n",
    "    be importable by the children. This is covered in Programming \n",
    "    guidelines however it is worth pointing out here. This means that \n",
    "    some examples, such as the multiprocessing.pool.Pool examples will \n",
    "    not work in the interactive interpreter. For example: ...\n",
    "\n",
    "If it were up to me this note would be at the beginning of the introduction, where everyone would certainly see it. As is, the reader is expected to at least scan over the entire introduction.\n",
    "\n",
    "* **macOS defaults to the spawn method**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "83af85dc-fc3e-44a8-8be6-d4e9b851f392",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hello bob\n"
     ]
    }
   ],
   "source": [
    "from multiprocessing import Process\n",
    "\n",
    "def f(name):\n",
    "    print('hello', name)\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    p = Process(target=f('bob'))\n",
    "    p.start()\n",
    "    p.join()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "8a15a3fa-9419-46ab-86fe-5a7253a8c9f6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Process name='Process-4' pid=14664 parent=21148 stopped exitcode=0>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80c3056f-0c59-47e7-acb5-aa316596554b",
   "metadata": {},
   "source": [
    "### $\\color{red}{\\text{Although exitcode = 0 above, but it is not multiprocessing !!! Check below!!!}}$ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2765dd84-066d-4b55-8ed7-06becfd9b15f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sleeping for 2 seconds\n",
      "Finished sleeping\n",
      "Sleeping for 2 seconds\n",
      "Finished sleeping\n",
      "Program finished in 4.269025802612305 seconds\n"
     ]
    }
   ],
   "source": [
    "import multiprocessing\n",
    "import time\n",
    " \n",
    "def task():\n",
    "    print('Sleeping for 2 seconds')\n",
    "    time.sleep(2)\n",
    "    print('Finished sleeping')\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    start_time = time.time()\n",
    " \n",
    "    # Creates two processes\n",
    "    p1 = multiprocessing.Process(target=task())\n",
    "    p2 = multiprocessing.Process(target=task())\n",
    " \n",
    "    # Starts both processes\n",
    "    p1.start()\n",
    "    p2.start()\n",
    "    \n",
    "    # p1.join()\n",
    "    # p2.join()\n",
    " \n",
    "    finish_time = time.time()\n",
    "    \n",
    "    print(f\"Program finished in {finish_time-start_time} seconds\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bf60b16b-e6d5-40f8-96f3-46fe83e147a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Process name='Process-1' pid=22404 parent=13508 stopped exitcode=0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "13a2dc41-106b-453f-bd76-51cc2f0740ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Process name='Process-2' pid=21608 parent=13508 stopped exitcode=0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37a3169f-8088-45ec-afb3-b2d5e24d1611",
   "metadata": {},
   "source": [
    "### $\\color{red}{\\text{Not Multiprocessing!!! Program executed one by one in order!!!}}$ "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "632df0a4-26a3-4c44-945e-ce629fcd828c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "square 4\n",
      "square 9\n",
      "square 64\n",
      "cube 8\n",
      "cube 27\n",
      "cube 512\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "def calc_square(numbers):\n",
    "    for n in numbers:\n",
    "        print('square ' + str(n*n))\n",
    "\n",
    "def calc_cube(numbers):\n",
    "    for n in numbers:\n",
    "        print('cube ' + str(n*n*n))\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    arr = [2,3,8]\n",
    "    p1 = multiprocessing.Process(target=calc_square(arr))\n",
    "    p2 = multiprocessing.Process(target=calc_cube(arr))\n",
    "\n",
    "    p1.start()\n",
    "    p2.start()\n",
    "\n",
    "    p1.join()\n",
    "    p2.join()\n",
    "\n",
    "    print(\"Done!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "41475057-589c-4b1f-bdf4-d19c28b4093c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Process name='Process-3' pid=18996 parent=13508 stopped exitcode=0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d3cd3539-815c-421b-901e-e5736516fb5e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Process name='Process-4' pid=8876 parent=13508 stopped exitcode=0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7fd2989-82a1-4f97-8da9-1a3f5471d365",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}