Multi-Processing and Lock.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import multiprocessing\n",
    "from multiprocessing import Pool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Pool (map and reduce: divide the task and aggregate after finish)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f(n):\n",
    "    sum=0\n",
    "    for x in range(1000):\n",
    "        sum+=x*x\n",
    "    return sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pool took:  2.861501932144165\n",
      "Serial Processing took:  10.514475107192993\n"
     ]
    }
   ],
   "source": [
    "if __name__=='__main__':\n",
    "    t1=time.time()\n",
    "    p=Pool(processes=12)\n",
    "    result = p.map(f,range(100000))\n",
    "    p.close()\n",
    "    p.join()\n",
    "    print('Pool took: ', time.time()-t1)\n",
    "    \n",
    "    t2=time.time()\n",
    "    result=[]\n",
    "    for x in range(100000):\n",
    "        result.append(f(x))\n",
    "    print('Serial Processing took: ', time.time()-t2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### normal multiple processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "square 4\n",
      "square 9\n",
      "cube 8\n",
      "square 64\n",
      "cube 27\n",
      "cube 512\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "def calc_square(numbers):\n",
    "    for n in numbers:\n",
    "        print('square ' + str(n*n))\n",
    "\n",
    "def calc_cube(numbers):\n",
    "    for n in numbers:\n",
    "        print('cube ' + str(n*n*n))\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    arr = [2,3,8]\n",
    "    p1 = multiprocessing.Process(target=calc_square, args=(arr,))\n",
    "    p2 = multiprocessing.Process(target=calc_cube, args=(arr,))\n",
    "\n",
    "    p1.start()\n",
    "    p2.start()\n",
    "\n",
    "    p1.join()\n",
    "    p2.join()\n",
    "\n",
    "    print(\"Done!\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### threading vs processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import multiprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "square 4\n",
      "square 9\n",
      "square 64\n",
      "square 81\n",
      "outside result[]\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "square_result=[]\n",
    "def calc_square(numbers):\n",
    "    global square_result\n",
    "    for n in numbers:\n",
    "        print('square ' + str(n*n))\n",
    "        square_result.append(n*n)\n",
    "if __name__=='__main__':\n",
    "    arr=[2,3,8,9]\n",
    "    p1=multiprocessing.Process(target=calc_square,args=(arr,))\n",
    "    p1.start()\n",
    "    p1.join()\n",
    "    print('outside result' + str(square_result))\n",
    "    print('Done!')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Comment**: \n",
    "When create a new process what will hapeen is it will create a new copy of square_result, global variable.\n",
    "**Every process has its own address space.** Thus program vaiables are not shared between two processes.\n",
    "**This is the key difference between multi-threading and multi-processing.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "square 4\n",
      "square 9\n",
      "square 64\n",
      "square 81\n",
      "within a process: result  [4, 9, 64, 81]\n",
      "outside result[]\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "square_result=[]\n",
    "def calc_square(numbers):\n",
    "    global square_result\n",
    "    for n in numbers:\n",
    "        print('square ' + str(n*n))\n",
    "        square_result.append(n*n)\n",
    "    print('within a process: result ', square_result)\n",
    "if __name__=='__main__':\n",
    "    arr=[2,3,8,9]\n",
    "    p1=multiprocessing.Process(target=calc_square,args=(arr,))\n",
    "    p1.start()\n",
    "    p1.join()\n",
    "    print('outside result' + str(square_result)) # this is outside the process\n",
    "    print('Done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "9\n",
      "25\n"
     ]
    }
   ],
   "source": [
    "# or use queue\n",
    "import multiprocessing\n",
    "\n",
    "def calc_square(numbers, q):\n",
    "    for n in numbers:\n",
    "        q.put(n*n)\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    numbers = [2,3,5]\n",
    "    q = multiprocessing.Queue()\n",
    "    p = multiprocessing.Process(target=calc_square, args=(numbers,q))\n",
    "\n",
    "    p.start()\n",
    "    p.join()\n",
    "\n",
    "    while q.empty() is False:\n",
    "        print(q.get())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "multiprocessing queue\n",
    "import multiprocessing\n",
    "q=multiprocessing.Queue()\n",
    ". lives in shared memory\n",
    ". used to share data between processes\n",
    "\n",
    "queue module\n",
    "import queue\n",
    "q=queue.Queue()\n",
    ".live in in-process memory\n",
    ". used to share data between threads"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### lock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def deposit(balance, lock):\n",
    "    for i in range(100):\n",
    "        time.sleep(0.01)\n",
    "        lock.acquire()\n",
    "        balance.value = balance.value + 1\n",
    "        lock.release()\n",
    "\n",
    "def withdraw(balance, lock):\n",
    "    for i in range(100):\n",
    "        time.sleep(0.01)\n",
    "        lock.acquire()\n",
    "        balance.value = balance.value - 1\n",
    "        lock.release()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    balance = multiprocessing.Value('i', 200) # share between different processes !!!\n",
    "    lock = multiprocessing.Lock()\n",
    "    d = multiprocessing.Process(target=deposit, args=(balance,lock))\n",
    "    w = multiprocessing.Process(target=withdraw, args=(balance,lock))\n",
    "    d.start()\n",
    "    w.start()\n",
    "    d.join()\n",
    "    w.join()\n",
    "    print(balance.value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### If without using lock, the result will be different each time !!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "194\n"
     ]
    }
   ],
   "source": [
    "def deposit(balance):\n",
    "    for i in range(100):\n",
    "        time.sleep(0.01)\n",
    "        balance.value = balance.value + 1\n",
    "\n",
    "def withdraw(balance):\n",
    "    for i in range(100):\n",
    "        time.sleep(0.01)\n",
    "        balance.value = balance.value - 1\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    balance = multiprocessing.Value('i', 200)\n",
    "    d = multiprocessing.Process(target=deposit, args=(balance,))\n",
    "    w = multiprocessing.Process(target=withdraw, args=(balance,))\n",
    "    d.start()\n",
    "    w.start()\n",
    "    d.join()\n",
    "    w.join()\n",
    "    print(balance.value)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### multiple threading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "calculate square numberscalculate cube of numbers\n",
      "\n",
      "cube:square: 8\n",
      " 4\n",
      "square:cube: 9\n",
      " 27\n",
      "cube:square: 512\n",
      " 64\n",
      "cube:square: 729\n",
      " 81\n",
      "done in :  4.0178937911987305\n",
      "Hah... I am done with all my work now!\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import threading\n",
    "\n",
    "def calc_square(numbers):\n",
    "    print(\"calculate square numbers\")\n",
    "    for n in numbers:\n",
    "        time.sleep(1)\n",
    "        print('square:',n*n)\n",
    "\n",
    "def calc_cube(numbers):\n",
    "    print(\"calculate cube of numbers\")\n",
    "    for n in numbers:\n",
    "        time.sleep(1)\n",
    "        print('cube:',n*n*n)\n",
    "\n",
    "arr = [2,3,8,9]\n",
    "\n",
    "t = time.time()\n",
    "\n",
    "t1= threading.Thread(target=calc_square, args=(arr,))\n",
    "t2= threading.Thread(target=calc_cube, args=(arr,))\n",
    "\n",
    "t1.start()\n",
    "t2.start()\n",
    "\n",
    "t1.join()\n",
    "t2.join()\n",
    "\n",
    "print(\"done in : \",time.time()-t)\n",
    "print(\"Hah... I am done with all my work now!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}