Final_Metrics_DenseNet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Final Metrics DenseNet.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOtHFTf7q4V2FgPIBoF8hMv",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ManasChandan/Pneumonia-Detection/blob/master/Final_Metrics_DenseNet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Mn0yhrj912d"
      },
      "source": [
        "from google.colab import drive"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "c6mtqE0HMzVa"
      },
      "source": [
        "!pip install -q kaggle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "adpRJ31EM1wv",
        "colab": {
          "resources": {
            "http://localhost:8080/nbextensions/google.colab/files.js": {
              "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7CgpmdW5jdGlvbiBfdXBsb2FkRmlsZXMoaW5wdXRJZCwgb3V0cHV0SWQpIHsKICBjb25zdCBzdGVwcyA9IHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCk7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICAvLyBDYWNoZSBzdGVwcyBvbiB0aGUgb3V0cHV0RWxlbWVudCB0byBtYWtlIGl0IGF2YWlsYWJsZSBmb3IgdGhlIG5leHQgY2FsbAogIC8vIHRvIHVwbG9hZEZpbGVzQ29udGludWUgZnJvbSBQeXRob24uCiAgb3V0cHV0RWxlbWVudC5zdGVwcyA9IHN0ZXBzOwoKICByZXR1cm4gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpOwp9CgovLyBUaGlzIGlzIHJvdWdobHkgYW4gYXN5bmMgZ2VuZXJhdG9yIChub3Qgc3VwcG9ydGVkIGluIHRoZSBicm93c2VyIHlldCksCi8vIHdoZXJlIHRoZXJlIGFyZSBtdWx0aXBsZSBhc3luY2hyb25vdXMgc3RlcHMgYW5kIHRoZSBQeXRob24gc2lkZSBpcyBnb2luZwovLyB0byBwb2xsIGZvciBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcC4KLy8gVGhpcyB1c2VzIGEgUHJvbWlzZSB0byBibG9jayB0aGUgcHl0aG9uIHNpZGUgb24gY29tcGxldGlvbiBvZiBlYWNoIHN0ZXAsCi8vIHRoZW4gcGFzc2VzIHRoZSByZXN1bHQgb2YgdGhlIHByZXZpb3VzIHN0ZXAgYXMgdGhlIGlucHV0IHRvIHRoZSBuZXh0IHN0ZXAuCmZ1bmN0aW9uIF91cGxvYWRGaWxlc0NvbnRpbnVlKG91dHB1dElkKSB7CiAgY29uc3Qgb3V0cHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKG91dHB1dElkKTsKICBjb25zdCBzdGVwcyA9IG91dHB1dEVsZW1lbnQuc3RlcHM7CgogIGNvbnN0IG5leHQgPSBzdGVwcy5uZXh0KG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSk7CiAgcmV0dXJuIFByb21pc2UucmVzb2x2ZShuZXh0LnZhbHVlLnByb21pc2UpLnRoZW4oKHZhbHVlKSA9PiB7CiAgICAvLyBDYWNoZSB0aGUgbGFzdCBwcm9taXNlIHZhbHVlIHRvIG1ha2UgaXQgYXZhaWxhYmxlIHRvIHRoZSBuZXh0CiAgICAvLyBzdGVwIG9mIHRoZSBnZW5lcmF0b3IuCiAgICBvdXRwdXRFbGVtZW50Lmxhc3RQcm9taXNlVmFsdWUgPSB2YWx1ZTsKICAgIHJldHVybiBuZXh0LnZhbHVlLnJlc3BvbnNlOwogIH0pOwp9CgovKioKICogR2VuZXJhdG9yIGZ1bmN0aW9uIHdoaWNoIGlzIGNhbGxlZCBiZXR3ZWVuIGVhY2ggYXN5bmMgc3RlcCBvZiB0aGUgdXBsb2FkCiAqIHByb2Nlc3MuCiAqIEBwYXJhbSB7c3RyaW5nfSBpbnB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIGlucHV0IGZpbGUgcGlja2VyIGVsZW1lbnQuCiAqIEBwYXJhbSB7c3RyaW5nfSBvdXRwdXRJZCBFbGVtZW50IElEIG9mIHRoZSBvdXRwdXQgZGlzcGxheS4KICogQHJldHVybiB7IUl0ZXJhYmxlPCFPYmplY3Q+fSBJdGVyYWJsZSBvZiBuZXh0IHN0ZXBzLgogKi8KZnVuY3Rpb24qIHVwbG9hZEZpbGVzU3RlcChpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IGlucHV0RWxlbWVudCA9IGRvY3VtZW50LmdldEVsZW1lbnRCeUlkKGlucHV0SWQpOwogIGlucHV0RWxlbWVudC5kaXNhYmxlZCA9IGZhbHNlOwoKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIG91dHB1dEVsZW1lbnQuaW5uZXJIVE1MID0gJyc7CgogIGNvbnN0IHBpY2tlZFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgaW5wdXRFbGVtZW50LmFkZEV2ZW50TGlzdGVuZXIoJ2NoYW5nZScsIChlKSA9PiB7CiAgICAgIHJlc29sdmUoZS50YXJnZXQuZmlsZXMpOwogICAgfSk7CiAgfSk7CgogIGNvbnN0IGNhbmNlbCA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2J1dHRvbicpOwogIGlucHV0RWxlbWVudC5wYXJlbnRFbGVtZW50LmFwcGVuZENoaWxkKGNhbmNlbCk7CiAgY2FuY2VsLnRleHRDb250ZW50ID0gJ0NhbmNlbCB1cGxvYWQnOwogIGNvbnN0IGNhbmNlbFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgY2FuY2VsLm9uY2xpY2sgPSAoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9OwogIH0pOwoKICAvLyBXYWl0IGZvciB0aGUgdXNlciB0byBwaWNrIHRoZSBmaWxlcy4KICBjb25zdCBmaWxlcyA9IHlpZWxkIHsKICAgIHByb21pc2U6IFByb21pc2UucmFjZShbcGlja2VkUHJvbWlzZSwgY2FuY2VsUHJvbWlzZV0pLAogICAgcmVzcG9uc2U6IHsKICAgICAgYWN0aW9uOiAnc3RhcnRpbmcnLAogICAgfQogIH07CgogIGNhbmNlbC5yZW1vdmUoKTsKCiAgLy8gRGlzYWJsZSB0aGUgaW5wdXQgZWxlbWVudCBzaW5jZSBmdXJ0aGVyIHBpY2tzIGFyZSBub3QgYWxsb3dlZC4KICBpbnB1dEVsZW1lbnQuZGlzYWJsZWQgPSB0cnVlOwoKICBpZiAoIWZpbGVzKSB7CiAgICByZXR1cm4gewogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgICAgfQogICAgfTsKICB9CgogIGZvciAoY29uc3QgZmlsZSBvZiBmaWxlcykgewogICAgY29uc3QgbGkgPSBkb2N1bWVudC5jcmVhdGVFbGVtZW50KCdsaScpOwogICAgbGkuYXBwZW5kKHNwYW4oZmlsZS5uYW1lLCB7Zm9udFdlaWdodDogJ2JvbGQnfSkpOwogICAgbGkuYXBwZW5kKHNwYW4oCiAgICAgICAgYCgke2ZpbGUudHlwZSB8fCAnbi9hJ30pIC0gJHtmaWxlLnNpemV9IGJ5dGVzLCBgICsKICAgICAgICBgbGFzdCBtb2RpZmllZDogJHsKICAgICAgICAgICAgZmlsZS5sYXN0TW9kaWZpZWREYXRlID8gZmlsZS5sYXN0TW9kaWZpZWREYXRlLnRvTG9jYWxlRGF0ZVN0cmluZygpIDoKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgJ24vYSd9IC0gYCkpOwogICAgY29uc3QgcGVyY2VudCA9IHNwYW4oJzAlIGRvbmUnKTsKICAgIGxpLmFwcGVuZENoaWxkKHBlcmNlbnQpOwoKICAgIG91dHB1dEVsZW1lbnQuYXBwZW5kQ2hpbGQobGkpOwoKICAgIGNvbnN0IGZpbGVEYXRhUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICAgIGNvbnN0IHJlYWRlciA9IG5ldyBGaWxlUmVhZGVyKCk7CiAgICAgIHJlYWRlci5vbmxvYWQgPSAoZSkgPT4gewogICAgICAgIHJlc29sdmUoZS50YXJnZXQucmVzdWx0KTsKICAgICAgfTsKICAgICAgcmVhZGVyLnJlYWRBc0FycmF5QnVmZmVyKGZpbGUpOwogICAgfSk7CiAgICAvLyBXYWl0IGZvciB0aGUgZGF0YSB0byBiZSByZWFkeS4KICAgIGxldCBmaWxlRGF0YSA9IHlpZWxkIHsKICAgICAgcHJvbWlzZTogZmlsZURhdGFQcm9taXNlLAogICAgICByZXNwb25zZTogewogICAgICAgIGFjdGlvbjogJ2NvbnRpbnVlJywKICAgICAgfQogICAgfTsKCiAgICAvLyBVc2UgYSBjaHVua2VkIHNlbmRpbmcgdG8gYXZvaWQgbWVzc2FnZSBzaXplIGxpbWl0cy4gU2VlIGIvNjIxMTU2NjAuCiAgICBsZXQgcG9zaXRpb24gPSAwOwogICAgd2hpbGUgKHBvc2l0aW9uIDwgZmlsZURhdGEuYnl0ZUxlbmd0aCkgewogICAgICBjb25zdCBsZW5ndGggPSBNYXRoLm1pbihmaWxlRGF0YS5ieXRlTGVuZ3RoIC0gcG9zaXRpb24sIE1BWF9QQVlMT0FEX1NJWkUpOwogICAgICBjb25zdCBjaHVuayA9IG5ldyBVaW50OEFycmF5KGZpbGVEYXRhLCBwb3NpdGlvbiwgbGVuZ3RoKTsKICAgICAgcG9zaXRpb24gKz0gbGVuZ3RoOwoKICAgICAgY29uc3QgYmFzZTY0ID0gYnRvYShTdHJpbmcuZnJvbUNoYXJDb2RlLmFwcGx5KG51bGwsIGNodW5rKSk7CiAgICAgIHlpZWxkIHsKICAgICAgICByZXNwb25zZTogewogICAgICAgICAgYWN0aW9uOiAnYXBwZW5kJywKICAgICAgICAgIGZpbGU6IGZpbGUubmFtZSwKICAgICAgICAgIGRhdGE6IGJhc2U2NCwKICAgICAgICB9LAogICAgICB9OwogICAgICBwZXJjZW50LnRleHRDb250ZW50ID0KICAgICAgICAgIGAke01hdGgucm91bmQoKHBvc2l0aW9uIC8gZmlsZURhdGEuYnl0ZUxlbmd0aCkgKiAxMDApfSUgZG9uZWA7CiAgICB9CiAgfQoKICAvLyBBbGwgZG9uZS4KICB5aWVsZCB7CiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICB9CiAgfTsKfQoKc2NvcGUuZ29vZ2xlID0gc2NvcGUuZ29vZ2xlIHx8IHt9OwpzY29wZS5nb29nbGUuY29sYWIgPSBzY29wZS5nb29nbGUuY29sYWIgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYi5fZmlsZXMgPSB7CiAgX3VwbG9hZEZpbGVzLAogIF91cGxvYWRGaWxlc0NvbnRpbnVlLAp9Owp9KShzZWxmKTsK",
              "ok": true,
              "headers": [
                [
                  "content-type",
                  "application/javascript"
                ]
              ],
              "status": 200,
              "status_text": ""
            }
          },
          "base_uri": "https://localhost:8080/",
          "height": 91
        },
        "outputId": "fa9f8278-31ca-4b82-b894-5e82462002d8"
      },
      "source": [
        "from google.colab import files\n",
        "files.upload()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-11a06d0d-8df6-4966-aa3c-1f8c2bd4cd85\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-11a06d0d-8df6-4966-aa3c-1f8c2bd4cd85\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script src=\"/nbextensions/google.colab/files.js\"></script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Saving kaggle.json to kaggle.json\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'kaggle.json': b'{\"username\":\"manascbehera123\",\"key\":\"19e7fe4611ac8913c29a87790be44bca\"}'}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kmfjUaQuM3SE"
      },
      "source": [
        "! mkdir ~/.kaggle\n",
        "\n",
        "! cp kaggle.json ~/.kaggle/\n",
        "\n",
        "! chmod 600 ~/.kaggle/kaggle.json"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y17npKVyM6hU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        },
        "outputId": "f1860150-d394-49ad-8daf-6cfcda6fbe2d"
      },
      "source": [
        "!kaggle datasets download -d paultimothymooney/chest-xray-pneumonia"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading chest-xray-pneumonia.zip to /content\n",
            " 99% 2.28G/2.29G [00:43<00:00, 84.0MB/s]\n",
            "100% 2.29G/2.29G [00:43<00:00, 56.7MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "25T76hyoM8ua"
      },
      "source": [
        "!unzip -qq -o chest-xray-pneumonia.zip"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xKXZcYw7M_Hs"
      },
      "source": [
        "test_path = 'chest_xray/test'\n",
        "valid_path = 'chest_xray/val'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zlp-Z6Sz94Hi",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 124
        },
        "outputId": "de469ff0-d7b0-4507-ecf7-48b6ad019735"
      },
      "source": [
        "drive.mount(\"/content/gdrive\")"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4UbLZe2Y0qrL"
      },
      "source": [
        "model_path = \"/content/gdrive/My Drive/IOT LAB FILES/Pneumonia-DENSENET.h5\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2zhQKj27064Y"
      },
      "source": [
        "from tensorflow.keras.models import load_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aP2GkZG71CC2"
      },
      "source": [
        "model = load_model(model_path)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vTpwVrum1Glw"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cpQJv0JGKoUp"
      },
      "source": [
        "import cv2\n",
        "import numpy as np"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-7xVK1REKxVw"
      },
      "source": [
        "import os"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H_lOp6fyK2u6"
      },
      "source": [
        "normal_photos = test_path + \"/NORMAL\"\n",
        "real_values = []\n",
        "predicted_values = []\n",
        "for img in os.listdir(normal_photos):\n",
        "  real_values.append(0)\n",
        "  path = os.path.join(normal_photos,img)\n",
        "  imgnew = cv2.imread(path)\n",
        "  imgnew = cv2.resize(imgnew, (224,224))\n",
        "  imgnew = np.expand_dims(imgnew ,axis = 0)\n",
        "  imgnew = imgnew / 255.0\n",
        "  ans = model.predict(imgnew).argmax(axis=1)\n",
        "  predicted_values.append(ans[0])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jeh17AmRRkHe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 235
        },
        "outputId": "9bd8151c-d5b6-4567-a710-4ceaa7d6a823"
      },
      "source": [
        "pneumonia_photos = test_path + \"/PNEUMONIA\"\n",
        "for img in os.listdir(pneumonia_photos):\n",
        "  real_values.append(1)\n",
        "  path = os.path.join(pneumonia_photos,img)\n",
        "  imgnew = cv2.imread(path)\n",
        "  imgnew = cv2.resize(imgnew, (224,224))\n",
        "  imgnew = np.expand_dims(imgnew ,axis = 0)\n",
        "  imgnew = imgnew / 255.0\n",
        "  ans = model.predict(imgnew).argmax(axis=1)\n",
        "  predicted_values.append(ans[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-1-049377551feb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpneumonia_photos\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtest_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/PNEUMONIA\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mimg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpneumonia_photos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m   \u001b[0mreal_values\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m   \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpneumonia_photos\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m   \u001b[0mimgnew\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mimread\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'test_path' is not defined"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xfKHbM1cSjVQ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "bd3f474c-1b80-432b-e1ba-52ae38d5f80d"
      },
      "source": [
        "print(len(real_values),len(predicted_values))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "624 624\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "etP7lS-aSoKr"
      },
      "source": [
        "from sklearn.metrics import classification_report , roc_curve , confusion_matrix"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pArVMIQqS4Wb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 173
        },
        "outputId": "220bcf88-527d-4043-e161-6efdd7880dc2"
      },
      "source": [
        "print(classification_report(real_values , predicted_values))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.96      0.84      0.89       234\n",
            "           1       0.91      0.98      0.94       390\n",
            "\n",
            "    accuracy                           0.92       624\n",
            "   macro avg       0.93      0.91      0.92       624\n",
            "weighted avg       0.93      0.92      0.92       624\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVTSzAilS_5S",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        },
        "outputId": "354527d0-21b0-4dde-b800-11db0b999f48"
      },
      "source": [
        "print(confusion_matrix(real_values,predicted_values))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[[196  38]\n",
            " [  9 381]]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0-oq188aPuOW",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "059a019a-3ffd-40ea-9af0-3dafc83e0219"
      },
      "source": [
        "sensitivity = (381 / (381+9))\n",
        "print(sensitivity)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9769230769230769\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PigW65gaQvgF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "27872b1b-4c39-4d04-fa23-4ae897251a12"
      },
      "source": [
        "specificity = (196/(196+38))\n",
        "print(specificity)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.8376068376068376\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uGpE4lPdRD_R"
      },
      "source": [
        "from sklearn.metrics import roc_curve, roc_auc_score"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xV70JMEWSxrx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d2efdc32-7482-4fee-efd0-facb9e8f8249"
      },
      "source": [
        "r_auc = roc_auc_score(real_values , predicted_values)\n",
        "print(r_auc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9072649572649573\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rDBYJv2mTY-S"
      },
      "source": [
        "r_fpr, r_tpr, _ = roc_curve(real_values , predicted_values)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pHmxU3WhTiqq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 313
        },
        "outputId": "7ca19ba9-9c81-4074-9d61-6e4998c14ed1"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.plot(r_fpr , r_tpr)\n",
        "# Title\n",
        "plt.title('ROC Plot')\n",
        "# Axis labels\n",
        "plt.xlabel('False Positive Rate')\n",
        "plt.ylabel('True Positive Rate')\n",
        "# Show legend\n",
        "plt.legend() # \n",
        "# Show plot\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "No handles with labels found to put in legend.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7QddX338ffnXJKT+x2EXEiCQYiIgAEC1haLF6QW7CNeqLa1dclTFXvRukori1q07WNtbWtLW1F5UKsi2tYnVpS2VsV6TkLCLRAuNvuQkEOISXbuCUnO5fv8MXNg51x3kjNn9t7zea21V+bym5nvJDDf+c1v5vdTRGBmZsXVlHcAZmaWLycCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMMuJpE2SXpN3HGZOBFYI6UX3OUkHJG2TdIekqQPKXCbpvyTtl7RX0rckLR9QZrqkv5b0dLqvUjo/d5jjhqSDadlnJH1KUvNxxn65pK7jP2uz6jgRWJH8YkRMBc4HLgD+oH+FpEuBfwf+H3A6sAR4GPixpKVpmQnA94CXAlcC04FLgTJw8QjHfXl63CuAXwbeM7anZXZynAiscCJiG3APSULo9+fAFyPibyJif0TsioibgNXAR9MyvwosAn4pIh6LiL6I2B4RH4uIu6s47hPAj4BzB66TNDGtWWxNf3+dLpsCfAc4Pa1VHJB0+smcv9lATgRWOJIWAG8ANqbzk4HLgK8PUfwu4LXp9GuA70bEgRM87nLgVcCDQ6z+CLCSJDm9nKSGcVNEHExj3RoRU9Pf1hM5vtlwnAisSL4paT+wBdgO/FG6fDbJ/wvPDrHNs0D/8/85w5QZzQOSdgPfAj4H/N8hyrwDuCWtYewA/hj4lRM4ltlxcyKwInlTREwDLgfO5oUL/G6gDzhtiG1OA3am0+VhyozmwoiYFRFnRsRNEdE3RJnTgc0V85vTZWaZcyKwwomIHwJ3AH+Rzh8EOoC3DFH8rSQNxAD/Cbw+fW4/1rYCZ1TML0qXAbiLYMuUE4EV1V8Dr5X08nT+RuDXJP2WpGmSZkn6OMlbQX+clvkSyWOlf5Z0tqQmSXMk/aGkq04ynq8CN0mal76KejPwT+m6nwJzJM04yWOYDcmJwAopfQ7/RZILLhHx38Drgf9F0g6wmeQV05+JiP9JyxwhaTB+AvgPYB9wH8kjpjUnGdLHgXXAeuAR4IF0Wf/bRl8FOiXt8VtDNtbkgWnMzIrNNQIzs4JzIjAzKzgnAjOzgnMiMDMruJa8Azhec+fOjcWLF+cdhplZXbn//vt3RsS8odbVXSJYvHgx69atyzsMM7O6ImnzcOv8aMjMrOCcCMzMCs6JwMys4OqujcDMrOi6u7vp6uri8OHDg9a1tbWxYMECWltbq96fE4GZWZ3p6upi2rRpLF68GEnPL48IyuUyXV1dLFmypOr9ZfZoSNLtkrZLenSY9ZL0aUkbJa2XdGFWsZiZNZLDhw8zZ86cY5IAgCTmzJkzZE1hJFm2EdxBMsD3cN4ALEt/1wP/kGEsZmYNZWASGG35SDJ7NBQR90paPEKRa0gGCw9gtaSZkk6LiBMZCtDMbFxFBN29QU9fHz19QU//dG/FdN8Q073pdH/ZvvTXWznf9/x23b1Bb7r+inNO5eULZ475ueTZRjCfZJCPfl3pskGJQNL1JLUGFi1aNC7BmdnJ6b9Q9vYF3X199Pamf6YXxO7eZLqyTOXFNFmXlumruID2Br0VF8juygtqxT5fuPhWbpfs85iL74AylRftY8oP2K4vhx78T5ne1nCJoGoRcRtwG8CKFSs8gII1hIgYfCc58E5xiAtU/8Wxskx337EXx8rtXrgIJ2WSi3Bavi+Gv0APsc9jLuyDLuiV2yW/8dbSJFqaRUtTU/pnMt3cJFqbRUtz0/NlmpuaaE2nJ09oObZ8s9J1Tcfuc9Cyivlj1g213YC4hinT2pzGm8bRv+/mJg1qGB7qMdCJjDGTZyJ4BlhYMb8gXWYGvHChPOYiM+Ciecyd4bB3dX2DthtUFa+srg+swg9RXa/2oj2ofMVdZR4Xyub0ovL8xaZZ6bIXLlD96yovcJNam2me2PJC+efXNT2/7IXt0ovY8/tvOma71qZjy/eXGbxdUmbwBX2IC/sQF8pG1tbWRrlcHtRg3P/WUFtb23HtL89EsAq4QdKdwCXAXrcPHJ+I9E5tUDW34q6xb/CygdX1ngEX08qq+AvlB1fFB16g++8wewYeZ0CVeqgL+8Dy/ec13pr7L1JD3bH1X5wG3NU1N4m21iZaJrZUVX7wHekwd5jD3oUee3FM7iLTi+kId5gtBbpQNroFCxbQ1dXFjh07Bq3r/47geGSWCCR9FbgcmCupC/gjoBUgIv4RuBu4CtgIHAJ+PatYGsXRnj6u/rv/ZlP54PN3mOOtSQxZFa68Gxx4p9jS3MTEliamVGw3qPzzd4pNg+86h7jDrLz4DbzDHFS9HvKxwAtV7/4LdbNEU5MvlFb7Wltbj+s7gdFk+dbQdaOsD+D9WR2/ET3ctYcntu3njeedxqLZk4+5I2095k6x4s7wmIv2gDvMAXeVgy/QA8o0+UJp1ojqorHYEu0by0jw8Tedy8zJE/IOx8wahDudqyMdnTtZftp0JwEzG1NOBHXicHcvD2zew2Vnzsk7FDNrME4EdeKBzbs52tvHpU4EZjbGnAjqRHupTHOTuGjx7LxDMbMG40RQJzo6y7xs/gymtVXfx7iZWTWcCOrAgSM9PLzF7QNmlg0ngjqwdtMuevrC7QNmlgkngjqwulSmtVmsOMPtA2Y29pwI6kB7qcwFC2cxaUJz3qGYWQNyIqhxew91s2HrXj8WMrPMOBHUuDVPlekL3FBsZplxIqhxHZ1lJrY0cf6isR+VyMwMnAhqXkepzEWLZzOxxe0DZpYNJ4IaVj5whCe27Xf7gJllyomghq3u3AXgRGBmmXIiqGEdnTuZMqGZl82fkXcoZtbAnAhqWHupzMVLZtPa7H8mM8uOrzA16qf7DtO546AfC5lZ5pwIalRHqQzAZWfOzTkSM2t0TgQ1qqNUZnpbC+ecNj3vUMyswTkR1Kj2zp2sXDqH5iblHYqZNTgnghq0Zdchtux6zu0DZjYunAhqUEen2wfMbPw4EdSg1aUyc6ZM4KxTp+YdipkVgBNBjYkI2ktlVp45B8ntA2aWPSeCGrOpfIht+w5z6VK3D5jZ+HAiqDHtpZ2Axx8ws/HjRFBj2ktlTp0+kSVzp+QdipkVhBNBDYkIVpfKXHbmXLcPmNm4cSKoIT/56QHKB4/6+wEzG1eZJgJJV0p6UtJGSTcOsX6RpO9LelDSeklXZRlPretI2wfcUGxm4ymzRCCpGbgVeAOwHLhO0vIBxW4C7oqIC4C3A3+fVTz1oL1UZuHsSSycPTnvUMysQLKsEVwMbIyIzog4CtwJXDOgTAD9varNALZmGE9N6+0L1jy1y7UBMxt3WSaC+cCWivmudFmljwLvlNQF3A18YKgdSbpe0jpJ63bs2JFFrLl7/Nl97H2u291KmNm4y7ux+DrgjohYAFwFfEnSoJgi4raIWBERK+bNmzfuQY6H/vEH3FBsZuMty0TwDLCwYn5BuqzSu4G7ACKiA2gDCnlL3F7aydJ5Uzh1elveoZhZwWSZCNYCyyQtkTSBpDF41YAyTwNXAEg6hyQRNOaznxF09/Zxn9sHzCwnmSWCiOgBbgDuAR4neTtog6RbJF2dFvsQ8B5JDwNfBd4VEZFVTLXqkWf2cvBor9sHzCwXLVnuPCLuJmkErlx2c8X0Y8Ars4yhHvS3D6xcOjvnSMysiPJuLDaSRHD2i6YxZ+rEvEMxswJyIsjZkZ5e1m3exUq3D5hZTpwIcvbQ03s43N3nbqfNLDdOBDnr6CwjwSVLnAjMLB9OBDlrL5U59/QZzJjcmncoZlZQTgQ5eu5oLw89vcdfE5tZrpwIcnT/5t0c7e1zIjCzXDkR5Ki9tJPmJnHRYn8/YGb5cSLIUUdnmZcvmMHUiZl+12dmNiIngpwcONLD+q697lbCzHJXdSKQ5GGzxtDap3bR2xduHzCz3I2aCCRdJukx4Il0/uWSCj2k5FhoL+1kQnMTrzhjVt6hmFnBVVMj+Cvg9UAZICIeBn42y6CKoKOzzAWLZtLW2px3KGZWcFU9GoqILQMW9WYQS2HsOXSUDVv3uX3AzGpCNa+rbJF0GRCSWoHfJhlfwE7Qmqd2EeFhKc2sNlRTI/hN4P0kA88/A5wPvC/LoBpdR6lMW2sT5y+cmXcoZmZV1QheEhHvqFwg6ZXAj7MJqfF1lMpctHg2E1r89q6Z5a+aK9HfVrnMqrDzwBGe/Ol+PxYys5oxbI1A0qXAZcA8SR+sWDUd8KsuJ2h1ZzIspQeqN7NaMdKjoQnA1LTMtIrl+4BrswyqkbWXykyd2MLL5s/IOxQzM2CERBARPwR+KOmOiNg8jjE1tNWlMhcvmU1Ls9sHzKw2VNNYfEjSJ4GXAm39CyPi5zOLqkFt23uYzp0H+eVLFuUdipnZ86q5Lf0ySfcSS4A/BjYBazOMqWF1dO4E8ED1ZlZTqkkEcyLi80B3RPwwIn4DcG3gBLRvLDNjUivLT5uedyhmZs+r5tFQd/rns5J+AdgKeCSVE9DRWWbl0tk0NSnvUMzMnldNIvi4pBnAh0i+H5gO/E6mUTWgLbsO0bX7Od7zqqV5h2JmdoxRE0FE/Fs6uRd4NTz/ZbEdh45S+v2APyQzsxoz0gdlzcBbSfoY+m5EPCrpjcAfApOAC8YnxMbQXtrJ3KkTWHbK1LxDMTM7xkg1gs8DC4H7gE9L2gqsAG6MiG+OR3CNIiJoL5W59My5SG4fMLPaMlIiWAGcFxF9ktqAbcCZEVEen9AaR+fOg2zff8TdSphZTRrp9dGjEdEHEBGHgc7jTQKSrpT0pKSNkm4cpsxbJT0maYOkrxzP/utFe9o+cJnbB8ysBo1UIzhb0vp0WsCZ6byAiIjzRtpx2sZwK/BaoAtYK2lVRDxWUWYZ8AfAKyNit6RTTuJcatbqUpnTZrRxxpzJeYdiZjbISIngnJPc98XAxojoBJB0J3AN8FhFmfcAt0bEboCI2H6Sx6w5fX1BR2eZy18yz+0DZlaTRup07mQ7mpsPVI513AVcMqDMWQCSfkzStfVHI+K7A3ck6XrgeoBFi+qrn56fbN/ProNH3T5gZjUr7y4wW4BlwOXAdcBnJQ0avzEibouIFRGxYt68eeMc4slp3+jvB8ystmWZCJ4hef2034J0WaUuYFVEdEfEU8BPSBJDw+joLLNo9mQWzHL7gJnVpqoSgaRJkl5ynPteCyyTtETSBODtwKoBZb5JUhtA0lySR0Wdx3mcmtXbF6zuLPttITOraaMmAkm/CDwEfDedP1/SwAv6IBHRA9wA3AM8DtwVERsk3SLp6rTYPUBZ0mPA94EPN9J3Co9t3cf+wz1+LGRmNa2aTuc+SvIG0A8AIuIhSUuq2XlE3A3cPWDZzRXTAXww/TWc9lIy/oAbis2sllXzaKg7IvYOWBZZBNNoOjrLnDlvCqdMbxu9sJlZTqpJBBsk/TLQLGmZpL8F2jOOq+519/Zx31O7uOzMuXmHYmY2omoSwQdIxis+AnyFpDtqj0cwivVdezl0tNftA2ZW86ppIzg7Ij4CfCTrYBpJR8njE5tZfaimRvCXkh6X9DFJ52YeUYPo6Cxz9oumMXvKhLxDMTMb0aiJICJeTTIy2Q7gM5IekXRT5pHVsSM9vazbtNvtA2ZWF6r6oCwitkXEp4HfJPmm4OZRNim0B5/ew5GePrcPmFldqOaDsnMkfVTSIySD17eTdBdhw2gvlWkSXLxkdt6hmJmNqprG4tuBrwGvj4itGcfTEFaXyrxs/gxmTGrNOxQzs1GNmggi4tLxCKRRHDraw4NbdvMbP1PVx9dmZrkbNhFIuisi3po+Eqr8kriqEcqKat2m3XT3hhuKzaxujFQj+O30zzeORyCNoqOzTEuTWHHGrLxDMTOryrCNxRHxbDr5vojYXPkD3jc+4dWf9lKZ8xfOZMrEappfzMzyV83ro68dYtkbxjqQRrDvcDePdO3xa6NmVldGaiN4L8md/1JJ6ytWTQN+nHVg9WjtU7voCw9LaWb1ZaTnF18BvgP8GXBjxfL9EbEr06jqVEepzISWJi5c5PYBM6sfIyWCiIhNkt4/cIWk2U4Gg7WXyrxi0SzaWpvzDsXMrGqj1QjeCNxP8vqoKtYFsDTDuOrO7oNHeXzbPn73NWflHYqZ2XEZNhFExBvTP/1lVBXWPFUmAg9Ub2Z1p5q+hl4paUo6/U5Jn5K0KPvQ6ktHqcyk1mbOWzAz71DMzI5LNa+P/gNwSNLLgQ8BJeBLmUZVh9pLZS5aMpsJLVV16GpmVjOquWr1REQA1wB/FxG3krxCaqkd+4/wP9sPcKlHIzOzOlTN56/7Jf0B8CvAqyQ1Ae5Ws0JHZxlw+4CZ1adqagRvIxm4/jciYhvJWASfzDSqOtNRKjNtYgsvPX163qGYmR23aoaq3AZ8GZgh6Y3A4Yj4YuaR1ZGO0k4uWTqblma3D5hZ/anmraG3AvcBbwHeCqyRdG3WgdWLrXueY1P5ECvdPmBmdaqaNoKPABdFxHYASfOA/wS+kWVg9aKj1N8+4PEHzKw+VfMso6k/CaTKVW5XCB2dZWZNbuXsF/lFKjOrT9XUCL4r6R7gq+n824C7swupfkQEHaUyK5fOoalJo29gZlaDqmks/jDwGeC89HdbRPx+1oHVgy27nuOZPc/5tVEzq2sjjUewDPgL4EzgEeD3IuKZ8QqsHrSXdgIef8DM6ttINYLbgX8D3kzSA+nfHu/OJV0p6UlJGyXdOEK5N0sKSSuO9xh5ai+VmTdtImfOm5p3KGZmJ2ykNoJpEfHZdPpJSQ8cz44lNQO3kgx12QWslbQqIh4bUG4a8NvAmuPZf94igo7OMpcunYPk9gEzq18jJYI2SRfwwjgEkyrnI2K0xHAxsDEiOgEk3UnSX9FjA8p9DPgE8OHjjD1XpR0H2LH/iNsHzKzujZQIngU+VTG/rWI+gJ8fZd/zgS0V813AJZUFJF0ILIyIb0saNhFIuh64HmDRotroAbv/+wG3D5hZvRtpYJpXZ3ngtPO6TwHvGq1sRNwG3AawYsWKyDKuarWXysyfOYlFsyfnHYqZ2UnJ8sOwZ4CFFfML0mX9pgHnAj+QtAlYCayqhwbjvr5gdWfy/YDbB8ys3mWZCNYCyyQtkTQBeDuwqn9lROyNiLkRsTgiFgOrgasjYl2GMY2JJ7btZ/ehbrcPmFlDyCwRREQPcANwD/A4cFdEbJB0i6SrszrueOgff8DtA2bWCEbtYkLJs493AEsj4pZ0vOIXRcR9o20bEXczoDuKiLh5mLKXVxVxDego7WTxnMmcPnNS3qGYmZ20amoEfw9cClyXzu8n+T6gkHp6+1jTucu1ATNrGNV0OndJRFwo6UGAiNidPvMvpA1b97H/SA+XuttpM2sQ1dQIutOvhAOeH4+gL9Ooalh/+8DKpbNzjsTMbGxUkwg+DfwrcIqkPwH+G/jTTKOqYe2lMstOmcop09ryDsXMbEyM+mgoIr4s6X7gCpLuJd4UEY9nHlkNOtrTx7pNu7j2FQvyDsXMbMxU89bQIuAQ8K3KZRHxdJaB1aL1XXs4dLTX3w+YWUOpprH42yTtAwLagCXAk8BLM4yrJnWUykhwyRInAjNrHNU8GnpZ5XzaUdz7MouohrWXypzzounMmlLYl6bMrAEd95fFaffTl4xasMEc7u7l/qd3+7GQmTWcatoIPlgx2wRcCGzNLKIa9cDTuzna0+cPycys4VTTRjCtYrqHpM3gn7MJp3Z1lMo0N4mLl/j7ATNrLCMmgvRDsmkR8XvjFE/N6iiVOXf+DKa1teYdipnZmBq2jUBSS0T0Aq8cx3hq0sEjPTy0ZY/bB8ysIY1UI7iPpD3gIUmrgK8DB/tXRsS/ZBxbzVi3eTc9fcGlS50IzKzxVNNG0AaUScYo7v+eIIDCJIL20k5am8WKxbPyDsXMbMyNlAhOSd8YepQXEkC/mhg3eLysLpU5f+FMJk+oJm+amdWXkb4jaAampr9pFdP9v0LYd7ibR57Z626nzaxhjXSL+2xE3DJukdSo+zp30Re4fcDMGtZINQKNsK4w2ktlJrY0ccGimXmHYmaWiZESwRXjFkUN6+gs84ozZtHW2px3KGZmmRg2EUTErvEMpBbtOniUx5/d5+8HzKyhHXenc0WyJh2W0v0LmVkjcyIYQXupzOQJzZy3wO0DZta4nAhG0NFZ5qLFs2lt9l+TmTUuX+GGsX3fYTZuP+D2ATNreE4Ew+hI2wcu84dkZtbgnAiG0VEqM72theWnT887FDOzTDkRDKOjs8wlS+fQ3OTv6syssTkRDOGZPc+xuXzI3UqYWSE4EQyho5S2D7zYicDMGl+miUDSlZKelLRR0o1DrP+gpMckrZf0PUlnZBlPtdpLO5k9ZQJnnTJt9MJmZnUus0SQjnd8K/AGYDlwnaTlA4o9CKyIiPOAbwB/nlU81YoIOkplLl06hya3D5hZAWRZI7gY2BgRnRFxFLgTuKayQER8PyIOpbOrgQUZxlOVzeVDPLv3MCv9/YCZFUSWiWA+sKVivitdNpx3A98ZaoWk6yWtk7Rux44dYxjiYO397QNOBGZWEDXRWCzpncAK4JNDrY+I2yJiRUSsmDdvXqaxdHSWOWXaRJbOnZLpcczMakWWg/A+AyysmF+QLjuGpNcAHwF+LiKOZBjPqPrbB37mxXOQ3D5gZsWQZY1gLbBM0hJJE4C3A6sqC0i6APgMcHVEbM8wlqps3H6AnQeOuNtpMyuUzBJBRPQANwD3AI8Dd0XEBkm3SLo6LfZJYCrwdUkPSVo1zO7GxQvtA+5fyMyKI8tHQ0TE3cDdA5bdXDH9miyPf7w6SmXmz5zEwtmT8w7FzGzc1ERjcS3o6wtWP1X220JmVjhOBKnHt+1jz6Futw+YWeE4EaT6+xdyIjCzonEiSHWUyiyZO4XTZkzKOxQzs3HlRAD09Pax5qldrg2YWSE5EQCPbt3HgSM9big2s0JyIiDpdhpgpQeiMbMCciIgaR94yanTmDt1Yt6hmJmNu8IngqM9fazbtNvtA2ZWWIVPBA937eG57l4nAjMrrMIngvaNZSRYucSJwMyKqfCJoKNzJy89fTozJrfmHYqZWS4KnQgOd/fywOY9XOq3hcyswAqdCO7fvJujvX3udtrMCq3QiaCjVKa5SVy0ZHbeoZiZ5abQiaC9tJPzFsxg6sRMh2UwM6tphU0EB470sL5rr9sHzKzwCpsI1m7aRU9fuH3AzAqvsIlgdalMa7N4xRmz8g7FzCxXhU0E7aUyFyyaxaQJzXmHYmaWq0Imgr2Hutmw1e0DZmZQ0ESw5qkyfYHHHzAzo6CJoKOzzMSWJs5fNDPvUMzMclfMRFAqc9Hi2UxscfuAmVnhEkH5wBGe2Lbf3U6bmaUKlwhWd+4CcCIwM0sVLhF0dO5k6sQWzps/I+9QzMxqQuESQXupzEWLZ9HSXLhTNzMbUqGuhj/dd5jOHQfdrYSZWYVCJYKOUhlw+4CZWaXCJYIZk1o557TpeYdiZlYzMk0Ekq6U9KSkjZJuHGL9RElfS9evkbQ4y3jaO3dyyZLZNDcpy8OYmdWVzBKBpGbgVuANwHLgOknLBxR7N7A7Il4M/BXwiazi2bLrEFt2PeduJczMBsiyRnAxsDEiOiPiKHAncM2AMtcAX0invwFcISmT2/WOzv72ATcUm5lVyjIRzAe2VMx3pcuGLBMRPcBeYNAtu6TrJa2TtG7Hjh0nFMzMSa28bvmpnHXq1BPa3sysUdXFYL0RcRtwG8CKFSviRPbxupe+iNe99EVjGpeZWSPIskbwDLCwYn5BumzIMpJagBlAOcOYzMxsgCwTwVpgmaQlkiYAbwdWDSizCvi1dPpa4L8i4oTu+M3M7MRk9mgoInok3QDcAzQDt0fEBkm3AOsiYhXweeBLkjYCu0iShZmZjaNM2wgi4m7g7gHLbq6YPgy8JcsYzMxsZIX6stjMzAZzIjAzKzgnAjOzgnMiMDMrONXb25qSdgCbT3DzucDOMQynHvici8HnXAwnc85nRMS8oVbUXSI4GZLWRcSKvOMYTz7nYvA5F0NW5+xHQ2ZmBedEYGZWcEVLBLflHUAOfM7F4HMuhkzOuVBtBGZmNljRagRmZjaAE4GZWcE1ZCKQdKWkJyVtlHTjEOsnSvpaun6NpMXjH+XYquKcPyjpMUnrJX1P0hl5xDmWRjvninJvlhSS6v5Vw2rOWdJb03/rDZK+Mt4xjrUq/tteJOn7kh5M//u+Ko84x4qk2yVtl/ToMOsl6dPp38d6SRee9EEjoqF+JF1el4ClwATgYWD5gDLvA/4xnX478LW84x6Hc341MDmdfm8RzjktNw24F1gNrMg77nH4d14GPAjMSudPyTvucTjn24D3ptPLgU15x32S5/yzwIXAo8Osvwr4DiBgJbDmZI/ZiDWCi4GNEdEZEUeBO4FrBpS5BvhCOv0N4ApJGscYx9qo5xwR34+IQ+nsapIR4+pZNf/OAB8DPgEcHs/gMlLNOb8HuDUidgNExPZxjnGsVXPOAUxPp2cAW8cxvjEXEfeSjM8ynGuAL0ZiNTBT0mknc8xGTATzgS0V813psiHLREQPsBeYMy7RZaOac670bpI7ino26jmnVeaFEfHt8QwsQ9X8O58FnCXpx5JWS7py3KLLRjXn/FHgnZK6SMY/+cD4hJab4/3/fVR1MXi9jR1J7wRWAD+XdyxZktQEfAp4V86hjLcWksdDl5PU+u6V9LKI2JNrVNm6DrgjIv5S0qUkox6eGxF9eQdWLxqxRvAMsLBifkG6bMgyklpIqpPlcYkuG9WcM5JeA3wEuDoijoxTbFkZ7ZynAecCP5C0ieRZ6qo6bzCu5t+5C1gVEd0R8RTwE5LEUK+qOed3A3cBREQH0EbSOVujqur/9+PRiIlgLbBM0hJJE0gag1cNKLMK+LV0+lrgvyJthalTo56zpAuAz5AkgXp/bgyjnHNE7I2IuRGxOCIWk7SLXCbRPvkAAATeSURBVB0R6/IJd0xU89/2N0lqA0iaS/KoqHM8gxxj1Zzz08AVAJLOIUkEO8Y1yvG1CvjV9O2hlcDeiHj2ZHbYcI+GIqJH0g3APSRvHNweERsk3QKsi4hVwOdJqo8bSRpl3p5fxCevynP+JDAV+HraLv50RFydW9AnqcpzbihVnvM9wOskPQb0Ah+OiLqt7VZ5zh8CPivpd0kajt9Vzzd2kr5Kksznpu0efwS0AkTEP5K0g1wFbAQOAb9+0ses478vMzMbA434aMjMzI6DE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOB1SRJvZIeqvgtHqHsgTE43h2SnkqP9UD6herx7uNzkpan0384YF37ycaY7qf/7+VRSd+SNHOU8ufXe2+clj2/Pmo1SdKBiJg61mVH2McdwL9FxDckvQ74i4g47yT2d9IxjbZfSV8AfhIRfzJC+XeR9Lp6w1jHYo3DNQKrC5KmpuMoPCDpEUmDehqVdJqkeyvumF+VLn+dpI50269LGu0CfS/w4nTbD6b7elTS76TLpkj6tqSH0+VvS5f/QNIKSf8HmJTG8eV03YH0zzsl/UJFzHdIulZSs6RPSlqb9jH/v6v4a+kg7WxM0sXpOT4oqV3SS9IvcW8B3pbG8rY09tsl3ZeWHarHViuavPve9s+/oX4kX8U+lP7+leQr+OnpurkkX1X212gPpH9+CPhIOt1M0t/QXJIL+5R0+e8DNw9xvDuAa9PptwBrgFcAjwBTSL7K3gBcALwZ+GzFtjPSP39AOuZBf0wVZfpj/CXgC+n0BJJeJCcB1wM3pcsnAuuAJUPEeaDi/L4OXJnOTwda0unXAP+cTr8L+LuK7f8UeGc6PZOkL6Ipef97+5fvr+G6mLCG8VxEnN8/I6kV+FNJPwv0kdwJnwpsq9hmLXB7WvabEfGQpJ8jGazkx2nXGhNI7qSH8klJN5H0U/Nukv5r/jUiDqYx/AvwKuC7wF9K+gTJ46QfHcd5fQf4G0kTgSuBeyPiufRx1HmSrk3LzSDpLO6pAdtPkvRQev6PA/9RUf4LkpaRdLPQOszxXwdcLen30vk2YFG6LysoJwKrF+8A5gGviIhuJT2KtlUWiIh700TxC8Adkj4F7Ab+IyKuq+IYH46Ib/TPSLpiqEIR8RMlYx1cBXxc0vci4pZqTiIiDkv6AfB64G0kA61AMtrUByLinlF28VxEnC9pMkn/O+8HPk0yAM/3I+KX0ob1HwyzvYA3R8ST1cRrxeA2AqsXM4DtaRJ4NTBozGUl4zD/NCI+C3yOZLi/1cArJfU/858i6awqj/kj4E2SJkuaQvJY50eSTgcORcQ/kXTmN9SYsd1pzWQoXyPpKKy/dgHJRf29/dtIOis95pAiGW3ut4AP6YWu1Pu7In5XRdH9JI/I+t0DfEBp9UhJr7RWcE4EVi++DKyQ9Ajwq8ATQ5S5HHhY0oMkd9t/ExE7SC6MX5W0nuSx0NnVHDAiHiBpO7iPpM3gcxHxIPAy4L70Ec0fAR8fYvPbgPX9jcUD/DvJwED/Gcnwi5AkrseAB5QMWv4ZRqmxp7GsJxmY5c+BP0vPvXK77wPL+xuLSWoOrWlsG9J5Kzi/PmpmVnCuEZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRWcE4GZWcE5EZiZFdz/By79ZTbMTVBMAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yUgpSIkMTzgl"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}