Semi_supervised_acrtive_learning.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wm9tgHaPTQRe"
      },
      "source": [
        "# 1. Import backage\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "S4VQtg66TQRg",
        "outputId": "1a0498b5-f32f-4afb-ea4b-bdb14a009849"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Number of replicas: 1\n",
            "2.8.2\n"
          ]
        }
      ],
      "source": [
        "import re\n",
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.model_selection import train_test_split\n",
        "from keras.preprocessing import image\n",
        "\n",
        "# Try to use TPU strategy, fall back to default strategy if not available\n",
        "try:\n",
        "    # Create a TPUClusterResolver and use it to initialize the TPU system\n",
        "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
        "    print('Device:', tpu.master())\n",
        "    tf.config.experimental_connect_to_cluster(tpu)\n",
        "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
        "\n",
        "    # Create a TPUStrategy using the TPUClusterResolver\n",
        "    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
        "except:\n",
        "    # Fall back to the default strategy if a TPU is not available\n",
        "    strategy = tf.distribute.get_strategy()\n",
        "\n",
        "# Print the number of replicas in the strategy\n",
        "print('Number of replicas:', strategy.num_replicas_in_sync)\n",
        "\n",
        "# Print the TensorFlow version\n",
        "print(tf.__version__)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vTiOKdTUTQRi",
        "outputId": "0edd02ca-e262-4ef8-f592-23b8ac814629"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "# Mount Google Drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "# Set the AUTOTUNE setting for the tf.data API\n",
        "AUTOTUNE = tf.data.experimental.AUTOTUNE\n",
        "\n",
        "# Get the path to the drive directory\n",
        "PATH = \"/content/drive/MyDrive\"\n",
        "\n",
        "# Set the batch size using the number of replicas in the strategy\n",
        "BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n",
        "\n",
        "# Set the image size\n",
        "IMAGE_SIZE = [180, 180]\n",
        "\n",
        "# Set the number of epochs\n",
        "EPOCHS = 100"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9yqY_H8CTQRi"
      },
      "source": [
        "# 2. Load Input data : L labelled dataset, U unlabelled dataset\n",
        "\n",
        "### Download the data  from Kaggle : https://www.kaggle.com/datasets/paultimothymooney/chest-xray-pneumonia/code?datasetId=17810&sortBy=voteCount\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9Wj7RC2UTQRi"
      },
      "outputs": [],
      "source": [
        "# Get a list of filenames in the training and validation directories\n",
        "filenames = tf.io.gfile.glob(str(PATH + '/chest_xray/train/*/*'))\n",
        "filenames.extend(tf.io.gfile.glob(str(PATH + '/chest_xray/val/*/*')))\n",
        "\n",
        "# Split the filenames into a training set and a validation set\n",
        "train_filenames, val_filenames = train_test_split(filenames, test_size=0.2)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ctahyaaCV4Dc"
      },
      "outputs": [],
      "source": [
        "# Split the training filenames into a labelled set and an unlabelled set\n",
        "L_labelled_filenames, U_unlabelled_filenames = train_test_split(train_filenames, test_size=0.8)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NlIIZCERTQRi"
      },
      "source": [
        "Run the following cell to see how many healthy/normal chest X-rays we have and how many pneumonia chest X-rays we have."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "W7H3qbfPTQRj",
        "outputId": "abd34390-46b7-40ec-e8e8-4283e8ae5220"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Normal images count in training set: 57\n",
            "Pneumonia images count in training set: 636\n"
          ]
        }
      ],
      "source": [
        "# Count the number of normal images in the labelled training set\n",
        "COUNT_NORMAL = len([filename for filename in L_labelled_filenames if \"NORMAL\" in filename])\n",
        "print(\"Normal images count in training set: \" + str(COUNT_NORMAL))\n",
        "\n",
        "# Count the number of pneumonia images in the labelled training set\n",
        "COUNT_PNEUMONIA = len([filename for filename in L_labelled_filenames if \"PNEUMONIA\" in filename])\n",
        "print(\"Pneumonia images count in training set: \" + str(COUNT_PNEUMONIA))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yBAeFFWjTQRj"
      },
      "source": [
        "Notice that the there are way more images that are classified as pneumonia than normal. This shows that we have a imbalance in our data. We will correct for this imbalance later on in our notebook."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hx9sUeUKTQRj"
      },
      "outputs": [],
      "source": [
        "# Create a dataset from the list of training filenames\n",
        "train_list_ds = tf.data.Dataset.from_tensor_slices(L_labelled_filenames)\n",
        "\n",
        "# Create a dataset from the list of validation filenames\n",
        "val_list_ds = tf.data.Dataset.from_tensor_slices(val_filenames)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JBa1s71gTQRk"
      },
      "source": [
        "Run the following cell to see how many images we have in our training dataset and how many images we have in our validation set. Verify that the ratio of images is 80:20."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tG7VbsmeTQRk",
        "outputId": "4ca0b7bb-3053-4fa9-e567-54aa45368c10"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Training images count: 693\n",
            "Validating images count: 867\n"
          ]
        }
      ],
      "source": [
        "# Get the number of training images\n",
        "TRAIN_IMG_COUNT = tf.data.experimental.cardinality(train_list_ds).numpy()\n",
        "print(\"Training images count: \" + str(TRAIN_IMG_COUNT))\n",
        "\n",
        "# Get the number of validation images\n",
        "VAL_IMG_COUNT = tf.data.experimental.cardinality(val_list_ds).numpy()\n",
        "print(\"Validating images count: \" + str(VAL_IMG_COUNT))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MdBFaT83TQRk"
      },
      "source": [
        "As expected, we have two labels for our images."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jc_7EN0xTQRk"
      },
      "outputs": [],
      "source": [
        "CLASS_NAMES = np.array([str(tf.strings.split(item, os.path.sep)[-1].numpy())[2:-1]\n",
        "                        for item in tf.io.gfile.glob(str(PATH + \"/chest_xray/train/*\"))])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZTXFLLd9TQRl"
      },
      "source": [
        "Currently our dataset is just a list of filenames. We want to map each filename to the corresponding (image, label) pair. The following methods will help us do that.\n",
        "\n",
        "As we only have two labels, we will rewrite the label so that `1` or `True` indicates pneumonia and `0` or `False` indicates normal."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6w7eq_hHTQRl"
      },
      "outputs": [],
      "source": [
        "def get_label(file_path):\n",
        "    # convert the path to a list of path components\n",
        "    parts = tf.strings.split(file_path, os.path.sep)\n",
        "    # The second to last is the class-directory\n",
        "    return parts[-2] == \"PNEUMONIA\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0zOSOd38TQRl"
      },
      "source": [
        "The images originally have values that range from [0, 255]. CNNs work better with smaller numbers so we will scale this down."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xaYWODYQTQRl"
      },
      "outputs": [],
      "source": [
        "def decode_img(img):\n",
        "  # convert the compressed string to a 3D uint8 tensor\n",
        "  img = tf.image.decode_jpeg(img, channels=3)\n",
        "  # Use `convert_image_dtype` to convert to floats in the [0,1] range.\n",
        "  img = tf.image.convert_image_dtype(img, tf.float32)\n",
        "  # resize the image to the desired size.\n",
        "  return tf.image.resize(img, IMAGE_SIZE)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LAfMTApzTQRl"
      },
      "outputs": [],
      "source": [
        "def process_path(file_path):\n",
        "    label = get_label(file_path)\n",
        "    # load the raw data from the file as a string\n",
        "    img = tf.io.read_file(file_path)\n",
        "    img = decode_img(img)\n",
        "    return img, label"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GRtXfRu1TQRm"
      },
      "outputs": [],
      "source": [
        "# Create a dataset from the training filenames by applying the `process_path` function to each element\n",
        "train_ds = train_list_ds.map(process_path, num_parallel_calls=AUTOTUNE)\n",
        "\n",
        "# Create a dataset from the validation filenames by applying the `process_path` function to each element\n",
        "val_ds = val_list_ds.map(process_path, num_parallel_calls=AUTOTUNE)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xi2enkpWTQRm"
      },
      "source": [
        "Let's visualize the shape of an (image, label) pair."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PAS5gRLpTQRm"
      },
      "source": [
        "Load and format the test data as well."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oOQ-AAIITQRm",
        "outputId": "9f328b58-1b3c-40e6-c13f-46aa20364359"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "624"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "# Create a dataset of filenames in the test directory\n",
        "test_list_ds = tf.data.Dataset.list_files(str(PATH + '/chest_xray/test/*/*'))\n",
        "\n",
        "# Get a list of filenames in the test directory\n",
        "test_filenames=tf.io.gfile.glob(str(PATH + '/chest_xray/test/*/*'))\n",
        "\n",
        "# Get the number of test images\n",
        "TEST_IMAGE_COUNT = tf.data.experimental.cardinality(test_list_ds).numpy()\n",
        "\n",
        "# Create a dataset from the test filenames by applying the `process_path` function to each element\n",
        "test_ds = test_list_ds.map(process_path, num_parallel_calls=AUTOTUNE)\n",
        "\n",
        "# Batch the test dataset\n",
        "test_ds = test_ds.batch(BATCH_SIZE)\n",
        "\n",
        "# Print the number of test images\n",
        "TEST_IMAGE_COUNT"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "d1s4v2aaTQRn"
      },
      "outputs": [],
      "source": [
        "def prepare_for_training(ds, cache=True, shuffle_buffer_size=1000):\n",
        "    # This is a small dataset, only load it once, and keep it in memory.\n",
        "    # use `.cache(filename)` to cache preprocessing work for datasets that don't\n",
        "    # fit in memory.\n",
        "    if cache:\n",
        "        if isinstance(cache, str):\n",
        "            ds = ds.cache(cache)\n",
        "        else:\n",
        "            ds = ds.cache()\n",
        "\n",
        "    ds = ds.shuffle(buffer_size=shuffle_buffer_size)\n",
        "\n",
        "    # Repeat forever\n",
        "    ds = ds.repeat()\n",
        "\n",
        "    ds = ds.batch(BATCH_SIZE)\n",
        "\n",
        "    # `prefetch` lets the dataset fetch batches in the background while the model\n",
        "    # is training.\n",
        "    ds = ds.prefetch(buffer_size=AUTOTUNE)\n",
        "\n",
        "    return ds"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oYhDXzRRTQRn"
      },
      "source": [
        "Call the next batch iteration of the training data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "i51JGWiTTQRn"
      },
      "outputs": [],
      "source": [
        "train_ds = prepare_for_training(train_ds)\n",
        "val_ds = prepare_for_training(val_ds)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3SDZaCbqTQRn"
      },
      "source": [
        "Define the method to show the images in the batch."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F0SlJ4djTQRo"
      },
      "source": [
        "# 4. Build the CNN\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UsZLQPjhTQRp"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers import Input, Conv2D, BatchNormalization, Dense\n",
        "from tensorflow.keras.layers import AvgPool2D, GlobalAveragePooling2D, MaxPool2D\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import ReLU, concatenate\n",
        "import tensorflow.keras.backend as K\n",
        "# Creating Densenet121\n",
        "def densenet(input_shape, n_classes, filters = 32):\n",
        "    \n",
        "    #batch norm + relu + conv\n",
        "    def bn_rl_conv(x,filters,kernel=1,strides=1):\n",
        "        \n",
        "        x = BatchNormalization()(x)\n",
        "        x = ReLU()(x)\n",
        "        x = Conv2D(filters, kernel, strides=strides,padding = 'same')(x)\n",
        "        return x\n",
        "    \n",
        "    def dense_block(x, repetition):\n",
        "        \n",
        "        for _ in range(repetition):\n",
        "            y = bn_rl_conv(x, 4*filters)\n",
        "            y = bn_rl_conv(y, filters, 3)\n",
        "            x = concatenate([y,x])\n",
        "        return x\n",
        "        \n",
        "    def transition_layer(x):\n",
        "        \n",
        "        x = bn_rl_conv(x, K.int_shape(x)[-1] //2 )\n",
        "        x = AvgPool2D(2, strides = 2, padding = 'same')(x)\n",
        "        return x\n",
        "    \n",
        "    input = Input (input_shape)\n",
        "    x = Conv2D(64, 7, strides = 2, padding = 'same')(input)\n",
        "    x = MaxPool2D(3, strides = 2, padding = 'same')(x)\n",
        "    \n",
        "    for repetition in [6,12,24,16]:\n",
        "        \n",
        "        d = dense_block(x, repetition)\n",
        "        x = transition_layer(d)\n",
        "    x = GlobalAveragePooling2D()(d)\n",
        "    output = Dense(n_classes, activation = 'softmax')(x)\n",
        "    \n",
        "    model = Model(input, output)\n",
        "    return model\n",
        "input_shape = 180, 180, 3\n",
        "n_classes = 1\n",
        "\n",
        "\n",
        "def build_model():\n",
        "    \n",
        "    \n",
        "    return  densenet(input_shape,n_classes)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ai-bRb9jTQRp"
      },
      "source": [
        "# 5. Correct for data imbalance\n",
        "\n",
        "We saw earlier in this notebook that the data was imbalanced, with more images classified as pneumonia than normal. We will correct for that in this following section."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4rjDlWFETQRp",
        "outputId": "0c3750e0-9af8-4afb-c3d6-6b7e33082bdc"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([2.4121473])"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "initial_bias = np.log([COUNT_PNEUMONIA/COUNT_NORMAL])\n",
        "initial_bias"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ba3SgvJ2TQRp",
        "outputId": "5e502026-854c-477b-dc82-a0683d5a7ea4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Weight for class 0: 6.08\n",
            "Weight for class 1: 0.54\n"
          ]
        }
      ],
      "source": [
        "weight_for_0 = (1 / COUNT_NORMAL)*(TRAIN_IMG_COUNT)/2.0 \n",
        "weight_for_1 = (1 / COUNT_PNEUMONIA)*(TRAIN_IMG_COUNT)/2.0\n",
        "\n",
        "class_weight = {0: weight_for_0, 1: weight_for_1}\n",
        "\n",
        "print('Weight for class 0: {:.2f}'.format(weight_for_0))\n",
        "print('Weight for class 1: {:.2f}'.format(weight_for_1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-aa9zTJmTQRp"
      },
      "source": [
        "The weight for class `0` (Normal) is a lot higher than the weight for class `1` (Pneumonia). Because there are less normal images, each normal image will be weighted more to balance the data as the CNN works best when the training data is balanced."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QrBL0NPtTQRq"
      },
      "source": [
        "# Step 1: Train a DenseNet using an initial labelled training set L"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KwdzBAs-TQRq"
      },
      "outputs": [],
      "source": [
        "def model_cloner(learning_rate):\n",
        "    model = build_model()\n",
        "\n",
        "    METRICS = [\n",
        "        'accuracy',\n",
        "        tf.keras.metrics.Precision(name='precision'),\n",
        "        tf.keras.metrics.Recall(name='recall')\n",
        "    ]\n",
        "    \n",
        "    model.compile(\n",
        "        optimizer = tf.keras.optimizers.Adam(learning_rate=learning_rate),\n",
        "        loss='binary_crossentropy',\n",
        "        metrics=METRICS\n",
        "    )\n",
        "    return model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "E1j7eNZUWkwr"
      },
      "outputs": [],
      "source": [
        "checkpoint_cb = tf.keras.callbacks.ModelCheckpoint(\"xray_model.h5\",\n",
        "                                                    save_best_only=True)\n",
        "\n",
        "early_stopping_cb = tf.keras.callbacks.EarlyStopping(patience=10,\n",
        "                                                     restore_best_weights=True)\n",
        "\n",
        "def exponential_decay(lr0, s):\n",
        "    def exponential_decay_fn(epoch):\n",
        "        return lr0 * 0.1 **(epoch / s)\n",
        "    return exponential_decay_fn\n",
        "\n",
        "exponential_decay_fn = exponential_decay(0.01, 20)\n",
        "\n",
        "lr_scheduler = tf.keras.callbacks.LearningRateScheduler(exponential_decay_fn)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZRG2eFTFWz3p"
      },
      "source": [
        "# Step 3: Select samples from U using a query function Q,"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N8Wb4zIZWppb"
      },
      "outputs": [],
      "source": [
        "from keras.preprocessing import image\n",
        "\n",
        "img_width, img_height = 180, 180\n",
        "threshold=len(U_unlabelled_filenames)\n",
        "def new_labelled_dataset(U_unlabelled_filenames,model):\n",
        "\n",
        "  df=pd.DataFrame(columns=[\"url\", \"score\"])\n",
        "  for url in U_unlabelled_filenames:\n",
        "    \n",
        "    img = image.load_img(url, target_size = (img_width, img_height))\n",
        "    img = image.img_to_array(img)\n",
        "    img = np.expand_dims(img, axis = 0)\n",
        "    df.loc[len(df)] = [url,model.predict(img)[0][0]]\n",
        "\n",
        "  df=df.sort_values(by='score', ascending=False)\n",
        "  new_labelled_filenames=df.head(int(threshold*0.2)).url.tolist()\n",
        "\n",
        "  \n",
        "  U_unlabelled_filenames = [ele for ele in U_unlabelled_filenames if ele not in new_labelled_filenames]\n",
        "  return U_unlabelled_filenames,new_labelled_filenames\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s_2_iOeGW3ZZ"
      },
      "source": [
        "# Step 4: request the labels for the samples selected in step 3 from the expert A,\n",
        "\n",
        "#### we don't need this step because we have all the labels"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wOkfp5XaW655"
      },
      "source": [
        "# Step 5: remove the selected samples from the dataset U and add the selected samples to L"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ov6tHp-MW90X"
      },
      "outputs": [],
      "source": [
        "def update_L_labelled_filenames(new_labelled_filenames):\n",
        "  return L_labelled_filenames + new_labelled_filenames"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SknJjuBQXA6j"
      },
      "source": [
        "\\# Step 6: retain the DenseNet using the dataset L"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Learning rate list\n",
        "learning_rates=[0.1,0.05,0.01,0.005,0.001]"
      ],
      "metadata": {
        "id": "4k0XcHgIH5yR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "learning_rates"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ARerSxC4VJkG",
        "outputId": "db1f2460-e5cc-4d37-fcd9-e995b9ed07bb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.1, 0.05, 0.01, 0.005, 0.001]"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JQJ9ixD5XEmN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3bfc3687-fc08-437a-989b-b5e092ebc516"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch 1/100\n",
            "43/43 [==============================] - 52s 485ms/step - loss: 93.2785 - accuracy: 0.9186 - precision: 0.9186 - recall: 1.0000 - val_loss: 2481929798549504.0000 - val_accuracy: 0.9178 - val_precision: 0.9178 - val_recall: 1.0000\n",
            "Epoch 2/100\n",
            "43/43 [==============================] - 10s 244ms/step - loss: 5.7193 - accuracy: 0.9172 - precision: 0.9172 - recall: 1.0000 - val_loss: 74221776.0000 - val_accuracy: 0.9190 - val_precision: 0.9190 - val_recall: 1.0000\n",
            "Epoch 3/100\n",
            " 5/43 [==>...........................] - ETA: 8s - loss: 0.8327 - accuracy: 0.9250 - precision: 0.9250 - recall: 1.0000"
          ]
        }
      ],
      "source": [
        "accuracy=[]\n",
        "\n",
        "for learning_rate in learning_rates:\n",
        "    x=[]\n",
        "    L_labelled_filenames, U_unlabelled_filenames = train_test_split(train_filenames, test_size=0.8)\n",
        "    x.append(len(L_labelled_filenames))\n",
        "    learning_rate_accuracy=[]\n",
        "    model=model_cloner(learning_rate)\n",
        "\n",
        "    history = model.fit(\n",
        "        train_ds,\n",
        "        steps_per_epoch=TRAIN_IMG_COUNT // BATCH_SIZE,\n",
        "        epochs=EPOCHS,\n",
        "        validation_data=val_ds,\n",
        "        validation_steps=VAL_IMG_COUNT // BATCH_SIZE,\n",
        "        class_weight=class_weight,\n",
        "        callbacks=[checkpoint_cb, early_stopping_cb]\n",
        "    )\n",
        "    loss, acc, prec, rec = model.evaluate(test_ds)\n",
        "    learning_rate_accuracy.append(acc)\n",
        "    step = 0\n",
        "    \n",
        "    while U_unlabelled_filenames!= []:\n",
        "    \n",
        "      print( \"Start of step : \" + str(step))\n",
        "\n",
        "      # Update training data\n",
        "      \n",
        "      U_unlabelled_filenames,new_labelled_filenames= new_labelled_dataset(U_unlabelled_filenames,model)\n",
        "      L_labelled_filenames = update_L_labelled_filenames(new_labelled_filenames)\n",
        "      x.append(len(L_labelled_filenames))\n",
        "      # Count number of NORMAL and PNEUMONIA  image\n",
        "\n",
        "      COUNT_NORMAL = len([filename for filename in L_labelled_filenames if \"NORMAL\" in filename])\n",
        "      COUNT_PNEUMONIA = len([filename for filename in L_labelled_filenames if \"PNEUMONIA\" in filename])\n",
        "\n",
        "      train_list_ds = tf.data.Dataset.from_tensor_slices(L_labelled_filenames)\n",
        "      train_ds = train_list_ds.map(process_path, num_parallel_calls=AUTOTUNE)\n",
        "      train_ds = prepare_for_training(train_ds)\n",
        "\n",
        "      # Class weight\n",
        "\n",
        "      weight_for_0 = (1 / COUNT_NORMAL)*(TRAIN_IMG_COUNT)/2.0 \n",
        "      weight_for_1 = (1 / COUNT_PNEUMONIA)*(TRAIN_IMG_COUNT)/2.0\n",
        "      class_weight = {0: weight_for_0, 1: weight_for_1}\n",
        "\n",
        "      # Retrain model\n",
        "\n",
        "      history = model.fit(\n",
        "        train_ds,\n",
        "        steps_per_epoch=TRAIN_IMG_COUNT // BATCH_SIZE,\n",
        "        epochs=EPOCHS,\n",
        "        validation_data=val_ds,\n",
        "        validation_steps=VAL_IMG_COUNT // BATCH_SIZE,\n",
        "        class_weight=class_weight,\n",
        "         callbacks=[checkpoint_cb, early_stopping_cb]\n",
        "      )\n",
        "      # save model\n",
        "      loss, acc, prec, rec = model.evaluate(test_ds)\n",
        "      learning_rate_accuracy.append(acc)\n",
        "      print( \"End of step : \" + str(step))\n",
        "      step+=1\n",
        "    accuracy.append([learning_rate,x,learning_rate_accuracy])\n",
        "\n",
        "print( \"End of training\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# proposed algorithm Uncertainty Sampling (Least confidence (LC)) with different learning rates and labelled samples"
      ],
      "metadata": {
        "id": "Z5MT7wrqJSiQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# plot lines\n",
        "plt.plot(accuracy[0][1], accuracy[0][2], label = \"learning rate = \" + str(accuracy[0][0]))\n",
        "plt.plot(accuracy[1][1], accuracy[1][2], label = \"learning rate = \" + str(accuracy[1][0]))\n",
        "plt.plot(accuracy[2][1], accuracy[2][2], label = \"learning rate = \" + str(accuracy[2][0]))\n",
        "plt.plot(accuracy[3][1], accuracy[3][2], label = \"learning rate = \" + str(accuracy[3][0]))\n",
        "plt.plot(accuracy[4][1], accuracy[4][2], label = \"learning rate = \" + str(accuracy[4][0]))\n",
        "plt.xlabel(\"# Labelled Samples\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.grid()\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "ROKHcz6xWIN1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#  Accuracy of Uncertainty Sampling (Least confidence (LC)) on a single graph with same x-axis labelled samples with a learning rate 0.05,"
      ],
      "metadata": {
        "id": "bhqYCKMuctcv"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "plt.plot(accuracy[1][1], accuracy[1][2], label = \"learning rate = \" + str(accuracy[1][0]))\n",
        "plt.xlabel(\"# Labelled Samples\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.grid()\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "O5cQhadRX3ZG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# The accuracy gap in terms of labelled samples for the Uncertainty Sampling (Least confidence (LC))."
      ],
      "metadata": {
        "id": "z98eAWwReQno"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "L=[]\n",
        "for i in range(len(accuracy[1][2])):\n",
        "  if i == 0:\n",
        "    L.append(0)\n",
        "  else:\n",
        "    L.append(accuracy[1][2][i]-accuracy[1][2][i-1])\n",
        "\n",
        "plt.plot(accuracy[1][1], L, label = \"learning rate = \" + str(accuracy[1][0]))\n",
        "plt.xlabel(\"# Labelled Samples\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.grid()\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "X5QlmzlqeMzX"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Accuracy with cross validation"
      ],
      "metadata": {
        "id": "E6tn_RzvfusA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "n_folds=2\n",
        "\n",
        "learning_rate=1\n",
        "#save the model history in a list after fitting so that we can plot later\n",
        "model_history = [] \n",
        "accuracy=[]\n",
        "for i in range(n_folds):\n",
        "    x=[]\n",
        "    print(\"Training on Fold: \",i+1)\n",
        "    train_filenames, val_filenames = train_test_split(filenames, test_size=0.2,random_state = np.random.randint(1,1000, 1)[0])\n",
        "                                               \n",
        "\n",
        "    \n",
        "    L_labelled_filenames, U_unlabelled_filenames = train_test_split(train_filenames, test_size=0.8)\n",
        "    train_list_ds = tf.data.Dataset.from_tensor_slices(L_labelled_filenames)\n",
        "    val_list_ds = tf.data.Dataset.from_tensor_slices(val_filenames)\n",
        "    train_ds = train_list_ds.map(process_path, num_parallel_calls=AUTOTUNE)\n",
        "\n",
        "    val_ds = val_list_ds.map(process_path, num_parallel_calls=AUTOTUNE)\n",
        "    train_ds = prepare_for_training(train_ds)\n",
        "    val_ds = prepare_for_training(val_ds)\n",
        "\n",
        "\n",
        "    x.append(len(L_labelled_filenames))\n",
        "    learning_rate_accuracy=[]\n",
        "    model=model_cloner(learning_rate)\n",
        "\n",
        "    history = model.fit(\n",
        "        train_ds,\n",
        "        steps_per_epoch=TRAIN_IMG_COUNT // BATCH_SIZE,\n",
        "        epochs=EPOCHS,\n",
        "        validation_data=val_ds,\n",
        "        validation_steps=VAL_IMG_COUNT // BATCH_SIZE,\n",
        "        class_weight=class_weight,\n",
        "        callbacks=[checkpoint_cb, early_stopping_cb]\n",
        "    )\n",
        "    loss, acc, prec, rec = model.evaluate(test_ds)\n",
        "    learning_rate_accuracy.append(acc)\n",
        "    step = 0\n",
        "    \n",
        "    while U_unlabelled_filenames!= []:\n",
        "    \n",
        "      print( \"Start of step : \" + str(step))\n",
        "\n",
        "      # Update training data\n",
        "      \n",
        "      U_unlabelled_filenames,new_labelled_filenames= new_labelled_dataset(U_unlabelled_filenames,model)\n",
        "      print(len(U_unlabelled_filenames))\n",
        "      L_labelled_filenames = update_L_labelled_filenames(new_labelled_filenames)\n",
        "      x.append(len(L_labelled_filenames))\n",
        "      # Count number of NORMAL and PNEUMONIA  image\n",
        "\n",
        "      COUNT_NORMAL = len([filename for filename in L_labelled_filenames if \"NORMAL\" in filename])\n",
        "      COUNT_PNEUMONIA = len([filename for filename in L_labelled_filenames if \"PNEUMONIA\" in filename])\n",
        "\n",
        "      train_list_ds = tf.data.Dataset.from_tensor_slices(L_labelled_filenames)\n",
        "      train_ds = train_list_ds.map(process_path, num_parallel_calls=AUTOTUNE)\n",
        "      train_ds = prepare_for_training(train_ds)\n",
        "\n",
        "      # Class weight\n",
        "\n",
        "      weight_for_0 = (1 / COUNT_NORMAL)*(TRAIN_IMG_COUNT)/2.0 \n",
        "      weight_for_1 = (1 / COUNT_PNEUMONIA)*(TRAIN_IMG_COUNT)/2.0\n",
        "      class_weight = {0: weight_for_0, 1: weight_for_1}\n",
        "\n",
        "      # Retrain model\n",
        "\n",
        "      history = model.fit(\n",
        "        train_ds,\n",
        "        steps_per_epoch=TRAIN_IMG_COUNT // BATCH_SIZE,\n",
        "        epochs=EPOCHS,\n",
        "        validation_data=val_ds,\n",
        "        validation_steps=VAL_IMG_COUNT // BATCH_SIZE,\n",
        "        class_weight=class_weight,\n",
        "         callbacks=[checkpoint_cb, early_stopping_cb]\n",
        "      )\n",
        "      # save model\n",
        "      loss, acc, prec, rec = model.evaluate(test_ds)\n",
        "      learning_rate_accuracy.append(acc)\n",
        "      print( \"End of step : \" + str(step))\n",
        "      step+=1\n",
        "    accuracy.append([n_folds,x,learning_rate_accuracy])\n",
        "    model_history.append(history)\n",
        "\n",
        "print( \"End of training\")"
      ],
      "metadata": {
        "id": "yW09Dyrzfz8t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# \tTraining and validation accuracy  for proposed algorithm with a learning rate 0.05."
      ],
      "metadata": {
        "id": "5sjiW7k2kgsH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "plt.title('Train Accuracy vs Val Accuracy')\n",
        "plt.plot(model_history[0].history['accuracy'], label='Train Accuracy Fold 1', color='black')\n",
        "plt.plot(model_history[0].history['val_accuracy'], label='Val Accuracy Fold 1', color='black', linestyle = \"dashdot\")\n",
        "\n",
        "plt.legend()\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "XYUQBPGNf7EJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "v8v0IljZruoh"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}