BN_DRISHTI_DEMO.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/shaoncsecu/BN-DRISHTI/blob/main/BN_DRISHTI_DEMO.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "##UPLOAD -> An Image Containing Handwritings"
      ],
      "metadata": {
        "id": "tnUKF8GmKQkf"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Upload a single document image** from your device after running this cell. The image should contain handwritten lines and words which will be segmented and shown in the following cells..."
      ],
      "metadata": {
        "id": "I0qpxBJiKSQG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from google.colab import files\n",
        "path15 = \"/content/1_1.jpg\"\n",
        "if os.path.exists(path15) == True:\n",
        "  !rm 1_1.jpg\n",
        "\n",
        "uploaded_image = files.upload()"
      ],
      "metadata": {
        "id": "108maeNUCVQo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "img = list(uploaded_image.keys())\n",
        "given_img_lbl = str(img[0])\n",
        "img1 = '1_1.jpg'\n",
        "\n",
        "old_name = '/content/'+ given_img_lbl\n",
        "new_name = \"/content/\" + img1\n",
        "os.rename(old_name, new_name)\n",
        "\n",
        "img2 = '/content/'+img1\n",
        "print(\"Image path: \",img2)"
      ],
      "metadata": {
        "id": "lwDSOnEyCjYS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FXQXRqjgMsD7"
      },
      "source": [
        "#**Defining Functions, Importing Libraries, Deleting Directory Contents**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WVu0zUTpMrvE"
      },
      "source": [
        "##Importing necessary libraries..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2jIXND5rkQds"
      },
      "outputs": [],
      "source": [
        "from os.path import join\n",
        "import numpy as np\n",
        "import operator\n",
        "import cv2\n",
        "import fnmatch\n",
        "import matplotlib.pyplot as plt\n",
        "import PIL\n",
        "from PIL import Image\n",
        "import glob\n",
        "import imutils\n",
        "from math import *\n",
        "from scipy.stats import mode\n",
        "from distutils.dir_util import copy_tree\n",
        "from IPython.display import Image\n",
        "from google.colab.patches import cv2_imshow\n",
        "import PIL.Image\n",
        "import os.path\n",
        "import shutil"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oeyhl7hVNtDM"
      },
      "source": [
        "##Removing previous files and folders from their directories in New run..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "N__WOJ1NJ4Z6"
      },
      "outputs": [],
      "source": [
        "# Removing previous files and folders from their directories....\n",
        "x = os.listdir(\"/content\")\n",
        "for i in x:\n",
        "  if i.endswith(('.png', '.jpg', '.jpeg', '.tiff', '.bmp', '.gif', 'JPG', 'JPEG', 'PNG')) == True:\n",
        "    img_label = i.split('.')[0]\n",
        "    final_path = \"/content/\"+img_label\n",
        "    if os.path.exists(final_path) == True:\n",
        "      shutil.rmtree(final_path)\n",
        "      # temp = \"/content/\"+i\n",
        "      # os.remove(temp)\n",
        "\n",
        "for j in x:\n",
        "  if j.endswith('.zip') == True:\n",
        "    !rm -rf /content/final_results.zip\n",
        "\n",
        "path1 = \"/content/DSkew\"\n",
        "if os.path.exists(path1) == True:\n",
        "  shutil.rmtree(path1)\n",
        "path2 = \"/content/HaughLine_Affine\"\n",
        "if os.path.exists(path2) == True:\n",
        "  shutil.rmtree(path2)\n",
        "path3 = \"/content/Rotated_line_by_HaughLine_Affine\"\n",
        "if os.path.exists(path3) == True:\n",
        "  shutil.rmtree(path3)\n",
        "path4 = \"/content/final_line_segmentation\"\n",
        "if os.path.exists(path4) == True:\n",
        "  shutil.rmtree(path4)\n",
        "path5 = \"/content/final_word_segmentation\"\n",
        "if os.path.exists(path5) == True:\n",
        "  shutil.rmtree(path5)\n",
        "path6 = \"/content/initial_line_segmantation\"\n",
        "if os.path.exists(path6) == True:\n",
        "  shutil.rmtree(path6)\n",
        "path7 = \"/content/sorted_Word_detection\"\n",
        "if os.path.exists(path7) == True:\n",
        "  shutil.rmtree(path7)\n",
        "path8 = \"/content/sorted_line_after_1st_detection\"\n",
        "if os.path.exists(path8) == True:\n",
        "  shutil.rmtree(path8)\n",
        "path9 = \"/content/yolov5/runs\"\n",
        "if os.path.exists(path9) == True:\n",
        "  shutil.rmtree(path9)\n",
        "path11 = \"/content/2nd line detection for rotated images\"\n",
        "if os.path.exists(path11) == True:\n",
        "  shutil.rmtree(path11)\n",
        "path12 = \"/content/2nd line detection for rotated images (labels)\"\n",
        "if os.path.exists(path12) == True:\n",
        "  shutil.rmtree(path12)\n",
        "path13 = \"/content/Original line images\"\n",
        "if os.path.exists(path13) == True:\n",
        "  shutil.rmtree(path13)\n",
        "path14 = \"/content/model\"\n",
        "if os.path.exists(path14) != True:\n",
        "  '''Downloading the model into CoLab temporary directory'''\n",
        "  !mkdir /content/model\n",
        "  !wget -P /content/model/ 'https://huggingface.co/crusnic/BN-DRISHTI/resolve/main/models/line_model_best.pt'\n",
        "  !wget -P /content/model/ 'https://huggingface.co/crusnic/BN-DRISHTI/resolve/main/models/word_model_best.pt'"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T2R2khd7Ucdl"
      },
      "source": [
        "##YoloV5 Setup..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4Eed9mPSUbM1"
      },
      "outputs": [],
      "source": [
        "path = \"/content/yolov5\"\n",
        "if os.path.exists(path) == False:\n",
        "  !git clone https://github.com/ultralytics/yolov5  # clone repo\n",
        "  %cd yolov5\n",
        "  %pip install -qr requirements.txt  # install dependencies\n",
        "\n",
        "  import torch\n",
        "  from IPython.display import Image, clear_output  # to display images\n",
        "\n",
        "  clear_output()\n",
        "  print('Setup complete. Using torch %s %s' % (torch.__version__, torch.cuda.get_device_properties(0) if torch.cuda.is_available() else 'CPU'))\n",
        "\n",
        "  %cd /content/\n",
        "else:\n",
        "  print(\"Yolo requirements are already exists!\")\n",
        "  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nfYJOTY-wxnE"
      },
      "source": [
        "##Unzipping files..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Yh18T2L28PPn"
      },
      "source": [
        "Defining a function to copy the dataset into colab local directory (for faster execution)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3P0Xu3HBwwW_"
      },
      "outputs": [],
      "source": [
        "def file_unzip(zip_file):\n",
        "  !unzip $zip_file -d \"/content\""
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wrIeF7wMi_TJ"
      },
      "source": [
        "##Sort line name of given list..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fz8JCWdX89VB"
      },
      "source": [
        "Defining a function to sort a list containing the name of images to visualize them serially. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Xd-wTz_lLzTb"
      },
      "outputs": [],
      "source": [
        "def line_sort(lines):\n",
        "    sort_lines = {}\n",
        "    for line in lines:\n",
        "        img_lb = line.split('.')[0]\n",
        "        lb = [int(i) for i in img_lb.split('_')]\n",
        "        new_lb = [ '0'+str(r) if r<10 else str(r) for r in lb]\n",
        "        if len(new_lb)==3:\n",
        "            items = int(new_lb[0]+new_lb[1]+new_lb[2])\n",
        "        if len(new_lb)==4:\n",
        "            items = int(new_lb[0]+new_lb[1]+new_lb[2]+new_lb[3])\n",
        "        sort_lines[items] = line\n",
        "    # print(sort_lines)\n",
        "    sort_lines = dict(sorted(sort_lines.items()))\n",
        "    new_lines = list(sort_lines.values())\n",
        "    return new_lines"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GaarKnu00UIU"
      },
      "source": [
        "##Showing All Transitions..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iTseHXFru5O5"
      },
      "source": [
        "Defining a function to draw images by iteration of a given directory. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OwKJD-2N0Y2_"
      },
      "outputs": [],
      "source": [
        "def show_transitions(f_name):\n",
        "  filename = f_name\n",
        "  crop_line = os.listdir(filename)\n",
        "  crop_lines = line_sort(crop_line)\n",
        "  print(crop_lines)\n",
        "  j = 1\n",
        "  for i in crop_lines:\n",
        "    print(\"Line no:\",j)\n",
        "    print(\"Image name:: \",i)\n",
        "    im = filename + \"/\" + i\n",
        "    crop_img = cv2.imread(im)\n",
        "    \n",
        "    width = crop_img.shape[1]\n",
        "    height = crop_img.shape[0]\n",
        "    print(\"Width, height: \",width, height)\n",
        "\n",
        "    plot_fig(crop_img)\n",
        "    j = j + 1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YFO_tls0uovo"
      },
      "source": [
        "Defining a function to plot figures."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NiP40j9jP5OB"
      },
      "outputs": [],
      "source": [
        "def plot_fig(img, size = 15):\n",
        "  plt.figure(figsize=(size, size))\n",
        "  plt.imshow(imutils.opencv2matplotlib(img))\n",
        "  plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UTRaB4mDvVG1"
      },
      "source": [
        "Defining a function to visually compare results of annotations and predictions of each line image by plotting them one after another."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mdDDyPkFKheW"
      },
      "outputs": [],
      "source": [
        "\n",
        "# def show_transitions_by_comparing(filename, gt_source_path, name1, name2):\n",
        "def show_transitions_by_comparing(filename, name2):\n",
        "    crop_line = os.listdir(filename)\n",
        "    crop_lines = line_sort(crop_line)\n",
        "    print(\"Cropped Line images: \",crop_lines)\n",
        "    print()\n",
        "    j = 1\n",
        "    for i in crop_lines:\n",
        "      print(\"Line no:\",j)\n",
        "      print(\"Image name:: \",i)\n",
        "      im = filename + \"/\" + i\n",
        "      crop_img = cv2.imread(im)\n",
        "      \n",
        "      width = crop_img.shape[1]\n",
        "      height = crop_img.shape[0]\n",
        "      \n",
        "      # # print(\"Annotated Line or GroundTruth ->\")\n",
        "      # print(name1)\n",
        "      # # img_gd = gt_line_img_dir + i\n",
        "      # img_gd = gt_source_path + i\n",
        "      # img_gd1 = cv2.imread(img_gd)\n",
        "      # plot_fig(img_gd1)\n",
        "\n",
        "      # print(\"Predicted Line ->\")\n",
        "      print(name2)\n",
        "      plot_fig(crop_img)\n",
        "      print()\n",
        "      j = j + 1"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vj5nqA3nQi5f"
      },
      "outputs": [],
      "source": [
        "def show_transitions_by_comparing_1(filename, gt_source_path, name1, name2):\n",
        "    crop_line = os.listdir(filename)\n",
        "    crop_lines = line_sort(crop_line)\n",
        "    print(\"Cropped Line images: \",crop_lines)\n",
        "    print()\n",
        "    j = 1\n",
        "    for i in crop_lines:\n",
        "      print(\"Line no:\",j)\n",
        "      print(\"Image name:: \",i)\n",
        "      im = filename + \"/\" + i\n",
        "      crop_img = cv2.imread(im)\n",
        "      \n",
        "      width = crop_img.shape[1]\n",
        "      height = crop_img.shape[0]\n",
        "      \n",
        "      # print(\"Annotated Line or GroundTruth ->\")\n",
        "      print(name1)\n",
        "      # img_gd = gt_line_img_dir + i\n",
        "      img_gd = gt_source_path + i\n",
        "      img_gd1 = cv2.imread(img_gd)\n",
        "      plot_fig(img_gd1)\n",
        "\n",
        "      # print(\"Predicted Line ->\")\n",
        "      print(name2)\n",
        "      plot_fig(crop_img)\n",
        "      print()\n",
        "      j = j + 1"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vAHj6AsXiUYS"
      },
      "source": [
        "##Drawing bounding box..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e636IVvnvvW9"
      },
      "source": [
        "Defining a function to draw bounding boxes on images with their given labels."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fTprGMPjiXQa"
      },
      "outputs": [],
      "source": [
        "def draw_BB(img_path, label_path, flag):\n",
        "  img = cv2.imread(img_path)\n",
        "  dh, dw, _ = img.shape\n",
        "\n",
        "  lb = open(label_path, 'r')\n",
        "  data = lb.readlines()\n",
        "  lb.close()\n",
        "\n",
        "  for dt in data:\n",
        "    if flag == 0:\n",
        "      _, x, y, w, h = map(float, dt.split(' '))\n",
        "    else:\n",
        "      _, x, y, w, h, conf = map(float, dt.split(' '))\n",
        "    l = int((x - w / 2) * dw)\n",
        "    r = int((x + w / 2) * dw)\n",
        "    t = int((y - h / 2) * dh)\n",
        "    b = int((y + h / 2) * dh)\n",
        "    if flag == 0:\n",
        "      cv2.rectangle(img, (l, t), (r, b), (0, 250, 0), 2)\n",
        "    else:\n",
        "      cv2.rectangle(img, (l, t), (r, b), (0, 0, 250), 2)\n",
        "    \n",
        "  # plot_fig(img)\n",
        "  return img\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kT40IqFNk5x9"
      },
      "source": [
        "#**Detecting lines with YOLO** (Upload the given trained Line Model weights on your drive and assign its directory to the --weights directory.)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KX_Ttz_Ev9Oq"
      },
      "source": [
        "Defining a function where we pass necessary attributes to detect lines by YOLO."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-WRT-NnJkyKW"
      },
      "outputs": [],
      "source": [
        "# Yolo Detection...\n",
        "def yolo_detection(img_path, img_size, conf):\n",
        "  # %cd yolov5\n",
        "  !python /content/yolov5/detect.py --weights /content/model/line_model_best.pt --img $img_size --conf $conf --source $img_path --save-conf --save-txt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "irXlLcrro76F"
      },
      "source": [
        "##Sorting & Filtering lines & words ( labels ) after their 1st detection..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V4bFk8fz3-VA"
      },
      "source": [
        "Defining a function to sort and filter -\n",
        "\n",
        "* lines based on their y-axis's attribute and confidence, respectively. \n",
        "* words based on their x-axis's attribute and width, respectively.  \n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YYh7x7Rfo69t"
      },
      "outputs": [],
      "source": [
        "class Line_sort:\n",
        "    def __init__(self, txt_files, txt_loc, sort_label, flag):\n",
        "        self.txt_files = txt_files\n",
        "        self.txt_loc = txt_loc\n",
        "        self.sort_label = sort_label\n",
        "        self.flag = flag\n",
        "        self.read_file()\n",
        "\n",
        "    def read_file(self):\n",
        "        files = self.txt_files\n",
        "        # os.mkdir('/content/sorted_line_after_1st_detection')\n",
        "        os.mkdir(self.sort_label)\n",
        "        for file in files:\n",
        "            txt_file = []\n",
        "            file_loc = self.txt_loc+file\n",
        "            with open(file_loc, 'r' , encoding='utf-8',errors='ignore') as lines:\n",
        "                for line in lines:\n",
        "                    token = line.split()\n",
        "                    \n",
        "                    _, x, y, w, h, conf = map(float, line.split(' '))\n",
        "                    # print(\"width -> \",w)\n",
        "                    # print(\"confidence -> \",conf)\n",
        "                    if self.flag == 0: # 1st line detection lavel\n",
        "                      if w > 0.50 and conf < 0.50:\n",
        "                        continue\n",
        "                      else:\n",
        "                        txt_file.append(token)\n",
        "                    else: # Word detection lavel\n",
        "                      # if w > 0.50:\n",
        "                      #   continue\n",
        "                      # else:\n",
        "                        txt_file.append(token)\n",
        "\n",
        "            if self.flag == 0: # 1st line detection lavel\n",
        "               sorted_txt_file = sorted(txt_file, key=operator.itemgetter(2))\n",
        "            else: # Word detection lavel\n",
        "               sorted_txt_file = sorted(txt_file, key=operator.itemgetter(1))\n",
        "\n",
        "            # lenght = len(sorted_txt_file[0])\n",
        "            self.file_write(sorted_txt_file, file)\n",
        "\n",
        "    def file_write(self,txt_file, file_name):\n",
        "        # loc = '/content/sorted_line_after_1st_detection/'+file_name\n",
        "        loc = self.sort_label+file_name\n",
        "        with open(loc, 'w') as f: \n",
        "            c=0\n",
        "            for line in txt_file:  \n",
        "                for l in line:\n",
        "                    c+=1\n",
        "                    if c == len(line):\n",
        "                        f.write('%s' % l)\n",
        "                    else:\n",
        "                        f.write('%s ' % l)\n",
        "                f.write(\"\\n\")\n",
        "                c=0\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SN80A0_1pq2F"
      },
      "outputs": [],
      "source": [
        "def sort_detection_label(txt_loc, sort_label, flag):\n",
        "  txt_files = os.listdir(txt_loc)\n",
        "  obj = Line_sort(txt_files, txt_loc, sort_label, flag)\n",
        "  "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "w6YALdh8TKXK"
      },
      "source": [
        "##1st Line Segment from Sorted line ( labels ) after 1st detection..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8A3aiY1B5osH"
      },
      "source": [
        "Defining a function to segment lines with YOLO's 1st line detection where we take full document image's width for a line if detected lines width ranges between 50% to 80%; otherwise, the lines are segmented as per detection."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4TfIxm5iTL75"
      },
      "outputs": [],
      "source": [
        "def line_segmantation_1(img, img_lb, label, segmented_img_path, flag = 0):\n",
        "# def line_segmantation_1(img, img_lb, flag = 0):\n",
        "  pred_lb = os.listdir(label)\n",
        "  print(pred_lb)\n",
        "  pred_lb2 = str(pred_lb[0])\n",
        "  pred_lb3 = label + pred_lb[0]\n",
        "\n",
        "  dir = segmented_img_path\n",
        "  # dir = \"/content/initial_line_segmantation\"\n",
        "  os.mkdir(dir)\n",
        "  img1 = cv2.imread(img)\n",
        "  dh, dw, _ = img1.shape\n",
        "  txt_lb = open(pred_lb3, 'r')\n",
        "  txt_lb_data = txt_lb.readlines()\n",
        "  txt_lb.close()\n",
        "  img_lb2 = img_lb.split('.')[0]\n",
        "  \n",
        "  k=1\n",
        "  for dt in txt_lb_data:\n",
        "      if flag != 0:\n",
        "        _, x, y, w, h = map(float, dt.split(' '))\n",
        "      else:\n",
        "        _, x, y, w, h, conf = map(float, dt.split(' '))\n",
        "        \n",
        "      if w > 0.50 and w < 0.80 and flag == 0:\n",
        "        x = 0.5\n",
        "        w = 1.0\n",
        "      l = int((x - w / 2) * dw)\n",
        "      r = int((x + w / 2) * dw)\n",
        "      t = int((y - h / 2) * dh)\n",
        "      b = int((y + h / 2) * dh)\n",
        "\n",
        "      crop = img1[t:b, l:r]\n",
        "      cv2.imwrite(\"{}/{}_{}.jpg\".format(dir, img_lb2, k), crop)\n",
        "      k += 1\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vRGq-eDyatWE"
      },
      "source": [
        "#**Rotation**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7pLXkMLsauVq"
      },
      "source": [
        "##DSkew by HaughLine..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nJQBXMDI_xMY"
      },
      "source": [
        "Defining functions to do necessary preprocessing and calculating a line image's dimension skew (DSkew) using Probabilistic Hough Line Transform (PHT). And correcting it by using the Affine Transform if the DSkew angle detected by PHT is more than 0; otherwise, keep the image without DSkew by setting the DSkew angle as 0.  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bXtNd9h6a3mm"
      },
      "outputs": [],
      "source": [
        "class ImgCorrect():\n",
        "    def __init__(self, img):\n",
        "        self.img = img\n",
        "        self.h, self.w, self.channel = self.img.shape\n",
        "        # print(\"Original images h & w -> | w: \",self.w, \"| h: \",self.h)\n",
        "        if self.w <= self.h:\n",
        "            self.scale = 700 / self.w\n",
        "            self.img = cv2.resize(self.img, (0, 0), fx=self.scale, fy=self.scale, interpolation=cv2.INTER_NEAREST)\n",
        "        else:\n",
        "            self.scale = 700 / self.h\n",
        "            self.img = cv2.resize(self.img, (0, 0), fx=self.scale, fy=self.scale, interpolation=cv2.INTER_NEAREST)\n",
        "        print(\"Resized Image by Padding and Scaling:\")\n",
        "        plot_fig(self.img)\n",
        "        self.gray = cv2.cvtColor(self.img, cv2.COLOR_BGR2GRAY)\n",
        "\n",
        "    def img_lines(self):\n",
        "        print(\"Gray Image:\")\n",
        "        plot_fig(self.gray)\n",
        "        ret, binary = cv2.threshold(self.gray, 0, 255, cv2.THRESH_BINARY_INV | cv2.THRESH_OTSU)\n",
        "        # cv2.imshow(\"bin\",binary)\n",
        "        print(\"Inverse Binary:\")\n",
        "        plot_fig(binary)\n",
        "        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3))  # rectangular structure\n",
        "        # print(\"Kernel for dialation:\")\n",
        "        # print(kernel)\n",
        "        binary = cv2.dilate(binary, kernel)  # dilate\n",
        "        print(\"Dilated Binary:\")\n",
        "        plot_fig(binary)\n",
        "        edges = cv2.Canny(binary, 50, 200)\n",
        "        print(\"Canny edged detection:\")\n",
        "        plot_fig(edges)\n",
        "\n",
        "        # print(\"Edge 1: \")\n",
        "        # cv2.imshow(\"edges\", edges)\n",
        "\n",
        "        self.lines = cv2.HoughLinesP(edges, 1, np.pi / 180, 100, minLineLength=100, maxLineGap=20)\n",
        "        # print(self.lines)\n",
        "        if self.lines is None:\n",
        "            print(\"Line segment not found\")\n",
        "            return None\n",
        "\n",
        "        lines1 = self.lines[:, 0, :]  # Extract as 2D\n",
        "        # print(lines1)\n",
        "        imglines = self.img.copy()\n",
        "        for x1, y1, x2, y2 in lines1[:]:\n",
        "            cv2.line(imglines, (x1, y1), (x2, y2), (0, 255, 0), 3)\n",
        "        print(\"Probabilistic Hough Lines:\")\n",
        "        plot_fig(imglines)\n",
        "        return imglines\n",
        "\n",
        "    def search_lines(self):\n",
        "      lines = self.lines[:, 0, :]  # extract as 2D\n",
        "    \n",
        "      number_inexist_k = 0\n",
        "      sum_pos_k45 = number_pos_k45 = 0\n",
        "      sum_pos_k90 = number_pos_k90 = 0\n",
        "      sum_neg_k45 = number_neg_k45 = 0\n",
        "      sum_neg_k90 = number_neg_k90 = 0\n",
        "      sum_zero_k = number_zero_k = 0\n",
        "\n",
        "      for x in lines:\n",
        "          if x[2] == x[0]:\n",
        "              number_inexist_k += 1\n",
        "              continue\n",
        "          #print(degrees(atan((x[3] - x[1]) / (x[2] - x[0]))), \"pos:\", x[0], x[1], x[2], x[3], \"Slope:\",(x[3] - x[1]) / (x[2] - x[0]))\n",
        "          degree = degrees(atan((x[3] - x[1]) / (x[2] - x[0])))\n",
        "          # print(\"Degree or Slope of detected lines : \",degree)\n",
        "          if 0 < degree < 45:\n",
        "              number_pos_k45 += 1\n",
        "              sum_pos_k45 += degree\n",
        "          if 45 <= degree < 90:\n",
        "              number_pos_k90 += 1\n",
        "              sum_pos_k90 += degree\n",
        "          if -45 < degree < 0:\n",
        "              number_neg_k45 += 1\n",
        "              sum_neg_k45 += degree\n",
        "          if -90 < degree <= -45:\n",
        "              number_neg_k90 += 1\n",
        "              sum_neg_k90 += degree\n",
        "          if x[3] == x[1]:\n",
        "              number_zero_k += 1\n",
        "\n",
        "      max_number = max(number_inexist_k, number_pos_k45, number_pos_k90, number_neg_k45,number_neg_k90, number_zero_k)\n",
        "      # print(\"Num of lines in different Degree range ->\")\n",
        "      # print(\"Not a Line: \",number_inexist_k, \"| 0 to 45: \",number_pos_k45, \"| 45 to 90: \",number_pos_k90, \"| -45 to 0: \",number_neg_k45, \"| -90 to -45: \",number_neg_k90, \"| Line where y1 equals y2 :\",number_zero_k)\n",
        "    \n",
        "      if max_number == number_inexist_k:\n",
        "          return 90\n",
        "      if max_number == number_pos_k45:\n",
        "          return sum_pos_k45 / number_pos_k45\n",
        "      if max_number == number_pos_k90:\n",
        "          return sum_pos_k90 / number_pos_k90\n",
        "      if max_number == number_neg_k45:\n",
        "          return sum_neg_k45 / number_neg_k45\n",
        "      if max_number == number_neg_k90:\n",
        "          return sum_neg_k90 / number_neg_k90\n",
        "      if max_number == number_zero_k:\n",
        "          return 0\n",
        "\n",
        "    def rotate_image(self, degree):\n",
        "        \"\"\"\n",
        "        Positive angle counterclockwise rotation\n",
        "        :param degree:\n",
        "        :return:\n",
        "        \"\"\"\n",
        "        # print(\"degree:\", degree)\n",
        "        if -45 <= degree <= 0:\n",
        "            degree = degree  # #negative angle clockwise\n",
        "        if -90 <= degree < -45:\n",
        "            degree = 90 + degree  # positive angle counterclockwise\n",
        "        if 0 < degree <= 45:\n",
        "            degree = degree  # positive angle counterclockwise\n",
        "        if 45 < degree <= 90:\n",
        "            degree = degree - 90  # negative angle clockwise\n",
        "        print(\"DSkew angle: \", degree)\n",
        "\n",
        "        # degree = degree - 90\n",
        "        height, width = self.img.shape[:2]\n",
        "        heightNew = int(width * fabs(sin(radians(degree))) + height * fabs(\n",
        "            cos(radians(degree))))  # This formula refers to the previous content\n",
        "        widthNew = int(height * fabs(sin(radians(degree))) + width * fabs(cos(radians(degree))))\n",
        "        # print(\"Height :\",height)\n",
        "        # print(\"Width :\",width)\n",
        "        # print(\"HeightNew :\",heightNew)\n",
        "        # print(\"WidthNew :\",widthNew)\n",
        "\n",
        "        matRotation = cv2.getRotationMatrix2D((width / 2, height / 2), degree, 1)  # rotate degree counterclockwise\n",
        "        # print(\"Mat Rotation (Before): \",matRotation)\n",
        "        matRotation[0, 2] += (widthNew - width) / 2\n",
        "        # Because after rotation, the origin of the coordinate system is the upper left corner of the new image, so it needs to be converted according to the original image\n",
        "        matRotation[1, 2] += (heightNew - height) / 2\n",
        "        # print(\"Mat Rotation (After): \",matRotation)\n",
        "\n",
        "        # Affine transformation, the background color is filled with white\n",
        "        imgRotation = cv2.warpAffine(self.img, matRotation, (widthNew, heightNew), borderValue=(255, 255, 255))\n",
        "\n",
        "        # Padding\n",
        "        pad_image_rotate = cv2.warpAffine(self.img, matRotation, (widthNew, heightNew), borderValue=(0, 255, 0))\n",
        "        plot_fig(pad_image_rotate)\n",
        "\n",
        "        return imgRotation\n",
        "\n",
        "def dskew(line_path, img):\n",
        "    img_loc = line_path + img\n",
        "    im = cv2.imread(img_loc)\n",
        "\n",
        "    # Padding\n",
        "    bg_color = [255, 255, 255]\n",
        "    pad_img = cv2.copyMakeBorder(im,100,100,100,100,cv2.BORDER_CONSTANT,value=bg_color)\n",
        "\n",
        "    imgcorrect = ImgCorrect(pad_img)\n",
        "    lines_img = imgcorrect.img_lines()\n",
        "    # print(type(lines_img))\n",
        "    \n",
        "    if lines_img is None:\n",
        "        rotate = imgcorrect.rotate_image(0)\n",
        "    else:\n",
        "        degree = imgcorrect.search_lines()\n",
        "        rotate = imgcorrect.rotate_image(degree)\n",
        "\n",
        "\n",
        "    return rotate\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UrhXDItra-Wl"
      },
      "source": [
        "## HoughLine and Affine Transform..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yLD-XcQ8-hfR"
      },
      "source": [
        "Defining functions to do necessary preprocessing and calculating the skew of an image's main handwritten line using Standard Hough Line Transform (SHT). And correcting it by using Affine Transform if the skew angle detected by SHT is more than 0; otherwise, send the image for DSkew. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cZVvh3sqbLsD"
      },
      "outputs": [],
      "source": [
        "rotate_line = \"/content/Rotated_line_by_HaughLine_Affine/\"\n",
        "# os.mkdir(rotate_line)\n",
        "\n",
        "rotate_line_Dskew = \"/content/DSkew/\"\n",
        "# os.mkdir(rotate_line_Dskew)\n",
        "\n",
        "rotate_line_Haughline = \"/content/HaughLine_Affine/\"\n",
        "# os.mkdir(rotate_line_Haughline)\n",
        "\n",
        "exception = []\n",
        "\n",
        "# Degree conversion\n",
        "def DegreeTrans(theta):\n",
        "    res = theta / np.pi * 180\n",
        "    # print(res)\n",
        "    return res\n",
        "\n",
        "# Rotate the image degree counterclockwise (original size)\n",
        "def rotateImage(src, degree):\n",
        "    # The center of rotation is the center of the image\n",
        "    h, w = src.shape[:2]\n",
        "    # Calculate the two-dimensional rotating affine transformation matrix\n",
        "    RotateMatrix = cv2.getRotationMatrix2D((w / 2.0, h / 2.0), degree, 1)\n",
        "    # print(\"Rotate Matrix: \")\n",
        "    # print(RotateMatrix)\n",
        "\n",
        "    # Affine transformation, the background color is filled with GREEN so that the rotation can be easily understood\n",
        "    rotate1 = cv2.warpAffine(src, RotateMatrix, (w, h), borderValue=(0, 255, 0))\n",
        "    plot_fig(rotate1)\n",
        "    # Affine transformation, the background color is filled with white\n",
        "    rotate = cv2.warpAffine(src, RotateMatrix, (w, h), borderValue=(255, 255, 255))\n",
        "\n",
        "    # Padding\n",
        "    bg_color = [255, 255, 255]\n",
        "    pad_image_rotate = cv2.copyMakeBorder(rotate,100,100,100,100,cv2.BORDER_CONSTANT,value=bg_color)\n",
        "\n",
        "    return pad_image_rotate\n",
        "\n",
        "# Calculate angle by Hough transform\n",
        "def CalcDegree(srcImage,canny_img):\n",
        "    lineimage = srcImage.copy()\n",
        "    lineimg = srcImage.copy()\n",
        "    # Detect straight lines by Hough transform\n",
        "    # The fourth parameter is the threshold, the greater the threshold, the higher the detection accuracy\n",
        "    try:\n",
        "        lines = cv2.HoughLines(canny_img, 1, np.pi / 180, 200)\n",
        "        # print(\"HoughLines: \")\n",
        "        # cv2_imshow(lines)\n",
        "        # Due to different images, the threshold is not easy to set, because the threshold is set too high, so that the line cannot be detected, the threshold is too low, the line is too much, the speed is very slow\n",
        "        theta_sum = 0\n",
        "        rho_sum = 0\n",
        "        sum_x1 = sum_x2 = sum_y1 = sum_y2 = 0\n",
        "        # Draw each line segment in turn\n",
        "        for i in range(len(lines)):\n",
        "            for rho, theta in lines[i]:\n",
        "                # print(\"theta:\", theta, \" rho:\", rho)\n",
        "                a = np.cos(theta)\n",
        "                b = np.sin(theta)\n",
        "                x0 = a * rho\n",
        "                y0 = b * rho\n",
        "                x1 = int(round(x0 + 1000 * (-b)))\n",
        "                y1 = int(round(y0 + 1000 * a))\n",
        "                x2 = int(round(x0 - 1000 * (-b)))\n",
        "                y2 = int(round(y0 - 1000 * a))\n",
        "                # print(\"a: \",a, \" b: \",b, \" x0: \",x0, \" y0: \",y0, \" x1: \",x1, \" y1: \",y1, \" x2: \",x2, \" y2: \",y2)\n",
        "                # Only select the smallest angle as the rotation angle\n",
        "                sum_x1+=x1\n",
        "                sum_x2+=x2\n",
        "                sum_y1+=y1\n",
        "                sum_y2+=y2\n",
        "                rho_sum += rho\n",
        "                theta_sum += theta\n",
        "                cv2.line(lineimage, (x1, y1), (x2, y2), (0, 0, 255), 1, cv2.LINE_AA)\n",
        "\n",
        "        \n",
        "        print(\"HoughLines: \")\n",
        "        plot_fig(lineimage)\n",
        "        print()\n",
        "\n",
        "        pt1 = (sum_x1//len(lines), sum_y1//len(lines))\n",
        "        pt2 = (sum_x2//len(lines), sum_y2//len(lines))\n",
        "        \n",
        "        # print(\"Sum of thetas: \",theta_sum)\n",
        "        # print(\"lines: \",lines)\n",
        "        average = theta_sum / len(lines)\n",
        "        # print(\"Avg. Theta: \",average)\n",
        "        angle = DegreeTrans(average) - 90\n",
        "        # print(\"Avg. Angle: \",angle)\n",
        "        print(\"Skewed Angle: \",angle)\n",
        "        average_rho = rho_sum / len(lines)\n",
        "        # print(\"Avg. rho: \",average_rho)\n",
        "\n",
        "        # print(pt1,pt2)\n",
        "        print('Draw best fit line with full:')\n",
        "        # h, w = lineimg.shape[:2]\n",
        "        # pt2 = (w,h)\n",
        "        # print(\"Cordinates of the best fit line: \",pt1,pt2)\n",
        "        cv2.line(lineimg, pt1, pt2, (0,0,255), 2)\n",
        "        plot_fig(lineimg)\n",
        "        # cv2_imshow(lineimg)\n",
        "\n",
        "        return angle\n",
        "    except:\n",
        "        angle = 0.0\n",
        "        return angle\n",
        "\n",
        "def ready_for_rotate(line_path, img):\n",
        "    print()\n",
        "    print(\"Image :: \",img)\n",
        "    img_loc = line_path + img\n",
        "    image = cv2.imread(img_loc)\n",
        "\n",
        "    org_width = image.shape[1]\n",
        "    org_height = image.shape[0]\n",
        "\n",
        "    img1 = image\n",
        "    im_gray = cv2.cvtColor(img1, cv2.COLOR_BGR2GRAY)\n",
        "    print(\"Gray Image: \")\n",
        "    plot_fig(im_gray)\n",
        "\n",
        "    edges = cv2.Canny(im_gray,50,150,apertureSize=3)\n",
        "    print(\"Canny Image: \")\n",
        "    plot_fig(edges)\n",
        "\n",
        "    degree = CalcDegree(image,edges)\n",
        "    \n",
        "    if degree == 0.0:\n",
        "        rotate = dskew(line_path, img)\n",
        "        print(\"Rotated Image by DSkew: \")\n",
        "        plot_fig(rotate)\n",
        "        print()\n",
        "        \n",
        "        filename1 = rotate_line_Dskew + img\n",
        "        cv2.imwrite(filename1, rotate)\n",
        "        filename = rotate_line + img\n",
        "        cv2.imwrite(filename, rotate)\n",
        "    else:\n",
        "        rotate = rotateImage(image, degree)\n",
        "        print(\"Rotated Image by Houghline Affine transform: \")\n",
        "        plot_fig(rotate)\n",
        "        print()\n",
        "\n",
        "        filename2 = rotate_line_Haughline + img\n",
        "        cv2.imwrite(filename2, rotate)\n",
        "        filename = rotate_line + img\n",
        "        cv2.imwrite(filename, rotate)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NaBsaedbcOG_"
      },
      "outputs": [],
      "source": [
        "def rotate_lines(first_detection):\n",
        "  line_path = first_detection\n",
        "  line_dir = line_sort(os.listdir(line_path))\n",
        "  print(line_dir)\n",
        "\n",
        "  for img in line_dir:\n",
        "      ready_for_rotate(line_path, img)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B95GQhgIDsfK"
      },
      "source": [
        "#**Final Line Segmentation**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "POoMuSzjAvB0"
      },
      "source": [
        "##Find undetected image in Yolo 2nd detection..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cVF-hwXFCymn"
      },
      "source": [
        "Defining a function to get those images that are not getting 2nd YOLO detection for their low dimension and putting those images with the final segmented images.  "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "yRuA-ABVA1gY"
      },
      "outputs": [],
      "source": [
        "undetected_images_path = []\n",
        "\n",
        "def find_undetected_images(img, label):\n",
        "  # img_path = \"/content/Rotated_line_by_HaughLine_Affine/\"\n",
        "  img_path = img\n",
        "  # detect_lb_path = \"/content/yolov5/runs/detect/exp2/labels/\"\n",
        "  detect_lb_path = label\n",
        "  undetect_img_path = \"/content/final_line_segmentation/\"\n",
        "\n",
        "  def take_valid_img(images):\n",
        "      image = []\n",
        "      valid_img_ext = [\"jpg\", \"JPG\", \"jpeg\", \"JPEG\", \"png\", \"PNG\"]\n",
        "      for img in images:\n",
        "          try:\n",
        "              ext = img.split('.')[1]\n",
        "              if ext not in valid_img_ext :\n",
        "                  continue\n",
        "              else:\n",
        "                  image.append(img)\n",
        "          except:\n",
        "              continue\n",
        "      return image\n",
        "\n",
        "  img1 = os.listdir(img_path)\n",
        "  img = take_valid_img(img1)\n",
        "  detect_lb = os.listdir(detect_lb_path)\n",
        "\n",
        "  def find_undetect_img(img,detect_lb):\n",
        "      img_lb = [im.split('.')[0] for im in img]\n",
        "      dt_lb = [dt.split('.')[0] for dt in detect_lb]\n",
        "      undt_lb = list(set(img_lb).difference(dt_lb))\n",
        "      undetect_img = []\n",
        "      detect_img = []\n",
        "      for lb in undt_lb:\n",
        "          for im in img:\n",
        "              im_lb = im.split('.')[0]\n",
        "              if lb == im_lb:\n",
        "                  undetect_img.append(im)\n",
        "              else:\n",
        "                  detect_img.append(im)\n",
        "      print(\"Undetect image: \",undetect_img)\n",
        "      write_image(undetect_img)\n",
        "\n",
        "  def write_image(undt_img):\n",
        "      for im in undt_img:\n",
        "          filename = undetect_img_path+im\n",
        "          img = cv2.imread(img_path+im)\n",
        "          cv2.imwrite(filename,img)\n",
        "          undetected_images_path.append(filename)\n",
        "\n",
        "  find_undetect_img(img,detect_lb)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FvoftpP5JZz_"
      },
      "source": [
        "## 2nd Line Segmentation after Rotation..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ABAZ6Rh8DySL"
      },
      "source": [
        "Defining a function to select the main handwritten line from line images followed by segmenting it using YOLO's 2nd line detection. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Jsv1wlDkrKBH"
      },
      "outputs": [],
      "source": [
        "def crop_image(bb_data, destination, image, img_lb, dh, dw):\n",
        "    x = float(bb_data[1])\n",
        "    y = float(bb_data[2])\n",
        "    w = float(bb_data[3])\n",
        "    h = float(bb_data[4])\n",
        "  \n",
        "    # x = 0.5\n",
        "    # w  = 1.0\n",
        "    l = int((x - w / 2) * dw)\n",
        "    r = int((x + w / 2) * dw)\n",
        "    t = int((y - h / 2) * dh)\n",
        "    b = int((y + h / 2) * dh)\n",
        "\n",
        "    crop = image[t:b, l:r]\n",
        "    filename = destination+img_lb\n",
        "    cv2.imwrite(filename,crop)\n",
        "    print(\"Segmented successfully!\\n\")\n",
        "\n",
        "def line_segmantation_2(img, img_path, label, label_path, segmented_img_path):\n",
        "  dir = segmented_img_path\n",
        "  print(\"Image path -> \",img_path)\n",
        "  img1 = cv2.imread(img_path)\n",
        "  dh, dw, _ = img1.shape\n",
        "  txt_lb = open(label_path, 'r')\n",
        "  txt_lb_data = txt_lb.readlines()\n",
        "  txt_lb.close()\n",
        "  img_name = img\n",
        "  \n",
        "  max_w = 0\n",
        "  data1 = []\n",
        "  for line in txt_lb_data:\n",
        "      token = line.split()\n",
        "      data1.append(token)\n",
        "  \n",
        "  if len(data1)==1:\n",
        "      bb_data = data1[0]\n",
        "      wdth = float(bb_data[3])\n",
        "      if wdth>0.4:\n",
        "          crop_image(bb_data,dir,img1,img_name,dh,dw,)\n",
        "      else:\n",
        "          filename = dir+img_name\n",
        "          cv2.imwrite(filename,img1)\n",
        "  elif len(data1)==2:\n",
        "      bb_data1 = data1[0]\n",
        "      bb_data2 = data1[1]\n",
        "      w1 = float(bb_data1[3]) \n",
        "      w2 = float(bb_data2[3])\n",
        "      c1 = float(bb_data1[5]) \n",
        "      c2 = float(bb_data2[5])\n",
        "      if w1 <= 0.5 and w2 <= 0.5:\n",
        "        if c1 >= 0.8 and c2 >= 0.8:\n",
        "          sorted_bb_data = sorted(data1, key=operator.itemgetter(5))\n",
        "          bb_data = sorted_bb_data[-1]\n",
        "          crop_image(bb_data,dir,img1,img_name,dh,dw,)\n",
        "        else:\n",
        "          filename = dir+img_name\n",
        "          cv2.imwrite(filename,img1)\n",
        "      else:\n",
        "        sorted_bb_data = sorted(data1, key=operator.itemgetter(3))\n",
        "        bb_data = sorted_bb_data[-1]\n",
        "        crop_image(bb_data,dir,img1,img_name,dh,dw,)\n",
        "  elif len(data1)==3:\n",
        "      sorted_bb_data = sorted(data1, key=operator.itemgetter(2))\n",
        "      bb_data = sorted_bb_data[1]\n",
        "      crop_image(bb_data,dir,img1,img_name,dh,dw,) \n",
        "  else:\n",
        "      sorted_bb_data = sorted(data1, key=operator.itemgetter(3))\n",
        "      bb_data = sorted_bb_data[-1]\n",
        "      crop_image(bb_data,dir,img1,img_name,dh,dw,)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1QVZUHX_8lSX"
      },
      "source": [
        "#**Word Segmentation**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tdngiWC7GNyL"
      },
      "source": [
        "Defining a function to segment the words of line images by using YOLO's word detection."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "mkMTHFDv8mk2"
      },
      "outputs": [],
      "source": [
        "def word_segmentation(line_images, word_labels):\n",
        "  line_img = os.listdir(line_images)\n",
        "  word_label = os.listdir(word_labels)\n",
        "  print(line_img)\n",
        "  print(word_label)\n",
        "\n",
        "  for i in word_label:\n",
        "    for j in line_img:\n",
        "      fn_i = i.split(\".\")\n",
        "      fn_j = j.split(\".\")\n",
        "      if fn_i[0] ==  fn_j[0]:\n",
        "        # print(\"yes\")\n",
        "        dir = \"/content/final_word_segmentation/\"+fn_i[0]\n",
        "        os.mkdir(dir)\n",
        "\n",
        "        img = cv2.imread(line_images + j)\n",
        "        dh, dw, _ = img.shape\n",
        "        txt_lb = open(word_labels + i, 'r')\n",
        "        txt_lb_data = txt_lb.readlines()\n",
        "        txt_lb.close()\n",
        "        img_lb = fn_i[0]\n",
        "        \n",
        "        k=1\n",
        "        for dt in txt_lb_data:\n",
        "            # _, x, y, w, h = map(float, dt.split(' '))\n",
        "            _, x, y, w, h, conf = map(float, dt.split(' '))\n",
        "            l = int((x - w / 2) * dw)\n",
        "            r = int((x + w / 2) * dw)\n",
        "            t = int((y - h / 2) * dh)\n",
        "            b = int((y + h / 2) * dh)\n",
        "            if w > 0.50:\n",
        "                continue\n",
        "            crop = img[t:b, l:r]\n",
        "            cv2.imwrite(\"{}/{}_{}.jpg\".format(dir, img_lb, k), crop)\n",
        "            k += 1\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G2lYFDvp-v5Z"
      },
      "source": [
        "#**Calculating IoU, DR, RA, FM by comparing Sorted line (labels) and GroundTruth**"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rvnPmg0cfoHf"
      },
      "outputs": [],
      "source": [
        "def arrange_img_gt_pred_labels(images, gt, pred):\n",
        "    images_lables = {}\n",
        "    for i in range(len(images)):\n",
        "        img_lb = images[i].split('.')[0]\n",
        "        value = []\n",
        "        for j in range(len(pred)):\n",
        "            pred_lb = pred[j].split(\".\")[0]\n",
        "            if img_lb == pred_lb:\n",
        "                value.append(pred[j])\n",
        "                break\n",
        "        for k in range(len(gt)):\n",
        "            gt_lb = gt[k].split(\".\")[0]\n",
        "            if img_lb == gt_lb:\n",
        "                value.append(gt[k])\n",
        "                break\n",
        "        if len(value)==2:\n",
        "            images_lables[images[i]] = value\n",
        "    return images_lables"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Wpd_BuGTfrfd"
      },
      "outputs": [],
      "source": [
        "# Convert Yolo format to Pascal voc format\n",
        "def yolo_to_voc(width, height, x, y, w, h):\n",
        "    xmax = int((x * width) + (w * width) / 2.0)\n",
        "    xmin = int((x * width) - (w * width) / 2.0)\n",
        "    ymax = int((y * height) + (h * height) / 2.0)\n",
        "    ymin = int((y * height) - (h * height) / 2.0)\n",
        "    return (xmin, ymin, xmax, ymax)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pZUAv3GDfvI6"
      },
      "outputs": [],
      "source": [
        "# take all Ground truth image shape in a list\n",
        "def get_img_shape(image_path,images):\n",
        "    img_shape = []\n",
        "    for im in images:\n",
        "        shape = []\n",
        "        img_path = image_path+im\n",
        "        img = cv2.imread(img_path)\n",
        "        dh,dw,_ = img.shape\n",
        "        shape.append(dh)\n",
        "        shape.append(dw)\n",
        "        img_shape.append(shape)\n",
        "    return img_shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "gir3cZrifxvv"
      },
      "outputs": [],
      "source": [
        "'''\n",
        "    This function takes the predicted bounding box and ground truth bounding box and\n",
        "    return the IoU ratio\n",
        "'''\n",
        "def calc_iou(gt_bbox,pred_bbox):\n",
        "\n",
        "    x_topleft_gt, y_topleft_gt, x_bottomright_gt, y_bottomright_gt = gt_bbox\n",
        "    #print(x_topleft_gt, y_topleft_gt, x_bottomright_gt, y_bottomright_gt)\n",
        "\n",
        "    x_topleft_p, y_topleft_p, x_bottomright_p, y_bottomright_p = pred_bbox\n",
        "\n",
        "    if (x_topleft_gt > x_bottomright_gt) or (y_topleft_gt > y_bottomright_gt):\n",
        "        raise AssertionError(\"Ground Truth Bounding Box is not correct\")\n",
        "    if (x_topleft_p > x_bottomright_p) or (y_topleft_p > y_bottomright_p):\n",
        "        raise AssertionError(\"Predicted Bounding Box is not correct\", x_topleft_p, x_bottomright_p, y_topleft_p,y_bottomright_gt)\n",
        "\n",
        "    # if the GT bbox and predcited BBox do not overlap then iou=0\n",
        "\n",
        "    # If bottom right of x-coordinate  GT  bbox is less than or above the top left of x coordinate of  the predicted BBox\n",
        "    if (x_bottomright_gt < x_topleft_p): \n",
        "        return 0.0\n",
        "    \n",
        "    # If bottom right of y-coordinate  GT  bbox is less than or above the top left of y coordinate of  the predicted BBox    \n",
        "    if (y_bottomright_gt < y_topleft_p):  \n",
        "        return 0.0\n",
        "    \n",
        "    # If bottom right of x-coordinate  GT  bbox is greater than or below the bottom right  of x coordinate of  the predcited BBox\n",
        "    if (x_topleft_gt > x_bottomright_p):  \n",
        "        return 0.0\n",
        "\n",
        "    # If bottom right of y-coordinate  GT  bbox is greater than or below the bottom right  of y coordinate of  the predcited BBox\n",
        "    if (y_topleft_gt > y_bottomright_p):  \n",
        "        return 0.0\n",
        "\n",
        "    GT_bbox_area = (x_bottomright_gt - x_topleft_gt + 1) * (y_bottomright_gt - y_topleft_gt + 1)\n",
        "    Pred_bbox_area = (x_bottomright_p - x_topleft_p + 1) * (y_bottomright_p - y_topleft_p + 1)\n",
        "\n",
        "    x_top_left = np.max([x_topleft_gt, x_topleft_p])\n",
        "    y_top_left = np.max([y_topleft_gt, y_topleft_p])\n",
        "    x_bottom_right = np.min([x_bottomright_gt, x_bottomright_p])\n",
        "    y_bottom_right = np.min([y_bottomright_gt, y_bottomright_p])\n",
        "\n",
        "    intersection_area = (x_bottom_right - x_top_left + 1) * (y_bottom_right - y_top_left + 1)\n",
        "\n",
        "    union_area = (GT_bbox_area + Pred_bbox_area - intersection_area)\n",
        "\n",
        "    return intersection_area / union_area"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LolAGsexf0E1"
      },
      "outputs": [],
      "source": [
        "def save_result(iou_thr, file_name, per_img_result, final_iou, avg_dr, avg_ra, FM):\n",
        "    #file = '/content/overall_result.txt'\n",
        "    with open(file_name, 'w') as f:\n",
        "        for result in per_img_result:\n",
        "            for res in result:\n",
        "                f.write(\"%s  \" %res)\n",
        "            f.write('\\n')\n",
        "        f.write('\\nOverall IoU score @{} = {} \\nDetection Rate(Recall)/DR= {} \\nRecognition Rate(Precision)/RA = {} \\nFinal Performance(f1 score)/FM = {} \\n'.format(iou_thr, final_iou,avg_dr,avg_ra,FM))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IUpjYGVPf3J5"
      },
      "outputs": [],
      "source": [
        "\"\"\"\n",
        "  This function take images, Ground truth labels, and Predicted labels then calculate IOU value, \n",
        "  Detection Rate(Recall), Recognition Rate(Precision), Final Performance(f1 score)  \n",
        "\"\"\"\n",
        "def multiple_image_result(file_path, iou_thr, image_gt_pred_data, image_shape, gt_path, pred_path):\n",
        "    avg_iou = []\n",
        "    #print(iou_thr)\n",
        "    DR = []\n",
        "    RA = []\n",
        "    per_img_result = []\n",
        "\n",
        "    id = 0\n",
        "    for key in image_gt_pred_data:\n",
        "\n",
        "        gt_lb_path = gt_path + image_gt_pred_data[key][1]\n",
        "        pred_lb_path = pred_path + image_gt_pred_data[key][0]\n",
        "\n",
        "        # print(gt_lb_path)\n",
        "        # print(pred_lb_path)\n",
        "\n",
        "        fl = open(gt_lb_path, 'r')\n",
        "        f2 = open(pred_lb_path, 'r')\n",
        "        gt_lb_data = fl.readlines()\n",
        "        pred_lb_data = f2.readlines()\n",
        "        fl.close()\n",
        "        f2.close()\n",
        "        \n",
        "        height = image_shape[id][0]\n",
        "        width = image_shape[id][1]\n",
        "        # print(height,width)\n",
        "         \n",
        "        all_gt_lb = []\n",
        "        all_pred_lb = []\n",
        "\n",
        "        for dt in gt_lb_data:\n",
        "            class_id, x, y, w, h = map(float, dt.split(' '))\n",
        "            new_label = list(yolo_to_voc(width, height, x, y, w, h))\n",
        "            all_gt_lb.append(new_label)\n",
        "\n",
        "        for dt in pred_lb_data:\n",
        "            class_id, x, y, w, h, conf = map(float, dt.split(' '))\n",
        "            new_label = list(yolo_to_voc(width, height, x, y, w, h))\n",
        "            all_pred_lb.append(new_label)\n",
        "\n",
        "        ious = []\n",
        "        for ipb, pred_box in enumerate(all_pred_lb):\n",
        "            for igb, gt_box in enumerate(all_gt_lb):\n",
        "                iou = calc_iou(gt_box, pred_box)\n",
        "                if iou > iou_thr:\n",
        "                    ious.append(iou)\n",
        "\n",
        "        o2o_per_img = len(ious)\n",
        "        N_per_img = len(all_gt_lb)\n",
        "        M_per_img = len(all_pred_lb)\n",
        "\n",
        "        per_img_iou = 0\n",
        "        if N_per_img != 0:\n",
        "            per_line_iou = 0\n",
        "            for i in range(len(ious)):\n",
        "                per_line_iou += ious[i]\n",
        "            per_img_iou = per_line_iou / N_per_img\n",
        "            avg_iou.append(per_img_iou)\n",
        "        else:\n",
        "            avg_iou.append(per_img_iou)\n",
        "\n",
        "        dr_per_img = o2o_per_img / N_per_img\n",
        "        ra_per_img = o2o_per_img / M_per_img\n",
        "\n",
        "        DR.append(dr_per_img)\n",
        "        RA.append(ra_per_img)\n",
        "        \n",
        "        result = []\n",
        "        result.append(\"image=\"+key)\n",
        "        result.append(\"o2o=\"+str(o2o_per_img))\n",
        "        result.append(\"N=\"+str(N_per_img))\n",
        "        result.append(\"M=\"+str(M_per_img))\n",
        "        result.append(\"iou=\"+str(per_img_iou))\n",
        "        per_img_result.append(result)\n",
        "        id+=1\n",
        "    #print(id)\n",
        "    avg_sum_iou = 0\n",
        "    for i in range(len(avg_iou)):\n",
        "        avg_sum_iou += avg_iou[i]\n",
        "\n",
        "    avg_sum_dr = 0\n",
        "    for i in range(len(DR)):\n",
        "        avg_sum_dr += DR[i]\n",
        "\n",
        "    avg_sum_ra = 0\n",
        "    for i in range(len(RA)):\n",
        "        avg_sum_ra += RA[i]\n",
        "\n",
        "    # print(avg_sum_iou,type(avg_sum_iou),len(avg_iou))\n",
        "    # print(avg_sum_dr,len(DR))\n",
        "    # print(avg_sum_ra,len(RA))\n",
        "\n",
        "    final_iou = avg_sum_iou / len(avg_iou)\n",
        "    avg_dr = avg_sum_dr / len(DR)\n",
        "    avg_ra = avg_sum_ra / len(RA)\n",
        "    \n",
        "    if avg_dr==0.0 and avg_ra==0.0:\n",
        "        FM = 0.00\n",
        "    else:\n",
        "        FM = ((2 * avg_dr) * avg_ra) / (avg_ra + avg_dr)\n",
        "\n",
        "    print(\"For {} IoU Score :: {}\".format(iou_thr,final_iou))\n",
        "    print(\"Detection Rate(Recall) :: {}\".format(avg_dr))\n",
        "    print(\"Recognition Rate(Precision) :: {}\".format(avg_ra))\n",
        "    print(\"Final Performance(f1 score) :: {}\".format(FM))\n",
        "    print()\n",
        "\n",
        "    file_name = file_path + \"overall_result_@\"+str(iou_thr)+\".txt\"\n",
        "    save_result(iou_thr, file_name, per_img_result, final_iou, avg_dr, avg_ra, FM)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3-PX6Ubcf5oy"
      },
      "outputs": [],
      "source": [
        "# this function read data for given path and call the other function which is showing our test result\n",
        "def read_data(file_path, image_path, gt_path, pred_path):\n",
        "\n",
        "    image_data = os.listdir(image_path)\n",
        "    gt_label_data = os.listdir(gt_path)\n",
        "    pred_label_data = os.listdir(pred_path)\n",
        "\n",
        "\n",
        "    image_gt_pred_data = arrange_img_gt_pred_labels(image_data, gt_label_data, pred_label_data)\n",
        "    #print(image_gt_pred_data)\n",
        "\n",
        "    images = list(image_gt_pred_data.keys())\n",
        "    #print(images)\n",
        "\n",
        "    image_shape = get_img_shape(image_path,images)\n",
        "    #print(image_shape)\n",
        "\n",
        "    custom_tresh = [0.50, 0.70, 0.80, 0.85, 0.90]\n",
        "    for thr in custom_tresh:\n",
        "        multiple_image_result(file_path, thr, image_gt_pred_data, image_shape, gt_path, pred_path)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y0v0VccCIoFw"
      },
      "source": [
        "#**Main ( Transition ) Starts from here**"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oK7zd7EaKE9a"
      },
      "source": [
        "##Given input document image... "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "k_MkxuWVIwue"
      },
      "outputs": [],
      "source": [
        "# Making final folder...\n",
        "img_lbl = img1.split('.')[0]\n",
        "final_dir = '/content/' + img_lbl\n",
        "os.mkdir(final_dir)\n",
        "\n",
        "# Copying uploaded image to final folder...\n",
        "to_dir = final_dir+\"/Uploaded image\"\n",
        "os.mkdir(to_dir)\n",
        "test_img = cv2.imread(img2)\n",
        "cv2.imwrite(os.path.join(to_dir, img1),test_img)\n",
        "\n",
        "# Showing uploaded image...\n",
        "plot_fig(test_img)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xiB-qJmgKbCV"
      },
      "source": [
        "Detecting 1st handwritten line of the given document by YOLO."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ZYHtpOlhKerV"
      },
      "outputs": [],
      "source": [
        "# 1st detection...\n",
        "img_path = img2\n",
        "img_size = 640\n",
        "conf = 0.30\n",
        "yolo_detection(img_path, img_size, conf)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7aP05xnjfcVw"
      },
      "source": [
        "Displaying the 1st handwritten line detection of the given document image."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MPfGLT5cdvzS"
      },
      "outputs": [],
      "source": [
        "# Showing 1st Detection result...\n",
        "first_det = \"/content/yolov5/runs/detect/exp/\"\n",
        "x = os.listdir(first_det)\n",
        "x.remove('labels')\n",
        "org_img = first_det + x[0]\n",
        "print(\"First Yolo detection :::\")\n",
        "print(\"Image path : \",org_img)\n",
        "org_img1 = cv2.imread(org_img)\n",
        "plot_fig(org_img1)\n",
        "\n",
        "# Copying 1st detection Result...\n",
        "to_dir = final_dir+\"/1st line detection for uploaded image\"\n",
        "shutil.copytree(first_det, to_dir)\n",
        "\n",
        "# Sorting Labels of 1st detection on the basis of y...\n",
        "txt_loc = \"/content/yolov5/runs/detect/exp/labels/\"\n",
        "new_sort_label = '/content/sorted_line_after_1st_detection/'\n",
        "flag = 0\n",
        "sort_detection_label(txt_loc, new_sort_label, flag)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W-7bf56ZAznq"
      },
      "source": [
        "##Displaying multiple images...\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hU0ohkLDc8t7"
      },
      "source": [
        "Displaying the document's line detection output, where -\n",
        "* In the first row, we have the uploaded document and its 1st handwritten line detection by YOLO.\n",
        "* In the second row, we have the annotated lines on the uploaded document and its final 1st handwritten line detection by YOLO. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I1l0saOvIWnw"
      },
      "outputs": [],
      "source": [
        "# create figure\n",
        "fig = plt.figure(figsize=(40, 60))\n",
        "\n",
        "# setting values to rows and column variables\n",
        "rows = 2\n",
        "columns = 2\n",
        "\n",
        "# Adds a subplot at the 1st position\n",
        "fig.add_subplot(rows, columns, 1)\n",
        "\n",
        "# showing image\n",
        "plt.imshow(test_img)\n",
        "plt.title(\"\\nUploaded Image\\n\",  fontsize=40)\n",
        "\n",
        "# Adds a subplot at the 2nd position\n",
        "fig.add_subplot(rows, columns, 2)\n",
        "\n",
        "# showing image\n",
        "plt.imshow(cv2.cvtColor(org_img1, cv2.COLOR_BGR2RGB))\n",
        "plt.title(\"Initial Predicted Lines by YOLO\\n\", fontsize=40)\n",
        "\n",
        "# Adds a subplot at the 3rd position\n",
        "fig.add_subplot(rows, columns, 3)\n",
        "\n",
        "\n",
        "# # showing image\n",
        "# flag = 0\n",
        "# label_path = gt_line_dir + img1.split('.')[0] + \".txt\"\n",
        "# img_g = draw_BB(img_path, label_path, flag)\n",
        "# plt.imshow(img_g)\n",
        "# plt.title(\"\\nAnnotated Lines\\n\",  fontsize=40)\n",
        "\n",
        "# # Adds a subplot at the 4th position\n",
        "# fig.add_subplot(rows, columns, 4)\n",
        "\n",
        "# showing image\n",
        "flag = 1\n",
        "label_path = \"/content/sorted_line_after_1st_detection/\" + img1.split('.')[0] + \".txt\"\n",
        "img_p = draw_BB(img_path, label_path, flag)\n",
        "plt.imshow(cv2.cvtColor(img_p, cv2.COLOR_BGR2RGB))\n",
        "plt.title(\"Final Predicted Lines by YOLO\\n\", fontsize=40)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lOlMrQlL6Liy"
      },
      "outputs": [],
      "source": [
        "img_pth = \"/content/203_8/\" + img1\n",
        "cv2.imwrite(img_pth, img_p)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zk9jtEiXKczf"
      },
      "source": [
        "##1st Line segmentation results..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IjdMgYNVcjeT"
      },
      "source": [
        "Displaying the 1st line segmentation output with the annotated line segmentation of the given document."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CcR16ZYYRdOp"
      },
      "outputs": [],
      "source": [
        "# Line Segmentation after 1st Detection...\n",
        "img_path = '/content/'+img1\n",
        "sorted_label = \"/content/sorted_line_after_1st_detection/\"\n",
        "filename = \"/content/initial_line_segmantation/\"\n",
        "line_segmantation_1(img_path, img1, sorted_label, filename)\n",
        "\n",
        "print(\"Segmented Lines from 1st Yolo detection :::\")\n",
        "content1 = \"Annotated Line or GroundTruth ->\"\n",
        "content2 = \"Predicted Line ->\"\n",
        "# show_transitions_by_comparing(filename, gt_line_img_dir, content1, content2)\n",
        "show_transitions_by_comparing(filename, content2)\n",
        "\n",
        "# Copying line Segmentation after 1st Detection...\n",
        "to_dir = final_dir+\"/Initial line segmentation\"\n",
        "shutil.copytree(filename, to_dir)\n",
        "\n",
        "# Copying sorted label...\n",
        "to_dir = final_dir+\"/Initial line segmentation/Sorted label (based on y axis)\"\n",
        "shutil.copytree(sorted_label, to_dir)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t_wKbLjdKlrt"
      },
      "source": [
        "##Rotation Process..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c8BYxqg7cHI7"
      },
      "source": [
        "Displaying the transition of line image's rotation process by Hough Line and Affine Transform."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UxgtW4iUuX7q"
      },
      "outputs": [],
      "source": [
        "# Roation..Roatating image after 1st line Segmentation...\n",
        "rotate_line = \"/content/Rotated_line_by_HaughLine_Affine/\"\n",
        "os.mkdir(rotate_line)\n",
        "\n",
        "rotate_line_Dskew = \"/content/DSkew/\"\n",
        "os.mkdir(rotate_line_Dskew)\n",
        "\n",
        "rotate_line_Haughline = \"/content/HaughLine_Affine/\"\n",
        "os.mkdir(rotate_line_Haughline)\n",
        "\n",
        "rotate_lines(filename)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cRk_e1P5W-ko"
      },
      "outputs": [],
      "source": [
        "x = final_dir+\"/Rotation\"\n",
        "os.mkdir(x)\n",
        "\n",
        "# Copying DSkew...\n",
        "to_dir = final_dir+\"/Rotation/DSkew\"\n",
        "shutil.copytree(rotate_line_Dskew, to_dir)\n",
        "\n",
        "# Copying Houghline...\n",
        "to_dir = final_dir+\"/Rotation/Houghline\"\n",
        "shutil.copytree(rotate_line_Haughline, to_dir)\n",
        "\n",
        "# Copying Dskew & Houghline Affine; thus final rotated lines...\n",
        "to_dir = final_dir+\"/Rotation/Final rotated lines (Houghline & DSkew)\"\n",
        "shutil.copytree(rotate_line, to_dir)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "utlw9M5BQCtK"
      },
      "source": [
        "##Final Rotation results..."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1o8pFyKJQon8"
      },
      "outputs": [],
      "source": [
        "print(\"Final Rotation results are shown below :::\")\n",
        "content1 = \"Before Rotation (Initial Segmented Lines) ->\"\n",
        "content2 = \"After Rotation ->\"\n",
        "filename_1 = final_dir + \"/Rotation/Final rotated lines (Houghline & DSkew)/\"\n",
        "show_transitions_by_comparing_1(filename_1, filename, content1, content2)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "It5LpocLK8vh"
      },
      "source": [
        "##Detecting of rotated image with Yolo..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "18YnisFHbs_m"
      },
      "source": [
        "Detecting main handwritten line of rotated line images in 2nd line detection by YOLO."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sRk9em_PCcoU"
      },
      "outputs": [],
      "source": [
        "# Image Path from unziped file...\n",
        "# Yolo 2nd Detection...\n",
        "rotated_img_path = \"/content/Rotated_line_by_HaughLine_Affine/\"\n",
        "img_size = 640\n",
        "conf = 0.50\n",
        "yolo_detection(rotated_img_path, img_size, conf)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6VxRz7i9LTnY"
      },
      "source": [
        "##2nd Line detection output..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hcEzRw6zIm9K"
      },
      "source": [
        "Displaying the line images with 2nd line detection by YOLO. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u3GIaIvDhNts"
      },
      "outputs": [],
      "source": [
        "# Showing 2nd Detection result...\n",
        "second_det = \"/content/yolov5/runs/detect/exp2/\"\n",
        "x = os.listdir(second_det)\n",
        "x.remove('labels')\n",
        "x = line_sort(x)\n",
        "print(\"Showing 2nd Detection result after Rotation :::\")\n",
        "for i in x:\n",
        "  temp = second_det + i\n",
        "  temp1 = cv2.imread(temp)\n",
        "  print()\n",
        "  print(\"Image: \",i)\n",
        "  plot_fig(temp1)\n",
        "\n",
        "label_path1 = second_det + \"labels\"\n",
        "if os.path.exists(label_path1) == True:  # Copying and removeg 2nd detection label if exists...\n",
        "  to_dir = \"/content/2nd line detection for rotated images (labels)/\"\n",
        "  shutil.copytree(label_path1, to_dir)\n",
        "\n",
        "to_dir = final_dir+\"/2nd line detection for rotated images\"\n",
        "shutil.copytree(second_det, to_dir)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IP3pQpWkH-VR"
      },
      "source": [
        "Segmenting line image using YOLO's 2nd detection and the segmented lines are our final lines of the given document."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e0MnDMZRKiku"
      },
      "outputs": [],
      "source": [
        "# Croping Rotated image with 2nd detection label...\n",
        "# Target Images\n",
        "target_image_path1 = \"/content/Rotated_line_by_HaughLine_Affine/\"\n",
        "target_image_path = \"/content/2nd line detection for rotated images/\"\n",
        "if os.path.exists(target_image_path1) == True:  # Copying and removeg 2nd detection label if exists...\n",
        "  to_dir = target_image_path\n",
        "  shutil.copytree(target_image_path1, to_dir)\n",
        "\n",
        "# Target Images labels\n",
        "target_label_path = \"/content/2nd line detection for rotated images (labels)/\"\n",
        "\n",
        "target_image = os.listdir(target_image_path)\n",
        "target_label = os.listdir(target_label_path)\n",
        "\n",
        "new_dir = \"/content/final_line_segmentation/\"\n",
        "os.mkdir(new_dir)\n",
        "\n",
        "for i in target_image:\n",
        "  for j in target_label:\n",
        "    fn_i = i.split(\".\")\n",
        "    fn_j = j.split(\".\")\n",
        "    if fn_i[0] ==  fn_j[0]:\n",
        "      # Line Segmentation after 1st Detection...\n",
        "      # Final Line Segmentation...\n",
        "      img_path = target_image_path + i\n",
        "      img = i\n",
        "      sorted_label = j\n",
        "      sorted_label_path = target_label_path + j\n",
        "      line_segmantation_2(img, img_path, sorted_label, sorted_label_path, new_dir)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSt25tOELtdd"
      },
      "source": [
        "##Undetected image labels in 2nd detection..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "x6qCPkhuHt8D"
      },
      "source": [
        "Displaying undetected line images in 2nd YOLO detection."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YjcfhxcMCKva"
      },
      "outputs": [],
      "source": [
        "print(\"List of undetected images in 2nd detection ->\")\n",
        "find_undetected_images(target_image_path, target_label_path)\n",
        "print()\n",
        "print(\"Undetected images are shown below -> \\n\")\n",
        "for i in list(set(undetected_images_path)):\n",
        "  print(\"Line image path: \",i)\n",
        "  undit_img = cv2.imread(i)\n",
        "  plot_fig(undit_img)\n",
        "  print()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "REIMcJiS3yGR"
      },
      "source": [
        "##Trimming the DSkewed line images & replacing them in the final segmented image..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DfGijyyN478P"
      },
      "source": [
        "Displaying (before & after trimming) and trimming those DSkewed images that didn't go through the 2nd line detection; because the additional size creates the problem by detecting false bounding box in Word detection."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LopgJdTQ1DOK"
      },
      "outputs": [],
      "source": [
        "def trim_original_image(rotate, org_w, org_h):\n",
        "  org_width = org_w\n",
        "  org_height = org_h\n",
        "\n",
        "  img1 = rotate\n",
        "  width = img1.shape[1]\n",
        "  height = img1.shape[0]\n",
        "  print(\"Original height -> \",org_height)\n",
        "  print(\"Original width -> \",org_width)\n",
        "\n",
        "  start_row = 60\n",
        "  end_row = height - 60\n",
        "\n",
        "  start_col = 60\n",
        "  end_col = width - 60\n",
        "  img_new = img1[start_row:end_row, start_col:end_col]\n",
        "\n",
        "  width1 = img_new.shape[1]\n",
        "  height1 = img_new.shape[0]\n",
        "  print(\"New height -> \",height1)\n",
        "  print(\"New width -> \",width1)\n",
        "    \n",
        "  return img_new\n",
        "\n",
        "\n",
        "final_line_segment = \"/content/final_line_segmentation/\"\n",
        "rotate_line_Dskew = \"/content/DSkew/\"\n",
        "dskew_img_list = os.listdir(rotate_line_Dskew)\n",
        "print(dskew_img_list)\n",
        "for i in dskew_img_list:\n",
        "  print(\"Target image -> \",i)\n",
        "  temp = final_line_segment + i\n",
        "  print(\"Path -> \",temp)\n",
        "  img1 = cv2.imread(temp)\n",
        "  height, width, channels = img1.shape\n",
        "  temp2 = rotate_line_Dskew + i\n",
        "  img2 = cv2.imread(temp2)\n",
        "  height2, width2, channels = img2.shape\n",
        "  if height >= height2:\n",
        "    # os.remove(temp)\n",
        "    temp3 = trim_original_image(img1, width, height)\n",
        "    cv2.imwrite(temp, temp3)\n",
        "    print(\"Original ->\")\n",
        "    plot_fig(img1)\n",
        "    print(\"New ->\")\n",
        "    plot_fig(temp3)\n",
        "    print()\n",
        "  else:\n",
        "    print(\"Original ->\")\n",
        "    plot_fig(img1)\n",
        "    print(\"No need for change as its gone through the 2nd line detection!\\n\")\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gCD39GNeL2m3"
      },
      "source": [
        "##Final Line segmentation after 2nd detection..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sCiDtAE8JhUj"
      },
      "source": [
        "Displaying the final segmented line images of the document with their ground truth images."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R9pziQ5qC3E1"
      },
      "outputs": [],
      "source": [
        "# Final Line Segmentation after 2nd Detection...\n",
        "filename_1 = \"/content/final_line_segmentation\"\n",
        "\n",
        "print(\"Final Line Segmentation of Yolo detection :::\")\n",
        "print()\n",
        "\n",
        "content1 = \"Annotated Line or GroundTruth ->\"\n",
        "content2 = \"Predicted Line ->\"\n",
        "# show_transitions_by_comparing(filename_1, gt_line_img_dir, content1, content2)\n",
        "show_transitions_by_comparing(filename_1, content2)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PntJOyh2kd4C"
      },
      "outputs": [],
      "source": [
        "# Copying Final line segmentation...\n",
        "to_dir = final_dir+\"/Final line segmentation\"\n",
        "shutil.copytree(new_dir, to_dir)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mouP89jVMHpX"
      },
      "source": [
        "##**Detecting Word from final segmented lines** "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n3DBScgQKp-r"
      },
      "source": [
        "Detecting words of the final segmented lines using our trained YOLO model."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "FA2aAZZ4eUh8"
      },
      "outputs": [],
      "source": [
        "!python /content/yolov5/detect.py --weights /content/model/word_model_best.pt --img 640 --conf 0.40 --source /content/final_line_segmentation --save-conf --save-txt"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1eWXze9pMfto"
      },
      "source": [
        "##Final Word detection results..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "n5Ja3Hrx_9IL"
      },
      "source": [
        "Displaying YOLO's prediction for Word."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "B5CXlHt4jixA"
      },
      "outputs": [],
      "source": [
        "# Showing Word Detection result...\n",
        "word_det = \"/content/yolov5/runs/detect/exp3/\"\n",
        "x = os.listdir(word_det)\n",
        "x.remove('labels')\n",
        "x = line_sort(x)\n",
        "print(\"Showing Word Detection result after 2nd Segmentation :::\")\n",
        "for i in x:\n",
        "  temp = word_det + i\n",
        "  temp1 = cv2.imread(temp)\n",
        "  print()\n",
        "  print(\"Image: \",i)\n",
        "  # flag = 0\n",
        "  # img_path = gt_line_img_dir + i\n",
        "  # label_path = gt_word_dir + i.split('.')[0] + \".txt\"\n",
        "  # img_g = draw_BB(img_path, label_path, flag)\n",
        "  # print(\"Annotated Words ->\")\n",
        "  # plot_fig(img_g)\n",
        "  print(\"Predicted Words by YOLO ->\")\n",
        "  plot_fig(temp1)\n",
        "\n",
        "# Copying Word Detection result...\n",
        "to_dir = final_dir+\"/Word detection for final segmented lines\"\n",
        "shutil.copytree(word_det, to_dir)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HMuCH2T2Mnlt"
      },
      "source": [
        "Sorting Labels of Word detection on the basis of x axis.\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8qVgFJ9pHQVW"
      },
      "outputs": [],
      "source": [
        "# Sorting Labels of Word detection on the basis of x...\n",
        "txt_loc = \"/content/yolov5/runs/detect/exp3/labels/\"\n",
        "new_sort_label = '/content/sorted_Word_detection/'\n",
        "flag = 1\n",
        "sort_detection_label(txt_loc, new_sort_label, flag)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MHMjBWZxMmeb"
      },
      "source": [
        "Segmenting the words by using YOLO's word predictions."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0rwTRAS09sO4"
      },
      "outputs": [],
      "source": [
        "# Word Segmentation... \n",
        "word_labels = \"/content/sorted_Word_detection/\"\n",
        "line_images = \"/content/final_line_segmentation/\"\n",
        "final_word_dir = \"/content/final_word_segmentation/\"\n",
        "os.mkdir(final_word_dir)\n",
        "word_segmentation(line_images, word_labels)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5e9xSU0iAIwL"
      },
      "source": [
        "Displaying final word detection results with their ground truth."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "g3INBPYe7C4k"
      },
      "outputs": [],
      "source": [
        "# Showing Word Detection result...\n",
        "word_det = \"/content/yolov5/runs/detect/exp3/\"\n",
        "x = os.listdir(word_det)\n",
        "x.remove('labels')\n",
        "x = line_sort(x)\n",
        "print(\"Showing Word Detection result after 2nd Segmentation :::\")\n",
        "for i in x:\n",
        "  print()\n",
        "  print(\"Image: \",i)\n",
        "  # flag = 0\n",
        "  # img_path = gt_line_img_dir + i\n",
        "  # label_path = gt_word_dir + i.split('.')[0] + \".txt\"\n",
        "  # img_g = draw_BB(img_path, label_path, flag)\n",
        "  # print(\"Annotated Words ->\")\n",
        "  # plot_fig(img_g)\n",
        "  \n",
        "  # flag = 1\n",
        "  # img_path1 = \"/content/final_line_segmentation/\" + i\n",
        "  # label_path1 = \"/content/sorted_Word_detection/\" + i.split('.')[0] + \".txt\"\n",
        "  # img_p = draw_BB(img_path1, label_path1, flag)\n",
        "  # print(\"Final predicted Words by YOLO ->\")\n",
        "  # plot_fig(img_p)\n",
        "\n",
        "  flag = 1\n",
        "  img_path1 = \"/content/final_line_segmentation/\" + i\n",
        "  label_path1 = \"/content/sorted_Word_detection/\" + i.split('.')[0] + \".txt\"\n",
        "  if os.path.exists(img_path1) == True and os.path.exists(label_path1) == True: \n",
        "    img_p = draw_BB(img_path1, label_path1, flag)\n",
        "    print(\"Final predicted Words by YOLO ->\")\n",
        "    plot_fig(img_p)\n",
        "  else:\n",
        "    continue\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SBcdBt8neEk4"
      },
      "outputs": [],
      "source": [
        "# Copying Final word segmentation...\n",
        "to_dir = final_dir+\"/Final word segmentation\"\n",
        "shutil.copytree(final_word_dir, to_dir)\n",
        "\n",
        "# Copying sorted label of final word detection...\n",
        "to_dir = final_dir+\"/Final word segmentation/Sorted label (based on x axis)\"\n",
        "shutil.copytree(word_labels, to_dir)\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QFnI6k0uMsUA"
      },
      "source": [
        "##Final Word segmentation results..."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fd8dGTIiNHm6"
      },
      "source": [
        "Displaying the final word segmentation results."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "s5AkxFQcN5fz"
      },
      "outputs": [],
      "source": [
        "print(\"Word segmentation completed!\")\n",
        "word_list = os.listdir(final_word_dir)\n",
        "word_lists = line_sort(word_list)\n",
        "for i in word_lists:\n",
        "  print()\n",
        "  print(\"Folder name: \",i)\n",
        "  temp =  final_word_dir+i\n",
        "  temp_list = os.listdir(temp)\n",
        "  temp_lists = line_sort(temp_list)\n",
        "  # print(temp_lists)\n",
        "  for j in temp_lists:\n",
        "    print(j)\n",
        "    temp1 = temp+'/'+j\n",
        "    crop_word = cv2.imread(temp1)\n",
        "    plot_fig(crop_word, 5)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q2ICy8gX6HN1"
      },
      "source": [
        "#**Saving and Download Results** "
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Zipping results..."
      ],
      "metadata": {
        "id": "4173wQ8SGap8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "!zip -r /content/final_results.zip /content/1_1/"
      ],
      "metadata": {
        "id": "CYNMIs8JGWWo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "To download the generated results for the uploaded image, please remove comments from the below cell and run the cell. The resulted zip file will automatically download and this may take less than a minute."
      ],
      "metadata": {
        "id": "xMj9AzMDGj1h"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# files.download(\"/content/final_results.zip\")"
      ],
      "metadata": {
        "id": "qpqlQz2v-enJ"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}