ICIP_Text2Vec.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ICIP_Text2Vec.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "314iP3CKscAW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd \n",
        "import os\n",
        "from scipy import stats\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "# path_data = \"/content/drive/My Drive/ICIP/\"\n",
        "# path_imgs = \"/content/drive/My Drive/ICIP/train_imgs/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mm9cjXccDxSy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#TEST DATA\n",
        "test_path_data=\"/content/drive/My Drive/ICIP/TEST/\"\n",
        "# test_path_imgs=\"/content/drive/My Drive/ICIP/TEST/test_imgs/\""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i_0SMvFkstjh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_img_info1 = pd.read_csv(test_path_data+'test_img_info1.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2oY3pjbIsv5U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# headers = pd.read_csv(path_data+'headers_TRAIN.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WvHBImRTPvbR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Y=pd.read_csv(path_data+'Clusters.csv')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WSX3mQDfsxaf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 606
        },
        "outputId": "5db400c5-43a0-4f80-d880-4c37081a2066"
      },
      "source": [
        "img_info1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>FlickrId</th>\n",
              "      <th>Size</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>NumSets</th>\n",
              "      <th>NumGroups</th>\n",
              "      <th>AvgGroupsMemb</th>\n",
              "      <th>AvgGroupPhotos</th>\n",
              "      <th>Tags</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>621</th>\n",
              "      <td>23647573678</td>\n",
              "      <td>0</td>\n",
              "      <td>image</td>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>164</th>\n",
              "      <td>23647582078</td>\n",
              "      <td>0</td>\n",
              "      <td>20171004_215401</td>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>57</th>\n",
              "      <td>23647582388</td>\n",
              "      <td>1166400</td>\n",
              "      <td>Always good to see Sergio at the Italian pavil...</td>\n",
              "      <td>on Instagram &lt;a href= http://ift.tt/2xia4c9  r...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>112</th>\n",
              "      <td>23647583748</td>\n",
              "      <td>0</td>\n",
              "      <td>2017 - September - AESHM - Mary Alice Gallery-...</td>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>clothing textiles museum aeshm alice amd mary ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>544</th>\n",
              "      <td>23647586378</td>\n",
              "      <td>2457600</td>\n",
              "      <td>Grand Elk.  CP 14</td>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20292</th>\n",
              "      <td>49108928407</td>\n",
              "      <td>288300</td>\n",
              "      <td>Người Nhật làm phim live-action về tình bạn gi...</td>\n",
              "      <td>via WordPress &lt;a href= https://ift.tt/2qF8Ydh ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>phim \\u1ea3nh ch\\u00faa \\u0111\\u1ee9c ph\\u1ead...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20038</th>\n",
              "      <td>49108930512</td>\n",
              "      <td>24160256</td>\n",
              "      <td>SAVE_20191122_233459</td>\n",
              "      <td></td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td></td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20009</th>\n",
              "      <td>49108933202</td>\n",
              "      <td>17735042</td>\n",
              "      <td>Orange Revolution</td>\n",
              "      <td></td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>2019 2020 wikipedia</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20289</th>\n",
              "      <td>49108933542</td>\n",
              "      <td>202500</td>\n",
              "      <td>Review tay cầm xbox 360 chính hãng giá rẻ.</td>\n",
              "      <td>via Smart New &lt;a href= https://ift.tt/2D609vU ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>admin review</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20161</th>\n",
              "      <td>49108934937</td>\n",
              "      <td>0</td>\n",
              "      <td>Industry</td>\n",
              "      <td>Looking towards the Tacoma tideflats taken fro...</td>\n",
              "      <td>1</td>\n",
              "      <td>11</td>\n",
              "      <td>11083.090909</td>\n",
              "      <td>908502.909091</td>\n",
              "      <td>tacoma tideflats portoftacoma commencementbay ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>20337 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "          FlickrId  ...                                               Tags\n",
              "621    23647573678  ...                                                   \n",
              "164    23647582078  ...                                                   \n",
              "57     23647582388  ...                                          instagram\n",
              "112    23647583748  ...  clothing textiles museum aeshm alice amd mary ...\n",
              "544    23647586378  ...                                                   \n",
              "...            ...  ...                                                ...\n",
              "20292  49108928407  ...  phim \\u1ea3nh ch\\u00faa \\u0111\\u1ee9c ph\\u1ead...\n",
              "20038  49108930512  ...                                                   \n",
              "20009  49108933202  ...                                2019 2020 wikipedia\n",
              "20289  49108933542  ...                                       admin review\n",
              "20161  49108934937  ...  tacoma tideflats portoftacoma commencementbay ...\n",
              "\n",
              "[20337 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8GpSevZqEGuG",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 760
        },
        "outputId": "36647892-2331-4bdc-f9e0-bbdc021936a7"
      },
      "source": [
        "test_img_info1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>FlickrId</th>\n",
              "      <th>Size</th>\n",
              "      <th>Title</th>\n",
              "      <th>Description</th>\n",
              "      <th>NumSets</th>\n",
              "      <th>NumGroups</th>\n",
              "      <th>AvgGroupsMemb</th>\n",
              "      <th>AvgGroupPhotos</th>\n",
              "      <th>Tags</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>49098581833</td>\n",
              "      <td>15980544</td>\n",
              "      <td>Clariant International Ltd: Clariant launches ...</td>\n",
              "      <td>Clariant launches antimony-trioxide-free laser...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>prf clariantinternationalltd clapr1634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>49098582593</td>\n",
              "      <td>19961856</td>\n",
              "      <td>MK Seiko Big Top Salio</td>\n",
              "      <td>MK Seiko Big Top Salio car wash system with ye...</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>456.000000</td>\n",
              "      <td>1.217800e+04</td>\n",
              "      <td>carwash</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>49098582698</td>\n",
              "      <td>0</td>\n",
              "      <td>Pano Experiment No. 2, Northville, MI, Septemb...</td>\n",
              "      <td>NAP_Canon EOS 5DS R_20190923__13A3294_0002-Pan...</td>\n",
              "      <td>0</td>\n",
              "      <td>69</td>\n",
              "      <td>41385.347826</td>\n",
              "      <td>2.480120e+06</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>49098582733</td>\n",
              "      <td>0</td>\n",
              "      <td>Green Heron</td>\n",
              "      <td>Butorides viriscens</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>2458.250000</td>\n",
              "      <td>7.432825e+04</td>\n",
              "      <td>greenheron usa wildlife butoridesvirescens us ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>49098584118</td>\n",
              "      <td>409600</td>\n",
              "      <td>Studying beetles at Naturalis this week; Endle...</td>\n",
              "      <td>via Instagram &lt;a href= https://ift.tt/2KD7gA0 ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7688</th>\n",
              "      <td>49104539732</td>\n",
              "      <td>2650383</td>\n",
              "      <td>Volleyball Mug Lover I Like Funny Gift Idea Fo...</td>\n",
              "      <td>&lt;a href= https://www.amazon.com/dp/B07L7V5BWM ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>volleyball mug lover i like funny gift idea fo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7689</th>\n",
              "      <td>49104542697</td>\n",
              "      <td>1964800</td>\n",
              "      <td>Poetry</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>9716.000000</td>\n",
              "      <td>6.995257e+05</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7690</th>\n",
              "      <td>49104544567</td>\n",
              "      <td>14625792</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Processed with...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7691</th>\n",
              "      <td>49104545392</td>\n",
              "      <td>18917874</td>\n",
              "      <td>Z8271928.jpg</td>\n",
              "      <td>Olympus digital camera</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7692</th>\n",
              "      <td>49104546122</td>\n",
              "      <td>12192768</td>\n",
              "      <td>Ecuador y Peru en la Westfalia</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7693 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         FlickrId  ...                                               Tags\n",
              "0     49098581833  ...             prf clariantinternationalltd clapr1634\n",
              "1     49098582593  ...                                            carwash\n",
              "2     49098582698  ...                                                NaN\n",
              "3     49098582733  ...  greenheron usa wildlife butoridesvirescens us ...\n",
              "4     49098584118  ...                                    ifttt instagram\n",
              "...           ...  ...                                                ...\n",
              "7688  49104539732  ...  volleyball mug lover i like funny gift idea fo...\n",
              "7689  49104542697  ...                                                NaN\n",
              "7690  49104544567  ...                                                NaN\n",
              "7691  49104545392  ...                                                NaN\n",
              "7692  49104546122  ...                                                NaN\n",
              "\n",
              "[7693 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zb8I0cLpENTq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 194
        },
        "outputId": "1aad4479-97d4-4f0d-a4bf-b97523235e63"
      },
      "source": [
        "test_img_info1.isnull().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "FlickrId             0\n",
              "Size                 0\n",
              "Title              954\n",
              "Description       4815\n",
              "NumSets              0\n",
              "NumGroups            0\n",
              "AvgGroupsMemb        0\n",
              "AvgGroupPhotos       0\n",
              "Tags              4945\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fJLER9ESEVKf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "outputId": "43dd5c79-0437-4b38-9ec0-8425fab105aa"
      },
      "source": [
        "test_img_info1.Tags.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "instagram ifttt                                                                                                                                                                   256\n",
              "ifttt instagram                                                                                                                                                                   251\n",
              "firestorm secondlife                                                                                                                                                               34\n",
              "instagram                                                                                                                                                                          33\n",
              "ssr                                                                                                                                                                                16\n",
              "                                                                                                                                                                                 ... \n",
              "naked nainen nude redhead terhi tfcd woman curlygirl topless                                                                                                                        1\n",
              "director tshirt coworker gift idea funny gag unisex tee joke funnygift unique giftidea giftideas funnygifts custom handmade giftsforher giftsforhim coworkergift coworkergifts      1\n",
              "hungary sail landscape nature water harbor balaton holiday yacht lake blue hungarian sailboat beautiful travel scene dock sky sport boat ship europe yachting vacation              1\n",
              "nadiapalik                                                                                                                                                                          1\n",
              "urbex abbandonato isoladellecorrenti abbandono sicilia                                                                                                                              1\n",
              "Name: Tags, Length: 1985, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pxF-tOWSE5lq",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "outputId": "fe50734c-8e52-4f5a-ddf8-cad42b952200"
      },
      "source": [
        "test_img_info1.Title.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Untitled                                                                                                                                                                                                                                                           67\n",
              "Photo                                                                                                                                                                                                                                                              42\n",
              "5 Star Review                                                                                                                                                                                                                                                      25\n",
              ".                                                                                                                                                                                                                                                                   9\n",
              "...                                                                                                                                                                                                                                                                 6\n",
              "                                                                                                                                                                                                                                                                   ..\n",
              "Проект застройка для зоны фуд-корта в ТЦ Ясень (г.Иваново). Подвесные потолки,  монтажная высота 8м. Стойка, декор, монтаж и работа со смежниками. Срок реализации проекта составил 3 недели под ключ.                                                              1\n",
              "100% american muscle                                                                                                                                                                                                                                                1\n",
              "Juana Serra                                                                                                                                                                                                                                                         1\n",
              "Moving some bikes around, and highlighting some serious close outs we have on models from Brodie, Opus, Bombtrack... and we have some sweet deals on previously enjoyed bikes. Plenty of wine tour rentals left! The winter forecast is calling for a serious S     1\n",
              "hot choccie time                                                                                                                                                                                                                                                    1\n",
              "Name: Title, Length: 6437, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GxbM_DqgFCk3",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "outputId": "fbc86a58-8c04-4986-9a5a-0d6a8eca1bdf"
      },
      "source": [
        "test_img_info1.Description.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                                                                                                                          93\n",
              "                                                                                                                                                                          16\n",
              "Olympus digital camera                                                                                                                                                    14\n",
              "dav                                                                                                                                                                       12\n",
              "by-[ChipVN]-Image-Uploader                                                                                                                                                 8\n",
              "                                                                                                                                                                          ..\n",
              "View on Instagram <a href= https://ift.tt/2D9DT44  rel= noreferrer nofollow >ift.tt/2D9DT44</a>                                                                            1\n",
              "Fairy Tail - Screenshot #233543                                                                                                                                            1\n",
              "Luật NQH Việt Nam – Luật Sư Của Mọi Nhà <a href= https://ift.tt/2O87tgS  rel= noreferrer nofollow >ift.tt/2O87tgS</a>                                                      1\n",
              "View on Instagram <a href= https://ift.tt/2qAD4P7  rel= noreferrer nofollow >ift.tt/2qAD4P7</a>                                                                            1\n",
              "via JewelryVID.com 💎 Best Jewelry Videos Inspirations from all your favorite videostars! <a href= https://ift.tt/2OxrzjD  rel= noreferrer nofollow >ift.tt/2OxrzjD</a>     1\n",
              "Name: Description, Length: 2639, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CrUxnyJQz5f9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img_info1.isnull().sum()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2y_GxziLrba7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img_info1.Tags.value_counts()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8RIeRipsg2W",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "img_info1[img_info1['Title']=='ineedhalloweenideas halloween makeup make up ideas for 2017 happy night before christmas october 31 autumn fall spooky body paint art creepy scary pumpkin boo artist goth gothic']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8aEALwOUvkuk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "d80af2a8-153c-4b32-aa23-e1f86cbb7fd1"
      },
      "source": [
        "img_info1['Tags'][img_info1['Description']=='from my tumblr blog'].isnull().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "20"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yEYRKpXos6Qt",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 80
        },
        "outputId": "70f8e40c-e3fb-4e9c-e867-a98765b36b06"
      },
      "source": [
        "headers[headers['FlickrId']==36871955293]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>FlickrId</th>\n",
              "      <th>UserId</th>\n",
              "      <th>URL</th>\n",
              "      <th>DatePosted</th>\n",
              "      <th>DateTaken</th>\n",
              "      <th>DateCrawl</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>16245</th>\n",
              "      <td>16245</td>\n",
              "      <td>36871955293</td>\n",
              "      <td>154077855@N05</td>\n",
              "      <td>https://farm5.staticflickr.com/4466/3687195529...</td>\n",
              "      <td>1507340508</td>\n",
              "      <td>1507340508</td>\n",
              "      <td>1507340747</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       Unnamed: 0     FlickrId  ...   DateTaken   DateCrawl\n",
              "16245       16245  36871955293  ...  1507340508  1507340747\n",
              "\n",
              "[1 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_mei6_-92No-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#MAKING iftt and insta same\n",
        "test_img_info1.Tags.replace('instagram ifttt','ifttt instagram',inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9iS1Y1vVzv84",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#FILLING NAN VALUES\n",
        "test_img_info1.Title.fillna('Untitled',inplace=True)\n",
        "test_img_info1.Description.fillna(' ',inplace=True)\n",
        "test_img_info1.Tags.fillna('ifttt instagram',inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wyH1r3r3_Sef",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import re\n",
        "import nltk\n",
        "import spacy\n",
        "import string"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1uySFHiW02p8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Arranging on basis of FlickrId"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RP88JmPF05vG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_img_info1.sort_values(by=['FlickrId'],inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dsw2UsyX_ZSk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Lower Casing"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "80StDXr8_qNk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_img_info1['Title_lower']=test_img_info1['Title'].str.lower()\n",
        "test_img_info1['Description_lower']=test_img_info1['Description'].str.lower()\n",
        "test_img_info1['Tags_lower']=test_img_info1['Tags'].str.lower()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LAHjf2zGAFyc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 247
        },
        "outputId": "1fb78ec6-9526-4101-e01d-3740741a38c4"
      },
      "source": [
        "test_img_info1.isnull().sum()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "FlickrId             0\n",
              "Size                 0\n",
              "Title                0\n",
              "Description          0\n",
              "NumSets              0\n",
              "NumGroups            0\n",
              "AvgGroupsMemb        0\n",
              "AvgGroupPhotos       0\n",
              "Tags                 0\n",
              "Title_lower          0\n",
              "Description_lower    0\n",
              "Tags_lower           0\n",
              "dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "73j-em0E0bC8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removing previous columns\n",
        "test_img_info1.drop(['Title'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['Description'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['Tags'],axis=1,inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Lts2CYI1lITd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removing URL because they also have punctuation so it might become a problem after that"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2o97UrZXlPbJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function for url's\n",
        "def remove_urls(text):\n",
        "    url_pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
        "    return url_pattern.sub(r'', text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x-i4D_ARlWWT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Passing the function to 'text_rare'\n",
        "test_img_info1['Title_lower'] = test_img_info1['Title_lower'].apply(remove_urls)\n",
        "test_img_info1['Description_lower'] = test_img_info1['Description_lower'].apply(remove_urls)\n",
        "test_img_info1['Tags_lower'] = test_img_info1['Tags_lower'].apply(remove_urls)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PD7w2fs0AS8b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removal of Punctuations"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDskymFbATBQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#removing punctuation, creating a new column\n",
        "test_img_info1['Title_punct'] = test_img_info1['Title_lower'].str.replace('[^\\w\\s]','')\n",
        "test_img_info1['Description_punct'] = test_img_info1['Description_lower'].str.replace('[^\\w\\s]','')\n",
        "test_img_info1['Tags_punct'] = test_img_info1['Tags_lower'].str.replace('[^\\w\\s]','')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m_ezHHG01Kr8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removing previous columns\n",
        "test_img_info1.drop(['Title_lower'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['Description_lower'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['Tags_lower'],axis=1,inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-KBcl08KFgb-",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 484
        },
        "outputId": "011378d5-7d9a-40e0-8de3-27e6815987df"
      },
      "source": [
        "test_img_info1.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>FlickrId</th>\n",
              "      <th>Size</th>\n",
              "      <th>NumSets</th>\n",
              "      <th>NumGroups</th>\n",
              "      <th>AvgGroupsMemb</th>\n",
              "      <th>AvgGroupPhotos</th>\n",
              "      <th>Title_punct</th>\n",
              "      <th>Description_punct</th>\n",
              "      <th>Tags_punct</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>49098581833</td>\n",
              "      <td>15980544</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>clariant international ltd clariant launches a...</td>\n",
              "      <td>clariant launches antimonytrioxidefree laserma...</td>\n",
              "      <td>prf clariantinternationalltd clapr1634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>49098582593</td>\n",
              "      <td>19961856</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>456.000000</td>\n",
              "      <td>1.217800e+04</td>\n",
              "      <td>mk seiko big top salio</td>\n",
              "      <td>mk seiko big top salio car wash system with ye...</td>\n",
              "      <td>carwash</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>49098582698</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>69</td>\n",
              "      <td>41385.347826</td>\n",
              "      <td>2.480120e+06</td>\n",
              "      <td>pano experiment no 2 northville mi september 2019</td>\n",
              "      <td>nap_canon eos 5ds r_20190923__13a3294_0002pano...</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>49098582733</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>2458.250000</td>\n",
              "      <td>7.432825e+04</td>\n",
              "      <td>green heron</td>\n",
              "      <td>butorides viriscens</td>\n",
              "      <td>greenheron usa wildlife butoridesvirescens us ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>49098584118</td>\n",
              "      <td>409600</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>studying beetles at naturalis this week endles...</td>\n",
              "      <td>via instagram a href   rel noreferrer nofollow...</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      FlickrId  ...                                         Tags_punct\n",
              "0  49098581833  ...             prf clariantinternationalltd clapr1634\n",
              "1  49098582593  ...                                            carwash\n",
              "2  49098582698  ...                                    ifttt instagram\n",
              "3  49098582733  ...  greenheron usa wildlife butoridesvirescens us ...\n",
              "4  49098584118  ...                                    ifttt instagram\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WpDknxo-r-Sg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#STOP WORDS REMOVAL"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ol1qLBWHFowy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.corpus import stopwords"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Rf0gcRajF4N9",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "458add43-6c19-4046-b17a-5c7fa44cd252"
      },
      "source": [
        "nltk.download('stopwords')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wWbqZDlqFx5O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "stop = set(stopwords.words('english'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xT96dwrQF79N",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Function to remove stopwords\n",
        "\n",
        "def removeStopWords(text):\n",
        "  return \" \".join([word for word in str(text).split() if word not in stop]) #since without tokenization"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BBgFZcYEGaqD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_img_info1[\"Title_stop\"] = test_img_info1[\"Title_punct\"].apply(removeStopWords)\n",
        "test_img_info1[\"Description_stop\"] = test_img_info1[\"Description_punct\"].apply(removeStopWords)\n",
        "test_img_info1[\"Tags_stop\"] = test_img_info1[\"Tags_punct\"].apply(removeStopWords)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ol5Cm0Tb1ZH1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removing previous columns\n",
        "test_img_info1.drop(['Title_punct'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['Description_punct'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['Tags_punct'],axis=1,inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "w9SJdWr3GycX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 517
        },
        "outputId": "eb7c7792-109b-4165-d05a-f24b5eddc670"
      },
      "source": [
        "test_img_info1.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>FlickrId</th>\n",
              "      <th>Size</th>\n",
              "      <th>NumSets</th>\n",
              "      <th>NumGroups</th>\n",
              "      <th>AvgGroupsMemb</th>\n",
              "      <th>AvgGroupPhotos</th>\n",
              "      <th>Title_stop</th>\n",
              "      <th>Description_stop</th>\n",
              "      <th>Tags_stop</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>49098581833</td>\n",
              "      <td>15980544</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>clariant international ltd clariant launches a...</td>\n",
              "      <td>clariant launches antimonytrioxidefree laserma...</td>\n",
              "      <td>prf clariantinternationalltd clapr1634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>49098582593</td>\n",
              "      <td>19961856</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>456.000000</td>\n",
              "      <td>1.217800e+04</td>\n",
              "      <td>mk seiko big top salio</td>\n",
              "      <td>mk seiko big top salio car wash system yellow ...</td>\n",
              "      <td>carwash</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>49098582698</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>69</td>\n",
              "      <td>41385.347826</td>\n",
              "      <td>2.480120e+06</td>\n",
              "      <td>pano experiment 2 northville mi september 2019</td>\n",
              "      <td>nap_canon eos 5ds r_20190923__13a3294_0002pano...</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>49098582733</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>2458.250000</td>\n",
              "      <td>7.432825e+04</td>\n",
              "      <td>green heron</td>\n",
              "      <td>butorides viriscens</td>\n",
              "      <td>greenheron usa wildlife butoridesvirescens us ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>49098584118</td>\n",
              "      <td>409600</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>studying beetles naturalis week endless patter...</td>\n",
              "      <td>via instagram href rel noreferrer nofollow ift...</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      FlickrId  ...                                          Tags_stop\n",
              "0  49098581833  ...             prf clariantinternationalltd clapr1634\n",
              "1  49098582593  ...                                            carwash\n",
              "2  49098582698  ...                                    ifttt instagram\n",
              "3  49098582733  ...  greenheron usa wildlife butoridesvirescens us ...\n",
              "4  49098584118  ...                                    ifttt instagram\n",
              "\n",
              "[5 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KPOANNXRT3IK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # Removing the frequent words\n",
        "# freq = set([w for (w, wc) in count.most_common(15)])\n",
        "\n",
        "# # function to remove the frequent words\n",
        "# def freqwords(text):\n",
        "#     return \" \".join([word for word in str(text).split() if word not \n",
        "# in freq])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TDArmANSc07H",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Passing the function freqwords\n",
        "# img_info1[\"Title_common\"] = img_info1[\"Title_stop\"].apply(freqwords)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QjA1OCMYdwgE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# count=Counter()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HOxN6BAUd0MV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# for text in img_info1[\"Description_stop\"].values:\n",
        "#     for word in text.split():\n",
        "#         count[word] += 1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uHIxNECtd5Zq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# count.most_common(10)   "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qm1DX7E9ferz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#SPELLING CORRECTION (PRETTY BIG DEAL)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OyryY91ufix6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # Spell check using text blob\n",
        "# from textblob import TextBlob\n",
        "# img_info1['Title_stop'].apply(lambda x: str(TextBlob(x).correct()))\n",
        "# img_info1['Description_stop'].apply(lambda x: str(TextBlob(x).correct()))\n",
        "# img_info1['Tags_stop'].apply(lambda x: str(TextBlob(x).correct()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yKmFuguhPdnS",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#CONVERTING EMOJIS TO WORDS"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oVTzZz4LSyDh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # Converting emojis to words\n",
        "# def convert_emojis(text):\n",
        "#     for emot in UNICODE_EMO:\n",
        "#         text = text.replace(emot, \"_\".join(UNICODE_EMO[emot].replace(\",\",\"\").replace(\":\",\"\").split()))\n",
        "#         return text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QSvqDHyES-mR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # Converting emoticons to words    \n",
        "# def convert_emoticons(text):\n",
        "#     for emot in EMOTICONS:\n",
        "#         text = re.sub(u'('+emot+')', \"_\".join(EMOTICONS[emot].replace(\",\",\"\").split()), text)\n",
        "#         return text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lYWlk-_fTCMP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# text = 'Happy :-)'\n",
        "# convert_emoticons(text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pDp6pgTATrdN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# text1 = \"Hilarious 😂\"\n",
        "# convert_emojis(text1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fv_MXLBVgMmJ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 107
        },
        "outputId": "ac177d3e-171a-4e5b-b715-418d4b08dbb1"
      },
      "source": [
        "!pip install emot"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting emot\n",
            "  Downloading https://files.pythonhosted.org/packages/49/07/20001ade19873de611b7b66a4d5e5aabbf190d65abea337d5deeaa2bc3de/emot-2.1-py3-none-any.whl\n",
            "Installing collected packages: emot\n",
            "Successfully installed emot-2.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OaXvaDG_P6p-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from emot.emo_unicode import UNICODE_EMO, EMOTICONS"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8j1plW9Whk_v",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function to remove emoji.\n",
        "def emoji(string):\n",
        "    emoji_pattern = re.compile(\"[\"\n",
        "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                           u\"\\U00002702-\\U000027B0\"\n",
        "                           u\"\\U000024C2-\\U0001F251\"\n",
        "                           \"]+\", flags=re.UNICODE)\n",
        "    return emoji_pattern.sub(r'', string)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6MIlrBOlhpfe",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function for removing emoticons\n",
        "def remove_emoticons(text):\n",
        "    emoticon_pattern = re.compile(u'(' + u'|'.join(k for k in EMOTICONS) + u')')\n",
        "    return emoticon_pattern.sub(r'', text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PHoEvR1Ohtx4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removing the emojis and emoticons from img_info1"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-4_3CVidhyck",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_img_info1['Title_stop'] = test_img_info1['Title_stop'].apply(emoji)\n",
        "test_img_info1['Title_stop'] = test_img_info1['Title_stop'].apply(remove_emoticons)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LKZMNFL5ie5V",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_img_info1['Description_stop'] = test_img_info1['Description_stop'].apply(emoji)\n",
        "test_img_info1['Description_stop'] = test_img_info1['Description_stop'].apply(remove_emoticons)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xlaeXvWxinkw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 250
        },
        "outputId": "b098b732-b6aa-4b5c-a4be-b829900172d0"
      },
      "source": [
        "test_img_info1['Description_stop'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              4939\n",
              "href rel noreferrer nofollow                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    30\n",
              "via instagram href rel noreferrer nofollow                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      16\n",
              "olympus digital camera                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          15\n",
              "dav                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             12\n",
              "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              ... \n",
              "via instagram href rel noreferrer nofollow ifttt2oycchfa                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         1\n",
              "href rel noreferrer nofollow bitly2xfpyrsa                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                       1\n",
              "serdecznie zapraszamy na pokaz kotów 23112019 w ch osowa będziemy z naszymi indy jupi indygirkapl przedstawiamy oryginalny wpis klubu cat club sopot pokaz kotów rasowych domowych w ch osowa w gdańsku 23112019 zarząd cat club sopot klubu miłośników kotów serdecznie zaprasza na event w centrum handlowym osowa w gdańsku w czasie trwania eventu odbędzie pokaz kotów rasowych domowych pokaz odbędzie się w dniu 23112019 w godz 1100 1800 w centrum handlowym osowa ul spacerowa 48 80299 gdańsk pokaz będzie odbywał się pod hasłem wspieranie kotów wolno bytujących podczas pokazu zostanie przeprowadzony konkurs na najładniejsze koty pokazu ch osowa 2019 1 głównym celem konkursu jest wyłonienie 3 najładniejszych kotów pokazu ch osowa 2019 przez publiczność 2 mioty nie biorą udziału w rywalizacji bezpośredniej tytuł najładniejszego kota pokazu konkurs na najładniejszy koci ogon warunki uczestnictwa 1 uczestnikami konkursu mogą być wyłącznie koty zgłoszone na pokaz kotów w ch osowa w gdańsku w dniu 23112019 zapisane na listę uczestników właściciel kota musi być zrzeszony 2 właściciel kota konkursu musi zgłosić dla organizatora dnia 20112019 godz 16 00 lub wyczerpania miejsc podając imię kota przydomek miejsce zamieszkania właściciela na adres biurocatclubsopotpl 3 na konkurs na najładniejszy koci ogon należy zgłosić kota godz 1400 u organizatora właściciel może zgłosić tylko jednego kota konkursu 4 właściciel osobiście prezentuje swojego kota 5 koty organizatora pokazu nie mogą brać udziału w konkursie nagrody w konkursie 1 najładniejszy kot pokazu otrzyma nagrodę w wysokości 100000 zł puchar 2 kot który zdobędzie ii miejsce otrzyma nagrodę w wysokości 70000 zł puchar 3 kot który zdobędzie iii miejsce otrzyma nagrodę w wysokości 50000 zł puchar 4 kot może otrzymać tylko jedną nagrodę 5 nagrody w konkursie nie można zamienić na inną nagrodę 6 najładniejszy miot otrzyma puchar żwirek 7 kot z najładniejszym ogonem otrzyma puchar żwirek antoni kondej prezes zarządu cat club sopot       1\n",
              "alphabet                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         1\n",
              "view instagram href rel noreferrer nofollow abadcheesecom2khcdhua                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                1\n",
              "Name: Description_stop, Length: 2562, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7MRAFSfhOIK6",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Filling the blank up"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VMUTBxj7OKjU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# img_info1.Description_stop.replace(' ','0',inplace=True)\n",
        "# img_info1.Tags_stop.replace(' ','0',inplace=True)\n",
        "# img_info1.Title_stop.replace(' ','0',inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q4rWBxScyU5d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Tokenization"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lOT0JmzVyVmN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # importing word_tokenize from nltk\n",
        "# from nltk.tokenize import word_tokenize"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-jIfHI78yYQy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # #In case the punkt package is missing \n",
        "# nltk.download('punkt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zfFOB_AuybQM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #Creating function for tokenization\n",
        "# def tokenization(text):\n",
        "#     text = re.split('\\W+', text)\n",
        "#     return text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P3-sLeYRyeXN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# # Passing the function to 'text_rare' and store into'text_token'\n",
        "# img_info1['Title_token'] = img_info1['Title_stop'].apply(lambda x: tokenization(x.lower()))\n",
        "# img_info1['Description_token'] = img_info1['Description_stop'].apply(lambda x: tokenization(x.lower()))\n",
        "# img_info1['Tags_token'] = img_info1['Tags_stop'].apply(lambda x: tokenization(x.lower()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NFptk3070QGZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Lemmatization"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n36Gg5IG0l1G",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "outputId": "ff458b22-383d-4d69-d644-09102f9c205b"
      },
      "source": [
        "nltk.download('wordnet')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/wordnet.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XyehsAh30TUK",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.corpus import wordnet\n",
        "from nltk.stem import WordNetLemmatizer\n",
        "\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "wordnet_map = {\"N\":wordnet.NOUN, \"V\":wordnet.VERB, \"J\":wordnet.ADJ, \"R\":wordnet.ADV} # Pos tag, used Noun, Verb, Adjective and Adverb"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G6ZF5OTM0ica",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function for lemmatization using POS tag\n",
        "def lemmatize_words(text):\n",
        "    pos_tagged_text = nltk.pos_tag(text.split())\n",
        "    return \" \".join([lemmatizer.lemmatize(word, wordnet_map.get(pos[0], wordnet.NOUN)) for word, pos in pos_tagged_text])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EJBgcjqh1QRY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 87
        },
        "outputId": "50950812-9d18-4025-b38b-4c85b97964f7"
      },
      "source": [
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MuSe7X7b0sn4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Passing the function to 'text_rare' and store in 'text_lemma'\n",
        "test_img_info1[\"Title_lemma\"] = test_img_info1[\"Title_stop\"].apply(lemmatize_words)\n",
        "test_img_info1[\"Description_lemma\"] = test_img_info1[\"Description_stop\"].apply(lemmatize_words)\n",
        "test_img_info1[\"Tags_lemma\"] = test_img_info1[\"Tags_stop\"].apply(lemmatize_words)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xRVtq1UL1d7d",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Removing previous columns\n",
        "test_img_info1.drop(['Title_stop'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['Description_stop'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['Tags_stop'],axis=1,inplace=True)\n",
        "# img_info1.drop(['Title_token'],axis=1,inplace=True)\n",
        "# img_info1.drop(['Description_token'],axis=1,inplace=True)\n",
        "# img_info1.drop(['Tags_token'],axis=1,inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uYpxLEMc1ah_",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 743
        },
        "outputId": "1f3c6130-a830-4f85-8d38-2f5f87004b81"
      },
      "source": [
        "test_img_info1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>FlickrId</th>\n",
              "      <th>Size</th>\n",
              "      <th>NumSets</th>\n",
              "      <th>NumGroups</th>\n",
              "      <th>AvgGroupsMemb</th>\n",
              "      <th>AvgGroupPhotos</th>\n",
              "      <th>Title_lemma</th>\n",
              "      <th>Description_lemma</th>\n",
              "      <th>Tags_lemma</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>49098581833</td>\n",
              "      <td>15980544</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>clariant international ltd clariant launch ant...</td>\n",
              "      <td>clariant launch antimonytrioxidefree lasermark...</td>\n",
              "      <td>prf clariantinternationalltd clapr1634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>49098582593</td>\n",
              "      <td>19961856</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>456.000000</td>\n",
              "      <td>1.217800e+04</td>\n",
              "      <td>mk seiko big top salio</td>\n",
              "      <td>mk seiko big top salio car wash system yellow ...</td>\n",
              "      <td>carwash</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>49098582698</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>69</td>\n",
              "      <td>41385.347826</td>\n",
              "      <td>2.480120e+06</td>\n",
              "      <td>pano experiment 2 northville mi september 2019</td>\n",
              "      <td>nap_canon eos 5ds r_20190923__13a3294_0002pano...</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>49098582733</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>2458.250000</td>\n",
              "      <td>7.432825e+04</td>\n",
              "      <td>green heron</td>\n",
              "      <td>butorides viriscens</td>\n",
              "      <td>greenheron usa wildlife butoridesvirescens u l...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>49098584118</td>\n",
              "      <td>409600</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>study beetle naturalis week endless pattern be...</td>\n",
              "      <td>via instagram href rel noreferrer nofollow ift...</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7688</th>\n",
              "      <td>49104539732</td>\n",
              "      <td>2650383</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>volleyball mug lover like funny gift idea hobb...</td>\n",
              "      <td>href rel noreferrer nofollow funny design make...</td>\n",
              "      <td>volleyball mug lover like funny gift idea hobb...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7689</th>\n",
              "      <td>49104542697</td>\n",
              "      <td>1964800</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>9716.000000</td>\n",
              "      <td>6.995257e+05</td>\n",
              "      <td>poetry</td>\n",
              "      <td></td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7690</th>\n",
              "      <td>49104544567</td>\n",
              "      <td>14625792</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>untitled</td>\n",
              "      <td>process vsco kp4 preset</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7691</th>\n",
              "      <td>49104545392</td>\n",
              "      <td>18917874</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>z8271928jpg</td>\n",
              "      <td>olympus digital camera</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7692</th>\n",
              "      <td>49104546122</td>\n",
              "      <td>12192768</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.000000</td>\n",
              "      <td>0.000000e+00</td>\n",
              "      <td>ecuador peru en la westfalia</td>\n",
              "      <td></td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7693 rows × 9 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         FlickrId  ...                                         Tags_lemma\n",
              "0     49098581833  ...             prf clariantinternationalltd clapr1634\n",
              "1     49098582593  ...                                            carwash\n",
              "2     49098582698  ...                                    ifttt instagram\n",
              "3     49098582733  ...  greenheron usa wildlife butoridesvirescens u l...\n",
              "4     49098584118  ...                                    ifttt instagram\n",
              "...           ...  ...                                                ...\n",
              "7688  49104539732  ...  volleyball mug lover like funny gift idea hobb...\n",
              "7689  49104542697  ...                                    ifttt instagram\n",
              "7690  49104544567  ...                                    ifttt instagram\n",
              "7691  49104545392  ...                                    ifttt instagram\n",
              "7692  49104546122  ...                                    ifttt instagram\n",
              "\n",
              "[7693 rows x 9 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 44
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bwkiQ5TLIH1T",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 230
        },
        "outputId": "f155d7e3-4925-4dfd-93c9-410db7d2fa73"
      },
      "source": [
        "test_img_info1['Description_lemma'].value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                                           4940\n",
              "href rel noreferrer nofollow                                                 30\n",
              "via instagram href rel noreferrer nofollow                                   16\n",
              "olympus digital camera                                                       15\n",
              "dav                                                                          12\n",
              "                                                                           ... \n",
              "via instagram href rel noreferrer nofollow ifttt343mdyha                      1\n",
              "polvillo dorado syrichtus proto                                               1\n",
              "via blogger href rel noreferrer nofollow ifttt2qrliova                        1\n",
              "via instagram href rel noreferrer nofollow ifttt2ocaa10a                      1\n",
              "delicious fresh lebanese food lovely decor atmosphere even though place       1\n",
              "Name: Description_lemma, Length: 2560, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 48
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P95-Q5-HP_Gg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_img_info1.drop(['Size'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['NumSets'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['NumGroups'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['AvgGroupsMemb'],axis=1,inplace=True)\n",
        "test_img_info1.drop(['AvgGroupPhotos'],axis=1,inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wD7hEUjHI6AK",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 490
        },
        "outputId": "2871cc33-b1c0-49a2-806a-0503a5cd736f"
      },
      "source": [
        "test_img_info1"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>FlickrId</th>\n",
              "      <th>Title_lemma</th>\n",
              "      <th>Description_lemma</th>\n",
              "      <th>Tags_lemma</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>49098581833</td>\n",
              "      <td>clariant international ltd clariant launch ant...</td>\n",
              "      <td>clariant launch antimonytrioxidefree lasermark...</td>\n",
              "      <td>prf clariantinternationalltd clapr1634</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>49098582593</td>\n",
              "      <td>mk seiko big top salio</td>\n",
              "      <td>mk seiko big top salio car wash system yellow ...</td>\n",
              "      <td>carwash</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>49098582698</td>\n",
              "      <td>pano experiment 2 northville mi september 2019</td>\n",
              "      <td>nap_canon eos 5ds r_20190923__13a3294_0002pano...</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>49098582733</td>\n",
              "      <td>green heron</td>\n",
              "      <td>butorides viriscens</td>\n",
              "      <td>greenheron usa wildlife butoridesvirescens u l...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>49098584118</td>\n",
              "      <td>study beetle naturalis week endless pattern be...</td>\n",
              "      <td>via instagram href rel noreferrer nofollow ift...</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7688</th>\n",
              "      <td>49104539732</td>\n",
              "      <td>volleyball mug lover like funny gift idea hobb...</td>\n",
              "      <td>href rel noreferrer nofollow funny design make...</td>\n",
              "      <td>volleyball mug lover like funny gift idea hobb...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7689</th>\n",
              "      <td>49104542697</td>\n",
              "      <td>poetry</td>\n",
              "      <td></td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7690</th>\n",
              "      <td>49104544567</td>\n",
              "      <td>untitled</td>\n",
              "      <td>process vsco kp4 preset</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7691</th>\n",
              "      <td>49104545392</td>\n",
              "      <td>z8271928jpg</td>\n",
              "      <td>olympus digital camera</td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7692</th>\n",
              "      <td>49104546122</td>\n",
              "      <td>ecuador peru en la westfalia</td>\n",
              "      <td></td>\n",
              "      <td>ifttt instagram</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>7693 rows × 4 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "         FlickrId  ...                                         Tags_lemma\n",
              "0     49098581833  ...             prf clariantinternationalltd clapr1634\n",
              "1     49098582593  ...                                            carwash\n",
              "2     49098582698  ...                                    ifttt instagram\n",
              "3     49098582733  ...  greenheron usa wildlife butoridesvirescens u l...\n",
              "4     49098584118  ...                                    ifttt instagram\n",
              "...           ...  ...                                                ...\n",
              "7688  49104539732  ...  volleyball mug lover like funny gift idea hobb...\n",
              "7689  49104542697  ...                                    ifttt instagram\n",
              "7690  49104544567  ...                                    ifttt instagram\n",
              "7691  49104545392  ...                                    ifttt instagram\n",
              "7692  49104546122  ...                                    ifttt instagram\n",
              "\n",
              "[7693 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T72vFVCs2ACP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_img_info1.to_csv(test_path_data+'test_textfeatures.csv',index=False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oucbgMqqyy3i",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #PRE-PROCESSING ON TEXT FEATURES\n",
        "# # from sklearn.feature_extraction.text import TfidfTransformer\n",
        "# from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "# # from sklearn.feature_extraction.text import CountVectorizer\n",
        "\n",
        "# vector=TfidfVectorizer(min_df=20)\n",
        "\n",
        "# # from sklearn.feature_extraction.text import HashingVectorizer\n",
        "# # vectorizer = HashingVectorizer(n_features=10000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XhlbelUKy9pb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #CONVERTING THE PANDAS INTO NUMPY AND THEN CONVERTING TO NUMERIC\n",
        "# X_tags=img_info1['Tags_lemma'].values\n",
        "# X_desc=img_info1['Description_lemma'].values\n",
        "# X_title=img_info1['Title_lemma'].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xd9yHttWzPed",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# X_tags=np.asarray(X_tags)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNNj4u7BzdM2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #Tags vectorization\n",
        "# tags_num = vector.fit_transform(X_tags)\n",
        "# print(tags_num.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4NbVcxnlZE9a",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# desc_num = vector.fit_transform(X_desc)\n",
        "# print(desc_num.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3gRb8WSJZKha",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# title_num=vector.fit_transform(X_title)\n",
        "# print(title_num.shape)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AAxthkTAL_nq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #Converting to numpy array\n",
        "# tags_num=tags_num.A\n",
        "# desc_num=desc_num.A\n",
        "# title_num=title_num.A"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xtz7KhXhL9Y2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#DIMENSION REDUCTION"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xARnIrDAXsfl",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from sklearn.decomposition import TruncatedSVD"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrdaLUgbXG-4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Y=Y.values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m4_9Q-eSXepF",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Y=Y.reshape((Y.shape[0],))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lho2uLTGXsjU",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Y.ravel()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eVQ_zMg4Y-ko",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Checking the variables importance"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8SZUX80nTgDf",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from sklearn.model_selection import train_test_split\n",
        "# Train_X,Test_X,Train_Y,Test_Y=train_test_split(title_num,Y,test_size=0.1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "blkTvTnKXOX3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #Preparing for Random Forest Classifier\n",
        "# from sklearn.ensemble.forest import RandomForestClassifier"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LmBgzXKJXRh7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# model=RandomForestClassifier(n_estimators=200,criterion='entropy',random_state=42)\n",
        "# model.fit(Train_X,Train_Y)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3SDJ7cXSYpUj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from sklearn import metrics\n",
        "\n",
        "# Pred_Y2=model.predict(Test_X) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkZ2ICQYjYeL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #title\n",
        "# print(metrics.accuracy_score(Test_Y,Pred_Y2))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dJGz9nNdgUZO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #description\n",
        "# print(metrics.accuracy_score(Test_Y,Pred_Y1))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YL0sc2atYumo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# #tags\n",
        "# print(metrics.accuracy_score(Test_Y,Pred_Y))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vWxYsJ-1M9U4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from sklearn.preprocessing import StandardScaler\n",
        "# tags_num=StandardScaler().fit_transform(tags_num)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LpTalVfvqWmP",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Tokenization"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "duMT_lNuwcy4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# from nltk.tokenize import word_tokenize"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wvg2IYetwy_7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# nltk.download('punkt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wgjfroOzw140",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# img_info1['token_tags'] = word_tokenize(img_info1['Tags_stop'])"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}