To_Perform_Word_Analysis_using_NLTK.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "#**To Perform Word Analysis using NLTK.**"
      ],
      "metadata": {
        "id": "4lbtqT8fALjf"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VAFp2T8pX-g1",
        "outputId": "4551b8eb-57b0-486e-8803-551826304989"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Counter({'google': 1,\n",
              "         'microsoft': 1,\n",
              "         'apple': 1,\n",
              "         'amazon': 1,\n",
              "         'infosys': 1,\n",
              "         'IBM': 1,\n",
              "         'orangeCart': 1,\n",
              "         'starterit': 2,\n",
              "         'mobtech': 1})"
            ]
          },
          "metadata": {},
          "execution_count": 1
        }
      ],
      "source": [
        "from collections import Counter\n",
        "\n",
        "Counter(['google', 'microsoft', 'apple', 'amazon', 'infosys', 'IBM', 'orangeCart', 'starterit', 'mobtech', 'starterit'])"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "Counter(['google', 'microsoft', 'apple','apple','apple'])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sNRUgrT7j_CX",
        "outputId": "5649e452-c581-4d2e-ad7b-af43e693d9f1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Counter({'google': 1, 'microsoft': 1, 'apple': 3})"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's do the word analysis for few sentamces:\n",
        "# We need to turn our text into a list of words. \n",
        "# We also need to do a little bit of cleanup - removing punctuation, making everything lowercase, just making sure the only things left are words.\n",
        "import re\n",
        "\n",
        "text = \"\"\"NLP is the study of excellent communication–both with yourself, and with others. It was developed by modeling \n",
        "excellent communicators and therapists who got results with their clients. NLP is a set of tools and techniques\"\"\"\n",
        "\n",
        "# Force to all be lowercase because FISH and fish and Fish are the same\n",
        "text = text.lower()\n",
        "\n",
        "# Remove anything that isn't a word character or a space\n",
        "# We could use .replace(\".\", \"\") but regex is a lot easier!\n",
        "text = re.sub(\"[^\\w ]\", \"\", text)\n",
        "\n",
        "print(\"Cleaned sentence is:\", text)\n",
        "\n",
        "words = text.split(\" \")\n",
        "Counter(words)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_yA2bzEGYD08",
        "outputId": "cf1c1e2f-e9fc-4133-831d-6255d3444ae7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cleaned sentence is: nlp is the study of excellent communicationboth with yourself and with others it was developed by modeling excellent communicators and therapists who got results with their clients nlp is a set of tools and techniques\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Counter({'nlp': 2,\n",
              "         'is': 2,\n",
              "         'the': 1,\n",
              "         'study': 1,\n",
              "         'of': 2,\n",
              "         'excellent': 2,\n",
              "         'communicationboth': 1,\n",
              "         'with': 3,\n",
              "         'yourself': 1,\n",
              "         'and': 3,\n",
              "         'others': 1,\n",
              "         'it': 1,\n",
              "         'was': 1,\n",
              "         'developed': 1,\n",
              "         'by': 1,\n",
              "         'modeling': 1,\n",
              "         'communicators': 1,\n",
              "         'therapists': 1,\n",
              "         'who': 1,\n",
              "         'got': 1,\n",
              "         'results': 1,\n",
              "         'their': 1,\n",
              "         'clients': 1,\n",
              "         'a': 1,\n",
              "         'set': 1,\n",
              "         'tools': 1,\n",
              "         'techniques': 1})"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#If you have a lot of text, you're usually only interested in the most common words.\n",
        "# If you just want the top words, so we use .most_common \n",
        "Counter(words).most_common(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZvvWE-loYI0T",
        "outputId": "466cb418-c3e2-4d20-ed49-66da490ebbd9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('with', 3),\n",
              " ('and', 3),\n",
              " ('nlp', 2),\n",
              " ('is', 2),\n",
              " ('of', 2),\n",
              " ('excellent', 2),\n",
              " ('the', 1),\n",
              " ('study', 1),\n",
              " ('communicationboth', 1),\n",
              " ('yourself', 1)]"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#Counting words in a book\n",
        "#Now that we know the basics of how to clean text and do text analysis with Counter, let's try it with an actual book!\n",
        "# I will be using NLP Book\n",
        "\n",
        "import requests\n",
        "\n",
        "response = requests.get('https://web.stanford.edu/~jurafsky/slp3/ed3book.pdf')\n",
        "text = response.text\n",
        "\n",
        "print(text[4100:4500])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QwJPJ0ucYPdb",
        "outputId": "1d0bd8c1-c12b-486d-9667-42922c37f188"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\r�L�Ǜk����P`j��\u0005�dQ�6�w~�\u0003����u��k�J�]\u001d}F�\u001av���[�ߐ�g4�ͫ`�\u001d�����bvz�4�]���[�\t�����e���\u0005�mk�\n",
            "endstream\n",
            "endobj\n",
            "595 0 obj\n",
            "<<\n",
            "/Type /ObjStm\n",
            "/N 100\n",
            "/First 864\n",
            "/Length 1307      \n",
            "/Filter /FlateDecode\n",
            ">>\n",
            "stream\n",
            "xڝVMs�6\u0010��W�19�!��L&3q�ĝ�]���gH�%6�pA2����@��⸤�\u0006�����{�B�JJHٔ��b)դ\u0013N��X\u00051e�jb��j�ٸZₑf\tq�`\u0015$\u00126�L��\n",
            "\u0002\u000e0d�w�H\n",
            "�qN�H��\u0014�X5)\u0011�-)�w\u0011��]\b|�L��b}P�\u00168�0�֒�Q�+9\u0019\u0003\\�(e�#5�\u0002�$�0�C�e\u0011C��f��$�l�i-꩔X��fx��\u001a�'\u0006��\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "\n",
        "response = requests.get('http://www.gutenberg.org/cache/epub/42671/pg42671.txt')\n",
        "text = response.text\n",
        "\n",
        "print(text[4100:4500])\n",
        "\n",
        "# Now let's us count works:\n",
        "import re\n",
        "text = text.lower()\n",
        "text = re.sub(\"[^\\w ]\", \"\", text)\n",
        "\n",
        "words = text.split(\" \")\n",
        "Counter(words).most_common(20)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KqW46_YVYVdb",
        "outputId": "25c10833-9324-4366-858d-763aaa7d3d7a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "d to be any thing extraordinary now. When a woman has\r\n",
            "five grown up daughters, she ought to give over thinking of her own\r\n",
            "beauty.\"\r\n",
            "\r\n",
            "\"In such cases, a woman has not often much beauty to think of.\"\r\n",
            "\r\n",
            "\"But, my dear, you must indeed go and see Mr. Bingley when he comes into\r\n",
            "the neighbourhood.\"\r\n",
            "\r\n",
            "\"It is more than I engage for, I assure you.\"\r\n",
            "\r\n",
            "\"But consider your daughters. Only think what an es\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('the', 3751),\n",
              " ('to', 3746),\n",
              " ('of', 3298),\n",
              " ('', 3289),\n",
              " ('and', 3113),\n",
              " ('her', 1811),\n",
              " ('a', 1745),\n",
              " ('in', 1679),\n",
              " ('i', 1655),\n",
              " ('was', 1622),\n",
              " ('she', 1385),\n",
              " ('that', 1325),\n",
              " ('it', 1294),\n",
              " ('not', 1278),\n",
              " ('he', 1148),\n",
              " ('you', 1145),\n",
              " ('be', 1101),\n",
              " ('his', 1061),\n",
              " ('as', 1052),\n",
              " ('had', 1036)]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Only extracting some words with regular expressions\n",
        "# Do men and women do different things in this book? Let's look at she ____ and he ____ to see what we can find out!\n",
        "\n",
        "# \\b marks a word boundary, otherwise the phrase \"she talks\" would match both she (\\w+) and he (\\w+)\n",
        "\n",
        "# Catch every word after 'she'\n",
        "she_words = re.findall(r\"\\b[Ss]he (\\w+)\", text)\n",
        "she_words[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LoHeojh_YWmD",
        "outputId": "a2893015-1af5-4172-8763-c1903219cb51"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['for', 'ought', 'is', 'was', 'was']"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Catch every word after 'he'\n",
        "he_words = re.findall(r\"\\b[Hh]e (\\w+)\", text)\n",
        "he_words[:5]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i4W67Ze0YZl6",
        "outputId": "97f876aa-02e2-4763-f7c3-932272dc0918"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['is', 'had', 'camedown', 'agreed', 'isto']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    }
  ]
}