ineuron_nlp.ipynb

{
  "metadata": {
    "kernelspec": {
      "language": "python",
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.10.13",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "kaggle": {
      "accelerator": "none",
      "dataSources": [],
      "dockerImageVersionId": 30746,
      "isInternetEnabled": true,
      "language": "python",
      "sourceType": "notebook",
      "isGpuEnabled": false
    },
    "colab": {
      "name": "ineuron_nlp.ipynb",
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat_minor": 0,
  "nbformat": 4,
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/daemonX10/Data-Science/blob/main/ineuron_nlp.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# This Python 3 environment comes with many helpful analytics libraries installed\n",
        "# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n",
        "# For example, here's several helpful packages to load\n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "# Input data files are available in the read-only \"../input/\" directory\n",
        "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
        "\n",
        "import os\n",
        "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
        "    for filename in filenames:\n",
        "        print(os.path.join(dirname, filename))\n",
        "\n",
        "# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\"\n",
        "# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session"
      ],
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "trusted": true,
        "id": "YJbzMdMmgfYI"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# annotator"
      ],
      "metadata": {
        "id": "iIifJ5HggfYL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "data = [\n",
        "    (\"I love this product!\", \"positive\"),\n",
        "    (\"This is not what I expected.\", \"negative\"),\n",
        "    (\"The customer service was excellent.\", \"positive\"),\n",
        "    (\"The quality of the item is poor.\", \"negative\")\n",
        "]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T05:44:35.632057Z",
          "iopub.execute_input": "2024-08-01T05:44:35.63252Z",
          "iopub.status.idle": "2024-08-01T05:44:35.638168Z",
          "shell.execute_reply.started": "2024-08-01T05:44:35.632484Z",
          "shell.execute_reply": "2024-08-01T05:44:35.636974Z"
        },
        "trusted": true,
        "id": "bEgQVMY-gfYM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Define the annotator\")\n",
        "\n",
        "def annotate_data(data):\n",
        "    annotate_data_arr = []\n",
        "    for text,label in data:\n",
        "        annotate_data_arr.append((text,label))\n",
        "    return annotate_data_arr"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T05:44:35.640124Z",
          "iopub.execute_input": "2024-08-01T05:44:35.640481Z",
          "iopub.status.idle": "2024-08-01T05:44:35.660808Z",
          "shell.execute_reply.started": "2024-08-01T05:44:35.64045Z",
          "shell.execute_reply": "2024-08-01T05:44:35.659128Z"
        },
        "trusted": true,
        "id": "FzyE8_5bgfYM",
        "outputId": "6afba783-2acc-4b1f-d55c-6b4f0303d7d2"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Define the annotator\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Annotate the data\")\n",
        "annotate_data_arr = annotate_data(data)\n",
        "for text,label in annotate_data_arr:\n",
        "    print(f\"Text : {text} \\t Labels:{label}\")"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T05:44:35.662298Z",
          "iopub.execute_input": "2024-08-01T05:44:35.662686Z",
          "iopub.status.idle": "2024-08-01T05:44:35.678348Z",
          "shell.execute_reply.started": "2024-08-01T05:44:35.662654Z",
          "shell.execute_reply": "2024-08-01T05:44:35.676968Z"
        },
        "trusted": true,
        "id": "59wJlux2gfYN",
        "outputId": "cc08b4e0-1f89-41d3-a4d8-c6db4569a81e"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Annotate the data\nText : I love this product! \t Labels:positive\nText : This is not what I expected. \t Labels:negative\nText : The customer service was excellent. \t Labels:positive\nText : The quality of the item is poor. \t Labels:negative\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Co-Occurrence"
      ],
      "metadata": {
        "id": "x1rvjAlKgfYN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "from collections import defaultdict\n",
        "from nltk.tokenize import word_tokenize,sent_tokenize\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T06:12:45.121742Z",
          "iopub.execute_input": "2024-08-01T06:12:45.12221Z",
          "iopub.status.idle": "2024-08-01T06:12:45.128739Z",
          "shell.execute_reply.started": "2024-08-01T06:12:45.122176Z",
          "shell.execute_reply": "2024-08-01T06:12:45.127165Z"
        },
        "trusted": true,
        "id": "4zIZtfz8gfYN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "UwGJyutRgfYN"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "text = \"Co-occurrence vectors capture word relationships in a text corpus. They are used for various NLP tasks.\""
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T06:12:46.830765Z",
          "iopub.execute_input": "2024-08-01T06:12:46.83125Z",
          "iopub.status.idle": "2024-08-01T06:12:46.837361Z",
          "shell.execute_reply.started": "2024-08-01T06:12:46.831213Z",
          "shell.execute_reply": "2024-08-01T06:12:46.836199Z"
        },
        "trusted": true,
        "id": "SS2er68VgfYO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"tokenize into sentence and words\")\n",
        "sentences = sent_tokenize(text)\n",
        "print(sentences)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T06:12:49.215423Z",
          "iopub.execute_input": "2024-08-01T06:12:49.215836Z",
          "iopub.status.idle": "2024-08-01T06:12:49.222559Z",
          "shell.execute_reply.started": "2024-08-01T06:12:49.215803Z",
          "shell.execute_reply": "2024-08-01T06:12:49.221436Z"
        },
        "trusted": true,
        "id": "vxY6s5AAgfYO",
        "outputId": "ac4fbb3a-1e5a-4ebb-9640-91225b7e219d"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "tokenize into sentence and words\n['Co-occurrence vectors capture word relationships in a text corpus.', 'They are used for various NLP tasks.']\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_sentences = [word_tokenize(sentence.lower()) for sentence in sentences]\n",
        "print(\"Token sentences\",tokenized_sentences)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T06:12:51.23922Z",
          "iopub.execute_input": "2024-08-01T06:12:51.23965Z",
          "iopub.status.idle": "2024-08-01T06:12:51.246906Z",
          "shell.execute_reply.started": "2024-08-01T06:12:51.239616Z",
          "shell.execute_reply": "2024-08-01T06:12:51.245114Z"
        },
        "trusted": true,
        "id": "6_-aq1vugfYO",
        "outputId": "66e5d1f4-3869-4c3e-b72e-edb02899ee79"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Token sentences [['co-occurrence', 'vectors', 'capture', 'word', 'relationships', 'in', 'a', 'text', 'corpus', '.'], ['they', 'are', 'used', 'for', 'various', 'nlp', 'tasks', '.']]\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## co-occurence matrix"
      ],
      "metadata": {
        "id": "ZnipVjucgfYO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Build co-occurence matrix\")\n",
        "window_size = 2\n",
        "co_occurence_matrix = defaultdict(lambda:defaultdict(int))\n",
        "\n",
        "for sentence in tokenized_sentences:\n",
        "    for i , target_word in enumerate(sentence):\n",
        "        for j in range(max(0,i-window_size),min(len(sentence),i+window_size+1)):\n",
        "            if i !=j:\n",
        "                context_word = sentence[j]\n",
        "                co_occurence_matrix[target_word][context_word] +=1"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T06:13:37.649878Z",
          "iopub.execute_input": "2024-08-01T06:13:37.650297Z",
          "iopub.status.idle": "2024-08-01T06:13:37.658946Z",
          "shell.execute_reply.started": "2024-08-01T06:13:37.650266Z",
          "shell.execute_reply": "2024-08-01T06:13:37.657533Z"
        },
        "trusted": true,
        "id": "A7UZuE_jgfYO",
        "outputId": "51b6d305-1830-437e-f0de-29a0d604080c"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "Build co-occurence matrix\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"co - occurrence matrix\")\n",
        "for target_word,context_word in co_occurence_matrix.items():\n",
        "    print(target_word,context_word)\n",
        "print()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-01T06:14:38.612971Z",
          "iopub.execute_input": "2024-08-01T06:14:38.613362Z",
          "iopub.status.idle": "2024-08-01T06:14:38.619966Z",
          "shell.execute_reply.started": "2024-08-01T06:14:38.613332Z",
          "shell.execute_reply": "2024-08-01T06:14:38.618893Z"
        },
        "trusted": true,
        "id": "Zvc1bdbMgfYP",
        "outputId": "64531e2d-4a77-4899-dc9e-6f13788536b9"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "co - occurrence matrix\nco-occurrence defaultdict(<class 'int'>, {'vectors': 1, 'capture': 1})\nvectors defaultdict(<class 'int'>, {'co-occurrence': 1, 'capture': 1, 'word': 1})\ncapture defaultdict(<class 'int'>, {'co-occurrence': 1, 'vectors': 1, 'word': 1, 'relationships': 1})\nword defaultdict(<class 'int'>, {'vectors': 1, 'capture': 1, 'relationships': 1, 'in': 1})\nrelationships defaultdict(<class 'int'>, {'capture': 1, 'word': 1, 'in': 1, 'a': 1})\nin defaultdict(<class 'int'>, {'word': 1, 'relationships': 1, 'a': 1, 'text': 1})\na defaultdict(<class 'int'>, {'relationships': 1, 'in': 1, 'text': 1, 'corpus': 1})\ntext defaultdict(<class 'int'>, {'in': 1, 'a': 1, 'corpus': 1, '.': 1})\ncorpus defaultdict(<class 'int'>, {'a': 1, 'text': 1, '.': 1})\n. defaultdict(<class 'int'>, {'text': 1, 'corpus': 1, 'nlp': 1, 'tasks': 1})\nthey defaultdict(<class 'int'>, {'are': 1, 'used': 1})\nare defaultdict(<class 'int'>, {'they': 1, 'used': 1, 'for': 1})\nused defaultdict(<class 'int'>, {'they': 1, 'are': 1, 'for': 1, 'various': 1})\nfor defaultdict(<class 'int'>, {'are': 1, 'used': 1, 'various': 1, 'nlp': 1})\nvarious defaultdict(<class 'int'>, {'used': 1, 'for': 1, 'nlp': 1, 'tasks': 1})\nnlp defaultdict(<class 'int'>, {'for': 1, 'various': 1, 'tasks': 1, '.': 1})\ntasks defaultdict(<class 'int'>, {'various': 1, 'nlp': 1, '.': 1})\n\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "uIzKZaUBgfYP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This imports the PorterStemmer class from the nltk.stem module. The Porter Stemmer is a popular algorithm used to reduce words to their base or root form (known as stemming). For example, \"running\" and \"runner\" might be reduced to \"run\". This process helps in normalizing words to their root forms, which can improve the performance of text analysis tasks.\n",
        "\n"
      ],
      "metadata": {
        "id": "U1WQoiS5gfYP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import re\n",
        "from nltk.tokenize import word_tokenize\n",
        "from nltk.corpus import stopwords\n",
        "from nltk.stem import PorterStemmer\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T08:43:58.625475Z",
          "iopub.execute_input": "2024-08-02T08:43:58.62585Z",
          "iopub.status.idle": "2024-08-02T08:44:00.023954Z",
          "shell.execute_reply.started": "2024-08-02T08:43:58.625818Z",
          "shell.execute_reply": "2024-08-02T08:44:00.022837Z"
        },
        "trusted": true,
        "id": "XgceMzJLgfYP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download()\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T08:44:27.492263Z",
          "iopub.execute_input": "2024-08-02T08:44:27.492751Z",
          "iopub.status.idle": "2024-08-02T08:45:56.475642Z",
          "shell.execute_reply.started": "2024-08-02T08:44:27.492714Z",
          "shell.execute_reply": "2024-08-02T08:45:56.474687Z"
        },
        "trusted": true,
        "id": "c3m0IVEcgfYP",
        "outputId": "0b94afe0-2f09-48e9-fa07-9ffef8091cc8"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "NLTK Downloader\n---------------------------------------------------------------------------\n    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n---------------------------------------------------------------------------\n",
          "output_type": "stream"
        },
        {
          "output_type": "stream",
          "name": "stdin",
          "text": "Downloader>  d\n"
        },
        {
          "name": "stdout",
          "text": "\nDownload which package (l=list; x=cancel)?\n",
          "output_type": "stream"
        },
        {
          "output_type": "stream",
          "name": "stdin",
          "text": "  Identifier>  l\n"
        },
        {
          "name": "stdout",
          "text": "Packages:\n  [ ] averaged_perceptron_tagger_eng Averaged Perceptron Tagger (JSON)\n  [ ] averaged_perceptron_tagger_ru Averaged Perceptron Tagger (Russian)\n  [ ] averaged_perceptron_tagger_rus Averaged Perceptron Tagger (Russian)\n  [ ] bcp47............... BCP-47 Language Tags\n  [ ] comparative_sentences Comparative Sentence Dataset\n  [ ] dolch............... Dolch Word List\n  [ ] extended_omw........ Extended Open Multilingual WordNet\n  [ ] framenet_v15........ FrameNet 1.5\n  [ ] framenet_v17........ FrameNet 1.7\n  [ ] maxent_ne_chunker_tab ACE Named Entity Chunker (Maximum entropy)\n  [ ] maxent_treebank_pos_tagger_tab Treebank Part of Speech Tagger (Maximum entropy)\n  [ ] mwa_ppdb............ The monolingual word aligner (Sultan et al.\n                           2015) subset of the Paraphrase Database.\n  [ ] nombank.1.0......... NomBank Corpus 1.0\n  [ ] nonbreaking_prefixes Non-Breaking Prefixes (Moses Decoder)\n  [ ] omw-1.4............. Open Multilingual Wordnet\n  [ ] panlex_swadesh...... PanLex Swadesh Corpora\n  [ ] pe08................ Cross-Framework and Cross-Domain Parser\n                           Evaluation Shared Task\n  [ ] perluniprops........ perluniprops: Index of Unicode Version 7.0.0\n                           character properties in Perl\n",
          "output_type": "stream"
        },
        {
          "output_type": "stream",
          "name": "stdin",
          "text": "Hit Enter to continue:  words\n"
        },
        {
          "name": "stdout",
          "text": "  [ ] punkt_tab........... Punkt Tokenizer Models\n  [ ] tagsets_json........ Help on Tagsets (JSON)\n  [ ] verbnet3............ VerbNet Lexicon, Version 3.3\n  [ ] wmt15_eval.......... Evaluation data from WMT15\n  [ ] wordnet2021......... Open English Wordnet 2021\n  [ ] wordnet2022......... Open English Wordnet 2022\n  [ ] wordnet31........... Wordnet 3.1\n\nCollections:\n  [P] all-corpora......... All the corpora\n  [P] all-nltk............ All packages available on nltk_data gh-pages\n                           branch\n  [P] all................. All packages\n  [P] book................ Everything used in the NLTK Book\n  [P] popular............. Popular packages\n  [P] tests............... Packages for running tests\n  [ ] third-party......... Third-party data packages\n\n([*] marks installed packages; [P] marks partially installed collections)\n\nDownload which package (l=list; x=cancel)?\n",
          "output_type": "stream"
        },
        {
          "output_type": "stream",
          "name": "stdin",
          "text": "  Identifier>  popular\n"
        },
        {
          "name": "stdout",
          "text": "    Downloading collection 'popular'\n       | \n       | Downloading package cmudict to /usr/share/nltk_data...\n       |   Package cmudict is already up-to-date!\n       | Downloading package gazetteers to /usr/share/nltk_data...\n       |   Package gazetteers is already up-to-date!\n       | Downloading package genesis to /usr/share/nltk_data...\n       |   Package genesis is already up-to-date!\n       | Downloading package gutenberg to /usr/share/nltk_data...\n       |   Package gutenberg is already up-to-date!\n       | Downloading package inaugural to /usr/share/nltk_data...\n       |   Package inaugural is already up-to-date!\n       | Downloading package movie_reviews to /usr/share/nltk_data...\n       |   Package movie_reviews is already up-to-date!\n       | Downloading package names to /usr/share/nltk_data...\n       |   Package names is already up-to-date!\n       | Downloading package shakespeare to /usr/share/nltk_data...\n       |   Package shakespeare is already up-to-date!\n       | Downloading package stopwords to /usr/share/nltk_data...\n       |   Package stopwords is already up-to-date!\n       | Downloading package treebank to /usr/share/nltk_data...\n       |   Package treebank is already up-to-date!\n       | Downloading package twitter_samples to\n       |     /usr/share/nltk_data...\n       |   Package twitter_samples is already up-to-date!\n       | Downloading package omw to /usr/share/nltk_data...\n       |   Package omw is already up-to-date!\n       | Downloading package omw-1.4 to /usr/share/nltk_data...\n       | Downloading package wordnet to /usr/share/nltk_data...\n       |   Package wordnet is already up-to-date!\n       | Downloading package wordnet2021 to /usr/share/nltk_data...\n       | Downloading package wordnet31 to /usr/share/nltk_data...\n       | Downloading package wordnet_ic to /usr/share/nltk_data...\n       |   Package wordnet_ic is already up-to-date!\n       | Downloading package words to /usr/share/nltk_data...\n       |   Package words is already up-to-date!\n       | Downloading package maxent_ne_chunker to\n       |     /usr/share/nltk_data...\n       |   Package maxent_ne_chunker is already up-to-date!\n       | Downloading package punkt to /usr/share/nltk_data...\n       |   Package punkt is already up-to-date!\n       | Downloading package snowball_data to /usr/share/nltk_data...\n       |   Package snowball_data is already up-to-date!\n       | Downloading package averaged_perceptron_tagger to\n       |     /usr/share/nltk_data...\n       |   Package averaged_perceptron_tagger is already up-to-date!\n       | \n     Done downloading collection popular\n\n---------------------------------------------------------------------------\n    d) Download   l) List    u) Update   c) Config   h) Help   q) Quit\n---------------------------------------------------------------------------\n",
          "output_type": "stream"
        },
        {
          "output_type": "stream",
          "name": "stdin",
          "text": "Downloader>  q\n"
        },
        {
          "execution_count": 4,
          "output_type": "execute_result",
          "data": {
            "text/plain": "True"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "url = \"https://en.wikipedia.org/wiki/Independence_Day_(India)\"\n",
        "response = requests.get(url)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T08:49:17.225837Z",
          "iopub.execute_input": "2024-08-02T08:49:17.226496Z",
          "iopub.status.idle": "2024-08-02T08:49:17.270267Z",
          "shell.execute_reply.started": "2024-08-02T08:49:17.226462Z",
          "shell.execute_reply": "2024-08-02T08:49:17.269467Z"
        },
        "trusted": true,
        "id": "nIJhhtkrgfYP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "soup = BeautifulSoup(response.content,'html.parser')\n"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T08:50:51.339382Z",
          "iopub.execute_input": "2024-08-02T08:50:51.339874Z",
          "iopub.status.idle": "2024-08-02T08:50:51.547293Z",
          "shell.execute_reply.started": "2024-08-02T08:50:51.339842Z",
          "shell.execute_reply": "2024-08-02T08:50:51.546287Z"
        },
        "trusted": true,
        "id": "QbzjT1bEgfYQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "paragraph = soup.find_all('p')\n",
        "paragraph"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T08:59:38.662452Z",
          "iopub.execute_input": "2024-08-02T08:59:38.662861Z",
          "iopub.status.idle": "2024-08-02T08:59:38.672702Z",
          "shell.execute_reply.started": "2024-08-02T08:59:38.66283Z",
          "shell.execute_reply": "2024-08-02T08:59:38.671479Z"
        },
        "trusted": true,
        "id": "wiUEXln4gfYQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "paragraph_text = [para.get_text() for para in paragraph]\n",
        "paragraph_text[1]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:00:08.603088Z",
          "iopub.execute_input": "2024-08-02T09:00:08.603472Z",
          "iopub.status.idle": "2024-08-02T09:00:08.61076Z",
          "shell.execute_reply.started": "2024-08-02T09:00:08.603443Z",
          "shell.execute_reply": "2024-08-02T09:00:08.609685Z"
        },
        "trusted": true,
        "id": "vRa9hTLhgfYQ",
        "outputId": "4380c5cf-a309-404f-92c3-f48e6c020248"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 17,
          "output_type": "execute_result",
          "data": {
            "text/plain": "\"Independence Day is celebrated annually on 15 August as a public holiday in India commemorating the nation's independence from the United Kingdom on 15 August 1947, the day when the provisions of the Indian Independence Act, which transferred legislative sovereignty to the Indian Constituent Assembly, came into effect. India retained King George\\xa0VI as head of state until its transition to a republic, when the Constitution of India came into effect on 26 January 1950 (celebrated as Indian Republic Day) and replaced the dominion prefix, Dominion of India, with the enactment of the sovereign law Constitution of India. India attained independence following the independence movement noted for largely non-violent resistance and civil disobedience led by Indian National Congress under the leadership of Mahatma Gandhi who adopted these values from one of the early movements[1] in India led by Ram Singh Kuka (quoted in one of the letters by Shaheed Bhagat Singh referring Guru Ram Singh as his Dada Guru).\\n\""
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cleaned_paragraphs = [re.sub(r'\\s+',' ',text).strip() for text in paragraph_text ]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:24:44.110694Z",
          "iopub.execute_input": "2024-08-02T09:24:44.111427Z",
          "iopub.status.idle": "2024-08-02T09:24:44.117588Z",
          "shell.execute_reply.started": "2024-08-02T09:24:44.111387Z",
          "shell.execute_reply": "2024-08-02T09:24:44.116426Z"
        },
        "trusted": true,
        "id": "OYyj9V3YgfYQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cleaned_paragraphs = [text for text in cleaned_paragraphs if text!='']"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:24:45.871277Z",
          "iopub.execute_input": "2024-08-02T09:24:45.871961Z",
          "iopub.status.idle": "2024-08-02T09:24:45.876297Z",
          "shell.execute_reply.started": "2024-08-02T09:24:45.871927Z",
          "shell.execute_reply": "2024-08-02T09:24:45.875331Z"
        },
        "trusted": true,
        "id": "2MZhdz9UgfYQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cleaned_paragraphs[0]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:30:01.757529Z",
          "iopub.execute_input": "2024-08-02T09:30:01.758189Z",
          "iopub.status.idle": "2024-08-02T09:30:01.7647Z",
          "shell.execute_reply.started": "2024-08-02T09:30:01.758158Z",
          "shell.execute_reply": "2024-08-02T09:30:01.763522Z"
        },
        "trusted": true,
        "id": "TSTelHGGgfYQ",
        "outputId": "6b8183b9-39c8-4654-b722-635de714ece1"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 54,
          "output_type": "execute_result",
          "data": {
            "text/plain": "\"Independence Day is celebrated annually on 15 August as a public holiday in India commemorating the nation's independence from the United Kingdom on 15 August 1947, the day when the provisions of the Indian Independence Act, which transferred legislative sovereignty to the Indian Constituent Assembly, came into effect. India retained King George VI as head of state until its transition to a republic, when the Constitution of India came into effect on 26 January 1950 (celebrated as Indian Republic Day) and replaced the dominion prefix, Dominion of India, with the enactment of the sovereign law Constitution of India. India attained independence following the independence movement noted for largely non-violent resistance and civil disobedience led by Indian National Congress under the leadership of Mahatma Gandhi who adopted these values from one of the early movements[1] in India led by Ram Singh Kuka (quoted in one of the letters by Shaheed Bhagat Singh referring Guru Ram Singh as his Dada Guru).\""
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lowercase_text = [text.lower() for text in cleaned_paragraphs]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:26:21.205566Z",
          "iopub.execute_input": "2024-08-02T09:26:21.206465Z",
          "iopub.status.idle": "2024-08-02T09:26:21.210903Z",
          "shell.execute_reply.started": "2024-08-02T09:26:21.206431Z",
          "shell.execute_reply": "2024-08-02T09:26:21.209815Z"
        },
        "trusted": true,
        "id": "stplj-AAgfYQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lowercase_text[0]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:29:57.078813Z",
          "iopub.execute_input": "2024-08-02T09:29:57.079591Z",
          "iopub.status.idle": "2024-08-02T09:29:57.085409Z",
          "shell.execute_reply.started": "2024-08-02T09:29:57.079555Z",
          "shell.execute_reply": "2024-08-02T09:29:57.084426Z"
        },
        "trusted": true,
        "id": "OYstGONxgfYQ",
        "outputId": "1e3e5b37-0e1d-4986-cf34-a05f15bfdbe6"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 53,
          "output_type": "execute_result",
          "data": {
            "text/plain": "\"independence day is celebrated annually on 15 august as a public holiday in india commemorating the nation's independence from the united kingdom on 15 august 1947, the day when the provisions of the indian independence act, which transferred legislative sovereignty to the indian constituent assembly, came into effect. india retained king george vi as head of state until its transition to a republic, when the constitution of india came into effect on 26 january 1950 (celebrated as indian republic day) and replaced the dominion prefix, dominion of india, with the enactment of the sovereign law constitution of india. india attained independence following the independence movement noted for largely non-violent resistance and civil disobedience led by indian national congress under the leadership of mahatma gandhi who adopted these values from one of the early movements[1] in india led by ram singh kuka (quoted in one of the letters by shaheed bhagat singh referring guru ram singh as his dada guru).\""
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cleaned_text = [re.sub('[^a-zA-Z0-9\\s]','',text) for text in lowercase_text]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:30:44.636601Z",
          "iopub.execute_input": "2024-08-02T09:30:44.637075Z",
          "iopub.status.idle": "2024-08-02T09:30:44.643023Z",
          "shell.execute_reply.started": "2024-08-02T09:30:44.63704Z",
          "shell.execute_reply": "2024-08-02T09:30:44.641918Z"
        },
        "trusted": true,
        "id": "nIjmxfP6gfYQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text = [word_tokenize(text) for text in cleaned_text]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:32:26.353429Z",
          "iopub.execute_input": "2024-08-02T09:32:26.35385Z",
          "iopub.status.idle": "2024-08-02T09:32:26.370905Z",
          "shell.execute_reply.started": "2024-08-02T09:32:26.353817Z",
          "shell.execute_reply": "2024-08-02T09:32:26.369924Z"
        },
        "trusted": true,
        "id": "UVptNTLIgfYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# tokenized_text[0]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:36:04.471852Z",
          "iopub.execute_input": "2024-08-02T09:36:04.472219Z",
          "iopub.status.idle": "2024-08-02T09:36:04.476581Z",
          "shell.execute_reply.started": "2024-08-02T09:36:04.47219Z",
          "shell.execute_reply": "2024-08-02T09:36:04.47549Z"
        },
        "trusted": true,
        "id": "oDSGiPHBgfYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stop_words = set(stopwords.words('english'))\n",
        "stremmer = PorterStemmer()"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:35:07.111557Z",
          "iopub.execute_input": "2024-08-02T09:35:07.111943Z",
          "iopub.status.idle": "2024-08-02T09:35:07.117833Z",
          "shell.execute_reply.started": "2024-08-02T09:35:07.111914Z",
          "shell.execute_reply": "2024-08-02T09:35:07.116815Z"
        },
        "trusted": true,
        "id": "ZstFFxkngfYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "filtered_text = [[word for word in tokens if word not in stop_words] for tokens in tokenized_text ]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:37:52.492513Z",
          "iopub.execute_input": "2024-08-02T09:37:52.493206Z",
          "iopub.status.idle": "2024-08-02T09:37:52.498257Z",
          "shell.execute_reply.started": "2024-08-02T09:37:52.49317Z",
          "shell.execute_reply": "2024-08-02T09:37:52.497119Z"
        },
        "trusted": true,
        "id": "bBz8bO25gfYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stemmer = PorterStemmer()\n",
        "stemmed_text = [[stemmer.stem(word) for word in tokens] for tokens in filtered_text]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:40:21.067669Z",
          "iopub.execute_input": "2024-08-02T09:40:21.06804Z",
          "iopub.status.idle": "2024-08-02T09:40:21.1116Z",
          "shell.execute_reply.started": "2024-08-02T09:40:21.068007Z",
          "shell.execute_reply": "2024-08-02T09:40:21.11071Z"
        },
        "trusted": true,
        "id": "uKFUZz-EgfYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "stemmed_text"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:40:22.397036Z",
          "iopub.execute_input": "2024-08-02T09:40:22.397699Z",
          "iopub.status.idle": "2024-08-02T09:40:22.422551Z",
          "shell.execute_reply.started": "2024-08-02T09:40:22.397661Z",
          "shell.execute_reply": "2024-08-02T09:40:22.421675Z"
        },
        "trusted": true,
        "id": "jO9x4Q2mgfYR",
        "outputId": "11e47efb-02b1-4015-bed1-16aa0f712d70"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 74,
          "output_type": "execute_result",
          "data": {
            "text/plain": "[['independ',\n  'day',\n  'celebr',\n  'annual',\n  '15',\n  'august',\n  'public',\n  'holiday',\n  'india',\n  'commemor',\n  'nation',\n  'independ',\n  'unit',\n  'kingdom',\n  '15',\n  'august',\n  '1947',\n  'day',\n  'provis',\n  'indian',\n  'independ',\n  'act',\n  'transfer',\n  'legisl',\n  'sovereignti',\n  'indian',\n  'constitu',\n  'assembl',\n  'came',\n  'effect',\n  'india',\n  'retain',\n  'king',\n  'georg',\n  'vi',\n  'head',\n  'state',\n  'transit',\n  'republ',\n  'constitut',\n  'india',\n  'came',\n  'effect',\n  '26',\n  'januari',\n  '1950',\n  'celebr',\n  'indian',\n  'republ',\n  'day',\n  'replac',\n  'dominion',\n  'prefix',\n  'dominion',\n  'india',\n  'enact',\n  'sovereign',\n  'law',\n  'constitut',\n  'india',\n  'india',\n  'attain',\n  'independ',\n  'follow',\n  'independ',\n  'movement',\n  'note',\n  'larg',\n  'nonviol',\n  'resist',\n  'civil',\n  'disobedi',\n  'led',\n  'indian',\n  'nation',\n  'congress',\n  'leadership',\n  'mahatma',\n  'gandhi',\n  'adopt',\n  'valu',\n  'one',\n  'earli',\n  'movements1',\n  'india',\n  'led',\n  'ram',\n  'singh',\n  'kuka',\n  'quot',\n  'one',\n  'letter',\n  'shahe',\n  'bhagat',\n  'singh',\n  'refer',\n  'guru',\n  'ram',\n  'singh',\n  'dada',\n  'guru'],\n ['independ',\n  'coincid',\n  'partit',\n  'india2',\n  'british',\n  'india',\n  'divid',\n  'dominion',\n  'india',\n  'pakistan',\n  'partit',\n  'accompani',\n  'violent',\n  'riot',\n  'mass',\n  'casualti',\n  'displac',\n  'nearli',\n  '15',\n  'million',\n  'peopl',\n  'due',\n  'religi',\n  'violenc',\n  '15',\n  'august',\n  '1947',\n  'first',\n  'prime',\n  'minist',\n  'india',\n  'jawaharl',\n  'nehru',\n  'rais',\n  'indian',\n  'nation',\n  'flag',\n  'lahori',\n  'gate',\n  'red',\n  'fort',\n  'delhi',\n  'subsequ',\n  'independ',\n  'day',\n  'incumb',\n  'prime',\n  'minist',\n  'customarili',\n  'rais',\n  'flag',\n  'give',\n  'address',\n  'nation3',\n  'entir',\n  'event',\n  'broadcast',\n  'doordarshan',\n  'india',\n  'nation',\n  'broadcast',\n  'usual',\n  'begin',\n  'shehnai',\n  'music',\n  'ustad',\n  'bismillah',\n  'khan',\n  'independ',\n  'day',\n  'observ',\n  'throughout',\n  'india',\n  'flaghoist',\n  'ceremoni',\n  'parad',\n  'cultur',\n  'event',\n  'nation',\n  'holiday456'],\n ['european',\n  'trader',\n  'establish',\n  'outpost',\n  'indian',\n  'subcontin',\n  'late',\n  '17th',\n  'centuri',\n  'overwhelm',\n  'militari',\n  'strength',\n  'east',\n  'india',\n  'compani',\n  'fought',\n  'annex',\n  'local',\n  'kingdom',\n  'establish',\n  'domin',\n  'forc',\n  '18th',\n  'centuri',\n  'follow',\n  'indian',\n  'rebellion',\n  '1857',\n  'govern',\n  'india',\n  'act',\n  '1858',\n  'led',\n  'british',\n  'crown',\n  'assum',\n  'direct',\n  'control',\n  'india',\n  'decad',\n  'follow',\n  'civic',\n  'societi',\n  'gradual',\n  'emerg',\n  'across',\n  'india',\n  'notabl',\n  'indian',\n  'nation',\n  'congress',\n  'parti',\n  'form',\n  '188578',\n  '123',\n  'period',\n  'world',\n  'war',\n  'mark',\n  'coloni',\n  'reform',\n  'montaguchelmsford',\n  'reform',\n  'also',\n  'wit',\n  'enact',\n  'unpopular',\n  'rowlatt',\n  'act',\n  'call',\n  'selfrul',\n  'indian',\n  'activist',\n  'discont',\n  'period',\n  'crystallis',\n  'nationwid',\n  'nonviol',\n  'movement',\n  'noncooper',\n  'civil',\n  'disobedi',\n  'led',\n  'mohanda',\n  'karamchand',\n  'gandhi8',\n  '167'],\n ['1930',\n  'reform',\n  'gradual',\n  'legisl',\n  'british',\n  'congress',\n  'victori',\n  'result',\n  'elections8',\n  '195197',\n  'next',\n  'decad',\n  'beset',\n  'polit',\n  'turmoil',\n  'indian',\n  'particip',\n  'world',\n  'war',\n  'ii',\n  'congress',\n  'final',\n  'push',\n  'noncooper',\n  'upsurg',\n  'muslim',\n  'nation',\n  'led',\n  'allindia',\n  'muslim',\n  'leagu',\n  'escal',\n  'polit',\n  'tension',\n  'cap',\n  'independ',\n  '1947',\n  'jubil',\n  'temper',\n  'bloodi',\n  'partit',\n  'coloni',\n  'india',\n  'india',\n  'pakistan8',\n  '203'],\n ['hasrat',\n  'mohani',\n  'first',\n  'person',\n  'indian',\n  'histori',\n  'demand',\n  'complet',\n  'independ',\n  'azadiekaamil',\n  '1929',\n  'session',\n  'indian',\n  'nation',\n  'congress',\n  'purna',\n  'swaraj',\n  'declar',\n  'declar',\n  'independ',\n  'india',\n  'promulgated9',\n  '26',\n  'januari',\n  'declar',\n  'independ',\n  'day',\n  '19309',\n  'congress',\n  'call',\n  'peopl',\n  'pledg',\n  'civil',\n  'disobedi',\n  'carri',\n  'congress',\n  'instruct',\n  'issu',\n  'time',\n  'time',\n  'india',\n  'attain',\n  'complet',\n  'independence10',\n  'celebr',\n  'independ',\n  'day',\n  'envis',\n  'stoke',\n  'nationalist',\n  'fervour',\n  'among',\n  'indian',\n  'citizen',\n  'forc',\n  'british',\n  'govern',\n  'consid',\n  'grant',\n  'independence11',\n  '19',\n  'congress',\n  'observ',\n  '26',\n  'januari',\n  'independ',\n  'day',\n  '1930',\n  '19461213',\n  'celebr',\n  'mark',\n  'meet',\n  'attend',\n  'took',\n  'pledg',\n  'independence11',\n  '1920',\n  'jawaharl',\n  'nehru',\n  'describ',\n  'autobiographi',\n  'meet',\n  'peac',\n  'solemn',\n  'without',\n  'speech',\n  'exhortation14',\n  'gandhi',\n  'envisag',\n  'besid',\n  'meet',\n  'day',\n  'would',\n  'spent',\n  'construct',\n  'work',\n  'whether',\n  'spin',\n  'servic',\n  'untouch',\n  'reunion',\n  'hindu',\n  'mussalman',\n  'prohibit',\n  'work',\n  'even',\n  'together15',\n  'follow',\n  'actual',\n  'independ',\n  '1947',\n  'constitut',\n  'india',\n  'came',\n  'effect',\n  '26',\n  'januari',\n  '1950',\n  'sinc',\n  '26',\n  'januari',\n  'celebr',\n  'republ',\n  'day'],\n ['1946',\n  'labour',\n  'govern',\n  'britain',\n  'exchequ',\n  'exhaust',\n  'recent',\n  'conclud',\n  'world',\n  'war',\n  'ii',\n  'realis',\n  'neither',\n  'mandat',\n  'home',\n  'intern',\n  'support',\n  'reliabl',\n  'nativ',\n  'forc',\n  'continu',\n  'maintain',\n  'control',\n  'increasingli',\n  'restless',\n  'india8',\n  '203',\n  '161718',\n  '20',\n  'februari',\n  '1947',\n  'prime',\n  'minist',\n  'clement',\n  'attle',\n  'announc',\n  'british',\n  'govern',\n  'would',\n  'grant',\n  'full',\n  'selfgovern',\n  'british',\n  'india',\n  'june',\n  '1948',\n  'latest19'],\n ['new',\n  'viceroy',\n  'lord',\n  'mountbatten',\n  'advanc',\n  'date',\n  'transfer',\n  'power',\n  'believ',\n  'continu',\n  'content',\n  'congress',\n  'muslim',\n  'leagu',\n  'might',\n  'lead',\n  'collaps',\n  'interim',\n  'government20',\n  'chose',\n  'second',\n  'anniversari',\n  'japan',\n  'surrend',\n  'world',\n  'war',\n  'ii',\n  '15',\n  'august',\n  'date',\n  'power',\n  'transfer20',\n  'british',\n  'govern',\n  'announc',\n  '3',\n  'june',\n  '1947',\n  'accept',\n  'idea',\n  'partit',\n  'british',\n  'india',\n  'two',\n  'states19',\n  'successor',\n  'govern',\n  'would',\n  'given',\n  'dominion',\n  'statu',\n  'would',\n  'implicit',\n  'right',\n  'seced',\n  'british',\n  'commonwealth',\n  'indian',\n  'independ',\n  'act',\n  '1947',\n  '10',\n  '11',\n  'geo',\n  '6',\n  'c',\n  '30',\n  'parliament',\n  'unit',\n  'kingdom',\n  'partit',\n  'british',\n  'india',\n  'two',\n  'new',\n  'independ',\n  'dominion',\n  'india',\n  'pakistan',\n  'includ',\n  'bangladesh',\n  'effect',\n  '15',\n  'august',\n  '1947',\n  'grant',\n  'complet',\n  'legisl',\n  'author',\n  'upon',\n  'respect',\n  'constitu',\n  'assembl',\n  'new',\n  'countries21',\n  'act',\n  'receiv',\n  'royal',\n  'assent',\n  '18',\n  'juli',\n  '1947'],\n ['million',\n  'muslim',\n  'sikh',\n  'hindu',\n  'refuge',\n  'trek',\n  'newli',\n  'drawn',\n  'border',\n  'month',\n  'surround',\n  'independence22',\n  'punjab',\n  'border',\n  'divid',\n  'sikh',\n  'region',\n  'halv',\n  'massiv',\n  'bloodsh',\n  'follow',\n  'bengal',\n  'bihar',\n  'mahatma',\n  'gandhi',\n  'presenc',\n  'assuag',\n  'commun',\n  'temper',\n  'violenc',\n  'mitig',\n  '250000',\n  '1000000',\n  'peopl',\n  'side',\n  'new',\n  'border',\n  'die',\n  'violence23',\n  'entir',\n  'nation',\n  'celebr',\n  'independ',\n  'day',\n  'gandhi',\n  'stay',\n  'calcutta',\n  'attempt',\n  'stem',\n  'carnage24',\n  '14',\n  'august',\n  '1947',\n  'independ',\n  'day',\n  'pakistan',\n  'new',\n  'dominion',\n  'pakistan',\n  'came',\n  'muhammad',\n  'ali',\n  'jinnah',\n  'sworn',\n  'first',\n  'governor',\n  'gener',\n  'karachi'],\n ['constitu',\n  'assembl',\n  'india',\n  'met',\n  'fifth',\n  'session',\n  '11',\n  'pm',\n  '14',\n  'august',\n  'constitut',\n  'hall',\n  'new',\n  'delhi25',\n  'session',\n  'chair',\n  'presid',\n  'rajendra',\n  'prasad',\n  'session',\n  'jawaharl',\n  'nehru',\n  'deliv',\n  'tryst',\n  'destini',\n  'speech',\n  'proclaim',\n  'india',\n  'independ'],\n ['long',\n  'year',\n  'ago',\n  'made',\n  'tryst',\n  'destini',\n  'time',\n  'come',\n  'shall',\n  'redeem',\n  'pledg',\n  'wholli',\n  'full',\n  'measur',\n  'substanti',\n  'stroke',\n  'midnight',\n  'hour',\n  'world',\n  'sleep',\n  'india',\n  'awak',\n  'life',\n  'freedom',\n  'moment',\n  'come',\n  'come',\n  'rare',\n  'histori',\n  'step',\n  'old',\n  'new',\n  'age',\n  'end',\n  'soul',\n  'nation',\n  'long',\n  'suppress',\n  'find',\n  'utter',\n  'fit',\n  'solemn',\n  'moment',\n  'take',\n  'pledg',\n  'dedic',\n  'servic',\n  'india',\n  'peopl',\n  'still',\n  'larger',\n  'caus',\n  'human'],\n ['member',\n  'assembl',\n  'formal',\n  'took',\n  'pledg',\n  'servic',\n  'countri',\n  'group',\n  'women',\n  'repres',\n  'women',\n  'india',\n  'formal',\n  'present',\n  'nation',\n  'flag',\n  'assembl'],\n ['dominion',\n  'india',\n  'becam',\n  'independ',\n  'countri',\n  'offici',\n  'ceremoni',\n  'took',\n  'place',\n  'new',\n  'delhi',\n  'nehru',\n  'assum',\n  'offic',\n  'first',\n  'prime',\n  'minist',\n  'viceroy',\n  'lord',\n  'mountbatten',\n  'continu',\n  'first',\n  'governor',\n  'general27',\n  '6',\n  'gandhi',\n  'name',\n  'invok',\n  'crowd',\n  'celebr',\n  'occas',\n  'gandhi',\n  'howev',\n  'took',\n  'part',\n  'offici',\n  'event',\n  'instead',\n  'mark',\n  'day',\n  '24hour',\n  'fast',\n  'spoke',\n  'crowd',\n  'calcutta',\n  'encourag',\n  'peac',\n  'hindu',\n  'muslims27',\n  '10'],\n ['0830',\n  'swear',\n  'governor',\n  'gener',\n  'minist',\n  'govern',\n  'house0940',\n  'process',\n  'minist',\n  'constitu',\n  'assembly0950',\n  'state',\n  'drive',\n  'constitu',\n  'assembly0955',\n  'royal',\n  'salut',\n  'governor',\n  'general1030',\n  'hoist',\n  'nation',\n  'flag',\n  'constitu',\n  'assembly1035',\n  'state',\n  'drive',\n  'govern',\n  'house0600',\n  'pm',\n  'flag',\n  'ceremoni',\n  'india',\n  'gate0700',\n  'pm',\n  'illuminations0745',\n  'pm',\n  'firework',\n  'display0845',\n  'pm',\n  'offici',\n  'dinner',\n  'govern',\n  'house1015',\n  'pm',\n  'recept',\n  'govern',\n  'offic'],\n ['day', 'programm', '15', 'august', '194727', '7'],\n ['independ',\n  'day',\n  'one',\n  'three',\n  'nation',\n  'holiday',\n  'india',\n  'two',\n  'republ',\n  'day',\n  '26',\n  'januari',\n  'mahatma',\n  'gandhi',\n  'birthday',\n  '2',\n  'octob',\n  'observ',\n  'indian',\n  'state',\n  'union',\n  'territori',\n  'eve',\n  'independ',\n  'day',\n  'presid',\n  'india',\n  'deliv',\n  'address',\n  'nation',\n  '15',\n  'august',\n  'prime',\n  'minist',\n  'hoist',\n  'indian',\n  'flag',\n  'rampart',\n  'histor',\n  'site',\n  'red',\n  'fort',\n  'delhi3',\n  'speech',\n  'prime',\n  'minist',\n  'highlight',\n  'past',\n  'year',\n  'achiev',\n  'rais',\n  'import',\n  'issu',\n  'call',\n  'develop',\n  'pay',\n  'tribut',\n  'leader',\n  'indian',\n  'independ',\n  'movement',\n  'indian',\n  'nation',\n  'anthem',\n  'jana',\n  'gana',\n  'mana',\n  'sung',\n  'speech',\n  'follow',\n  'march',\n  'past',\n  'divis',\n  'indian',\n  'arm',\n  'forc',\n  'paramilitari',\n  'forc',\n  'parad',\n  'pageant',\n  'showcas',\n  'scene',\n  'independ',\n  'struggl',\n  'india',\n  'divers',\n  'cultur',\n  'tradit',\n  'similar',\n  'event',\n  'take',\n  'place',\n  'state',\n  'capit',\n  'chief',\n  'minist',\n  'individu',\n  'state',\n  'unfurl',\n  'nation',\n  'flag',\n  'follow',\n  'parad',\n  'pageants2829',\n  '1973',\n  'governor',\n  'state',\n  'hoist',\n  'nation',\n  'flag',\n  'state',\n  'capit',\n  'februari',\n  '1974',\n  'chief',\n  'minist',\n  'tamil',\n  'nadu',\n  'karunanidhi',\n  'took',\n  'issu',\n  'prime',\n  'minist',\n  'indira',\n  'gandhi',\n  'chief',\n  'minist',\n  'like',\n  'prime',\n  'minist',\n  'allow',\n  'hoist',\n  'nation',\n  'flag',\n  'independ',\n  'day',\n  'sinc',\n  '1974',\n  'chief',\n  'minist',\n  'respect',\n  'state',\n  'allow',\n  'hoist',\n  'nation',\n  'flag',\n  'independ',\n  'day3031'],\n ['flaghoist',\n  'ceremoni',\n  'cultur',\n  'programm',\n  'take',\n  'place',\n  'government',\n  'nongovernment',\n  'institut',\n  'throughout',\n  'country32',\n  'school',\n  'colleg',\n  'conduct',\n  'flag',\n  'hoist',\n  'ceremoni',\n  'variou',\n  'cultur',\n  'event',\n  'government',\n  'nongovernment',\n  'institut',\n  'decor',\n  'premis',\n  'paper',\n  'balloon',\n  'decor',\n  'hang',\n  'freedom',\n  'fighter',\n  'portrait',\n  'wall',\n  'major',\n  'govern',\n  'build',\n  'often',\n  'adorn',\n  'string',\n  'lights33',\n  'delhi',\n  'citi',\n  'kite',\n  'fli',\n  'add',\n  'occasion3435',\n  'nation',\n  'flag',\n  'differ',\n  'size',\n  'use',\n  'abundantli',\n  'symbolis',\n  'allegi',\n  'country36',\n  'citizen',\n  'adorn',\n  'cloth',\n  'wristband',\n  'car',\n  'household',\n  'accessori',\n  'replica',\n  'tricolour36',\n  'period',\n  'time',\n  'celebr',\n  'chang',\n  'emphasi',\n  'nation',\n  'broader',\n  'celebr',\n  'thing',\n  'india3738'],\n ['indian',\n  'diaspora',\n  'celebr',\n  'independ',\n  'day',\n  'around',\n  'world',\n  'parad',\n  'pageant',\n  'particularli',\n  'region',\n  'higher',\n  'concentr',\n  'indian',\n  'immigrants39',\n  'locat',\n  'new',\n  'york',\n  'us',\n  'citi',\n  '15',\n  'august',\n  'becom',\n  'india',\n  'day',\n  'among',\n  'diaspora',\n  'local',\n  'populac',\n  'pageant',\n  'celebr',\n  'india',\n  'day',\n  'either',\n  '15',\n  'august',\n  'adjoin',\n  'weekend',\n  'day40'],\n ['kashmiri',\n  'throughout',\n  'kashmir',\n  'valley',\n  'observ',\n  'india',\n  'independ',\n  'day',\n  'black',\n  'day',\n  'sinc',\n  'late',\n  '1980',\n  'gener',\n  'strike',\n  'civil',\n  'curfew',\n  'organis',\n  'ceremoni',\n  'burn',\n  'indian',\n  'flag',\n  'shop',\n  'remain',\n  'close',\n  'reduc',\n  'traffic',\n  'seen',\n  'roads41424344',\n  'black',\n  'flag',\n  'hoist',\n  'build',\n  'throughout',\n  'indiaadminist',\n  'kashmir',\n  'purpos',\n  'send',\n  'messag',\n  'intern',\n  'commun',\n  'india',\n  'usurp',\n  'kashmiri',\n  'inalien',\n  'right',\n  'selfdetermin',\n  'use',\n  'force4546'],\n ['earli',\n  'three',\n  'year',\n  'independ',\n  'naga',\n  'nation',\n  'council',\n  'call',\n  'boycott',\n  'independ',\n  'day',\n  'northeast',\n  'india47',\n  'separatist',\n  'protest',\n  'region',\n  'intensifi',\n  '1980',\n  'call',\n  'boycott',\n  'terrorist',\n  'attack',\n  'insurg',\n  'organis',\n  'unit',\n  'liber',\n  'front',\n  'assam',\n  'nation',\n  'democrat',\n  'front',\n  'bodoland',\n  'mar',\n  'celebrations48',\n  'increas',\n  'insurg',\n  'jammu',\n  'kashmir',\n  'late',\n  '1980s49',\n  'separatist',\n  'protest',\n  'boycot',\n  'independ',\n  'day',\n  'bandh',\n  'strike',\n  'use',\n  'black',\n  'flag',\n  'flag',\n  'burning504251',\n  'terrorist',\n  'group',\n  'lashkaretaiba',\n  'hizbul',\n  'mujahideen',\n  'jaishemoham',\n  'issu',\n  'threat',\n  'carri',\n  'attack',\n  'around',\n  'independ',\n  'day52',\n  'boycot',\n  'celebr',\n  'also',\n  'advoc',\n  'insurg',\n  'maoist',\n  'rebel',\n  'organisations5354'],\n ['anticip',\n  'terrorist',\n  'attack',\n  'particularli',\n  'milit',\n  'secur',\n  'measur',\n  'intensifi',\n  'especi',\n  'major',\n  'citi',\n  'delhi',\n  'mumbai',\n  'troubl',\n  'state',\n  'jammu',\n  'kashmir5556',\n  'airspac',\n  'around',\n  'red',\n  'fort',\n  'declar',\n  'nofli',\n  'zone',\n  'prevent',\n  'aerial',\n  'attacks57',\n  'addit',\n  'polic',\n  'forc',\n  'deploy',\n  'cities58'],\n ['sinc',\n  'assassin',\n  'indira',\n  'gandhi',\n  'prime',\n  'minist',\n  'would',\n  'give',\n  'speech',\n  'behind',\n  'bulletproof',\n  'glass',\n  'panel59',\n  'sinc',\n  '2014',\n  'narendra',\n  'modi',\n  'elect',\n  'prime',\n  'minist',\n  'away',\n  'tradition60',\n  'nevertheless',\n  'addit',\n  'intens',\n  'measur',\n  'taken',\n  'ensur',\n  'secur',\n  'modi61'],\n ['independ',\n  'day',\n  'republ',\n  'day',\n  'patriot',\n  'song',\n  'region',\n  'languag',\n  'broadcast',\n  'televis',\n  'radio',\n  'channels62',\n  'also',\n  'play',\n  'alongsid',\n  'flaghoist',\n  'ceremonies62',\n  'patriot',\n  'film',\n  'broadcast32',\n  'decad',\n  'accord',\n  'time',\n  'india',\n  'number',\n  'film',\n  'broadcast',\n  'decreas',\n  'channel',\n  'report',\n  'audienc',\n  'oversatur',\n  'patriot',\n  'films63',\n  'popul',\n  'belong',\n  'gener',\n  'often',\n  'combin',\n  'nation',\n  'popular',\n  'cultur',\n  'celebr',\n  'mixtur',\n  'exemplifi',\n  'outfit',\n  'savouri',\n  'dy',\n  'tricolour',\n  'garment',\n  'repres',\n  'india',\n  'variou',\n  'cultur',\n  'traditions3764',\n  'shop',\n  'often',\n  'offer',\n  'independ',\n  'day',\n  'sale',\n  'promotions6566',\n  'news',\n  'report',\n  'decri',\n  'commercialism656768',\n  'indian',\n  'postal',\n  'servic',\n  'publish',\n  'commemor',\n  'stamp',\n  'depict',\n  'independ',\n  'movement',\n  'leader',\n  'nationalist',\n  'theme',\n  'defencerel',\n  'theme',\n  '15',\n  'august69'],\n ['independ',\n  'partit',\n  'inspir',\n  'literari',\n  'artist',\n  'creations70',\n  'creation',\n  'mostli',\n  'describ',\n  'human',\n  'cost',\n  'partit',\n  'limit',\n  'holiday',\n  'small',\n  'part',\n  'narrative7172',\n  'salman',\n  'rushdi',\n  'novel',\n  'midnight',\n  'children',\n  '1980',\n  'booker',\n  'prize',\n  'booker',\n  'booker',\n  'wove',\n  'narr',\n  'around',\n  'children',\n  'born',\n  'midnight',\n  '1415',\n  'august',\n  '1947',\n  'magic',\n  'abilities72',\n  'freedom',\n  'midnight',\n  '1975',\n  'nonfict',\n  'work',\n  'larri',\n  'collin',\n  'dominiqu',\n  'lapierr',\n  'chronicl',\n  'event',\n  'surround',\n  'first',\n  'independ',\n  'day',\n  'celebr',\n  '1947',\n  'film',\n  'centr',\n  'moment',\n  'independence737475',\n  'instead',\n  'highlight',\n  'circumst',\n  'partit',\n  'aftermath737677',\n  'internet',\n  'googl',\n  'commemor',\n  'independ',\n  'day',\n  'india',\n  'sinc',\n  '2003',\n  'special',\n  'doodl',\n  'indian',\n  'homepage78']]"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "final_text = [[word for word in tokens if word.strip()] for tokens in stemmed_text]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:47:15.432879Z",
          "iopub.execute_input": "2024-08-02T09:47:15.433775Z",
          "iopub.status.idle": "2024-08-02T09:47:15.438536Z",
          "shell.execute_reply.started": "2024-08-02T09:47:15.433738Z",
          "shell.execute_reply": "2024-08-02T09:47:15.437442Z"
        },
        "trusted": true,
        "id": "7fhoT9H_gfYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sentences = [' '.join(tokens) for tokens in final_text]"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:49:00.227511Z",
          "iopub.execute_input": "2024-08-02T09:49:00.227895Z",
          "iopub.status.idle": "2024-08-02T09:49:00.232736Z",
          "shell.execute_reply.started": "2024-08-02T09:49:00.227866Z",
          "shell.execute_reply": "2024-08-02T09:49:00.231638Z"
        },
        "trusted": true,
        "id": "b8zLHTbxgfYR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "processed_paragraphs = '\\n\\n'.join(sentences)"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:50:38.856945Z",
          "iopub.execute_input": "2024-08-02T09:50:38.857342Z",
          "iopub.status.idle": "2024-08-02T09:50:38.863289Z",
          "shell.execute_reply.started": "2024-08-02T09:50:38.85731Z",
          "shell.execute_reply": "2024-08-02T09:50:38.86225Z"
        },
        "trusted": true,
        "id": "tBcGdFMMgfYS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "processed_paragraphs"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:50:58.943747Z",
          "iopub.execute_input": "2024-08-02T09:50:58.944135Z",
          "iopub.status.idle": "2024-08-02T09:50:58.950514Z",
          "shell.execute_reply.started": "2024-08-02T09:50:58.944105Z",
          "shell.execute_reply": "2024-08-02T09:50:58.949536Z"
        },
        "trusted": true,
        "id": "9TSjHOJVgfYS",
        "outputId": "cb2a84c5-c8b8-4e2e-a438-15ba278f69c4"
      },
      "execution_count": null,
      "outputs": [
        {
          "execution_count": 80,
          "output_type": "execute_result",
          "data": {
            "text/plain": "'independ day celebr annual 15 august public holiday india commemor nation independ unit kingdom 15 august 1947 day provis indian independ act transfer legisl sovereignti indian constitu assembl came effect india retain king georg vi head state transit republ constitut india came effect 26 januari 1950 celebr indian republ day replac dominion prefix dominion india enact sovereign law constitut india india attain independ follow independ movement note larg nonviol resist civil disobedi led indian nation congress leadership mahatma gandhi adopt valu one earli movements1 india led ram singh kuka quot one letter shahe bhagat singh refer guru ram singh dada guru\\n\\nindepend coincid partit india2 british india divid dominion india pakistan partit accompani violent riot mass casualti displac nearli 15 million peopl due religi violenc 15 august 1947 first prime minist india jawaharl nehru rais indian nation flag lahori gate red fort delhi subsequ independ day incumb prime minist customarili rais flag give address nation3 entir event broadcast doordarshan india nation broadcast usual begin shehnai music ustad bismillah khan independ day observ throughout india flaghoist ceremoni parad cultur event nation holiday456\\n\\neuropean trader establish outpost indian subcontin late 17th centuri overwhelm militari strength east india compani fought annex local kingdom establish domin forc 18th centuri follow indian rebellion 1857 govern india act 1858 led british crown assum direct control india decad follow civic societi gradual emerg across india notabl indian nation congress parti form 188578 123 period world war mark coloni reform montaguchelmsford reform also wit enact unpopular rowlatt act call selfrul indian activist discont period crystallis nationwid nonviol movement noncooper civil disobedi led mohanda karamchand gandhi8 167\\n\\n1930 reform gradual legisl british congress victori result elections8 195197 next decad beset polit turmoil indian particip world war ii congress final push noncooper upsurg muslim nation led allindia muslim leagu escal polit tension cap independ 1947 jubil temper bloodi partit coloni india india pakistan8 203\\n\\nhasrat mohani first person indian histori demand complet independ azadiekaamil 1929 session indian nation congress purna swaraj declar declar independ india promulgated9 26 januari declar independ day 19309 congress call peopl pledg civil disobedi carri congress instruct issu time time india attain complet independence10 celebr independ day envis stoke nationalist fervour among indian citizen forc british govern consid grant independence11 19 congress observ 26 januari independ day 1930 19461213 celebr mark meet attend took pledg independence11 1920 jawaharl nehru describ autobiographi meet peac solemn without speech exhortation14 gandhi envisag besid meet day would spent construct work whether spin servic untouch reunion hindu mussalman prohibit work even together15 follow actual independ 1947 constitut india came effect 26 januari 1950 sinc 26 januari celebr republ day\\n\\n1946 labour govern britain exchequ exhaust recent conclud world war ii realis neither mandat home intern support reliabl nativ forc continu maintain control increasingli restless india8 203 161718 20 februari 1947 prime minist clement attle announc british govern would grant full selfgovern british india june 1948 latest19\\n\\nnew viceroy lord mountbatten advanc date transfer power believ continu content congress muslim leagu might lead collaps interim government20 chose second anniversari japan surrend world war ii 15 august date power transfer20 british govern announc 3 june 1947 accept idea partit british india two states19 successor govern would given dominion statu would implicit right seced british commonwealth indian independ act 1947 10 11 geo 6 c 30 parliament unit kingdom partit british india two new independ dominion india pakistan includ bangladesh effect 15 august 1947 grant complet legisl author upon respect constitu assembl new countries21 act receiv royal assent 18 juli 1947\\n\\nmillion muslim sikh hindu refuge trek newli drawn border month surround independence22 punjab border divid sikh region halv massiv bloodsh follow bengal bihar mahatma gandhi presenc assuag commun temper violenc mitig 250000 1000000 peopl side new border die violence23 entir nation celebr independ day gandhi stay calcutta attempt stem carnage24 14 august 1947 independ day pakistan new dominion pakistan came muhammad ali jinnah sworn first governor gener karachi\\n\\nconstitu assembl india met fifth session 11 pm 14 august constitut hall new delhi25 session chair presid rajendra prasad session jawaharl nehru deliv tryst destini speech proclaim india independ\\n\\nlong year ago made tryst destini time come shall redeem pledg wholli full measur substanti stroke midnight hour world sleep india awak life freedom moment come come rare histori step old new age end soul nation long suppress find utter fit solemn moment take pledg dedic servic india peopl still larger caus human\\n\\nmember assembl formal took pledg servic countri group women repres women india formal present nation flag assembl\\n\\ndominion india becam independ countri offici ceremoni took place new delhi nehru assum offic first prime minist viceroy lord mountbatten continu first governor general27 6 gandhi name invok crowd celebr occas gandhi howev took part offici event instead mark day 24hour fast spoke crowd calcutta encourag peac hindu muslims27 10\\n\\n0830 swear governor gener minist govern house0940 process minist constitu assembly0950 state drive constitu assembly0955 royal salut governor general1030 hoist nation flag constitu assembly1035 state drive govern house0600 pm flag ceremoni india gate0700 pm illuminations0745 pm firework display0845 pm offici dinner govern house1015 pm recept govern offic\\n\\nday programm 15 august 194727 7\\n\\nindepend day one three nation holiday india two republ day 26 januari mahatma gandhi birthday 2 octob observ indian state union territori eve independ day presid india deliv address nation 15 august prime minist hoist indian flag rampart histor site red fort delhi3 speech prime minist highlight past year achiev rais import issu call develop pay tribut leader indian independ movement indian nation anthem jana gana mana sung speech follow march past divis indian arm forc paramilitari forc parad pageant showcas scene independ struggl india divers cultur tradit similar event take place state capit chief minist individu state unfurl nation flag follow parad pageants2829 1973 governor state hoist nation flag state capit februari 1974 chief minist tamil nadu karunanidhi took issu prime minist indira gandhi chief minist like prime minist allow hoist nation flag independ day sinc 1974 chief minist respect state allow hoist nation flag independ day3031\\n\\nflaghoist ceremoni cultur programm take place government nongovernment institut throughout country32 school colleg conduct flag hoist ceremoni variou cultur event government nongovernment institut decor premis paper balloon decor hang freedom fighter portrait wall major govern build often adorn string lights33 delhi citi kite fli add occasion3435 nation flag differ size use abundantli symbolis allegi country36 citizen adorn cloth wristband car household accessori replica tricolour36 period time celebr chang emphasi nation broader celebr thing india3738\\n\\nindian diaspora celebr independ day around world parad pageant particularli region higher concentr indian immigrants39 locat new york us citi 15 august becom india day among diaspora local populac pageant celebr india day either 15 august adjoin weekend day40\\n\\nkashmiri throughout kashmir valley observ india independ day black day sinc late 1980 gener strike civil curfew organis ceremoni burn indian flag shop remain close reduc traffic seen roads41424344 black flag hoist build throughout indiaadminist kashmir purpos send messag intern commun india usurp kashmiri inalien right selfdetermin use force4546\\n\\nearli three year independ naga nation council call boycott independ day northeast india47 separatist protest region intensifi 1980 call boycott terrorist attack insurg organis unit liber front assam nation democrat front bodoland mar celebrations48 increas insurg jammu kashmir late 1980s49 separatist protest boycot independ day bandh strike use black flag flag burning504251 terrorist group lashkaretaiba hizbul mujahideen jaishemoham issu threat carri attack around independ day52 boycot celebr also advoc insurg maoist rebel organisations5354\\n\\nanticip terrorist attack particularli milit secur measur intensifi especi major citi delhi mumbai troubl state jammu kashmir5556 airspac around red fort declar nofli zone prevent aerial attacks57 addit polic forc deploy cities58\\n\\nsinc assassin indira gandhi prime minist would give speech behind bulletproof glass panel59 sinc 2014 narendra modi elect prime minist away tradition60 nevertheless addit intens measur taken ensur secur modi61\\n\\nindepend day republ day patriot song region languag broadcast televis radio channels62 also play alongsid flaghoist ceremonies62 patriot film broadcast32 decad accord time india number film broadcast decreas channel report audienc oversatur patriot films63 popul belong gener often combin nation popular cultur celebr mixtur exemplifi outfit savouri dy tricolour garment repres india variou cultur traditions3764 shop often offer independ day sale promotions6566 news report decri commercialism656768 indian postal servic publish commemor stamp depict independ movement leader nationalist theme defencerel theme 15 august69\\n\\nindepend partit inspir literari artist creations70 creation mostli describ human cost partit limit holiday small part narrative7172 salman rushdi novel midnight children 1980 booker prize booker booker wove narr around children born midnight 1415 august 1947 magic abilities72 freedom midnight 1975 nonfict work larri collin dominiqu lapierr chronicl event surround first independ day celebr 1947 film centr moment independence737475 instead highlight circumst partit aftermath737677 internet googl commemor independ day india sinc 2003 special doodl indian homepage78'"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "with open('processed_text.txt','w',encoding='utf-8') as file:\n",
        "    file.write(processed_paragraphs)\n",
        "print('done')"
      ],
      "metadata": {
        "execution": {
          "iopub.status.busy": "2024-08-02T09:52:43.969783Z",
          "iopub.execute_input": "2024-08-02T09:52:43.970176Z",
          "iopub.status.idle": "2024-08-02T09:52:43.976509Z",
          "shell.execute_reply.started": "2024-08-02T09:52:43.970144Z",
          "shell.execute_reply": "2024-08-02T09:52:43.975372Z"
        },
        "trusted": true,
        "id": "kryUChxBgfYS",
        "outputId": "507b6ed4-32e5-40a0-e5f9-f6d907df32b6"
      },
      "execution_count": null,
      "outputs": [
        {
          "name": "stdout",
          "text": "done\n",
          "output_type": "stream"
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "6Nomkp3UgfYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "rw98B1IBgfYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "gxn-0GOKgfYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "pRsR3F61gfYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "dEgW12_rgfYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "1dZ5qmc2gfYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "0r5OGSXogfYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ffx0OqcggfYV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "xc4blSpHgfYW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "vbdR78v3gfYW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "qTvERE1TgfYW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "wY_cNN_sgfYW"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}