Pre processing of a Text Document, stop word removal and stemming.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ddf6d22c-506b-4c3d-830c-db4f8e980bfe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in c:\\users\\somes\\anaconda3\\lib\\site-packages (3.9.1)\n",
      "Requirement already satisfied: click in c:\\users\\somes\\anaconda3\\lib\\site-packages (from nltk) (8.1.7)\n",
      "Requirement already satisfied: joblib in c:\\users\\somes\\anaconda3\\lib\\site-packages (from nltk) (1.4.2)\n",
      "Requirement already satisfied: regex>=2021.8.3 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from nltk) (2024.9.11)\n",
      "Requirement already satisfied: tqdm in c:\\users\\somes\\anaconda3\\lib\\site-packages (from nltk) (4.66.5)\n",
      "Requirement already satisfied: colorama in c:\\users\\somes\\anaconda3\\lib\\site-packages (from click->nltk) (0.4.6)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install --upgrade nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "6f29bbe7-42a4-42e7-9df4-2b585ab68ec4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\somes\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\somes\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "60138da2-70e6-465b-8b0d-70e4b35a0bb9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting spacy\n",
      "  Downloading spacy-3.8.3-cp312-cp312-win_amd64.whl.metadata (27 kB)\n",
      "Collecting spacy-legacy<3.1.0,>=3.0.11 (from spacy)\n",
      "  Downloading spacy_legacy-3.0.12-py2.py3-none-any.whl.metadata (2.8 kB)\n",
      "Collecting spacy-loggers<2.0.0,>=1.0.0 (from spacy)\n",
      "  Downloading spacy_loggers-1.0.5-py3-none-any.whl.metadata (23 kB)\n",
      "Collecting murmurhash<1.1.0,>=0.28.0 (from spacy)\n",
      "  Downloading murmurhash-1.0.11-cp312-cp312-win_amd64.whl.metadata (2.0 kB)\n",
      "Collecting cymem<2.1.0,>=2.0.2 (from spacy)\n",
      "  Downloading cymem-2.0.10-cp312-cp312-win_amd64.whl.metadata (8.6 kB)\n",
      "Collecting preshed<3.1.0,>=3.0.2 (from spacy)\n",
      "  Downloading preshed-3.0.9-cp312-cp312-win_amd64.whl.metadata (2.2 kB)\n",
      "Collecting thinc<8.4.0,>=8.3.0 (from spacy)\n",
      "  Downloading thinc-8.3.3-cp312-cp312-win_amd64.whl.metadata (15 kB)\n",
      "Collecting wasabi<1.2.0,>=0.9.1 (from spacy)\n",
      "  Downloading wasabi-1.1.3-py3-none-any.whl.metadata (28 kB)\n",
      "Collecting srsly<3.0.0,>=2.4.3 (from spacy)\n",
      "  Downloading srsly-2.5.0-cp312-cp312-win_amd64.whl.metadata (20 kB)\n",
      "Collecting catalogue<2.1.0,>=2.0.6 (from spacy)\n",
      "  Downloading catalogue-2.0.10-py3-none-any.whl.metadata (14 kB)\n",
      "Collecting weasel<0.5.0,>=0.1.0 (from spacy)\n",
      "  Downloading weasel-0.4.1-py3-none-any.whl.metadata (4.6 kB)\n",
      "Requirement already satisfied: typer<1.0.0,>=0.3.0 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from spacy) (0.9.0)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from spacy) (4.66.5)\n",
      "Requirement already satisfied: numpy>=1.19.0 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from spacy) (1.26.4)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from spacy) (2.32.3)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from spacy) (2.8.2)\n",
      "Requirement already satisfied: jinja2 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from spacy) (3.1.4)\n",
      "Requirement already satisfied: setuptools in c:\\users\\somes\\anaconda3\\lib\\site-packages (from spacy) (75.1.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from spacy) (24.1)\n",
      "Collecting langcodes<4.0.0,>=3.2.0 (from spacy)\n",
      "  Downloading langcodes-3.5.0-py3-none-any.whl.metadata (29 kB)\n",
      "Collecting language-data>=1.2 (from langcodes<4.0.0,>=3.2.0->spacy)\n",
      "  Downloading language_data-1.3.0-py3-none-any.whl.metadata (4.3 kB)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.20.1 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (2.20.1)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from pydantic!=1.8,!=1.8.1,<3.0.0,>=1.7.4->spacy) (4.11.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2024.12.14)\n",
      "Collecting blis<1.2.0,>=1.1.0 (from thinc<8.4.0,>=8.3.0->spacy)\n",
      "  Downloading blis-1.1.0-cp312-cp312-win_amd64.whl.metadata (7.9 kB)\n",
      "Collecting confection<1.0.0,>=0.0.1 (from thinc<8.4.0,>=8.3.0->spacy)\n",
      "  Downloading confection-0.1.5-py3-none-any.whl.metadata (19 kB)\n",
      "Requirement already satisfied: colorama in c:\\users\\somes\\anaconda3\\lib\\site-packages (from tqdm<5.0.0,>=4.38.0->spacy) (0.4.6)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from typer<1.0.0,>=0.3.0->spacy) (8.1.7)\n",
      "Collecting cloudpathlib<1.0.0,>=0.7.0 (from weasel<0.5.0,>=0.1.0->spacy)\n",
      "  Downloading cloudpathlib-0.20.0-py3-none-any.whl.metadata (14 kB)\n",
      "Requirement already satisfied: smart-open<8.0.0,>=5.2.1 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from weasel<0.5.0,>=0.1.0->spacy) (5.2.1)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\somes\\anaconda3\\lib\\site-packages (from jinja2->spacy) (2.1.3)\n",
      "Collecting marisa-trie>=1.1.0 (from language-data>=1.2->langcodes<4.0.0,>=3.2.0->spacy)\n",
      "  Downloading marisa_trie-1.2.1-cp312-cp312-win_amd64.whl.metadata (9.3 kB)\n",
      "Downloading spacy-3.8.3-cp312-cp312-win_amd64.whl (11.8 MB)\n",
      "   ---------------------------------------- 0.0/11.8 MB ? eta -:--:--\n",
      "   - -------------------------------------- 0.5/11.8 MB 2.1 MB/s eta 0:00:06\n",
      "   ---- ----------------------------------- 1.3/11.8 MB 2.8 MB/s eta 0:00:04\n",
      "   ------ --------------------------------- 1.8/11.8 MB 2.7 MB/s eta 0:00:04\n",
      "   -------- ------------------------------- 2.4/11.8 MB 2.6 MB/s eta 0:00:04\n",
      "   --------- ------------------------------ 2.9/11.8 MB 2.6 MB/s eta 0:00:04\n",
      "   ---------- ----------------------------- 3.1/11.8 MB 2.5 MB/s eta 0:00:04\n",
      "   ------------ --------------------------- 3.7/11.8 MB 2.6 MB/s eta 0:00:04\n",
      "   -------------- ------------------------- 4.2/11.8 MB 2.6 MB/s eta 0:00:03\n",
      "   ---------------- ----------------------- 4.7/11.8 MB 2.5 MB/s eta 0:00:03\n",
      "   ----------------- ---------------------- 5.2/11.8 MB 2.6 MB/s eta 0:00:03\n",
      "   ------------------- -------------------- 5.8/11.8 MB 2.5 MB/s eta 0:00:03\n",
      "   --------------------- ------------------ 6.3/11.8 MB 2.5 MB/s eta 0:00:03\n",
      "   ----------------------- ---------------- 6.8/11.8 MB 2.5 MB/s eta 0:00:02\n",
      "   ------------------------ --------------- 7.3/11.8 MB 2.5 MB/s eta 0:00:02\n",
      "   -------------------------- ------------- 7.9/11.8 MB 2.5 MB/s eta 0:00:02\n",
      "   ---------------------------- ----------- 8.4/11.8 MB 2.5 MB/s eta 0:00:02\n",
      "   ------------------------------ --------- 8.9/11.8 MB 2.5 MB/s eta 0:00:02\n",
      "   -------------------------------- ------- 9.4/11.8 MB 2.5 MB/s eta 0:00:01\n",
      "   --------------------------------- ------ 10.0/11.8 MB 2.5 MB/s eta 0:00:01\n",
      "   ----------------------------------- ---- 10.5/11.8 MB 2.5 MB/s eta 0:00:01\n",
      "   ------------------------------------- -- 11.0/11.8 MB 2.5 MB/s eta 0:00:01\n",
      "   ---------------------------------------  11.5/11.8 MB 2.5 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 11.8/11.8 MB 2.5 MB/s eta 0:00:00\n",
      "Downloading catalogue-2.0.10-py3-none-any.whl (17 kB)\n",
      "Downloading cymem-2.0.10-cp312-cp312-win_amd64.whl (39 kB)\n",
      "Downloading langcodes-3.5.0-py3-none-any.whl (182 kB)\n",
      "Downloading murmurhash-1.0.11-cp312-cp312-win_amd64.whl (25 kB)\n",
      "Downloading preshed-3.0.9-cp312-cp312-win_amd64.whl (122 kB)\n",
      "Downloading spacy_legacy-3.0.12-py2.py3-none-any.whl (29 kB)\n",
      "Downloading spacy_loggers-1.0.5-py3-none-any.whl (22 kB)\n",
      "Downloading srsly-2.5.0-cp312-cp312-win_amd64.whl (632 kB)\n",
      "   ---------------------------------------- 0.0/632.6 kB ? eta -:--:--\n",
      "   ---------------------------------------- 632.6/632.6 kB 3.4 MB/s eta 0:00:00\n",
      "Downloading thinc-8.3.3-cp312-cp312-win_amd64.whl (1.5 MB)\n",
      "   ---------------------------------------- 0.0/1.5 MB ? eta -:--:--\n",
      "   ------- -------------------------------- 0.3/1.5 MB ? eta -:--:--\n",
      "   ---------------------------- ----------- 1.0/1.5 MB 3.0 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 1.5/1.5 MB 2.7 MB/s eta 0:00:00\n",
      "Downloading wasabi-1.1.3-py3-none-any.whl (27 kB)\n",
      "Downloading weasel-0.4.1-py3-none-any.whl (50 kB)\n",
      "Downloading blis-1.1.0-cp312-cp312-win_amd64.whl (6.4 MB)\n",
      "   ---------------------------------------- 0.0/6.4 MB ? eta -:--:--\n",
      "   - -------------------------------------- 0.3/6.4 MB ? eta -:--:--\n",
      "   ---- ----------------------------------- 0.8/6.4 MB 2.4 MB/s eta 0:00:03\n",
      "   -------- ------------------------------- 1.3/6.4 MB 2.6 MB/s eta 0:00:02\n",
      "   ------------- -------------------------- 2.1/6.4 MB 2.7 MB/s eta 0:00:02\n",
      "   ---------------- ----------------------- 2.6/6.4 MB 2.6 MB/s eta 0:00:02\n",
      "   ------------------- -------------------- 3.1/6.4 MB 2.7 MB/s eta 0:00:02\n",
      "   ----------------------- ---------------- 3.7/6.4 MB 2.6 MB/s eta 0:00:02\n",
      "   -------------------------- ------------- 4.2/6.4 MB 2.6 MB/s eta 0:00:01\n",
      "   ----------------------------- ---------- 4.7/6.4 MB 2.6 MB/s eta 0:00:01\n",
      "   -------------------------------- ------- 5.2/6.4 MB 2.6 MB/s eta 0:00:01\n",
      "   ------------------------------------ --- 5.8/6.4 MB 2.5 MB/s eta 0:00:01\n",
      "   ---------------------------------------  6.3/6.4 MB 2.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 6.4/6.4 MB 2.5 MB/s eta 0:00:00\n",
      "Downloading cloudpathlib-0.20.0-py3-none-any.whl (52 kB)\n",
      "Downloading confection-0.1.5-py3-none-any.whl (35 kB)\n",
      "Downloading language_data-1.3.0-py3-none-any.whl (5.4 MB)\n",
      "   ---------------------------------------- 0.0/5.4 MB ? eta -:--:--\n",
      "   - -------------------------------------- 0.3/5.4 MB ? eta -:--:--\n",
      "   ------- -------------------------------- 1.0/5.4 MB 2.4 MB/s eta 0:00:02\n",
      "   ----------- ---------------------------- 1.6/5.4 MB 2.5 MB/s eta 0:00:02\n",
      "   ------------- -------------------------- 1.8/5.4 MB 2.5 MB/s eta 0:00:02\n",
      "   ------------------- -------------------- 2.6/5.4 MB 2.6 MB/s eta 0:00:02\n",
      "   ----------------------- ---------------- 3.1/5.4 MB 2.6 MB/s eta 0:00:01\n",
      "   --------------------------- ------------ 3.7/5.4 MB 2.7 MB/s eta 0:00:01\n",
      "   ------------------------------- -------- 4.2/5.4 MB 2.6 MB/s eta 0:00:01\n",
      "   ----------------------------------- ---- 4.7/5.4 MB 2.6 MB/s eta 0:00:01\n",
      "   -------------------------------------- - 5.2/5.4 MB 2.6 MB/s eta 0:00:01\n",
      "   ---------------------------------------- 5.4/5.4 MB 2.6 MB/s eta 0:00:00\n",
      "Downloading marisa_trie-1.2.1-cp312-cp312-win_amd64.whl (150 kB)\n",
      "Installing collected packages: cymem, wasabi, spacy-loggers, spacy-legacy, murmurhash, marisa-trie, cloudpathlib, catalogue, blis, srsly, preshed, language-data, langcodes, confection, weasel, thinc, spacy\n",
      "Successfully installed blis-1.1.0 catalogue-2.0.10 cloudpathlib-0.20.0 confection-0.1.5 cymem-2.0.10 langcodes-3.5.0 language-data-1.3.0 marisa-trie-1.2.1 murmurhash-1.0.11 preshed-3.0.9 spacy-3.8.3 spacy-legacy-3.0.12 spacy-loggers-1.0.5 srsly-2.5.0 thinc-8.3.3 wasabi-1.1.3 weasel-0.4.1\n",
      "Collecting en-core-web-sm==3.8.0\n",
      "  Downloading https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl (12.8 MB)\n",
      "     ---------------------------------------- 0.0/12.8 MB ? eta -:--:--\n",
      "     - -------------------------------------- 0.5/12.8 MB 4.2 MB/s eta 0:00:03\n",
      "     --- ------------------------------------ 1.0/12.8 MB 3.0 MB/s eta 0:00:04\n",
      "     ---- ----------------------------------- 1.6/12.8 MB 2.8 MB/s eta 0:00:05\n",
      "     ------ --------------------------------- 2.1/12.8 MB 2.7 MB/s eta 0:00:04\n",
      "     -------- ------------------------------- 2.6/12.8 MB 2.7 MB/s eta 0:00:04\n",
      "     --------- ------------------------------ 3.1/12.8 MB 2.6 MB/s eta 0:00:04\n",
      "     ----------- ---------------------------- 3.7/12.8 MB 2.6 MB/s eta 0:00:04\n",
      "     ------------- -------------------------- 4.2/12.8 MB 2.6 MB/s eta 0:00:04\n",
      "     -------------- ------------------------- 4.7/12.8 MB 2.6 MB/s eta 0:00:04\n",
      "     ---------------- ----------------------- 5.2/12.8 MB 2.6 MB/s eta 0:00:03\n",
      "     ------------------ --------------------- 5.8/12.8 MB 2.6 MB/s eta 0:00:03\n",
      "     ------------------- -------------------- 6.3/12.8 MB 2.5 MB/s eta 0:00:03\n",
      "     --------------------- ------------------ 6.8/12.8 MB 2.5 MB/s eta 0:00:03\n",
      "     ---------------------- ----------------- 7.3/12.8 MB 2.5 MB/s eta 0:00:03\n",
      "     ------------------------ --------------- 7.9/12.8 MB 2.5 MB/s eta 0:00:02\n",
      "     -------------------------- ------------- 8.4/12.8 MB 2.5 MB/s eta 0:00:02\n",
      "     --------------------------- ------------ 8.9/12.8 MB 2.5 MB/s eta 0:00:02\n",
      "     ----------------------------- ---------- 9.4/12.8 MB 2.5 MB/s eta 0:00:02\n",
      "     ------------------------------- -------- 10.0/12.8 MB 2.5 MB/s eta 0:00:02\n",
      "     -------------------------------- ------- 10.5/12.8 MB 2.5 MB/s eta 0:00:01\n",
      "     ---------------------------------- ----- 11.0/12.8 MB 2.5 MB/s eta 0:00:01\n",
      "     ------------------------------------ --- 11.5/12.8 MB 2.5 MB/s eta 0:00:01\n",
      "     ------------------------------------- -- 12.1/12.8 MB 2.5 MB/s eta 0:00:01\n",
      "     ---------------------------------------  12.6/12.8 MB 2.5 MB/s eta 0:00:01\n",
      "     ---------------------------------------- 12.8/12.8 MB 2.5 MB/s eta 0:00:00\n",
      "Installing collected packages: en-core-web-sm\n",
      "Successfully installed en-core-web-sm-3.8.0\n",
      "\u001b[38;5;2m[+] Download and installation successful\u001b[0m\n",
      "You can now load the package via spacy.load('en_core_web_sm')\n"
     ]
    }
   ],
   "source": [
    "!pip install spacy\n",
    "!python -m spacy download en_core_web_sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6042a27e-ba04-4cfe-935f-2eb71c2031d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "Text Preprocessing Theory in Brief\n",
    "\n",
    "1. Preprocessing Framework Overview\n",
    "\n",
    "   - Tokenization: Splitting text into smaller units like words or sentences. Challenges include handling punctuation, abbreviations, contractions, etc.\n",
    "   - Normalization: Standardizing text (e.g., lowercase conversion, removing numbers/punctuation, and stripping whitespace).\n",
    "   - Substitution: Replacing or removing elements, such as stop words, sparse terms, or noise.\n",
    "\n",
    "---\n",
    "\n",
    "2. Key Steps in Text Preprocessing\n",
    "\n",
    "1. Tokenization: Breaking text into tokens (e.g., words). \n",
    "\n",
    "- Example: \"Dr. Ford ran.\" → [\"Dr.\", \"Ford\", \"ran\"]\n",
    "   \n",
    "2. Normalization: Preparing text for analysis.\n",
    "   \n",
    "      - Convert to lowercase: \"HELLO\" → \"hello\"\n",
    "\n",
    "      - Remove punctuation: \"I, me!\" → \"I me\"\n",
    "\n",
    "      - Stemming: Reduce words to root form (e.g., \"running\" → \"run\").\n",
    "\n",
    "      - Lemmatization: Map words to their canonical form (e.g. \"better\" → \"good\").\n",
    "\n",
    "3. Stop Word Removal: Exclude common words (e.g., \"the\", \"and\") that add little meaning.\n",
    "   \n",
    "4. Noise Removal: Eliminate irrelevant elements, e.g., special characters, task-specific noise.\n",
    "\n",
    "---\n",
    "\n",
    "3. Stemming vs Lemmatization\n",
    "\n",
    "   - Stemming: Strips affixes; faster but less accurate.\n",
    "     \n",
    "   - \"running\" → \"run\"\n",
    "   \n",
    "   - Lemmatization: Uses linguistic rules; more accurate but slower.\n",
    "     \n",
    "   - \"better\" → \"good\"\n",
    "\n",
    "---\n",
    "\n",
    "Example usage\n",
    "\n",
    "text = \"I am running and eating. I like to play games, running and eating.\"\n",
    "\n",
    "preprocessed_text = preprocess_text(text)\n",
    "\n",
    "print(preprocessed_text)\n",
    "\n",
    "---\n",
    "\n",
    "5. Output Example\n",
    "\n",
    "Input: \"I am running and eating. I like to play games, running and eating.\"\n",
    "\n",
    "Output: \"run eat like play game run eat\"\n",
    "\n",
    "---\n",
    "\n",
    "6. Why Preprocess Text?\n",
    "\n",
    "- Simplifies text for machine learning or NLP tasks.\n",
    "- Reduces dimensionality by focusing on meaningful components.\n",
    "- Improves accuracy and efficiency in text-based models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6a7f8aa4-a9db-44f4-8fe1-0f081af64070",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "run eat like play game run eat\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "\n",
    "# Load the SpaCy model\n",
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "\n",
    "def preprocess_text(text):\n",
    "    doc = nlp(text)\n",
    "\n",
    "    # Tokenize, remove stop words, and lowercase\n",
    "    tokens = [token.lemma_.lower() for token in doc if not token.is_stop and token.is_alpha]\n",
    "\n",
    "    # Join tokens back into a single string\n",
    "    return \" \".join(tokens)\n",
    "\n",
    "text = \"I am running and eating. I like to play games, running and eating.\"\n",
    "preprocessed_text = preprocess_text(text)\n",
    "print(preprocessed_text)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}