create_new_faiss_index.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import faiss\n",
    "import pandas as pd\n",
    "import os\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>url</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Becurtovirus is a genus of viruses, in the fam...</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Becurtovirus</td>\n",
       "      <td>Becurtovirus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Cyprinivirus is a genus of viruses in the orde...</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Cyprinivirus</td>\n",
       "      <td>Cyprinivirus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Glossinavirus is a genus of viruses, in the fa...</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Glossinavirus</td>\n",
       "      <td>Glossinavirus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Ichtadenovirus is a genus of viruses, in the f...</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Ichtadenovirus</td>\n",
       "      <td>Ichtadenovirus</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Lambdatorquevirus is a genus of viruses, in th...</td>\n",
       "      <td>https://en.wikipedia.org/wiki/Lambdatorquevirus</td>\n",
       "      <td>Lambdatorquevirus</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text   \n",
       "0  Becurtovirus is a genus of viruses, in the fam...  \\\n",
       "1  Cyprinivirus is a genus of viruses in the orde...   \n",
       "2  Glossinavirus is a genus of viruses, in the fa...   \n",
       "3  Ichtadenovirus is a genus of viruses, in the f...   \n",
       "4  Lambdatorquevirus is a genus of viruses, in th...   \n",
       "\n",
       "                                               url              title  \n",
       "0       https://en.wikipedia.org/wiki/Becurtovirus       Becurtovirus  \n",
       "1       https://en.wikipedia.org/wiki/Cyprinivirus       Cyprinivirus  \n",
       "2      https://en.wikipedia.org/wiki/Glossinavirus      Glossinavirus  \n",
       "3     https://en.wikipedia.org/wiki/Ichtadenovirus     Ichtadenovirus  \n",
       "4  https://en.wikipedia.org/wiki/Lambdatorquevirus  Lambdatorquevirus  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1 = pd.read_parquet(\"./wikipedia stem/0_to_25000.parquet\")\n",
    "df1.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20829"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df1['text'].apply(lambda x: len(x.split(' '))).max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23ff2c666c964f4f85dc06496b92bc17",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)5dded/.gitattributes:   0%|          | 0.00/1.18k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c47ee9ca8bcc467f8d8c6e2b483b69e1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)_Pooling/config.json:   0%|          | 0.00/190 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "350371d19c6e490d91d02c2709168652",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)4d81d5dded/README.md:   0%|          | 0.00/10.6k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2cb0e6d81f2b4a959d6d0111974265a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)81d5dded/config.json:   0%|          | 0.00/573 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d2113af60044366992c6e01f472e80a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)ce_transformers.json:   0%|          | 0.00/116 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9244a7d988cd49efa0d253afae6b105c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)ded/data_config.json:   0%|          | 0.00/39.3k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f2a1dbd566ad4e1f9c8362137e4f1d34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading pytorch_model.bin:   0%|          | 0.00/134M [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4214ab602f594d69a500a5064e92063e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)nce_bert_config.json:   0%|          | 0.00/53.0 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "078704bf8b94421fafa39864cf1742db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92bca4f98a5b44bbb1788631dd114de2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)5dded/tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6bb57e1d92a4dc6be31d0796878c192",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/352 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d3a1f5968b94270b91b46fcf9739db3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)dded/train_script.py:   0%|          | 0.00/13.2k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a10c040df7e24e58adbaf4a69884febc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)4d81d5dded/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b3e773b1b5848d48c81929156b606c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)1d5dded/modules.json:   0%|          | 0.00/349 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c1102cec503448ca1356e630f4f41c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)okenizer_config.json:   0%|          | 0.00/352 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\Users\\86183\\anaconda3\\lib\\site-packages\\huggingface_hub\\file_download.py:133: UserWarning: `huggingface_hub` cache-system uses symlinks by default to efficiently store duplicated files but your machine does not support them in C:\\Users\\86183\\.cache\\huggingface\\hub. Caching files will still work but in a degraded version that might require more space on your disk. This warning can be disabled by setting the `HF_HUB_DISABLE_SYMLINKS_WARNING` environment variable. For more details, see https://huggingface.co/docs/huggingface_hub/how-to-cache#limitations.\n",
      "To support symlinks on Windows, you either need to activate Developer Mode or to run Python as an administrator. In order to see activate developer mode, see this article: https://docs.microsoft.com/en-us/windows/apps/get-started/enable-your-device-for-development\n",
      "  warnings.warn(message)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6dde4aa55be9436eb0a2a87f961f99cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7fe5756dc0c4adc875d32c980dd538d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/466k [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fbceff4bc43b48689b7a2bf9fcdef77f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "from transformers import AutoTokenizer\n",
    "\n",
    "model_name = 'sentence-transformers/all-MiniLM-L12-v2'\n",
    "model = SentenceTransformer(model_name)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_text_into_segments(text, max_length, stride, category):\n",
    "    tokens = tokenizer.tokenize(text)\n",
    "    segments = []\n",
    "    categories = []\n",
    "\n",
    "    start = 0\n",
    "    while start < len(tokens):\n",
    "        if start + max_length > len(tokens):\n",
    "            segment_tokens = tokens[start:]\n",
    "            segment = tokenizer.convert_tokens_to_string(segment_tokens)\n",
    "            segments.append(segment)\n",
    "            categories.append(category)\n",
    "            break\n",
    "        else:\n",
    "            end = start + max_length\n",
    "            segment_tokens = tokens[start:end]\n",
    "            segment = tokenizer.convert_tokens_to_string(segment_tokens)\n",
    "            segments.append(segment)\n",
    "            categories.append(category)\n",
    "            start = end - stride\n",
    "\n",
    "    return segments, categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['i', 'love', 'united', 'states', '.', 'this', 'is', 'a', 'great', 'country', '.']\n",
      "i love united states. this is a great country.\n"
     ]
    }
   ],
   "source": [
    "long_text = \"I love United States. This is a great country.\"\n",
    "print(tokenizer.tokenize(long_text))\n",
    "print(tokenizer.convert_tokens_to_string(tokenizer.tokenize(long_text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['becurtovirus is a genus of viruses, in the family geminiviridae. dicotyledonous plants serve as natural hosts. there are three species in this genus. taxonomy the following three species are assigned to the genus : beet curly top iran virus exomis microphylla latent virus spinach curly top arizona virus structure viruses in becurtovirus are non - enveloped, with icosahedral geometries, and t = 1 symmetry. genomes are circular and non - segmented, around 3. 0kb in length. life cycle viral replication is nuclear. entry into the host cell is achieved by penetration into the host cell. replication follows the ssdna rolling circle model. dna - templated transcription is the method of transcription. the virus exits the host cell by nuclear pore export, and tubule - guided viral movement. dicotyledonous plants serve as the natural host. references external links viralzone : becurtovirus ictv geminiviridae virus genera']\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "long_text = df1.loc[0, \"text\"]\n",
    "stride = 64 \n",
    "max_length = 384 \n",
    "\n",
    "segments, categories = split_text_into_segments(long_text, max_length, stride)\n",
    "print(segments)\n",
    "print(len(segments))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 25000/25000 [02:06<00:00, 197.94it/s]\n",
      "100%|██████████| 25000/25000 [02:27<00:00, 169.95it/s]\n",
      "100%|██████████| 6049/6049 [00:36<00:00, 166.09it/s]\n",
      "100%|██████████| 25000/25000 [02:14<00:00, 186.22it/s]\n",
      "100%|██████████| 25000/25000 [03:07<00:00, 133.10it/s]\n",
      "100%|██████████| 25000/25000 [02:28<00:00, 168.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "541776 541776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "chunked_context = []\n",
    "corresponding_categories = []\n",
    "\n",
    "for filename in os.listdir(\"./wikipedia stem/\"):\n",
    "    filepath = os.path.join(\"./wikipedia stem/\", filename)\n",
    "    print(f\"Splitting file {filepath}\")\n",
    "    df = pd.read_parquet(filepath)\n",
    "\n",
    "    for index, row in tqdm(df.iterrows(), total=len(df)):\n",
    "        context = row[\"text\"]\n",
    "        category = row[\"title\"]\n",
    "        max_length = 384\n",
    "        stride = 128\n",
    "\n",
    "        segments, categories = split_text_into_segments(context, max_length, stride, category)\n",
    "        chunked_context.extend(segments)\n",
    "        corresponding_categories.extend(categories)\n",
    "\n",
    "print(len(chunked_context), len(corresponding_categories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(541776, 2)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({\"context\": chunked_context, \"title\": corresponding_categories})\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 541776 entries, 0 to 541775\n",
      "Data columns (total 2 columns):\n",
      " #   Column   Non-Null Count   Dtype \n",
      "---  ------   --------------   ----- \n",
      " 0   context  541776 non-null  object\n",
      " 1   title    541776 non-null  object\n",
      "dtypes: object(2)\n",
      "memory usage: 8.3+ MB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}