basic tokenizer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7979ec82-186a-42b5-9df6-d90597f1a2f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3edad8ec-ba69-4386-a270-f82f0c90676e",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = ['The spacecraft drifted through the silent expanse of space, its path illuminated by distant stars.',\n",
    "'In the vastness of space, astronauts often ponder the mysteries of the universe.',\n",
    "'Space exploration requires advanced technology and meticulous planning to ensure the safety of the crew.',\n",
    "'The beauty of space lies in its infinite possibilities and the unknown realms it holds.',\n",
    "'Scientists study space to unlock the secrets of planets, stars, and galaxies far beyond our reach.'\n",
    "'As the space station orbited Earth, the crew marveled at the planets beauty from above.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "428aa3a0-6a5d-4a5c-8693-db9289d36f75",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = Tokenizer(num_words = 100, oov_token = \"<OOV>\")\n",
    "tokenizer.fit_on_texts(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "45a950af-882d-4a0d-a092-102efe9a65fa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<OOV>': 1, 'the': 2, 'of': 3, 'space': 4, 'and': 5, 'its': 6, 'stars': 7, 'in': 8, 'to': 9, 'crew': 10, 'beauty': 11, 'planets': 12, 'spacecraft': 13, 'drifted': 14, 'through': 15, 'silent': 16, 'expanse': 17, 'path': 18, 'illuminated': 19, 'by': 20, 'distant': 21, 'vastness': 22, 'astronauts': 23, 'often': 24, 'ponder': 25, 'mysteries': 26, 'universe': 27, 'exploration': 28, 'requires': 29, 'advanced': 30, 'technology': 31, 'meticulous': 32, 'planning': 33, 'ensure': 34, 'safety': 35, 'lies': 36, 'infinite': 37, 'possibilities': 38, 'unknown': 39, 'realms': 40, 'it': 41, 'holds': 42, 'scientists': 43, 'study': 44, 'unlock': 45, 'secrets': 46, 'galaxies': 47, 'far': 48, 'beyond': 49, 'our': 50, 'reach': 51, 'as': 52, 'station': 53, 'orbited': 54, 'earth': 55, 'marveled': 56, 'at': 57, 'from': 58, 'above': 59}\n"
     ]
    }
   ],
   "source": [
    "word_index = tokenizer.word_index\n",
    "print(word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b63e194d-183c-4435-94a7-507d5707693b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "59\n"
     ]
    }
   ],
   "source": [
    "print(word_index['above'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bad4a089-6192-44a2-bf83-be52ca073ee4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[2, 13, 14, 15, 2, 16, 17, 3, 4, 6, 18, 19, 20, 21, 7], [8, 2, 22, 3, 4, 23, 24, 25, 2, 26, 3, 2, 27], [4, 28, 29, 30, 31, 5, 32, 33, 9, 34, 2, 35, 3, 2, 10], [2, 11, 3, 4, 36, 8, 6, 37, 38, 5, 2, 39, 40, 41, 42], [43, 44, 4, 9, 45, 2, 46, 3, 12, 7, 5, 47, 48, 49, 50, 51, 52, 2, 4, 53, 54, 55, 2, 10, 56, 57, 2, 12, 11, 58, 59]]\n"
     ]
    }
   ],
   "source": [
    "sequences = tokenizer.texts_to_sequences(sentences)\n",
    "print(sequences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "309dc0ff-4eb9-488f-a1d5-717cf405deba",
   "metadata": {},
   "outputs": [],
   "source": [
    "new_sentences = ['Space missions push the boundaries of human knowledge and endurance.',\n",
    "'The silence of space is both eerie and awe-inspiring.']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a35ed740-04f6-4b62-bb98-4ac8a4646a6f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[4, 1, 1, 2, 1, 3, 1, 1, 5, 1], [2, 1, 3, 4, 1, 1, 1, 5, 1, 1]]\n"
     ]
    }
   ],
   "source": [
    "new_sequence = tokenizer.texts_to_sequences(new_sentences)\n",
    "print(new_sequence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99247f5a-7ed1-4acb-b4bd-d34b206d71fc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}