CrossOver_Alpha.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import nltk\n",
    "from nltk.tokenize import sent_tokenize,TreebankWordTokenizer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet,stopwords\n",
    "from nltk import pos_tag\n",
    "import texthero as hero\n",
    "from num2words import num2words\n",
    "import string\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from collections import Counter\n",
    "import language_tool_python,random,sys\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# file_1_path = './Data Files/ted_bundy_wiki_clean.txt'\n",
    "# file_2_path = './Data Files/lotr1.txt'\n",
    "path1 = './Data Files/harry_potter.pdf'\n",
    "path2 = './Data Files/percy_jackson.pdf'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "##################### READING TEXT FILES ###############################\n",
    "\n",
    "def read_data_files(path1,path2,threshold=1000):\n",
    "    file1 = open(path1,'r').read()\n",
    "    file2 = open(path2,'r').read()\n",
    "    \n",
    "    file1 = sent_tokenize(file1)\n",
    "    file2 = sent_tokenize(file2)\n",
    "    return file1[:min(len(file1),threshold)],file2[:min(len(file2),threshold)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################### READING PDFS ####################################\n",
    "\n",
    "import PyPDF2\n",
    "from io import StringIO\n",
    "\n",
    "from pdfminer.converter import TextConverter\n",
    "from pdfminer.layout import LAParams\n",
    "from pdfminer.pdfdocument import PDFDocument\n",
    "from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter\n",
    "from pdfminer.pdfpage import PDFPage\n",
    "from pdfminer.pdfparser import PDFParser\n",
    "\n",
    "def return_output_string(path):\n",
    "    output_string = StringIO()\n",
    "    with open(path, 'rb') as in_file:\n",
    "        parser = PDFParser(in_file)\n",
    "        doc = PDFDocument(parser)\n",
    "        rsrcmgr = PDFResourceManager()\n",
    "        device = TextConverter(rsrcmgr, output_string, laparams=LAParams())\n",
    "        interpreter = PDFPageInterpreter(rsrcmgr, device)\n",
    "        for page in PDFPage.create_pages(doc):\n",
    "            interpreter.process_page(page)\n",
    "    \n",
    "    return output_string.getvalue()\n",
    "\n",
    "def read_pdfs(path1,path2,threshold=1000):\n",
    "    file1 = return_output_string(path1)\n",
    "    file2 = return_output_string(path2)\n",
    "    \n",
    "    file1 = sent_tokenize(file1)\n",
    "    file2 = sent_tokenize(file2)\n",
    "    \n",
    "    return file1[5:min(len(file1),threshold)],file2[5:min(len(file2),threshold)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#corpus1,corpus2 = read_data_files(file_1_path,file_2_path)\n",
    "corpus1,corpus2 = read_pdfs(path1,path2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "995 995\n"
     ]
    }
   ],
   "source": [
    "print(len(corpus1),len(corpus2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def basic_preprocessing(series):\n",
    "    \n",
    "    # Removing brackets\n",
    "    series = hero.remove_brackets(series)\n",
    "    \n",
    "    # Removing diacritics\n",
    "    series = hero.remove_diacritics(series) # Words like Café, that top extra char will be removed\n",
    "    \n",
    "    # Removing whitespaces\n",
    "    series = hero.remove_whitespace(series)\n",
    "    \n",
    "    series = hero.remove_digits(series)\n",
    "                      \n",
    "    return series\n",
    "\n",
    "def custom_preprocessing(text):\n",
    "    text = re.sub(r\"[^A-Za-z0-9,!.']\", \" \", text)\n",
    "    text = re.sub(r\"(\\s)(\\s)+\",' ',text)\n",
    "    return text\n",
    "\n",
    "\n",
    "def list_to_string(sentence_list):\n",
    "    text = ''\n",
    "    for row in sentence_list:\n",
    "        if text is '':\n",
    "            text += row+'\\n'\n",
    "        else:\n",
    "            text += ' ' + row+'\\n'\n",
    "    return text\n",
    "\n",
    "def string_to_list(sentence_string):\n",
    "    sentences = sent_tokenize(sentence_string)\n",
    "    return sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "corpus1 = basic_preprocessing(pd.Series(corpus1)).tolist()\n",
    "corpus2 = basic_preprocessing(pd.Series(corpus2)).tolist()\n",
    "\n",
    "corpus1 = pd.Series(corpus1).apply(custom_preprocessing).tolist()\n",
    "corpus2 = pd.Series(corpus2).apply(custom_preprocessing).tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_words = 10000\n",
    "\n",
    "embeddings_index = {}\n",
    "f = open('glove.6B.50d.txt','r',encoding='utf-8')\n",
    "\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:],dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = TreebankWordTokenizer()\n",
    "\n",
    "def find_most_similar(input_sentence_embedding, target_sentence_embeddings, ignore_list):\n",
    "    similarity = [cosine_similarity([input_sentence_embedding], [target_sentence])[0][0] for target_sentence in target_sentence_embeddings]\n",
    "    if len(ignore_list) > 0 :\n",
    "        for i in ignore_list : \n",
    "            similarity[i] = -1\n",
    "    most_similar_index = np.argmax(similarity)\n",
    "    return most_similar_index, similarity[most_similar_index]\n",
    "\n",
    "def prepare_freq_table(corpus1,corpus2):\n",
    "    counter = Counter()\n",
    "    corpus1 = list_to_string(corpus1)\n",
    "    corpus2 = list_to_string(corpus2)\n",
    "    tokens1 = [word for word in tokenizer.tokenize(corpus1)]\n",
    "    tokens2 = [word for word in tokenizer.tokenize(corpus2)]\n",
    "    \n",
    "    counter.update(tokens1)\n",
    "    counter.update(tokens2)\n",
    "    return dict(counter)\n",
    "    \n",
    "\n",
    "\n",
    "def prepare_sentence_embeddings(corpus,index,freqs,dimensions=50,a=0.001):\n",
    "    total_freq = sum(freqs.values())\n",
    "    embeddings = []\n",
    "    stopwords_list = stopwords.words('english')\n",
    "    \n",
    "    for sentence in corpus:\n",
    "        sentence = sentence.lower()\n",
    "        sentence = re.sub(r\"[^A-Za-z]\", \" \", sentence)\n",
    "        sentence = re.sub(r\"(\\s)(\\s)+\",\" \",sentence)\n",
    "        tokens = list(set([word for word in tokenizer.tokenize(sentence) if word not in stopwords_list and word in index.keys()]))\n",
    "        weights = [a/(a+freqs.get(token,0)/total_freq) for token in tokens]\n",
    "        if len(tokens) == 0:\n",
    "            embeddings.append(np.zeros((dimensions,)))\n",
    "        else:\n",
    "            embedding = np.average([index[token] for token in tokens], axis=0, weights=weights)\n",
    "            embeddings.append(embedding)\n",
    "    \n",
    "    return embeddings\n",
    "    \n",
    "    \n",
    "def similarity_module(corpus1,corpus2,index):\n",
    "    freq_dict = prepare_freq_table(corpus1,corpus2)\n",
    "    corpus_1_embeddings = prepare_sentence_embeddings(corpus1,index,freq_dict)\n",
    "    corpus_2_embeddings = prepare_sentence_embeddings(corpus2,index,freq_dict)\n",
    "    similarity_df = pd.DataFrame(columns=['Source','Target','Similarity_Value'])\n",
    "    ignore_list = []\n",
    "    for i,e1 in enumerate(corpus_1_embeddings):\n",
    "        index,value = find_most_similar(e1,corpus_2_embeddings,ignore_list)\n",
    "        ignore_list.append(index)\n",
    "        similarity_df.loc[i,'Source'] = i\n",
    "        similarity_df.loc[i,'Target'] = index\n",
    "        similarity_df.loc[i,'Similarity_Value'] = value\n",
    "    \n",
    "    return similarity_df\n",
    "        \n",
    "    #print(find_most_similar(corpus_1_embeddings[2],corpus_2_embeddings))\n",
    "    #print(len(corpus1),len(corpus_1_embeddings))\n",
    "    #print(len(corpus2),len(corpus_2_embeddings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModelForTokenClassification\n",
    "from transformers import pipeline\n",
    "import re\n",
    "import heapq\n",
    "\n",
    "def prepare_freq_table(corpus1,corpus2):\n",
    "    counter = Counter()\n",
    "    corpus1 = list_to_string(corpus1)\n",
    "    corpus2 = list_to_string(corpus2)\n",
    "    tokens1 = [word for word in tokenizer.tokenize(corpus1)]\n",
    "    tokens2 = [word for word in tokenizer.tokenize(corpus2)]\n",
    "    \n",
    "    counter.update(tokens1)\n",
    "    counter.update(tokens2)\n",
    "    return dict(counter)\n",
    "\n",
    "def NER_tags_creator(sentence,model):\n",
    "    ner_results = model(sentence)\n",
    "    token = None\n",
    "    tag = None\n",
    "    ner_pairs = []\n",
    "    i = 0\n",
    "\n",
    "    while i < len(ner_results):\n",
    "        if tag is None:\n",
    "            token = ner_results[i]['word']\n",
    "            tag = ner_results[i]['entity'][2:]\n",
    "\n",
    "        elif 'B' in ner_results[i]['entity']:\n",
    "            ner_pairs.append((tag,token))\n",
    "            token = ner_results[i]['word']\n",
    "            tag = ner_results[i]['entity'][2:]\n",
    "\n",
    "        else:\n",
    "            token += ' '+ner_results[i]['word']   \n",
    "            token = re.sub(r'[#]+','',token)\n",
    "        i+= 1\n",
    "\n",
    "    ner_pairs.append((tag,token))\n",
    "    ner_pairs = list(set(ner_pairs))\n",
    "    return ner_pairs\n",
    "\n",
    "def build_list_for_heap(pairs,freq_table):\n",
    "    person_heap = []\n",
    "    org_loc_heap = []\n",
    "    \n",
    "    for tag,word in pairs:\n",
    "        freq = -1*freq_table.get(word,0)\n",
    "        if tag == \"PER\":\n",
    "            person_heap.append((freq,word))\n",
    "        \n",
    "        elif tag in ['LOC','ORG']:\n",
    "            org_loc_heap.append((freq,word))\n",
    "    \n",
    "    return person_heap,org_loc_heap\n",
    "\n",
    "def generate_swappers(s1,s2,model,freq_table): # Main function (CALL THIS FUNCTION) with two sentences s1,s2 which are similar(similarity module)\n",
    "    \n",
    "    s1_ner_pairs = NER_tags_creator(s1,model)\n",
    "    s2_ner_pairs = NER_tags_creator(s2,model)\n",
    "    \n",
    "    person_heap_s1, org_loc_heap_s1 = build_list_for_heap(s1_ner_pairs,freq_table)\n",
    "    person_heap_s2, org_loc_heap_s2 = build_list_for_heap(s2_ner_pairs,freq_table)\n",
    "    \n",
    "    heapq.heapify(person_heap_s1)\n",
    "    heapq.heapify(person_heap_s2)\n",
    "    \n",
    "    heapq.heapify(org_loc_heap_s1)\n",
    "    heapq.heapify(org_loc_heap_s2)\n",
    "    \n",
    "    new_sentences = []\n",
    "    \n",
    "    if len(person_heap_s1) != 0 and len(person_heap_s2) != 0:\n",
    "        word_s1 = heapq.heappop(person_heap_s1)[1]\n",
    "        word_s2 = heapq.heappop(person_heap_s2)[1]\n",
    "        \n",
    "        new_sentences.append(s1.replace(word_s1,word_s2))\n",
    "        new_sentences.append(s2.replace(word_s2,word_s1))\n",
    "    \n",
    "    \n",
    "    if len(org_loc_heap_s1) != 0 and len(org_loc_heap_s2) != 0:\n",
    "        word_s1 = heapq.heappop(org_loc_heap_s1)[1]\n",
    "        word_s2 = heapq.heappop(org_loc_heap_s2)[1]\n",
    "        \n",
    "        new_sentences.append(s1.replace(word_s1,word_s2))\n",
    "        new_sentences.append(s2.replace(word_s2,word_s1))\n",
    "    \n",
    "    return new_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_data_plus_ner_swap(corpus1,corpus2,sm_df,source,target,value_name,model,threshold=0.7):\n",
    "    freq_mapper =  prepare_freq_table(corpus1,corpus2)\n",
    "    sm_df = sm_df[sm_df[value_name] > threshold].sort_values(by=source)\n",
    "    final_data = []\n",
    "    ner_samples = []\n",
    "    indicator = []\n",
    "    for i,j in zip(sm_df[source],sm_df[target]):\n",
    "        s1 = corpus1[i]\n",
    "        s2 = corpus2[j]\n",
    "        samples = generate_swappers(s1,s2,nlp,freq_mapper)\n",
    "        if len(samples) > 0:\n",
    "            indicator.append((i,j))\n",
    "        final_data.append(s1) # S1 sentence\n",
    "        final_data.append(s2) # S2 sentence\n",
    "        final_data.extend(samples) # NER Swaps added\n",
    "        ner_samples.extend(samples) # All NER swaps samples\n",
    "    \n",
    "    ner_samples = [s for s in ner_samples if s not in corpus1 and s not in corpus2]\n",
    "    return final_data,ner_samples,indicator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(\"dslim/bert-base-NER\")\n",
    "model = AutoModelForTokenClassification.from_pretrained(\"dslim/bert-base-NER\")\n",
    "\n",
    "nlp = pipeline(\"ner\", model=model, tokenizer=tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Token indices sequence length is longer than the specified maximum sequence length for this model (17522 > 512). Running this sequence through the model will result in indexing errors\n"
     ]
    }
   ],
   "source": [
    "ss_df = similarity_module(corpus1,corpus2,embeddings_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_data,ner_samples,indicator = create_data_plus_ner_swap(corpus1,corpus2,ss_df,'Source','Target','Similarity_Value',nlp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 474),\n",
       " (1, 58),\n",
       " (2, 634),\n",
       " (5, 90),\n",
       " (7, 762),\n",
       " (8, 167),\n",
       " (11, 17),\n",
       " (15, 339),\n",
       " (21, 232),\n",
       " (22, 707)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "indicator[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The Dursleys had a small son called Dudley and in their opinion there was no finer boy anywhere.',\n",
       " \"something like The card was in fancy script, which was murder on my dyslexic eyes, but I finally made out Grover Underwood Keeper Half Blood Hill Long Island, New York 009 0009 What's Half Don't say it aloud! \",\n",
       " 'The Groversleys had a small son called Grovedley and in their opinion there was no finer boy anywhere.',\n",
       " \"something like The card was in fancy script, which was murder on my dyslexic eyes, but I finally made out Dur Underwood Keeper Half Blood Hill Long Island, New York 009 0009 What's Half Don't say it aloud! \",\n",
       " 'The Dursleys had everything they wanted, but they also had a secret, and their greatest fear was that somebody would discover it.']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_data[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The Groversleys had a small son called Grovedley and in their opinion there was no finer boy anywhere.',\n",
       " \"something like The card was in fancy script, which was murder on my dyslexic eyes, but I finally made out Dur Underwood Keeper Half Blood Hill Long Island, New York 009 0009 What's Half Don't say it aloud! \",\n",
       " 'The Doddrsleys had everything they wanted, but they also had a secret, and their greatest fear was that somebody would discover it.',\n",
       " 'I was trying to listen to what he had to say, because it was kind of interesting, but everybody around me was talking, and every time I told them to shut up, the other teacher chaperone, Mrs. Dus, would give me the evil eye.',\n",
       " \"They didn't think they could bear it if anyone found out about the Gabes.\"]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ner_samples[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['final_data']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import joblib\n",
    "\n",
    "joblib.dump(final_data,'final_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.17.0'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "joblib.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = joblib.load('final_data')\n",
    "type(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_training_data(data,maxlen=180,step=3):\n",
    "    data = list_to_string(data)\n",
    "    data = data.lower()\n",
    "    data = re.sub(r\"[^A-Za-z.']\", \" \", data)\n",
    "    data = re.sub(r\"(\\s)(\\s)+\",\" \",data)\n",
    "    \n",
    "    sentences = []\n",
    "    next_chars = []\n",
    "    for i in range(0,len(data)-maxlen,step):\n",
    "        sentences.append(data[i:i+maxlen])\n",
    "        next_chars.append(data[i+maxlen])\n",
    "    \n",
    "    print(f'The no of rows will be {len(sentences)}')\n",
    "    \n",
    "    chars = sorted(list(set(data))) # Consist of all unique characters in the file\n",
    "    char_indices = dict((char,chars.index(char)) for char in chars) # character to index mapping\n",
    "    print(f'The length of unique characters are {len(chars)}')\n",
    "    print(chars)\n",
    "    \n",
    "    x = np.zeros((len(sentences),maxlen,len(chars)))\n",
    "    y = np.zeros((len(sentences),len(chars)))\n",
    "    \n",
    "    for i,sentence in enumerate(sentences):\n",
    "        for t,char in enumerate(sentence):\n",
    "            x[i,t,char_indices[char]] = 1\n",
    "        y[i,char_indices[next_chars[i]]] = 1\n",
    "    return x,y,len(chars),chars,char_indices,sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The no of rows will be 45173\n",
      "The length of unique characters are 29\n",
      "[' ', \"'\", '.', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z']\n"
     ]
    }
   ],
   "source": [
    "maxlen=180\n",
    "X,y,len_chars,chars,char_indices,sentences = create_training_data(final_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MODEL BUILDING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential,load_model\n",
    "from keras.layers import LSTM,Dense,Dropout,GRU\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "model.add(GRU(256,input_shape=(maxlen,len_chars),recurrent_dropout=0.5))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Dense(len_chars,activation='softmax'))\n",
    "\n",
    "model.compile(optimizer='rmsprop',loss='categorical_crossentropy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "gru (GRU)                    (None, 256)               220416    \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 29)                7453      \n",
      "=================================================================\n",
      "Total params: 227,869\n",
      "Trainable params: 227,869\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "283/283 [==============================] - 218s 765ms/step - loss: 2.8473 - val_loss: 2.3597\n",
      "Epoch 2/100\n",
      "283/283 [==============================] - 218s 771ms/step - loss: 2.2989 - val_loss: 2.2203\n",
      "Epoch 3/100\n",
      "283/283 [==============================] - 227s 801ms/step - loss: 2.1912 - val_loss: 2.1488\n",
      "Epoch 4/100\n",
      "283/283 [==============================] - 235s 830ms/step - loss: 2.1243 - val_loss: 2.0905\n",
      "Epoch 5/100\n",
      "283/283 [==============================] - 237s 839ms/step - loss: 2.0699 - val_loss: 2.0400\n",
      "Epoch 6/100\n",
      "283/283 [==============================] - 229s 811ms/step - loss: 2.0245 - val_loss: 1.9975\n",
      "Epoch 7/100\n",
      "283/283 [==============================] - 230s 815ms/step - loss: 1.9745 - val_loss: 1.9659\n",
      "Epoch 8/100\n",
      "283/283 [==============================] - 229s 808ms/step - loss: 1.9426 - val_loss: 1.9321\n",
      "Epoch 9/100\n",
      "283/283 [==============================] - 229s 809ms/step - loss: 1.9088 - val_loss: 1.9093\n",
      "Epoch 10/100\n",
      "283/283 [==============================] - 230s 813ms/step - loss: 1.8833 - val_loss: 1.8863\n",
      "Epoch 11/100\n",
      "283/283 [==============================] - 229s 811ms/step - loss: 1.8516 - val_loss: 1.8633\n",
      "Epoch 12/100\n",
      "283/283 [==============================] - 233s 822ms/step - loss: 1.8373 - val_loss: 1.8428\n",
      "Epoch 13/100\n",
      "283/283 [==============================] - 233s 822ms/step - loss: 1.8123 - val_loss: 1.8330\n",
      "Epoch 14/100\n",
      "283/283 [==============================] - 232s 819ms/step - loss: 1.7974 - val_loss: 1.8146\n",
      "Epoch 15/100\n",
      "283/283 [==============================] - 228s 807ms/step - loss: 1.7769 - val_loss: 1.7983\n",
      "Epoch 16/100\n",
      "283/283 [==============================] - 230s 812ms/step - loss: 1.7618 - val_loss: 1.7860\n",
      "Epoch 17/100\n",
      "283/283 [==============================] - 228s 804ms/step - loss: 1.7453 - val_loss: 1.7718\n",
      "Epoch 18/100\n",
      "283/283 [==============================] - 226s 798ms/step - loss: 1.7326 - val_loss: 1.7662\n",
      "Epoch 19/100\n",
      "283/283 [==============================] - 224s 791ms/step - loss: 1.7174 - val_loss: 1.7619\n",
      "Epoch 20/100\n",
      "283/283 [==============================] - 224s 790ms/step - loss: 1.7022 - val_loss: 1.7502\n",
      "Epoch 21/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.6942 - val_loss: 1.7388\n",
      "Epoch 22/100\n",
      "283/283 [==============================] - 223s 789ms/step - loss: 1.6865 - val_loss: 1.7299\n",
      "Epoch 23/100\n",
      "283/283 [==============================] - 224s 793ms/step - loss: 1.6747 - val_loss: 1.7277\n",
      "Epoch 24/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.6577 - val_loss: 1.7217\n",
      "Epoch 25/100\n",
      "283/283 [==============================] - 224s 793ms/step - loss: 1.6526 - val_loss: 1.7131\n",
      "Epoch 26/100\n",
      "283/283 [==============================] - 225s 795ms/step - loss: 1.6522 - val_loss: 1.7103\n",
      "Epoch 27/100\n",
      "283/283 [==============================] - 231s 816ms/step - loss: 1.6375 - val_loss: 1.7034\n",
      "Epoch 28/100\n",
      "283/283 [==============================] - 229s 810ms/step - loss: 1.6334 - val_loss: 1.6969\n",
      "Epoch 29/100\n",
      "283/283 [==============================] - 230s 814ms/step - loss: 1.6210 - val_loss: 1.6960\n",
      "Epoch 30/100\n",
      "283/283 [==============================] - 228s 804ms/step - loss: 1.6170 - val_loss: 1.6887\n",
      "Epoch 31/100\n",
      "283/283 [==============================] - 231s 815ms/step - loss: 1.6120 - val_loss: 1.6831\n",
      "Epoch 32/100\n",
      "283/283 [==============================] - 233s 825ms/step - loss: 1.6064 - val_loss: 1.6823\n",
      "Epoch 33/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.5983 - val_loss: 1.6819\n",
      "Epoch 34/100\n",
      "283/283 [==============================] - 236s 834ms/step - loss: 1.5896 - val_loss: 1.6759\n",
      "Epoch 35/100\n",
      "283/283 [==============================] - 224s 791ms/step - loss: 1.5878 - val_loss: 1.6744\n",
      "Epoch 36/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.5762 - val_loss: 1.6704\n",
      "Epoch 37/100\n",
      "283/283 [==============================] - 224s 791ms/step - loss: 1.5757 - val_loss: 1.6668\n",
      "Epoch 38/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.5724 - val_loss: 1.6648\n",
      "Epoch 39/100\n",
      "283/283 [==============================] - 223s 789ms/step - loss: 1.5662 - val_loss: 1.6632\n",
      "Epoch 40/100\n",
      "283/283 [==============================] - 224s 793ms/step - loss: 1.5551 - val_loss: 1.6607\n",
      "Epoch 41/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.5600 - val_loss: 1.6565\n",
      "Epoch 42/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.5528 - val_loss: 1.6568\n",
      "Epoch 43/100\n",
      "283/283 [==============================] - 223s 790ms/step - loss: 1.5482 - val_loss: 1.6565\n",
      "Epoch 44/100\n",
      "283/283 [==============================] - 224s 790ms/step - loss: 1.5483 - val_loss: 1.6522\n",
      "Epoch 45/100\n",
      "283/283 [==============================] - 224s 793ms/step - loss: 1.5441 - val_loss: 1.6529\n",
      "Epoch 46/100\n",
      "283/283 [==============================] - 226s 797ms/step - loss: 1.5340 - val_loss: 1.6491\n",
      "Epoch 47/100\n",
      "283/283 [==============================] - 225s 795ms/step - loss: 1.5351 - val_loss: 1.6454\n",
      "Epoch 48/100\n",
      "283/283 [==============================] - 225s 795ms/step - loss: 1.5297 - val_loss: 1.6455\n",
      "Epoch 49/100\n",
      "283/283 [==============================] - 226s 800ms/step - loss: 1.5226 - val_loss: 1.6421\n",
      "Epoch 50/100\n",
      "283/283 [==============================] - 225s 796ms/step - loss: 1.5248 - val_loss: 1.6437\n",
      "Epoch 51/100\n",
      "283/283 [==============================] - 226s 800ms/step - loss: 1.5151 - val_loss: 1.6397\n",
      "Epoch 52/100\n",
      "283/283 [==============================] - 225s 795ms/step - loss: 1.5179 - val_loss: 1.6378\n",
      "Epoch 53/100\n",
      "283/283 [==============================] - 226s 800ms/step - loss: 1.5091 - val_loss: 1.6358\n",
      "Epoch 54/100\n",
      "283/283 [==============================] - 228s 807ms/step - loss: 1.5028 - val_loss: 1.6321\n",
      "Epoch 55/100\n",
      "283/283 [==============================] - 227s 803ms/step - loss: 1.5016 - val_loss: 1.6324\n",
      "Epoch 56/100\n",
      "283/283 [==============================] - 228s 806ms/step - loss: 1.4970 - val_loss: 1.6304\n",
      "Epoch 57/100\n",
      "283/283 [==============================] - 226s 800ms/step - loss: 1.5033 - val_loss: 1.6321\n",
      "Epoch 58/100\n",
      "283/283 [==============================] - 226s 798ms/step - loss: 1.5020 - val_loss: 1.6271\n",
      "Epoch 59/100\n",
      "283/283 [==============================] - 224s 793ms/step - loss: 1.4929 - val_loss: 1.6285\n",
      "Epoch 60/100\n",
      "283/283 [==============================] - 225s 794ms/step - loss: 1.4829 - val_loss: 1.6295\n",
      "Epoch 61/100\n",
      "283/283 [==============================] - 226s 799ms/step - loss: 1.4778 - val_loss: 1.6263\n",
      "Epoch 62/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.4916 - val_loss: 1.6258\n",
      "Epoch 63/100\n",
      "283/283 [==============================] - 224s 791ms/step - loss: 1.4860 - val_loss: 1.6244\n",
      "Epoch 64/100\n",
      "283/283 [==============================] - 224s 791ms/step - loss: 1.4856 - val_loss: 1.6227\n",
      "Epoch 65/100\n",
      "283/283 [==============================] - 224s 791ms/step - loss: 1.4834 - val_loss: 1.6247\n",
      "Epoch 66/100\n",
      "283/283 [==============================] - 225s 794ms/step - loss: 1.4829 - val_loss: 1.6218\n",
      "Epoch 67/100\n",
      "283/283 [==============================] - 224s 792ms/step - loss: 1.4783 - val_loss: 1.6222\n",
      "Epoch 68/100\n",
      "283/283 [==============================] - 225s 796ms/step - loss: 1.4733 - val_loss: 1.6185\n",
      "Epoch 69/100\n",
      "283/283 [==============================] - 226s 799ms/step - loss: 1.4765 - val_loss: 1.6189\n",
      "Epoch 70/100\n",
      "283/283 [==============================] - 227s 801ms/step - loss: 1.4684 - val_loss: 1.6213\n",
      "Epoch 71/100\n",
      "283/283 [==============================] - 227s 802ms/step - loss: 1.4638 - val_loss: 1.6184\n",
      "Epoch 72/100\n",
      "283/283 [==============================] - 227s 802ms/step - loss: 1.4606 - val_loss: 1.6138\n",
      "Epoch 73/100\n",
      "283/283 [==============================] - 227s 802ms/step - loss: 1.4539 - val_loss: 1.6182\n",
      "Epoch 74/100\n",
      "283/283 [==============================] - 227s 804ms/step - loss: 1.4648 - val_loss: 1.6141\n",
      "Epoch 75/100\n",
      "283/283 [==============================] - 227s 803ms/step - loss: 1.4570 - val_loss: 1.6143\n",
      "Epoch 76/100\n",
      "283/283 [==============================] - 228s 806ms/step - loss: 1.4556 - val_loss: 1.6146\n",
      "Epoch 77/100\n",
      "283/283 [==============================] - 227s 802ms/step - loss: 1.4530 - val_loss: 1.6159\n",
      "Epoch 78/100\n",
      "283/283 [==============================] - 227s 801ms/step - loss: 1.4466 - val_loss: 1.6141\n",
      "Epoch 79/100\n",
      "283/283 [==============================] - 226s 799ms/step - loss: 1.4484 - val_loss: 1.6103\n",
      "Epoch 80/100\n",
      "283/283 [==============================] - 227s 802ms/step - loss: 1.4440 - val_loss: 1.6080\n",
      "Epoch 81/100\n",
      "283/283 [==============================] - 225s 793ms/step - loss: 1.4520 - val_loss: 1.6115\n",
      "Epoch 82/100\n",
      "283/283 [==============================] - 225s 795ms/step - loss: 1.4483 - val_loss: 1.6046\n",
      "Epoch 83/100\n",
      "283/283 [==============================] - 225s 795ms/step - loss: 1.4369 - val_loss: 1.6085\n",
      "Epoch 84/100\n",
      "283/283 [==============================] - 224s 793ms/step - loss: 1.4453 - val_loss: 1.6054\n",
      "Epoch 85/100\n",
      "283/283 [==============================] - 225s 794ms/step - loss: 1.4367 - val_loss: 1.6061\n",
      "Epoch 86/100\n",
      "283/283 [==============================] - 224s 793ms/step - loss: 1.4357 - val_loss: 1.6070\n",
      "Epoch 87/100\n",
      "283/283 [==============================] - 225s 795ms/step - loss: 1.4317 - val_loss: 1.6082\n",
      "Epoch 88/100\n",
      "283/283 [==============================] - 226s 798ms/step - loss: 1.4277 - val_loss: 1.6021\n",
      "Epoch 89/100\n",
      "283/283 [==============================] - 226s 799ms/step - loss: 1.4261 - val_loss: 1.6051\n",
      "Epoch 90/100\n",
      "283/283 [==============================] - 226s 798ms/step - loss: 1.4242 - val_loss: 1.6057\n",
      "Epoch 91/100\n",
      "283/283 [==============================] - 226s 799ms/step - loss: 1.4207 - val_loss: 1.6072\n",
      "Epoch 92/100\n",
      "283/283 [==============================] - 225s 796ms/step - loss: 1.4230 - val_loss: 1.6057\n",
      "Epoch 93/100\n",
      "283/283 [==============================] - 225s 795ms/step - loss: 1.4211 - val_loss: 1.6008\n",
      "Epoch 94/100\n",
      "283/283 [==============================] - 226s 800ms/step - loss: 1.4331 - val_loss: 1.5982\n",
      "Epoch 95/100\n",
      "283/283 [==============================] - 226s 799ms/step - loss: 1.4215 - val_loss: 1.6001\n",
      "Epoch 96/100\n",
      "283/283 [==============================] - 226s 797ms/step - loss: 1.4213 - val_loss: 1.5962\n",
      "Epoch 97/100\n",
      "283/283 [==============================] - 225s 796ms/step - loss: 1.4136 - val_loss: 1.6013\n",
      "Epoch 98/100\n",
      "283/283 [==============================] - 225s 797ms/step - loss: 1.4180 - val_loss: 1.5997\n",
      "Epoch 99/100\n",
      "283/283 [==============================] - 226s 798ms/step - loss: 1.4150 - val_loss: 1.6044\n",
      "Epoch 100/100\n",
      "283/283 [==============================] - 227s 801ms/step - loss: 1.4162 - val_loss: 1.6004\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x28f8cd957c0>"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X,y,batch_size=128,epochs=100,validation_split=0.2,shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model.save('model_crossover_50_epochs_shuffle_false.h5')\n",
    "\n",
    "#model.save('model_crossover_100_gru_256_harry_percy.h5')\n",
    "\n",
    "#model = load_model('model_crossover_50_gru_256_harry_percy.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sample(preds,temperature=0.4):\n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    \n",
    "    preds = np.log(preds)/temperature # Here temperature acts like a amplifying factor. Lower value will amplify the\n",
    "                                      # most probable character and it will come again and again\n",
    "    exp_preds = np.exp(preds)\n",
    "    \n",
    "    exp_preds = exp_preds/np.sum(exp_preds)\n",
    "    \n",
    "    probas = np.random.multinomial(1,exp_preds,1)\n",
    "    \n",
    "    return np.argmax(probas)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.nn import softmax\n",
    "\n",
    "# def topK(preds,k=5):\n",
    "#     preds = np.asarray(preds).astype('float64')\n",
    "\n",
    "#     top_k_probs, top_k_indices = tf.math.top_k(preds, k=k, sorted=True)\n",
    "#     # will return an sorted array containing the probs of the top-k elements & the indices\n",
    "    \n",
    "#     top_k_probs_redistributed = np.asarray(softmax(top_k_probs)).astype(\"float\")\n",
    "    \n",
    "#     sampled_token = np.random.choice(top_k_indices, p=top_k_probs_redistributed)\n",
    "\n",
    "#     return sampled_token\n",
    "\n",
    "def topK(predictions, k=5):\n",
    "    top_k_probabilities, top_k_indices= tf.math.top_k(predictions, k=k, sorted=True)\n",
    "    top_k_indices = np.asarray(top_k_indices).astype(\"int32\")\n",
    "    top_k_redistributed_probability=softmax(np.log(top_k_probabilities))\n",
    "    top_k_redistributed_probability = np.asarray(top_k_redistributed_probability).astype(\"float32\")\n",
    "    sampled_token = np.random.choice(top_k_indices, p=top_k_redistributed_probability)\n",
    "    return sampled_token\n",
    "\n",
    "\n",
    "def topP(preds,p=.9):\n",
    "    preds = np.asarray(preds).astype('float64')\n",
    "    preds1 = np.copy(preds) \n",
    "    x,y=[],[] # x will contain indices, y will have values\n",
    "    small=True \n",
    "    while small==True:\n",
    "        p1 = np.argmax(preds1) # take index of highest value\n",
    "        x.append(p1) # append index of highest value\n",
    "        y.append(preds1[p1]) # append value of highest value\n",
    "        if sum(y) > p: \n",
    "            x=x[:-1] # if sum became greater than the threshold, just remoce the last values of x & y\n",
    "            y=y[:-1]\n",
    "            small=False\n",
    "        preds1[p1]=-1000000000000000000000000000000000\n",
    "    if len(x)>0 : \n",
    "        top_k_probs_redistributed = np.asarray(softmax(y)).astype(\"float\")\n",
    "    \n",
    "        sampled_token = np.random.choice(x, p=top_k_probs_redistributed)    \n",
    "        return sampled_token\n",
    "    else :\n",
    "        return np.argmax(preds)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## DYNAMIC TEMPERATURE SAMPLING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zeus did indeed feed kronor a mixture of mustard and wine which made him disgorge his other five children the books of the screaming at the only came to say band his had been his had a got on the first the said the fruit stand and started no look the saw and on shoot something a motorcycle Vernon and the little school Nancy benefit made me was all over the started to say in a very looked and a little was tell miss or Gabe a silly under. I wanted about the Nancy bob for he didn't be said band and strutting his rears instead he was to the end the book to he looked all over the started the cat the bark, and he wasn't be a street secs the said of the name in he couldn't be the suppose so Mrs. Dodds was the cat had been the sign they couldn't be a friend of the front start Dudley was the notice he was notes band and was when he had a landed to the on the stare the sign the strange and the bout to say, but it was before I had talking out of she said professor McGonagall got me the light the school but I was surprised it was the first the that he couldn't be said professor McGonagall the fruit starting the Dursley had a man and the might be and tried down he could the Dursley had a mother Mrs. Dodds said and down the cat one school in the EWS of his books and the cat to say to see the and and Grover and the stare and I was a look to say bother a landed at the bout what he was story the books a see that seen in the greet seating it was started to me. I was to Mon't be now because he was with the book and but I was the Dursley head proving harry with a good all over the Dursley his and Dudley door and the cat her under he looked in the glass said professor McGonagall pain that of the small and a fell my mom harry be in the fruit Percy back one a one that screwy didn't been she said Dudley Dan't he couldn't bean Mrs. Dudley had been a said and the other had a wasn't say the stare, and she said the master place had the maturing in the long that harry was the stare school in he was the stare things they could tell harry we could try to say back and the f.\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "#generate = \"He was provoked Professor Snape said Hagrid sticking his huge hairy face out from behind the tree.\"\n",
    "generate = \"Zeus did indeed feed Kronos a mixture of mustard and wine which made him disgorge his other five children\"\n",
    "generate = generate + ' '*(maxlen - len(generate)) # To make it as the same length\n",
    "generate = generate.lower()\n",
    "\n",
    "result = generate\n",
    "\n",
    "for i in range(2000):\n",
    "    sampled = np.zeros((1,maxlen,len_chars))\n",
    "    \n",
    "    for t,char in enumerate(generate):\n",
    "        sampled[0,t,char_indices[char]] = 1\n",
    "    \n",
    "    preds = model.predict(sampled)[0] \n",
    "    temperature = np.random.normal(0.5,0.1)\n",
    "    next_char = chars[sample(preds,temperature)]\n",
    "    generate += next_char\n",
    "    result += next_char\n",
    "    generate = generate[1:] # sliding window type. So you slide one step to the right and repeat\n",
    "    \n",
    "\n",
    "tool = language_tool_python.LanguageTool('en-US')\n",
    "result_clean = tool.correct(result)\n",
    "\n",
    "print(result_clean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TOP K SAMPLING (k=7) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zeus did indeed feed kronor a mixture of mustard and wine which made him disgorge his other five children about the small and a letter a stow his making of a ho went his one wrong. At the books his none of the Dursley a Food it was had a bit he'll be seen so scraps we tell him but the small. He sits the looked a little was her in I have a women range. Dudley of that all her cup she looked in the sudden the door there was said. I was the close a bold and the could gen the sign to tell the Food light. They made ted it was had being of the only of the said then a cool when he wanted to bead him. Nake main body would sea mast brand. The pitching a big said her up wet uncle Vernon seen he was the said part on the bad of the cat something and stared back on the call to there were the bus and tell the attended her like he looked it the like and get the cure cat had and storms the looked out. This Lady of the book to be a mound which really and she Dudley would he potter went talking old dating this ALD the cat a strong that with a harry the book a dare and the follows allowed i the they time. A fly his ladies the writing bowel we here be the the month. Thes bad a stupid when his beat about the took the table or CLAS station stranger held bad inst head to be as tally and he past and Hagrid get. My making out I want and street Dudley ARD the lightning the studies to lead back into his nothing in him. And crowed. Me front ideation Nancy book and grave the server the news world so mall hag ink his body wet me. him won't as a goo turning about the said back to her harry he thought to met harry little which wet about in the Dursley leading to be folded of cursed her in the bull, and he said professor. Shas went that was but they brent. At she wasn't like a moment but they could turn't be no called harry bard to and throw were state. Scards to a land and all she let to the fruit his here ALD me or good on she like moment it ate and they couldn't be a little mad to be of they wanted to have to be he looked under the dross still she wasn't think to be and how h.\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "#generate = \"He was provoked Professor Snape said Hagrid sticking his huge hairy face out from behind the tree.\"\n",
    "generate = \"Zeus did indeed feed Kronos a mixture of mustard and wine which made him disgorge his other five children\"\n",
    "generate = generate + ' '*(maxlen - len(generate)) # To make it as the same length\n",
    "generate = generate.lower()\n",
    "\n",
    "result = generate\n",
    "\n",
    "for i in range(2000):\n",
    "    sampled = np.zeros((1,maxlen,len_chars))\n",
    "    \n",
    "    for t,char in enumerate(generate):\n",
    "        sampled[0,t,char_indices[char]] = 1\n",
    "    \n",
    "    preds = model.predict(sampled)[0] \n",
    "    next_char = chars[topK(preds,7)]\n",
    "    generate += next_char\n",
    "    result += next_char\n",
    "    generate = generate[1:] # sliding window type. So you slide one step to the right and repeat\n",
    "    \n",
    "\n",
    "tool = language_tool_python.LanguageTool('en-US')\n",
    "result_clean = tool.correct(result)\n",
    "\n",
    "print(result_clean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TOP P SAMPLING (p = 0.9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Zeus did indeed feed kronor a mixture of mustard and wine which made him disgorge his other five children he as though its fave it I couldn't long but the but sibling all meters. Feal you want of the math about a segment it looked legal Gabe band't he was poached SNEP which he didn't her out was snake look was for old day on the Olympia. They reader moves. For the school do books up the lad, but you looked to know Mr. funner here peeped by fame ray. Selestars in Mrs. Dudley married to Dudley boy for the sown in came until smell he couldn't too longs pig him in all punter had in one a class. Wer moss if as every snake back world being spent ion Percy just from the it in Mr. Dursley was now mother non his scratch could standing ALD coming as walking which school you could be over seller began. We are outside me for mastermind new indie and the boy area on the bass prove Mrs. Dodds from his pickers Dudley odds that bought sick friend be the fairly it met ricing he mane of anything me and Cornish what each and old every the dishes out Mrs. Dursley later looking other click for been done no lottery the muggle poor SE looked his rare talking he carried out stirred or make on a had noting his pardon of the out as he makes a. out what been she living of make he but led. Mr. Brunner and neat trim dance lase. Harry had happening of cans. Class where been no older. A goon fitirtunin. People little which CLAS know when he head lotus my person but stirs. The Dursley who Sam might nor gold new you handy's haggling in she couldn't few on Mrs. For stated body fact poetic over heart as the events it's this he musters much again happy man of she penlight back of echo bowing Norfolk to fill back and who said back of parsing petunia the buss. Out of looking mash lighten it's as while party he parted at over for Mr. a mission a with smell now in a good my sent. As he pads sand in conically her and get the grants harry watched the top. Of chaplet to about was compared out with us in for good of the Nancy the wears upper from the gale with unusual you no life wetting out in a.\n"
     ]
    }
   ],
   "source": [
    "np.random.seed(42)\n",
    "\n",
    "#generate = \"He was provoked Professor Snape said Hagrid sticking his huge hairy face out from behind the tree.\"\n",
    "generate = \"Zeus did indeed feed Kronos a mixture of mustard and wine which made him disgorge his other five children\"\n",
    "generate = generate + ' '*(maxlen - len(generate)) # To make it as the same length\n",
    "generate = generate.lower()\n",
    "\n",
    "result = generate\n",
    "\n",
    "for i in range(2000):\n",
    "    sampled = np.zeros((1,maxlen,len_chars))\n",
    "    \n",
    "    for t,char in enumerate(generate):\n",
    "        sampled[0,t,char_indices[char]] = 1\n",
    "    \n",
    "    preds = model.predict(sampled)[0] \n",
    "    next_char = chars[topP(preds)]\n",
    "    generate += next_char\n",
    "    result += next_char\n",
    "    generate = generate[1:] # sliding window type. So you slide one step to the right and repeat\n",
    "    \n",
    "\n",
    "tool = language_tool_python.LanguageTool('en-US')\n",
    "result_clean = tool.correct(result)\n",
    "\n",
    "print(result_clean)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}