seq2seq-chatbot-keras-with-attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import pandas as pd \n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Attention Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "from tensorflow.python.keras.layers import Layer\n",
    "from tensorflow.python.keras import backend as K\n",
    "\n",
    "\n",
    "class AttentionLayer(Layer):\n",
    "    \"\"\"\n",
    "    This class implements Bahdanau attention (https://arxiv.org/pdf/1409.0473.pdf).\n",
    "    There are three sets of weights introduced W_a, U_a, and V_a\n",
    "     \"\"\"\n",
    "\n",
    "    def __init__(self, **kwargs):\n",
    "        super(AttentionLayer, self).__init__(**kwargs)\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        assert isinstance(input_shape, list)\n",
    "        # Create a trainable weight variable for this layer.\n",
    "\n",
    "        self.W_a = self.add_weight(name='W_a',\n",
    "                                   shape=tf.TensorShape((input_shape[0][2], input_shape[0][2])),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "        self.U_a = self.add_weight(name='U_a',\n",
    "                                   shape=tf.TensorShape((input_shape[1][2], input_shape[0][2])),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "        self.V_a = self.add_weight(name='V_a',\n",
    "                                   shape=tf.TensorShape((input_shape[0][2], 1)),\n",
    "                                   initializer='uniform',\n",
    "                                   trainable=True)\n",
    "\n",
    "        super(AttentionLayer, self).build(input_shape)  # Be sure to call this at the end\n",
    "\n",
    "    def call(self, inputs, verbose=False):\n",
    "        \"\"\"\n",
    "        inputs: [encoder_output_sequence, decoder_output_sequence]\n",
    "        \"\"\"\n",
    "        assert type(inputs) == list\n",
    "        encoder_out_seq, decoder_out_seq = inputs\n",
    "        if verbose:\n",
    "            print('encoder_out_seq>', encoder_out_seq.shape)\n",
    "            print('decoder_out_seq>', decoder_out_seq.shape)\n",
    "\n",
    "        def energy_step(inputs, states):\n",
    "            \"\"\" Step function for computing energy for a single decoder state\n",
    "            inputs: (batchsize * 1 * de_in_dim)\n",
    "            states: (batchsize * 1 * de_latent_dim)\n",
    "            \"\"\"\n",
    "\n",
    "            assert_msg = \"States must be an iterable. Got {} of type {}\".format(states, type(states))\n",
    "            assert isinstance(states, list) or isinstance(states, tuple), assert_msg\n",
    "\n",
    "            \"\"\" Some parameters required for shaping tensors\"\"\"\n",
    "            en_seq_len, en_hidden = encoder_out_seq.shape[1], encoder_out_seq.shape[2]\n",
    "            de_hidden = inputs.shape[-1]\n",
    "\n",
    "            \"\"\" Computing S.Wa where S=[s0, s1, ..., si]\"\"\"\n",
    "            # <= batch size * en_seq_len * latent_dim\n",
    "            W_a_dot_s = K.dot(encoder_out_seq, self.W_a)\n",
    "\n",
    "            \"\"\" Computing hj.Ua \"\"\"\n",
    "            U_a_dot_h = K.expand_dims(K.dot(inputs, self.U_a), 1)  # <= batch_size, 1, latent_dim\n",
    "            if verbose:\n",
    "                print('Ua.h>', U_a_dot_h.shape)\n",
    "\n",
    "            \"\"\" tanh(S.Wa + hj.Ua) \"\"\"\n",
    "            # <= batch_size*en_seq_len, latent_dim\n",
    "            Ws_plus_Uh = K.tanh(W_a_dot_s + U_a_dot_h)\n",
    "            if verbose:\n",
    "                print('Ws+Uh>', Ws_plus_Uh.shape)\n",
    "\n",
    "            \"\"\" softmax(va.tanh(S.Wa + hj.Ua)) \"\"\"\n",
    "            # <= batch_size, en_seq_len\n",
    "            e_i = K.squeeze(K.dot(Ws_plus_Uh, self.V_a), axis=-1)\n",
    "            # <= batch_size, en_seq_len\n",
    "            e_i = K.softmax(e_i)\n",
    "\n",
    "            if verbose:\n",
    "                print('ei>', e_i.shape)\n",
    "\n",
    "            return e_i, [e_i]\n",
    "\n",
    "        def context_step(inputs, states):\n",
    "            \"\"\" Step function for computing ci using ei \"\"\"\n",
    "\n",
    "            assert_msg = \"States must be an iterable. Got {} of type {}\".format(states, type(states))\n",
    "            assert isinstance(states, list) or isinstance(states, tuple), assert_msg\n",
    "\n",
    "            # <= batch_size, hidden_size\n",
    "            c_i = K.sum(encoder_out_seq * K.expand_dims(inputs, -1), axis=1)\n",
    "            if verbose:\n",
    "                print('ci>', c_i.shape)\n",
    "            return c_i, [c_i]\n",
    "\n",
    "        fake_state_c = K.sum(encoder_out_seq, axis=1)\n",
    "        fake_state_e = K.sum(encoder_out_seq, axis=2)  # <= (batch_size, enc_seq_len, latent_dim\n",
    "\n",
    "        \"\"\" Computing energy outputs \"\"\"\n",
    "        # e_outputs => (batch_size, de_seq_len, en_seq_len)\n",
    "        last_out, e_outputs, _ = K.rnn(\n",
    "            energy_step, decoder_out_seq, [fake_state_e],\n",
    "        )\n",
    "\n",
    "        \"\"\" Computing context vectors \"\"\"\n",
    "        last_out, c_outputs, _ = K.rnn(\n",
    "            context_step, e_outputs, [fake_state_c],\n",
    "        )\n",
    "\n",
    "        return c_outputs, e_outputs\n",
    "\n",
    "    def compute_output_shape(self, input_shape):\n",
    "        \"\"\" Outputs produced by the layer \"\"\"\n",
    "        return [\n",
    "            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[1][2])),\n",
    "            tf.TensorShape((input_shape[1][0], input_shape[1][1], input_shape[0][1]))\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "lines = open('../input/chatbot-data/cornell movie-dialogs corpus/movie_lines.txt', encoding='utf-8',\n",
    "             errors='ignore').read().split('\\n')\n",
    "\n",
    "convers = open('../input/chatbot-data/cornell movie-dialogs corpus/movie_conversations.txt', encoding='utf-8',\n",
    "             errors='ignore').read().split('\\n')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "304714"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lines)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Data Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "exchn = []\n",
    "for conver in convers:\n",
    "    exchn.append(conver.split(' +++$+++ ')[-1][1:-1].replace(\"'\", \" \").replace(\",\",\"\").split())\n",
    "\n",
    "diag = {}\n",
    "for line in lines:\n",
    "    diag[line.split(' +++$+++ ')[0]] = line.split(' +++$+++ ')[-1]\n",
    "\n",
    "\n",
    "\n",
    "## delete\n",
    "del(lines, convers, conver, line)\n",
    "\n",
    "\n",
    "\n",
    "questions = []\n",
    "answers = []\n",
    "\n",
    "for conver in exchn:\n",
    "    for i in range(len(conver) - 1):\n",
    "        questions.append(diag[conver[i]])\n",
    "        answers.append(diag[conver[i+1]])\n",
    "        \n",
    "        \n",
    "        \n",
    "\n",
    "## delete\n",
    "del(diag, exchn, conver, i)\n",
    "\n",
    "\n",
    "###############################\n",
    "#        max_len = 13         #\n",
    "###############################\n",
    "\n",
    "sorted_ques = []\n",
    "sorted_ans = []\n",
    "for i in range(len(questions)):\n",
    "    if len(questions[i]) < 13:\n",
    "        sorted_ques.append(questions[i])\n",
    "        sorted_ans.append(answers[i])\n",
    "\n",
    "\n",
    "\n",
    "###############################\n",
    "#                             #\n",
    "###############################\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def clean_text(txt):\n",
    "    txt = txt.lower()\n",
    "    txt = re.sub(r\"i'm\", \"i am\", txt)\n",
    "    txt = re.sub(r\"he's\", \"he is\", txt)\n",
    "    txt = re.sub(r\"she's\", \"she is\", txt)\n",
    "    txt = re.sub(r\"that's\", \"that is\", txt)\n",
    "    txt = re.sub(r\"what's\", \"what is\", txt)\n",
    "    txt = re.sub(r\"where's\", \"where is\", txt)\n",
    "    txt = re.sub(r\"\\'ll\", \" will\", txt)\n",
    "    txt = re.sub(r\"\\'ve\", \" have\", txt)\n",
    "    txt = re.sub(r\"\\'re\", \" are\", txt)\n",
    "    txt = re.sub(r\"\\'d\", \" would\", txt)\n",
    "    txt = re.sub(r\"won't\", \"will not\", txt)\n",
    "    txt = re.sub(r\"can't\", \"can not\", txt)\n",
    "    txt = re.sub(r\"[^\\w\\s]\", \"\", txt)\n",
    "    return txt\n",
    "\n",
    "clean_ques = []\n",
    "clean_ans = []\n",
    "\n",
    "for line in sorted_ques:\n",
    "    clean_ques.append(clean_text(line))\n",
    "        \n",
    "for line in sorted_ans:\n",
    "    clean_ans.append(clean_text(line))\n",
    "\n",
    "\n",
    "\n",
    "## delete\n",
    "del(answers, questions, line)\n",
    "\n",
    "\n",
    "###############################\n",
    "#                             #\n",
    "###############################\n",
    "\n",
    "\n",
    "for i in range(len(clean_ans)):\n",
    "    clean_ans[i] = ' '.join(clean_ans[i].split()[:11])\n",
    "\n",
    "\n",
    "\n",
    "###############################\n",
    "#                             #\n",
    "###############################\n",
    "\n",
    "del(sorted_ans, sorted_ques)\n",
    "\n",
    "\n",
    "## trimming\n",
    "clean_ans=clean_ans[:30000]\n",
    "clean_ques=clean_ques[:30000]\n",
    "## delete\n",
    "\n",
    "\n",
    "###  count occurences ###\n",
    "word2count = {}\n",
    "\n",
    "for line in clean_ques:\n",
    "    for word in line.split():\n",
    "        if word not in word2count:\n",
    "            word2count[word] = 1\n",
    "        else:\n",
    "            word2count[word] += 1\n",
    "for line in clean_ans:\n",
    "    for word in line.split():\n",
    "        if word not in word2count:\n",
    "            word2count[word] = 1\n",
    "        else:\n",
    "            word2count[word] += 1\n",
    "\n",
    "## delete\n",
    "del(word, line)\n",
    "\n",
    "\n",
    "###  remove less frequent ###\n",
    "thresh = 5\n",
    "\n",
    "vocab = {}\n",
    "word_num = 0\n",
    "for word, count in word2count.items():\n",
    "    if count >= thresh:\n",
    "        vocab[word] = word_num\n",
    "        word_num += 1\n",
    "        \n",
    "## delete\n",
    "del(word2count, word, count, thresh)       \n",
    "del(word_num)        \n",
    "\n",
    "\n",
    "\n",
    "for i in range(len(clean_ans)):\n",
    "    clean_ans[i] = '<SOS> ' + clean_ans[i] + ' <EOS>'\n",
    "\n",
    "\n",
    "\n",
    "tokens = ['<PAD>', '<EOS>', '<OUT>', '<SOS>']\n",
    "x = len(vocab)\n",
    "for token in tokens:\n",
    "    vocab[token] = x\n",
    "    x += 1\n",
    "    \n",
    "    \n",
    "\n",
    "vocab['cameron'] = vocab['<PAD>']\n",
    "vocab['<PAD>'] = 0\n",
    "\n",
    "## delete\n",
    "del(token, tokens) \n",
    "del(x)\n",
    "\n",
    "### inv answers dict ###\n",
    "inv_vocab = {w:v for v, w in vocab.items()}\n",
    "\n",
    "\n",
    "\n",
    "## delete\n",
    "del(i)\n",
    "\n",
    "\n",
    "\n",
    "encoder_inp = []\n",
    "for line in clean_ques:\n",
    "    lst = []\n",
    "    for word in line.split():\n",
    "        if word not in vocab:\n",
    "            lst.append(vocab['<OUT>'])\n",
    "        else:\n",
    "            lst.append(vocab[word])\n",
    "        \n",
    "    encoder_inp.append(lst)\n",
    "\n",
    "decoder_inp = []\n",
    "for line in clean_ans:\n",
    "    lst = []\n",
    "    for word in line.split():\n",
    "        if word not in vocab:\n",
    "            lst.append(vocab['<OUT>'])\n",
    "        else:\n",
    "            lst.append(vocab[word])        \n",
    "    decoder_inp.append(lst)\n",
    "\n",
    "### delete\n",
    "del(clean_ans, clean_ques, line, lst, word)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "encoder_inp = pad_sequences(encoder_inp, 13, padding='post', truncating='post')\n",
    "decoder_inp = pad_sequences(decoder_inp, 13, padding='post', truncating='post')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "decoder_final_output = []\n",
    "for i in decoder_inp:\n",
    "    decoder_final_output.append(i[1:]) \n",
    "\n",
    "decoder_final_output = pad_sequences(decoder_final_output, 13, padding='post', truncating='post')\n",
    "\n",
    "\n",
    "del(i)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30000, 13) (30000, 13) (30000, 13) 3027 3027 <PAD>\n"
     ]
    }
   ],
   "source": [
    "# decoder_final_output, decoder_final_input, encoder_final, vocab, inv_vocab\n",
    "\n",
    "VOCAB_SIZE = len(vocab)\n",
    "MAX_LEN = 13\n",
    "\n",
    "print(decoder_final_output.shape, decoder_inp.shape, encoder_inp.shape, len(vocab), len(inv_vocab), inv_vocab[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'they'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inv_vocab[16]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(len(decoder_final_input), MAX_LEN, VOCAB_SIZE)\n",
    "#decoder_final_input[0]\n",
    "#decoder_output_data = np.zeros((len(decoder_final_input), MAX_LEN, VOCAB_SIZE), dtype=\"float32\")\n",
    "#print(decoder_output_data.shape)\n",
    "#decoder_final_input[80]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.utils import to_categorical\n",
    "decoder_final_output = to_categorical(decoder_final_output, len(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(30000, 13, 3027)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "decoder_final_output.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Glove Embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Glove Loded!\n"
     ]
    }
   ],
   "source": [
    "\n",
    "embeddings_index = {}\n",
    "with open('../input/glove6b50d/glove.6B.50d.txt', encoding='utf-8') as f:\n",
    "    for line in f:\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        coefs = np.asarray(values[1:], dtype='float32')\n",
    "        embeddings_index[word] = coefs\n",
    "    f.close()\n",
    "\n",
    "print(\"Glove Loded!\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "embedding_dimention = 50\n",
    "def embedding_matrix_creater(embedding_dimention, word_index):\n",
    "    embedding_matrix = np.zeros((len(word_index)+1, embedding_dimention))\n",
    "    for word, i in word_index.items():\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None:\n",
    "          # words not found in embedding index will be all-zeros.\n",
    "            embedding_matrix[i] = embedding_vector\n",
    "    return embedding_matrix\n",
    "embedding_matrix = embedding_matrix_creater(50, word_index=vocab)    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "del(embeddings_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3028, 50)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embedding_matrix[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.layers import Dense, Embedding, LSTM, Input, Bidirectional, Concatenate, Dropout, Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "embed = Embedding(VOCAB_SIZE+1, \n",
    "                  50, \n",
    "                  \n",
    "                  input_length=13,\n",
    "                  trainable=True)\n",
    "\n",
    "embed.build((None,))\n",
    "embed.set_weights([embedding_matrix])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc_inp = Input(shape=(13, ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_2 (InputLayer)            [(None, 13)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_1 (InputLayer)            [(None, 13)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding (Embedding)           (None, 13, 50)       151400      input_1[0][0]                    \n",
      "                                                                 input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional (Bidirectional)   [(None, 13, 800), (N 1443200     embedding[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 800)          0           bidirectional[0][1]              \n",
      "                                                                 bidirectional[0][3]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 800)          0           bidirectional[0][2]              \n",
      "                                                                 bidirectional[0][4]              \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   [(None, 13, 800), (N 2723200     embedding[1][0]                  \n",
      "                                                                 concatenate[0][0]                \n",
      "                                                                 concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "attention_layer (AttentionLayer ((None, 13, 800), (N 1280800     bidirectional[0][0]              \n",
      "                                                                 lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 13, 1600)     0           lstm_1[0][0]                     \n",
      "                                                                 attention_layer[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 13, 3027)     4846227     concatenate_2[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 10,444,827\n",
      "Trainable params: 10,444,827\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#embed = Embedding(VOCAB_SIZE+1, 50, mask_zero=True, input_length=13)(enc_inp)\n",
    "enc_embed = embed(enc_inp)\n",
    "enc_lstm = Bidirectional(LSTM(400, return_state=True, dropout=0.05, return_sequences = True))\n",
    "\n",
    "encoder_outputs, forward_h, forward_c, backward_h, backward_c = enc_lstm(enc_embed)\n",
    "\n",
    "state_h = Concatenate()([forward_h, backward_h])\n",
    "state_c = Concatenate()([forward_c, backward_c])\n",
    "\n",
    "enc_states = [state_h, state_c]\n",
    "\n",
    "\n",
    "dec_inp = Input(shape=(13, ))\n",
    "dec_embed = embed(dec_inp)\n",
    "dec_lstm = LSTM(400*2, return_state=True, return_sequences=True, dropout=0.05)\n",
    "output, _, _ = dec_lstm(dec_embed, initial_state=enc_states)\n",
    "\n",
    "# attention\n",
    "attn_layer = AttentionLayer()\n",
    "attn_op, attn_state = attn_layer([encoder_outputs, output])\n",
    "decoder_concat_input = Concatenate(axis=-1)([output, attn_op])\n",
    "\n",
    "\n",
    "dec_dense = Dense(VOCAB_SIZE, activation='softmax')\n",
    "final_output = dec_dense(decoder_concat_input)\n",
    "\n",
    "model = Model([enc_inp, dec_inp], final_output)\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import keras\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['acc'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1063/1063 [==============================] - 54s 50ms/step - loss: 2.8887 - acc: 0.5147 - val_loss: 2.6757 - val_acc: 0.5333\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x7ff0d889ec90>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit([encoder_inp, decoder_inp], decoder_final_output, epochs=1, batch_size=24, validation_split=0.15)"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# inferece"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('chatbot.h5')\n",
    "model.save_weights('chatbot_weights.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "execution_count": null,
   "metadata": {},
   "source": [
    "# Attention Inference\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "enc_model = tf.keras.models.Model(enc_inp, [encoder_outputs, enc_states])\n",
    "\n",
    "\n",
    "decoder_state_input_h = tf.keras.layers.Input(shape=( 400 * 2,))\n",
    "decoder_state_input_c = tf.keras.layers.Input(shape=( 400 * 2,))\n",
    "\n",
    "decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]\n",
    "\n",
    "\n",
    "decoder_outputs, state_h, state_c = dec_lstm(dec_embed , initial_state=decoder_states_inputs)\n",
    "\n",
    "\n",
    "decoder_states = [state_h, state_c]\n",
    "\n",
    "#decoder_output = dec_dense(decoder_outputs)\n",
    "\n",
    "dec_model = tf.keras.models.Model([dec_inp, decoder_states_inputs],\n",
    "                                      [decoder_outputs] + decoder_states)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "##########################################\n",
      "#       start chatting ver. 1.0          #\n",
      "##########################################\n"
     ]
    },
    {
     "ename": "StdinNotImplementedError",
     "evalue": "raw_input was called, but this frontend does not support input requests.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mStdinNotImplementedError\u001b[0m                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-25-a7c003105828>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mprepro1\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'q'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mprepro1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"you : \"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     10\u001b[0m     \u001b[0mprepro\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mprepro1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/lib/python3.7/site-packages/ipykernel/kernelbase.py\u001b[0m in \u001b[0;36mraw_input\u001b[0;34m(self, prompt)\u001b[0m\n\u001b[1;32m    853\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_allow_stdin\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    854\u001b[0m             raise StdinNotImplementedError(\n\u001b[0;32m--> 855\u001b[0;31m                 \u001b[0;34m\"raw_input was called, but this frontend does not support input requests.\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    856\u001b[0m             )\n\u001b[1;32m    857\u001b[0m         return self._input_request(str(prompt),\n",
      "\u001b[0;31mStdinNotImplementedError\u001b[0m: raw_input was called, but this frontend does not support input requests."
     ]
    }
   ],
   "source": [
    "print(\"##########################################\")\n",
    "print(\"#       start chatting ver. 1.0          #\")\n",
    "print(\"##########################################\")\n",
    "\n",
    "\n",
    "prepro1 = \"\"\n",
    "while prepro1 != 'q':\n",
    "    \n",
    "    prepro1 = input(\"you : \")\n",
    "    prepro = [prepro1]\n",
    "    \n",
    "    try:\n",
    "        txt = []\n",
    "        for x in prepro:\n",
    "            lst = []\n",
    "            for y in x.split():\n",
    "                lst.append(vocab[y])\n",
    "            txt.append(lst)\n",
    "        txt = pad_sequences(txt, 13, padding='post')\n",
    "\n",
    "\n",
    "        ###\n",
    "        enc_op, stat = enc_model.predict( txt )\n",
    "\n",
    "        empty_target_seq = np.zeros( ( 1 , 1) )\n",
    "        empty_target_seq[0, 0] = vocab['<SOS>']\n",
    "        stop_condition = False\n",
    "        decoded_translation = ''\n",
    "\n",
    "\n",
    "        while not stop_condition :\n",
    "\n",
    "            dec_outputs , h , c = dec_model.predict([ empty_target_seq ] + stat )\n",
    "\n",
    "            ###\n",
    "            ###########################\n",
    "            attn_op, attn_state = attn_layer([enc_op, dec_outputs])\n",
    "            decoder_concat_input = Concatenate(axis=-1)([dec_outputs, attn_op])\n",
    "            decoder_concat_input = dec_dense(decoder_concat_input)\n",
    "            ###########################\n",
    "\n",
    "            sampled_word_index = np.argmax( decoder_concat_input[0, -1, :] )\n",
    "\n",
    "            sampled_word = inv_vocab[sampled_word_index] + ' '\n",
    "\n",
    "            if sampled_word != '<EOS> ':\n",
    "                decoded_translation += sampled_word           \n",
    "\n",
    "\n",
    "            if sampled_word == '<EOS> ' or len(decoded_translation.split()) > 13:\n",
    "                stop_condition = True\n",
    "\n",
    "            empty_target_seq = np.zeros( ( 1 , 1 ) )  \n",
    "            empty_target_seq[ 0 , 0 ] = sampled_word_index\n",
    "            stat = [ h , c ] \n",
    "    except:\n",
    "        pass\n",
    "\n",
    "    print(\"chatbot attention : \", decoded_translation )\n",
    "    print(\"==============================================\")\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}