lab-12-5-seq-to-seq-keras-eager.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 12-5 sequence to sequence (Keras + eager version)\n",
    "\n",
    "### simple neural machine translation training\n",
    "\n",
    "* sequence to sequence\n",
    "  \n",
    "### Reference\n",
    "* [Sequence to Sequence Learning with Neural Networks](https://arxiv.org/abs/1409.3215)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tnxXKDjq3jEL"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "# Import TensorFlow >= 1.10 and enable eager execution\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "from matplotlib import font_manager, rc\n",
    "\n",
    "rc('font', family='AppleGothic') #for mac\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tensorflow import keras\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "from pprint import pprint\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "sources = [['I', 'feel', 'hungry'],\n",
    "     ['tensorflow', 'is', 'very', 'difficult'],\n",
    "     ['tensorflow', 'is', 'a', 'framework', 'for', 'deep', 'learning'],\n",
    "     ['tensorflow', 'is', 'very', 'fast', 'changing']]\n",
    "targets = [['나는', '배가', '고프다'],\n",
    "           ['텐서플로우는', '매우', '어렵다'],\n",
    "           ['텐서플로우는', '딥러닝을', '위한', '프레임워크이다'],\n",
    "           ['텐서플로우는', '매우', '빠르게', '변화한다']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<pad>': 0,\n",
      " 'I': 1,\n",
      " 'a': 2,\n",
      " 'changing': 3,\n",
      " 'deep': 4,\n",
      " 'difficult': 5,\n",
      " 'fast': 6,\n",
      " 'feel': 7,\n",
      " 'for': 8,\n",
      " 'framework': 9,\n",
      " 'hungry': 10,\n",
      " 'is': 11,\n",
      " 'learning': 12,\n",
      " 'tensorflow': 13,\n",
      " 'very': 14}\n"
     ]
    }
   ],
   "source": [
    "# vocabulary for sources\n",
    "s_vocab = list(set(sum(sources, [])))\n",
    "s_vocab.sort()\n",
    "s_vocab = ['<pad>'] + s_vocab\n",
    "source2idx = {word : idx for idx, word in enumerate(s_vocab)}\n",
    "idx2source = {idx : word for idx, word in enumerate(s_vocab)}\n",
    "\n",
    "pprint(source2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'<bos>': 1,\n",
      " '<eos>': 2,\n",
      " '<pad>': 0,\n",
      " '고프다': 3,\n",
      " '나는': 4,\n",
      " '딥러닝을': 5,\n",
      " '매우': 6,\n",
      " '배가': 7,\n",
      " '변화한다': 8,\n",
      " '빠르게': 9,\n",
      " '어렵다': 10,\n",
      " '위한': 11,\n",
      " '텐서플로우는': 12,\n",
      " '프레임워크이다': 13}\n"
     ]
    }
   ],
   "source": [
    "# vocabulary for targets\n",
    "t_vocab = list(set(sum(targets, [])))\n",
    "t_vocab.sort()\n",
    "t_vocab = ['<pad>', '<bos>', '<eos>'] + t_vocab\n",
    "target2idx = {word : idx for idx, word in enumerate(t_vocab)}\n",
    "idx2target = {idx : word for idx, word in enumerate(t_vocab)}\n",
    "\n",
    "pprint(target2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(sequences, max_len, dic, mode = 'source'):\n",
    "    assert mode in ['source', 'target'], 'source와 target 중에 선택해주세요.'\n",
    "    \n",
    "    if mode == 'source':\n",
    "        # preprocessing for source (encoder)\n",
    "        s_input = list(map(lambda sentence : [dic.get(token) for token in sentence], sequences))\n",
    "        s_len = list(map(lambda sentence : len(sentence), s_input))\n",
    "        s_input = pad_sequences(sequences = s_input, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        return s_len, s_input\n",
    "    \n",
    "    elif mode == 'target':\n",
    "        # preprocessing for target (decoder)\n",
    "        # input\n",
    "        t_input = list(map(lambda sentence : ['<bos>'] + sentence + ['<eos>'], sequences))\n",
    "        t_input = list(map(lambda sentence : [dic.get(token) for token in sentence], t_input))\n",
    "        t_len = list(map(lambda sentence : len(sentence), t_input))\n",
    "        t_input = pad_sequences(sequences = t_input, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        \n",
    "        # output\n",
    "        t_output = list(map(lambda sentence : sentence + ['<eos>'], sequences))\n",
    "        t_output = list(map(lambda sentence : [dic.get(token) for token in sentence], t_output))\n",
    "        t_output = pad_sequences(sequences = t_output, maxlen = max_len, padding = 'post', truncating = 'post')\n",
    "        \n",
    "        return t_len, t_input, t_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 4, 7, 5] [[ 1  7 10  0  0  0  0  0  0  0]\n",
      " [13 11 14  5  0  0  0  0  0  0]\n",
      " [13 11  2  9  8  4 12  0  0  0]\n",
      " [13 11 14  6  3  0  0  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# preprocessing for source\n",
    "s_max_len = 10\n",
    "s_len, s_input = preprocess(sequences = sources,\n",
    "                            max_len = s_max_len, dic = source2idx, mode = 'source')\n",
    "print(s_len, s_input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 5, 6, 6] [[ 1  4  7  3  2  0  0  0  0  0  0  0]\n",
      " [ 1 12  6 10  2  0  0  0  0  0  0  0]\n",
      " [ 1 12  5 11 13  2  0  0  0  0  0  0]\n",
      " [ 1 12  6  9  8  2  0  0  0  0  0  0]] [[ 4  7  3  2  0  0  0  0  0  0  0  0]\n",
      " [12  6 10  2  0  0  0  0  0  0  0  0]\n",
      " [12  5 11 13  2  0  0  0  0  0  0  0]\n",
      " [12  6  9  8  2  0  0  0  0  0  0  0]]\n"
     ]
    }
   ],
   "source": [
    "# preprocessing for target\n",
    "t_max_len = 12\n",
    "t_len, t_input, t_output = preprocess(sequences = targets,\n",
    "                                      max_len = t_max_len, dic = target2idx, mode = 'target')\n",
    "print(t_len, t_input, t_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# hyper-param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hyper-parameters\n",
    "epochs = 100\n",
    "batch_size = 4\n",
    "learning_rate = .005\n",
    "total_step = epochs / batch_size\n",
    "buffer_size = 100\n",
    "n_batch = buffer_size//batch_size\n",
    "embedding_dim = 32\n",
    "units = 32\n",
    "\n",
    "# input\n",
    "data = tf.data.Dataset.from_tensor_slices((s_len, s_input, t_len, t_input, t_output))\n",
    "data = data.shuffle(buffer_size = buffer_size)\n",
    "data = data.batch(batch_size = batch_size)\n",
    "# s_mb_len, s_mb_input, t_mb_len, t_mb_input, t_mb_output = iterator.get_next()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gru(units):\n",
    "  # If you have a GPU, we recommend using CuDNNGRU(provides a 3x speedup than GRU)\n",
    "  # the code automatically does that.\n",
    "    if tf.test.is_gpu_available():\n",
    "        return tf.keras.layers.CuDNNGRU(units, \n",
    "                                        return_sequences=True, \n",
    "                                        return_state=True, \n",
    "                                        recurrent_initializer='glorot_uniform')\n",
    "    else:\n",
    "        return tf.keras.layers.GRU(units, \n",
    "                                   return_sequences=True, \n",
    "                                   return_state=True, \n",
    "                                   recurrent_activation='sigmoid', \n",
    "                                   recurrent_initializer='glorot_uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.enc_units)\n",
    "        \n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "        \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.dec_units)\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "                \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        \n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "        \n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        \n",
    "        # output shape == (batch_size * 1, vocab)\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, state\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.dec_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder(len(source2idx), embedding_dim, units, batch_size)\n",
    "decoder = Decoder(len(target2idx), embedding_dim, units, batch_size)\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = 1 - np.equal(real, 0)\n",
    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
    "    \n",
    "#     print(\"real: {}\".format(real))\n",
    "#     print(\"pred: {}\".format(pred))\n",
    "#     print(\"mask: {}\".format(mask))\n",
    "#     print(\"loss: {}\".format(tf.reduce_mean(loss_)))\n",
    "    \n",
    "    return tf.reduce_mean(loss_)\n",
    "\n",
    "# creating optimizer\n",
    "optimizer = tf.train.AdamOptimizer()\n",
    "\n",
    "# creating check point (Object-based saving)\n",
    "checkpoint_dir = './data_out/training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, 'ckpt')\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                encoder=encoder,\n",
    "                                decoder=decoder)\n",
    "\n",
    "# create writer for tensorboard\n",
    "summary_writer = tf.contrib.summary.create_file_writer(logdir=checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0 Loss 0.0396 Batch Loss 0.9894\n",
      "Epoch 10 Loss 0.0387 Batch Loss 0.9682\n",
      "Epoch 20 Loss 0.0375 Batch Loss 0.9379\n",
      "Epoch 30 Loss 0.0353 Batch Loss 0.8830\n",
      "Epoch 40 Loss 0.0315 Batch Loss 0.7864\n",
      "Epoch 50 Loss 0.0281 Batch Loss 0.7034\n",
      "Epoch 60 Loss 0.0249 Batch Loss 0.6222\n",
      "Epoch 70 Loss 0.0221 Batch Loss 0.5528\n",
      "Epoch 80 Loss 0.0195 Batch Loss 0.4887\n",
      "Epoch 90 Loss 0.0172 Batch Loss 0.4290\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 100\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    \n",
    "    hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for i, (s_len, s_input, t_len, t_input, t_output) in enumerate(data):\n",
    "        loss = 0\n",
    "        with tf.GradientTape() as tape:\n",
    "            enc_output, enc_hidden = encoder(s_input, hidden)\n",
    "\n",
    "            dec_hidden = enc_hidden\n",
    "            \n",
    "            dec_input = tf.expand_dims([target2idx['<bos>']] * batch_size, 1)\n",
    "            \n",
    "            #Teacher Forcing: feeding the target as the next input\n",
    "            for t in range(1, t_input.shape[1]):\n",
    "                \n",
    "                predictions, dec_hidden = decoder(dec_input, dec_hidden, enc_output)\n",
    "                \n",
    "                loss += loss_function(t_input[:, t], predictions)\n",
    "            \n",
    "                dec_input = tf.expand_dims(t_input[:, t], 1) #using teacher forcing\n",
    "                \n",
    "        batch_loss = (loss / int(t_input.shape[1]))\n",
    "        \n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        variables = encoder.variables + decoder.variables\n",
    "        \n",
    "        gradient = tape.gradient(loss, variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(gradient, variables))\n",
    "        \n",
    "    if epoch % 10 == 0:\n",
    "        #save model every 10 epoch\n",
    "        print('Epoch {} Loss {:.4f} Batch Loss {:.4f}'.format(epoch,\n",
    "                                            total_loss / n_batch,\n",
    "                                            batch_loss.numpy()))\n",
    "        checkpoint.save(file_prefix = checkpoint_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.checkpointable.util.CheckpointLoadStatus at 0x1171349b0>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#restore checkpoint\n",
    "\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence = 'I feel hungry'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I feel hungry\n",
      "텐서플로우는 매우 어렵다 <eos> \n"
     ]
    }
   ],
   "source": [
    "def prediction(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    \n",
    "    inputs = [inp_lang[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "        \n",
    "    result = ''\n",
    "    \n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "        \n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang['<bos>']], 0)\n",
    "    \n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden = decoder(dec_input, dec_hidden, enc_out)\n",
    "        \n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += idx2target[predicted_id] + ' '\n",
    "\n",
    "        if idx2target.get(predicted_id) == '<eos>':\n",
    "            return result, sentence\n",
    "        \n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)    \n",
    "    \n",
    "    return result, sentence\n",
    "    \n",
    "result, output_sentence = prediction(sentence, encoder, decoder, source2idx, target2idx, s_max_len, t_max_len)\n",
    "\n",
    "print(sentence)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "nmt_with_attention.ipynb",
   "private_outputs": true,
   "provenance": [
    {
     "file_id": "1C4fpM7_7IL8ZzF7Gc5abywqQjeQNS2-U",
     "timestamp": 1527858391290
    },
    {
     "file_id": "1pExo6aUuw0S6MISFWoinfJv0Ftm9V4qv",
     "timestamp": 1527776041613
    }
   ],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}