Model Architecture and Training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import json\n",
    "import collections\n",
    "import pickle\n",
    "import time\n",
    "import os\n",
    "import cv2\n",
    "import gensim\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.applications.resnet50 import ResNet50,preprocess_input,decode_predictions\n",
    "from tensorflow.keras.preprocessing import image\n",
    "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
    "from tensorflow.keras.layers import Dense,Flatten,Embedding,Dropout,Input,LSTM,add\n",
    "from tensorflow.keras.models import Model,load_model\n",
    "from tensorflow.keras.utils import to_categorical \n",
    "#%xmode verbose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Loading all descriptions\n",
    "with open('./resources/descriptions.txt', 'r') as f:\n",
    "    descriptions = f.read()\n",
    "descriptions = json.loads(descriptions)\n",
    "\n",
    "## Loading vocab\n",
    "with open('./resources/vocab.pkl', 'rb') as f:\n",
    "    vocab = pickle.load(f)\n",
    "\n",
    "## Loading training descriptions    \n",
    "with open('./resources/train_descriptions.txt','r') as f:\n",
    "    train_descriptions=f.read()\n",
    "train_descriptions = json.loads(train_descriptions)\n",
    "\n",
    "## Loading word to index mapping\n",
    "with open('./resources/word2idx.pkl','rb') as f:\n",
    "    word2idx=pickle.load(f)\n",
    "\n",
    "## Loading index to word mapping\n",
    "with open('./resources/idx2word.pkl','rb') as f:\n",
    "    idx2word=pickle.load(f)\n",
    "\n",
    "## Loading ResNet50 training image features \n",
    "with open(\"./resources/encoded_train_images.pkl\",\"rb\") as f:\n",
    "    encoded_train_images = pickle.load(f)\n",
    "    \n",
    "## Loading ResNet50 testing image features     \n",
    "# with open(\"./resources/encoded_test_images.pkl\",\"rb\") as f:    \n",
    "#     encoded_test_images = pickle.load(f)\n",
    "\n",
    "## Loading Glove embeddings for all words    \n",
    "with open('./resources/embedding_matrix.pkl','rb') as f:\n",
    "     embedding_matrix = pickle.load(f)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84\n"
     ]
    }
   ],
   "source": [
    "## Getting max length of a caption from the training captions\n",
    "maxlen=0\n",
    "for item in train_descriptions.keys():\n",
    "    for cap in train_descriptions[item]:\n",
    "        if len(cap.split()) > maxlen:\n",
    "            maxlen=len(cap.split())\n",
    "print(maxlen)            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_data(train_descriptions,encoded_train,word2idx,max_len,batch_size):\n",
    "    x1=[]\n",
    "    x2=[]\n",
    "    y=[]\n",
    "    n=0\n",
    "    while True:\n",
    "        for img,cap_list in train_descriptions.items():\n",
    "            n+=1\n",
    "            photo = encoded_train[img]\n",
    "            for cap in cap_list:\n",
    "                seq = [word2idx[word] for word in cap.split() if word in word2idx]\n",
    "                #print(seq)\n",
    "                for i in range(1,len(seq)):\n",
    "                    xi = seq[:i]\n",
    "                    yi = seq[i]\n",
    "                    \n",
    "                    xi = pad_sequences([xi],maxlen=max_len,value=0,padding='post')[0]\n",
    "                    yi = to_categorical([yi],num_classes=len(word2idx)+1)[0]\n",
    "                    \n",
    "                    x1.append(photo)\n",
    "                    x2.append(xi)\n",
    "                    y.append(yi)       \n",
    "            if n==batch_size:\n",
    "                yield ([np.array(x1),np.array(x2)],np.array(y))\n",
    "            \n",
    "                x1=[]\n",
    "                x2=[]\n",
    "                y=[]\n",
    "                n=0\n",
    "#batch1=generate_data(train_descriptions,encoded_train_images,word2idx,maxlen,8)                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5176\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(embedding_matrix)\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Architecture "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Image \n",
    "input_image_features = Input(shape=(2048,))\n",
    "inp1 = Dropout(0.3)(input_image_features)\n",
    "inp2 = Dense(256,activation='relu')(inp1)\n",
    "\n",
    "## Caption\n",
    "\n",
    "input_captions = Input(shape=(maxlen,))\n",
    "cap1 = Embedding(input_dim=vocab_size,output_dim=50,mask_zero=True)(input_captions)\n",
    "cap2 = Dropout(0.3)(cap1)\n",
    "cap3 = LSTM(256)(cap2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_2\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_6 (InputLayer)            [(None, 84)]         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_5 (InputLayer)            [(None, 2048)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_2 (Embedding)         (None, 84, 50)       258800      input_6[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 2048)         0           input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)             (None, 84, 50)       0           embedding_2[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "dense_6 (Dense)                 (None, 256)          524544      dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "unified_lstm_2 (UnifiedLSTM)    (None, 256)          314368      dropout_5[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "add_2 (Add)                     (None, 256)          0           dense_6[0][0]                    \n",
      "                                                                 unified_lstm_2[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dense_7 (Dense)                 (None, 256)          65792       add_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dense_8 (Dense)                 (None, 5176)         1330232     dense_7[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 2,493,736\n",
      "Trainable params: 2,493,736\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "decoder1 = add([inp2,cap3])\n",
    "decoder2 = Dense(256,activation='relu')(decoder1)\n",
    "outputs = Dense(vocab_size,activation='softmax')(decoder2)\n",
    "\n",
    "image_caption_model = Model(inputs=[input_image_features,input_captions],outputs=outputs)\n",
    "image_caption_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "image_caption_model.layers[2].set_weights([embedding_matrix])\n",
    "image_caption_model.layers[2].trainable=False\n",
    "\n",
    "image_caption_model.compile(loss='categorical_crossentropy',optimizer='adam')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 1\n",
    "batch_size = 32\n",
    "steps = len(train_descriptions)//batch_size\n",
    "\n",
    "def train():\n",
    "    for i in range(epochs):\n",
    "        generator = generate_data(train_descriptions,encoded_train_images,word2idx,maxlen,batch_size)\n",
    "        image_caption_model.fit_generator(generator,epochs=1,steps_per_epoch=steps)\n",
    "        image_caption_model.save('./models_weights/model_{}.h5'.format(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}