Develop model.ipynb

{
  "cells": [
    {
      "metadata": {
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "trusted": true
      },
      "cell_type": "code",
      "source": "!ls",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": "__notebook_source__.ipynb\r\n",
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
        "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
        "trusted": true
      },
      "cell_type": "code",
      "source": "from numpy import array\nfrom pickle import load\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils import to_categorical\nfrom keras.utils import plot_model\nfrom keras.models import Model\nfrom keras.layers import Input\nfrom keras.layers import Dense\nfrom keras.layers import LSTM\nfrom keras.layers import Embedding\nfrom keras.layers import Dropout\nfrom keras.layers.merge import add\nfrom keras.callbacks import ModelCheckpoint\n\n# load doc into memory\ndef load_doc(filename):\n    # open the file as read only\n    file = open(filename, 'r')\n    # read all text\n    text = file.read()\n    # close the file\n    file.close()\n    return text\n\n# load a pre-defined list of photo identifiers\ndef load_set(filename):\n    doc = load_doc(filename)\n    dataset = list()\n    # process line by line\n    for line in doc.split('\\n'):\n        # skip empty lines\n        if len(line) < 1:\n            continue\n        # get the image identifier\n        identifier = line.split('.')[0]\n        dataset.append(identifier)\n    return set(dataset)\n\n# load clean descriptions into memory\ndef load_clean_descriptions(filename, dataset):\n    # load document\n    doc = load_doc(filename)\n    descriptions = dict()\n    for line in doc.split('\\n'):\n        # split line by white space\n        tokens = line.split()\n        # split id from description\n        image_id, image_desc = tokens[0], tokens[1:]\n        # skip images not in the set\n        if image_id in dataset:\n            # create list\n            if image_id not in descriptions:\n                descriptions[image_id] = list()\n            # wrap description in tokens\n            desc = 'startseq ' + ' '.join(image_desc) + ' endseq'\n            # store\n            descriptions[image_id].append(desc)\n    return descriptions\n\n# load photo features\ndef load_photo_features(filename, dataset):\n    # load all features\n    all_features = load(open(filename, 'rb'))\n    # filter features\n    features = {k: all_features[k] for k in dataset}\n    return features\n\n# covert a dictionary of clean descriptions to a list of descriptions\ndef to_lines(descriptions):\n    all_desc = list()\n    for key in descriptions.keys():\n        [all_desc.append(d) for d in descriptions[key]]\n    return all_desc\n\n# fit a tokenizer given caption descriptions\ndef create_tokenizer(descriptions):\n    lines = to_lines(descriptions)\n    tokenizer = Tokenizer()\n    tokenizer.fit_on_texts(lines)\n    return tokenizer\n\n# calculate the length of the description with the most words\ndef max_length(descriptions):\n    lines = to_lines(descriptions)\n    return max(len(d.split()) for d in lines)\n\n# create sequences of images, input sequences and output words for an image\ndef create_sequences(tokenizer, max_length, desc_list, photo):\n    X1, X2, y = list(), list(), list()\n    # walk through each description for the image\n    for desc in desc_list:\n        # encode the sequence\n        seq = tokenizer.texts_to_sequences([desc])[0]\n        # split one sequence into multiple X,y pairs\n        for i in range(1, len(seq)):\n            # split into input and output pair\n            in_seq, out_seq = seq[:i], seq[i]\n            # pad input sequence\n            in_seq = pad_sequences([in_seq], maxlen=max_length)[0]\n            # encode output sequence\n            out_seq = to_categorical([out_seq], num_classes=vocab_size)[0]\n            # store\n            X1.append(photo)\n            X2.append(in_seq)\n            y.append(out_seq)\n    return array(X1), array(X2), array(y)\n\n# define the captioning model\ndef define_model(vocab_size, max_length):\n    # feature extractor model\n    inputs1 = Input(shape=(2048,))\n    fe1 = Dropout(0.5)(inputs1)\n    fe2 = Dense(256, activation='relu')(fe1)\n    # sequence model\n    inputs2 = Input(shape=(max_length,))\n    se1 = Embedding(vocab_size, 256, mask_zero=True)(inputs2)\n    se2 = Dropout(0.5)(se1)\n    se3 = LSTM(256)(se2)\n    # decoder model\n    decoder1 = add([fe2, se3])\n    decoder2 = Dense(256, activation='relu')(decoder1)\n    decoder3 = Dropout(0.5)(decoder2)\n    outputs = Dense(vocab_size, activation='softmax')(decoder2)\n    # tie it together [image, seq] [word]\n    model = Model(inputs=[inputs1, inputs2], outputs=outputs)\n    # compile model\n    model.compile(loss='categorical_crossentropy', optimizer='adam')\n    # summarize model\n    model.summary()\n    plot_model(model, to_file='model.png', show_shapes=True)\n    return model\n\n# data generator, intended to be used in a call to model.fit_generator()\ndef data_generator(descriptions, photos, tokenizer, max_length):\n    # loop for ever over images\n    while 1:\n        for key, desc_list in descriptions.items():\n            # retrieve the photo feature\n            photo = photos[key][0]\n            in_img, in_seq, out_word = create_sequences(tokenizer, max_length, desc_list, photo)\n            yield [[in_img, in_seq], out_word]\n\n# load training dataset (6K)\nfilename = '../input/8kflickrfeature/flickr8k_text/Flickr_8k.trainImages.txt'\ntrain = load_set(filename)\nprint('Dataset: %d' % len(train))\n# descriptions\ntrain_descriptions = load_clean_descriptions('../input/text-data-exploxe/descriptions.txt', train)\nprint('Descriptions: train=%d' % len(train_descriptions))\n# photo features\ntrain_features = load_photo_features('../input/flirck-8k-dataset-explore-image-resnet/features.pkl', train)\nprint('Photos: train=%d' % len(train_features))\n# prepare tokenizer\ntokenizer = create_tokenizer(train_descriptions)\nvocab_size = len(tokenizer.word_index) + 1\nprint('Vocabulary Size: %d' % vocab_size)\n# determine the maximum sequence length\nmax_length = max_length(train_descriptions)\nprint('Description Length: %d' % max_length)\n\n# define the model\nmodel = define_model(vocab_size, max_length)\n# train the model, run epochs manually and save after each epoch\nepochs = 10\nsteps = len(train_descriptions)\nfor i in range(epochs):\n    # create the data generator\n    generator = data_generator(train_descriptions, train_features, tokenizer, max_length)\n    # fit for one epoch\n    model.fit_generator(generator, epochs=1, steps_per_epoch=steps, verbose=1)\n    # save model\n    model.save('model_' + str(i) + '.h5')",
      "execution_count": null,
      "outputs": []
    },
    {
      "metadata": {
        "trusted": true,
        "_uuid": "cb9b33711ae14deda9497712bfc67b32f818ebf2"
      },
      "cell_type": "code",
      "source": "",
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "name": "python",
      "version": "3.6.6",
      "mimetype": "text/x-python",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 1
}