vgg_16_solution.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook code is addapted from jiao dong's script for state farm competition\n",
    "Which in turn is based on code from zfturbo \n",
    "\n",
    "I hope it would serve as a good starting point for anyone who would like to use the pre-trained vgg16 model\n",
    "(or would like to adapt it to any other pre-trained model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/sklearn/cross_validation.py:44: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n",
      "Using Theano backend.\n",
      "Using gpu device 0: GeForce GTX 1080 (CNMeM is disabled, cuDNN 5105)\n",
      "/usr/local/lib/python2.7/dist-packages/theano/sandbox/cuda/__init__.py:600: UserWarning: Your cuDNN version is more recent than the one Theano officially supports. If you see any problems, try updating Theano or downgrading cuDNN to version 5.\n",
      "  warnings.warn(warn)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import cv2\n",
    "import math\n",
    "import pickle\n",
    "import datetime\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.cross_validation import KFold\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Flatten\n",
    "from keras.layers.convolutional import Convolution2D, MaxPooling2D, \\\n",
    "                                       ZeroPadding2D\n",
    "\n",
    "# from keras.layers.normalization import BatchNormalization\n",
    "# from keras.optimizers import Adam\n",
    "from keras.optimizers import SGD\n",
    "from keras.utils import np_utils\n",
    "from keras.models import model_from_json\n",
    "# from sklearn.metrics import log_loss\n",
    "from numpy.random import permutation\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.random.seed(2016)\n",
    "use_cache = 1\n",
    "# color type: 1 - grey, 3 - rgb\n",
    "color_type_global = 3\n",
    "img_rows = 224\n",
    "img_cols = 224\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_im(path, img_rows, img_cols, color_type=3):\n",
    "    # Load as color\n",
    "    if color_type == 1:\n",
    "        img = cv2.imread(path, 0)\n",
    "    elif color_type == 3:\n",
    "        img = cv2.imread(path)\n",
    "    # Reduce size\n",
    "    resized = cv2.resize(img, (img_cols, img_rows))\n",
    "    # mean_pixel = [103.939, 116.799, 123.68]\n",
    "    # resized = resized.astype(np.float32, copy=False)\n",
    "\n",
    "    # for c in range(3):\n",
    "    #    resized[:, :, c] = resized[:, :, c] - mean_pixel[c]\n",
    "    # resized = resized.transpose((2, 0, 1))\n",
    "    # resized = np.expand_dims(img, axis=0)\n",
    "    return resized"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder\n",
    "le = LabelEncoder()\n",
    "encoded = le.fit(['LAG', 'YFT', 'OTHER', 'DOL', 'SHARK', 'NoF', 'BET', 'ALB'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_train(img_rows=224, img_cols=224, color_type=3):    \n",
    "    X_train = []\n",
    "    y_train = []\n",
    "    print('Read train images')\n",
    "    for j in ['LAG', 'YFT', 'OTHER', 'DOL', 'SHARK', 'NoF', 'BET', 'ALB']:\n",
    "        print('Load folder {}'.format(j))\n",
    "        path = os.path.join('..', 'input', 'train', j , '*.jpg')\n",
    "        files = glob.glob(path)\n",
    "        for fl in files:\n",
    "            flbase = os.path.basename(fl)\n",
    "            img = get_im(fl, img_rows, img_cols, color_type)\n",
    "            X_train.append(img)\n",
    "            y_train.append(j)\n",
    "    y_train = le.transform(y_train)\n",
    "    return X_train, y_train\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cache_data(data, path):\n",
    "    if not os.path.isdir('cache'):\n",
    "        os.mkdir('cache')\n",
    "    if os.path.isdir(os.path.dirname(path)):\n",
    "        file = open(path, 'wb')\n",
    "        pickle.dump(data, file)\n",
    "        file.close()\n",
    "    else:\n",
    "        print('Directory doesnt exists')\n",
    "        \n",
    "def restore_data(path):\n",
    "    data = dict()\n",
    "    if os.path.isfile(path):\n",
    "        print('Restore data from pickle....')\n",
    "        file = open(path, 'rb')\n",
    "        data = pickle.load(file)\n",
    "    return data\n",
    "\n",
    "def save_model(model, index, cross=''):\n",
    "    json_string = model.to_json()\n",
    "    if not os.path.isdir('cache'):\n",
    "        os.mkdir('cache')\n",
    "    json_name = 'architecture' + str(index) + cross + '.json'\n",
    "    weight_name = 'model_weights' + str(index) + cross + '.h5'\n",
    "    open(os.path.join('cache', json_name), 'w').write(json_string)\n",
    "    model.save_weights(os.path.join('cache', weight_name), overwrite=True)\n",
    "\n",
    "\n",
    "def read_model(index, cross=''):\n",
    "    json_name = 'architecture' + str(index) + cross + '.json'\n",
    "    weight_name = 'model_weights' + str(index) + cross + '.h5'\n",
    "    model = model_from_json(open(os.path.join('cache', json_name)).read())\n",
    "    model.load_weights(os.path.join('cache', weight_name))\n",
    "    return model\n",
    "\n",
    "\n",
    "def split_validation_set(train, target, test_size):\n",
    "    random_state = 1234\n",
    "    X_train, X_test, y_train, y_test = \\\n",
    "        train_test_split(train, target,\n",
    "                         test_size=test_size,\n",
    "                         random_state=random_state)\n",
    "    return X_train, X_test, y_train, y_test\n",
    "\n",
    "\n",
    "def dict_to_list(d):\n",
    "    ret = []\n",
    "    for i in d.items():\n",
    "        ret.append(i[1])\n",
    "    return ret\n",
    "\n",
    "\n",
    "def merge_several_folds_mean(data, nfolds):\n",
    "    a = np.array(data[0])\n",
    "    for i in range(1, nfolds):\n",
    "        a += np.array(data[i])\n",
    "    a /= nfolds\n",
    "    return a.tolist()\n",
    "\n",
    "\n",
    "def merge_several_folds_geom(data, nfolds):\n",
    "    a = np.array(data[0])\n",
    "    for i in range(1, nfolds):\n",
    "        a *= np.array(data[i])\n",
    "    a = np.power(a, 1/nfolds)\n",
    "    return a.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_test(img_rows, img_cols, color_type=1):\n",
    "    print('Read test images')\n",
    "    path = os.path.join('..', 'input','test_stg1', '*.jpg')\n",
    "    files = glob.glob(path)\n",
    "    X_test = []\n",
    "    X_test_id = []\n",
    "    total = 0\n",
    "    thr = math.floor(len(files)/100)\n",
    "    for fl in files:\n",
    "        flbase = os.path.basename(fl)\n",
    "        img = get_im(fl, img_rows, img_cols, color_type)\n",
    "        X_test.append(img)\n",
    "        X_test_id.append(flbase)\n",
    "        total += 1\n",
    "        if total % thr == 0:\n",
    "            print('Read {} images from {}'.format(total, len(files)))\n",
    "\n",
    "    return X_test, X_test_id\n",
    "\n",
    "\n",
    "def create_submission(predictions, test_id, info):\n",
    "    result1 = pd.DataFrame(predictions, columns=['image', 'LAG', 'YFT', 'OTHER', 'DOL', 'SHARK', 'NoF', 'BET', 'ALB'])\n",
    "    result1.loc[:, 'image'] = pd.Series(test_id, index=result1.index)\n",
    "    now = datetime.datetime.now()\n",
    "    if not os.path.isdir('subm'):\n",
    "        os.mkdir('subm')\n",
    "    suffix = info + '_' + str(now.strftime(\"%Y-%m-%d-%H-%M\"))\n",
    "    sub_file = os.path.join('subm', 'submission_' + suffix + '.csv')\n",
    "    result1.to_csv(sub_file, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vgg_std16_model(img_rows, img_cols, color_type=1):\n",
    "    model = Sequential()\n",
    "    model.add(ZeroPadding2D((1, 1), input_shape=(color_type,\n",
    "                                                 img_rows, img_cols)))\n",
    "    model.add(Convolution2D(64, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(64, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\n",
    "\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(128, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(128, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\n",
    "\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(256, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(256, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(256, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\n",
    "\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\n",
    "\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(ZeroPadding2D((1, 1)))\n",
    "    model.add(Convolution2D(512, 3, 3, activation='relu'))\n",
    "    model.add(MaxPooling2D((2, 2), strides=(2, 2)))\n",
    "\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(4096, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(4096, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(1000, activation='softmax'))\n",
    "\n",
    "    model.load_weights('../pre-trained-model-weights/vgg16_weights.h5')\n",
    "\n",
    "    # Code above loads pre-trained data and\n",
    "    model.layers.pop()\n",
    "    model.add(Dense(8, activation='softmax'))\n",
    "    # Learning rate is changed to 0.001\n",
    "    sgd = SGD(lr=1e-3, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "    model.compile(optimizer=sgd, loss='categorical_crossentropy')\n",
    "    print('model loading and compilation finished succesfully')\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def read_and_normalize_and_shuffle_train_data(img_rows=224, img_cols=224,\n",
    "                                              color_type=3):\n",
    "\n",
    "    cache_path = os.path.join('cache', 'train_r_' + str(img_rows) +\n",
    "                              '_c_' + str(img_cols) + '_t_' +\n",
    "                              str(color_type) + '.dat')\n",
    "\n",
    "    if not os.path.isfile(cache_path) or use_cache == 0:\n",
    "        train_data, train_target = load_train(img_rows, img_cols, color_type)\n",
    "        cache_data((train_data, train_target),cache_path)\n",
    "    else:\n",
    "        print('Restore train from cache!')\n",
    "        (train_data, train_target) = restore_data(cache_path)\n",
    "        \n",
    "    train_data = np.array(train_data, dtype=np.uint8)    \n",
    "    train_target = np.array(train_target, dtype=np.uint8)\n",
    "\n",
    "    train_data = train_data.transpose((0, 3, 1, 2))\n",
    "\n",
    "    train_target = np_utils.to_categorical(train_target, 8)\n",
    "    train_data = train_data.astype('float32')\n",
    "    \n",
    "    ## check mean pixel value\n",
    "    mean_pixel = [103.939, 116.779, 123.68]\n",
    "    for c in range(3):\n",
    "        train_data[:, c, :, :] = train_data[:, c, :, :] - mean_pixel[c]\n",
    "    # train_data /= 255\n",
    "    perm = permutation(len(train_target))\n",
    "    train_data = train_data[perm]\n",
    "    train_target = train_target[perm]\n",
    "    print('Train shape:', train_data.shape)\n",
    "    print(train_data.shape[0], 'train samples')\n",
    "    return train_data, train_target\n",
    "\n",
    "\n",
    "def read_and_normalize_test_data(img_rows=224, img_cols=224, color_type=3):\n",
    "    cache_path = os.path.join('cache', 'test_r_' + str(img_rows) +\n",
    "                              '_c_' + str(img_cols) + '_t_' +\n",
    "                              str(color_type) + '.dat')\n",
    "    if not os.path.isfile(cache_path) or use_cache == 0:\n",
    "        test_data, test_id = load_test(img_rows, img_cols, color_type)\n",
    "        cache_data((test_data, test_id), cache_path)\n",
    "    else:\n",
    "        print('Restore test from cache!')\n",
    "        (test_data, test_id) = restore_data(cache_path)\n",
    "\n",
    "    test_data = np.array(test_data, dtype=np.uint8)\n",
    "\n",
    "    test_data = test_data.transpose((0, 3, 1, 2))\n",
    "\n",
    "    test_data = test_data.astype('float32')\n",
    "    mean_pixel = [103.939, 116.779, 123.68]\n",
    "    for c in range(3):\n",
    "        test_data[:, c, :, :] = test_data[:, c, :, :] - mean_pixel[c]\n",
    "    # test_data /= 255\n",
    "    print('Test shape:', test_data.shape)\n",
    "    print(test_data.shape[0], 'test samples')\n",
    "    return test_data, test_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def run_cross_validation(nfolds=10, nb_epoch=10, split=0.2, modelStr='initial'):\n",
    "\n",
    "    # Now it loads color image\n",
    "    # input image dimensions\n",
    "    img_rows, img_cols = 224, 224\n",
    "    batch_size = 16\n",
    "    random_state = 20\n",
    "\n",
    "    train_data, train_target = \\\n",
    "        read_and_normalize_and_shuffle_train_data(224, 224,3)\n",
    "\n",
    "    # ishuf_train_data = []\n",
    "    # shuf_train_target = []\n",
    "    # index_shuf = range(len(train_target))\n",
    "    # shuffle(index_shuf)\n",
    "    # for i in index_shuf:\n",
    "    #     shuf_train_data.append(train_data[i])\n",
    "    #     shuf_train_target.append(train_target[i])\n",
    "\n",
    "    # yfull_train = dict()\n",
    "    # yfull_test = []\n",
    "    num_fold = 0\n",
    "    kf = KFold(len(train_target), n_folds=nfolds,\n",
    "               shuffle=True, random_state=random_state)\n",
    "    for train_imgs, test_imgs in kf:\n",
    "        num_fold += 1\n",
    "        print('Start KFold number {} from {}'.format(num_fold, nfolds))\n",
    "        # print('Split train: ', len(X_train), len(Y_train))\n",
    "        # print('Split valid: ', len(X_valid), len(Y_valid))\n",
    "        # print('Train drivers: ', unique_list_train)\n",
    "        # print('Test drivers: ', unique_list_valid)\n",
    "        # model = create_model_v1(img_rows, img_cols, color_type_global)\n",
    "        # model = vgg_bn_model(img_rows, img_cols, color_type_global)\n",
    "        model = vgg_std16_model(224, 224, 3)\n",
    "        # correct validation to kfold rather than singel fold\n",
    "        model.fit(train_data, train_target, batch_size=batch_size,\n",
    "                  nb_epoch=nb_epoch,\n",
    "                  show_accuracy=True, verbose=1,\n",
    "                  validation_split=split, shuffle=True)\n",
    "\n",
    "        # print('losses: ' + hist.history.losses[-1])\n",
    "\n",
    "        # print('Score log_loss: ', score[0])\n",
    "\n",
    "        save_model(model, num_fold, modelStr)\n",
    "\n",
    "        # predictions_valid = model.predict(X_valid, batch_size=128, verbose=1)\n",
    "        # score = log_loss(Y_valid, predictions_valid)\n",
    "        # print('Score log_loss: ', score)\n",
    "        # Store valid predictions\n",
    "        # for i in range(len(test_index)):\n",
    "        #    yfull_train[test_index[i]] = predictions_valid[i]\n",
    "\n",
    "    print('Start testing............')\n",
    "    test_data, test_id = read_and_normalize_test_data(224, 224,3)\n",
    "    yfull_test = []\n",
    "\n",
    "    for index in range(1, num_fold + 1):\n",
    "        # 1,2,3,4,5\n",
    "        # Store test predictions\n",
    "        model = read_model(index, modelStr)\n",
    "        test_prediction = model.predict(test_data, batch_size=128, verbose=1)\n",
    "        yfull_test.append(test_prediction)\n",
    "\n",
    "    info_string = 'loss_' + modelStr \\\n",
    "                  + '_r_' + str(img_rows) \\\n",
    "                  + '_c_' + str(img_cols) \\\n",
    "                  + '_folds_' + str(nfolds) \\\n",
    "                  + '_ep_' + str(nb_epoch)\n",
    "\n",
    "    test_res = merge_several_folds_mean(yfull_test, nfolds)\n",
    "    create_submission(test_res, test_id, info_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Restore train from cache!\n",
      "Restore data from pickle....\n",
      "('Train shape:', (3777, 3, 224, 224))\n",
      "(3777, 'train samples')\n",
      "Start KFold number 1 from 3\n",
      "model loading and compilation finished succesfully\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/dist-packages/keras/models.py:603: UserWarning: The \"show_accuracy\" argument is deprecated, instead you should pass the \"accuracy\" metric to the model at compile time:\n",
      "`model.compile(optimizer, loss, metrics=[\"accuracy\"])`\n",
      "  warnings.warn('The \"show_accuracy\" argument is deprecated, '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 3210 samples, validate on 567 samples\n",
      "Epoch 1/20\n",
      "3210/3210 [==============================] - 132s - loss: 1.8861 - val_loss: 1.7390\n",
      "Epoch 2/20\n",
      "1344/3210 [===========>..................] - ETA: 73s - loss: 1.7059"
     ]
    }
   ],
   "source": [
    "from keras import backend as K\n",
    "K.set_image_dim_ordering('th')\n",
    "# nfolds, nb_epoch, split\n",
    "run_cross_validation(3, 20, 0.15, '_vgg_16_2x20')\n",
    "\n",
    "# nb_epoch, split\n",
    "# run_one_fold_cross_validation(10, 0.1)\n",
    "\n",
    "# test_model_and_submit(1, 10, 'high_epoch')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}