tf_keras_RCNN_physionet_2017_cross_val.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "tf.keras RCNN physionet 2017 cross val.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hadaev8/physionet_2017_rcrnn/blob/master/tf_keras_RCNN_physionet_2017_cross_val.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vNfhgRt03J9j",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "!rsync -Cav physionet.org::challenge-2017/validation ./ 2>&1 >/dev/null\n",
        "!rsync -Cav physionet.org::challenge-2017/training ./ 2>&1 >/dev/null"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7u1EdpULhuBJ",
        "colab_type": "code",
        "outputId": "87d1f405-165e-4a31-e3b6-fedd8fc6a684",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "np.random.seed(1488)\n",
        "import os\n",
        "import tensorflow as tf\n",
        "tf.logging.set_verbosity('ERROR')\n",
        "print(tf.VERSION)\n",
        "import tensorflow.keras as keras\n",
        "import tensorflow.keras.backend as K\n",
        "tf.set_random_seed(1488)\n",
        "\n",
        "import scipy.io\n",
        "from shutil import copy\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "from sklearn.utils import class_weight\n",
        "from sklearn.metrics import classification_report, f1_score\n",
        "from sklearn.model_selection import StratifiedKFold\n",
        "\n",
        "batch_size = 128\n",
        "    \n",
        "le = LabelEncoder()\n",
        "data = pd.read_csv('training/REFERENCE-v3.csv', header=None, names=['name', 'label'])\n",
        "y = le.fit_transform(data.label.values)\n",
        "class_weights = class_weight.compute_class_weight('balanced', np.unique(y), y)\n",
        "y = y.reshape(-1, 1)\n",
        "\n",
        "def get_data(file):\n",
        "    data = scipy.io.loadmat(file)['val'][0]\n",
        "    return data.astype(float)\n",
        "\n",
        "X = np.array([get_data('training/{}.mat'.format(i)) for i in data.name.values])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.13.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rkaMqJ2YI-Iw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "X = keras.preprocessing.sequence.pad_sequences(X, dtype='float32')\n",
        "X = X.reshape(-1, X.shape[1], 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-6RUuB4Qo9z-",
        "colab_type": "code",
        "outputId": "2c5cc404-1c8c-4fb0-d201-5766b5692508",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "source": [
        "print('labels encoding for 0 1 2 3:', le.inverse_transform([0, 1, 2, 3]))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "labels encoding for 0 1 2 3: ['A' 'N' 'O' '~']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LkfDWpcYQR55",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def conv(i, filters=16, kernel_size=9, strides=1):\n",
        "    i = keras.layers.Conv1D(filters=filters, kernel_size=kernel_size, strides=strides, padding='same')(i)#, activity_regularizer=keras.regularizers.l2(1e-6)\n",
        "    i = keras.layers.BatchNormalization()(i)\n",
        "    i = keras.layers.LeakyReLU()(i)\n",
        "    i = keras.layers.SpatialDropout1D(0.1)(i)\n",
        "    return i\n",
        "\n",
        "def residual_unit(x, filters, layers=3):\n",
        "    inp = x\n",
        "    for i in range(layers):\n",
        "        x = conv(x, filters)\n",
        "    return keras.layers.add([x, inp])\n",
        "\n",
        "def conv_block(x, filters, strides):\n",
        "    x = conv(x, filters)\n",
        "    #x = dense_residual_unit(x, filters)\n",
        "    x = residual_unit(x, filters)\n",
        "    if strides > 1:\n",
        "        x = keras.layers.AveragePooling1D(strides, strides)(x)\n",
        "    return x\n",
        "\n",
        "def get_model():\n",
        "    inp = keras.layers.Input(shape=(X.shape[1], 1), dtype=tf.float32)\n",
        "\n",
        "    x = inp\n",
        "    x = conv_block(x, 16, 4)\n",
        "    x = conv_block(x, 16, 4)\n",
        "    x = conv_block(x, 32, 4)\n",
        "    x = conv_block(x, 32, 4)\n",
        "    x = keras.layers.Masking(mask_value=0)(x)\n",
        "    x = keras.layers.GRU(32, recurrent_dropout=0.1)(x)\n",
        "    x = keras.layers.Dense(4, activation='softmax')(x)\n",
        "\n",
        "    model = keras.models.Model(inp, x)\n",
        "    return model\n",
        "\n",
        "def make_dataset(X, y):\n",
        "    data = tf.data.Dataset.from_tensor_slices((X, y))\n",
        "    data = data.apply(tf.data.experimental.shuffle_and_repeat(buffer_size=len(y)))\n",
        "    data = data.batch(batch_size, drop_remainder=False)\n",
        "    data = data.prefetch(2)\n",
        "    return data"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LPxot7ux0ifX",
        "colab_type": "code",
        "outputId": "ec464b96-9da7-4ddb-daa0-3d6ac9ebe313",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1303
        }
      },
      "source": [
        "folds = list(StratifiedKFold(n_splits=5, shuffle=True, random_state=1).split(X, y))\n",
        "\n",
        "scores = []\n",
        "for j, (train_idx, test_idx) in enumerate(folds):\n",
        "    X_train, y_train, X_test, y_test = X[train_idx], y[train_idx], X[test_idx], y[test_idx]\n",
        "    print('started {} fold'.format(j))\n",
        "    K.clear_session()\n",
        "    train_data = make_dataset(X_train, y_train)\n",
        "    test_data = make_dataset(X_test, y_test)\n",
        "    model = get_model()\n",
        "    model.compile(loss='sparse_categorical_crossentropy', optimizer=keras.optimizers.Adam(clipnorm=1.), metrics=['accuracy'])\n",
        "\n",
        "    h = model.fit(train_data,\n",
        "              steps_per_epoch=len(y_train)//batch_size,\n",
        "              validation_data=test_data,\n",
        "              validation_steps=len(y_test)//batch_size,\n",
        "              epochs=1000,\n",
        "              verbose=0,\n",
        "              class_weight=class_weights,\n",
        "              callbacks=[\n",
        "                keras.callbacks.TerminateOnNaN(),\n",
        "                keras.callbacks.EarlyStopping(patience=50, verbose=1),\n",
        "                keras.callbacks.ReduceLROnPlateau(factor=0.9, patience=2, verbose=0, min_lr=1e-6),\n",
        "              ])\n",
        "    model.save('final_model_{}.h5'.format(j))\n",
        "    #copy('final_model_{}.h5'.format(j), 'drive/My Drive')\n",
        "    y_pred = np.argmax(model.predict(X_test), axis=-1).reshape(-1, 1)\n",
        "    print(np.equal(y_test, y_pred).mean())\n",
        "    print(classification_report(y_test, y_pred))\n",
        "    scores.append([np.equal(y_test, y_pred).mean(), f1_score(y_test, y_pred, average=None)])"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "started 0 fold\n",
            "Epoch 00107: early stopping\n",
            "0.8588166373755126\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.75      0.80      0.78       152\n",
            "           1       0.90      0.93      0.92      1016\n",
            "           2       0.81      0.76      0.78       483\n",
            "           3       0.69      0.61      0.65        56\n",
            "\n",
            "    accuracy                           0.86      1707\n",
            "   macro avg       0.79      0.77      0.78      1707\n",
            "weighted avg       0.86      0.86      0.86      1707\n",
            "\n",
            "started 1 fold\n",
            "Epoch 00154: early stopping\n",
            "0.8511137162954279\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.77      0.86      0.81       152\n",
            "           1       0.90      0.92      0.91      1015\n",
            "           2       0.79      0.75      0.77       483\n",
            "           3       0.62      0.50      0.55        56\n",
            "\n",
            "    accuracy                           0.85      1706\n",
            "   macro avg       0.77      0.76      0.76      1706\n",
            "weighted avg       0.85      0.85      0.85      1706\n",
            "\n",
            "started 2 fold\n",
            "Epoch 00104: early stopping\n",
            "0.8528722157092614\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.78      0.74      0.76       152\n",
            "           1       0.89      0.94      0.91      1015\n",
            "           2       0.80      0.73      0.77       483\n",
            "           3       0.71      0.57      0.63        56\n",
            "\n",
            "    accuracy                           0.85      1706\n",
            "   macro avg       0.80      0.75      0.77      1706\n",
            "weighted avg       0.85      0.85      0.85      1706\n",
            "\n",
            "started 3 fold\n",
            "Epoch 00104: early stopping\n",
            "0.866275659824047\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.76      0.81      0.78       151\n",
            "           1       0.90      0.95      0.92      1015\n",
            "           2       0.83      0.76      0.79       483\n",
            "           3       0.71      0.48      0.57        56\n",
            "\n",
            "    accuracy                           0.87      1705\n",
            "   macro avg       0.80      0.75      0.77      1705\n",
            "weighted avg       0.86      0.87      0.86      1705\n",
            "\n",
            "started 4 fold\n",
            "Epoch 00109: early stopping\n",
            "0.8697183098591549\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.81      0.89      0.85       151\n",
            "           1       0.90      0.94      0.92      1015\n",
            "           2       0.83      0.73      0.78       483\n",
            "           3       0.83      0.69      0.75        55\n",
            "\n",
            "    accuracy                           0.87      1704\n",
            "   macro avg       0.84      0.81      0.82      1704\n",
            "weighted avg       0.87      0.87      0.87      1704\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YlJt2_Pxodva",
        "colab_type": "code",
        "outputId": "2618e9b8-455c-472b-b18e-5e1dd5b42108",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        }
      },
      "source": [
        "print('train set cross val metrics')\n",
        "print('accuracy:', np.transpose(np.array(scores))[:1].mean())\n",
        "print('F1 per class:', np.transpose(np.array(scores))[1:].mean())\n",
        "print('F1 mean:', np.transpose(np.array(scores))[1:].mean()[:-1].mean())"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "train set cross val metrics\n",
            "accuracy: 0.8597593078126808\n",
            "F1 per class: [0.79611215 0.9169862  0.77750201 0.63253624]\n",
            "F1 mean: 0.8302001192185081\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "murCRATqjJh7",
        "colab_type": "code",
        "outputId": "c9ac9f6a-fd81-4454-9d2e-bcc8a236d719",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        }
      },
      "source": [
        "data = pd.read_csv('validation/REFERENCE-v3.csv', header=None, names=['name', 'label'])\n",
        "y_val = le.transform(data.label.values).reshape(-1, 1)\n",
        "X_val = np.array([get_data('validation/{}.mat'.format(i)) for i in data.name.values])\n",
        "X_val = keras.preprocessing.sequence.pad_sequences(X_val, maxlen=X.shape[1], dtype='float32')\n",
        "X_val = X_val.reshape(-1, X_val.shape[1], 1)\n",
        "\n",
        "y_val_pred = []\n",
        "model = get_model()\n",
        "for i in range(5):\n",
        "    model.load_weights('final_model_{}.h5'.format(i))\n",
        "    y_val_pred.append(model.predict(X_val))\n",
        "y_val_pred = np.argmax(np.sum(np.array(y_val_pred), axis=0), axis=-1).reshape(-1, 1)\n",
        "print('validation set metrics:')\n",
        "print('accuracy:', np.equal(y_val, y_val_pred).mean())\n",
        "print('F1 mean:', f1_score(y_val, y_val_pred, average=None)[:-1].mean())\n",
        "print('classification report:\\n', classification_report(y_val, y_val_pred))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "validation set metrics:\n",
            "accuracy: 0.9266666666666666\n",
            "F1 mean: 0.9139330718278087\n",
            "classification report:\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.94      0.96      0.95        47\n",
            "           1       0.95      0.95      0.95       148\n",
            "           2       0.84      0.86      0.85        65\n",
            "           3       1.00      0.93      0.96        40\n",
            "\n",
            "    accuracy                           0.93       300\n",
            "   macro avg       0.93      0.92      0.93       300\n",
            "weighted avg       0.93      0.93      0.93       300\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}