importance sampling experiment(not ready).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import print_function\n",
    "import keras\n",
    "import pandas as pd\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers import Conv2D, MaxPooling2D\n",
    "from keras import backend as K\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from time import time\n",
    "from sklearn.metrics import roc_auc_score as rocauc\n",
    "from sklearn.metrics import accuracy_score as acc\n",
    "from sklearn.model_selection import train_test_split\n",
    "from __future__ import print_function  \n",
    "import seaborn as sns\n",
    "sns.set_style(\"whitegrid\")\n",
    "np.set_printoptions(precision=10)\n",
    "np.set_printoptions(suppress=True)\n",
    "input_shape = (28, 28, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def D1_problem_to_C1_problem(arr, fractions):\n",
    "    number_of_C0_in_D0, number_of_C0_in_D1, number_of_C1_in_D0, number_of_C1_in_D1 = fractions\n",
    "    alpha = number_of_C0_in_D0 / (number_of_C0_in_D0 + number_of_C1_in_D0)\n",
    "    beta = number_of_C0_in_D1 / (number_of_C0_in_D1 + number_of_C1_in_D1)\n",
    "    arr = (arr - beta/(alpha + beta))*(((1 - beta) / (2 - alpha - beta) - beta / (alpha + beta)) ** -1)\n",
    "    arr = np.clip(arr,0,1) # значение недообученного классификатора на предыдущем этапе может выйти за границы 0 и 1 \n",
    "    return arr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_dataset(C0_values, fractions):\n",
    "    (x_train, y_train), (x_test, y_test) = mnist.load_data()\n",
    "    x = np.append(x_train, x_test, axis=0)\n",
    "    y = np.append(y_train, y_test)\n",
    "    x = x.reshape(x.shape[0], 28, 28, 1)\n",
    "    y_C1 = np.array([int(elem not in C0_values) for elem in y])\n",
    "\n",
    "    NUMBER_OF_C0_IN_D0, NUMBER_OF_C0_IN_D1, NUMBER_OF_C1_IN_D0, NUMBER_OF_C1_IN_D1 = fractions\n",
    "    i, X, Y_C1, Y_D1 = 0, [], [], []\n",
    "\n",
    "    while (NUMBER_OF_C0_IN_D0, NUMBER_OF_C0_IN_D1, NUMBER_OF_C1_IN_D0, NUMBER_OF_C1_IN_D1) != (0,0,0,0):\n",
    "        if y_C1[i] == 1 and NUMBER_OF_C1_IN_D0 != 0:\n",
    "            Y_C1.append(1)\n",
    "            Y_D1.append(0)\n",
    "            X.append(x[i])\n",
    "            NUMBER_OF_C1_IN_D0 -= 1\n",
    "            i += 1\n",
    "        elif y_C1[i] == 0 and NUMBER_OF_C0_IN_D0 != 0:\n",
    "            Y_C1.append(0)\n",
    "            Y_D1.append(0)\n",
    "            X.append(x[i])\n",
    "            NUMBER_OF_C0_IN_D0 -= 1\n",
    "            i += 1\n",
    "        elif y_C1[i] == 1 and NUMBER_OF_C1_IN_D1 != 0:\n",
    "            Y_C1.append(1)\n",
    "            Y_D1.append(1)\n",
    "            X.append(x[i])\n",
    "            NUMBER_OF_C1_IN_D1 -= 1\n",
    "            i += 1\n",
    "        elif y_C1[i] == 0 and NUMBER_OF_C0_IN_D1 != 0:\n",
    "            Y_C1.append(0)\n",
    "            Y_D1.append(1)\n",
    "            X.append(x[i])\n",
    "            NUMBER_OF_C0_IN_D1 -= 1\n",
    "            i += 1\n",
    "        else:\n",
    "            i += 1\n",
    "    X = np.array(X)\n",
    "    Y_C1 = np.array(Y_C1)\n",
    "    Y_D1 = np.array(Y_D1)\n",
    "    \n",
    "    return train_test_split(X, np.expand_dims(Y_C1, axis=1), np.expand_dims(Y_D1, axis=1), test_size=0.4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LogReg(Sequential):\n",
    "    def __init__(self):\n",
    "        Sequential.__init__(self)\n",
    "        self.add(Flatten(input_shape=input_shape))\n",
    "        self.add(Dense(1, activation='sigmoid'))\n",
    "        self.compile(loss=keras.losses.binary_crossentropy,\n",
    "                   optimizer=keras.optimizers.Adam())\n",
    "        \n",
    "    def fit(self, dataset):\n",
    "        X_train, X_test, Y_C1_train, Y_C1_test, Y_D1_train, Y_D1_test = dataset\n",
    "        \n",
    "        saver_logreg = keras.callbacks.ModelCheckpoint(filepath='/tmp/logreg.hdf5', monitor='val_loss', verbose=0, save_best_only=True)\n",
    "        lr_decreaser_logreg = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=10, factor=0.1, epsilon=0.001, verbose=0)\n",
    "\n",
    "        Sequential.fit(self, X_train, Y_D1_train, validation_data = (X_test, Y_D1_test),\n",
    "                   verbose=0, batch_size=5000, epochs=2, \n",
    "                   callbacks=[saver_logreg, lr_decreaser_logreg])\n",
    "        \n",
    "    def rocauc(self, dataset, fractions):\n",
    "        X_train, X_test, Y_C1_train, Y_C1_test, Y_D1_train, Y_D1_test = dataset\n",
    "        \n",
    "        return rocauc(Y_C1_test, D1_problem_to_C1_problem(self.predict(X_test), fractions).flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Cnn(Sequential):\n",
    "    def __init__(self):\n",
    "        Sequential.__init__(self)\n",
    "        self.add(Conv2D(8, kernel_size=(3, 3),\n",
    "                         activation='relu',\n",
    "                         input_shape=input_shape))\n",
    "        self.add(Conv2D(8, (3, 3), activation='relu'))\n",
    "        self.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        self.add(Conv2D(16, (3, 3), activation='relu'))\n",
    "        self.add(Conv2D(16, (3, 3), activation='relu'))\n",
    "        self.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "        self.add(Flatten())\n",
    "        self.add(Dense(128, activation='relu'))\n",
    "        self.add(Dense(1, activation='sigmoid'))\n",
    "        self.compile(loss=keras.losses.binary_crossentropy,\n",
    "                                optimizer=keras.optimizers.Adam())\n",
    "        \n",
    "    def fit(self, dataset, epochs, importance_sampling_classifier=None, importance_sampling_constant=1):\n",
    "        X_train, X_test, Y_C1_train, Y_C1_test, Y_D1_train, Y_D1_test = dataset\n",
    "        \n",
    "        if importance_sampling_classifier is None:\n",
    "            name = 'cnn_pure'\n",
    "            Sequential.fit(self, X_train, Y_D1_train, #validation_data = (X_test, Y_D1_test),\n",
    "                            verbose=1, epochs=epochs)\n",
    "        else:\n",
    "            importance_sampling_classifier.load_weights('/tmp/logreg.hdf5')\n",
    "            \n",
    "            weights = np.array(importance_sampling_classifier.predict(X_train).flat)\n",
    "            weights = importance_sampling_constant * np.abs(weights - 0.5)\n",
    "            weights = (lambda x: np.exp(x) / np.sum(np.exp(x), axis=0))(weights)\n",
    "            \n",
    "            for i in range(13): #range(len(X_train) // 1000):\n",
    "                mask = list(np.random.choice(np.arange(len(X_train)), size=1000, p=weights))\n",
    "                batch_X = X_train[mask]\n",
    "                batch_Y = Y_D1_train[mask]\n",
    "                Sequential.fit(self, batch_X, batch_Y, #validation_data = (X_test, Y_D1_test),\n",
    "                            verbose=1, epochs=1)\n",
    "            \n",
    "            name = 'cnn_aux'\n",
    "        \n",
    "        #saver_cnn = keras.callbacks.ModelCheckpoint(filepath='/tmp/'+name, monitor='val_loss', verbose=0, save_best_only=True)\n",
    "        #lr_decreaser_cnn = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', patience=3, factor=0.5, epsilon=0.001, verbose=0)\n",
    "\n",
    "        \n",
    "    def rocauc(self, dataset, fractions):\n",
    "        X_train, X_test, Y_C1_train, Y_C1_test, Y_D1_train, Y_D1_test = dataset\n",
    "        \n",
    "        return rocauc(Y_C1_test, D1_problem_to_C1_problem(self.predict(X_test), fractions).flat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def experiment(C0_values, fractions, importance_sampling_constant):\n",
    "    number_of_C0_in_D0, number_of_C0_in_D1, number_of_C1_in_D0, number_of_C1_in_D1 = fractions\n",
    "    alpha = number_of_C0_in_D0 / (number_of_C0_in_D0 + number_of_C1_in_D0)\n",
    "    beta = number_of_C0_in_D1 / (number_of_C0_in_D1 + number_of_C1_in_D1)\n",
    "    \n",
    "    experiment_logs = {'C0 values': C0_values, 'number of C0 in D0':number_of_C0_in_D0, 'number of C0 in D1':number_of_C0_in_D1, 'number of C1 in D0':number_of_C1_in_D0, 'number of C1 in D1':number_of_C1_in_D1}\n",
    "    experiment_logs['alpha'] = alpha\n",
    "    experiment_logs['beta'] = beta\n",
    "    experiment_logs['importance_sampling_constant'] = importance_sampling_constant\n",
    "    \n",
    "    dataset = generate_dataset(C0_values, fractions)\n",
    "    logreg, cnn_aux, cnn_pure = LogReg(), Cnn(), Cnn()\n",
    "    \n",
    "    t0 = time()\n",
    "    logreg.fit(dataset)\n",
    "    experiment_logs['logreg time'] = time() - t0\n",
    "    experiment_logs['logreg rocauc'] = logreg.rocauc(dataset, fractions)\n",
    "    \n",
    "    t0 = time()\n",
    "    cnn_aux.fit(dataset, 1, importance_sampling_classifier=logreg, importance_sampling_constant=importance_sampling_constant)\n",
    "    experiment_logs['cnn_aux time'] = time() - t0\n",
    "    experiment_logs['cnn_aux rocauc'] = cnn_aux.rocauc(dataset, fractions)\n",
    "    #cnn_aux.fit(dataset, 1, auxiliary_classifier=logreg)\n",
    "    #experiment_logs['cnn_aux rocauc plus epoch'] = cnn_aux.rocauc(dataset, fractions)\n",
    "    \n",
    "    t0 = time()\n",
    "    cnn_pure.fit(dataset, 1)\n",
    "    experiment_logs['cnn_pure time'] = time() - t0\n",
    "    experiment_logs['cnn_pure rocauc'] = cnn_pure.rocauc(dataset, fractions)\n",
    "    #cnn_pure.fit(dataset, 1)\n",
    "    #experiment_logs['cnn_pure rocauc plus epoch'] = cnn_pure.rocauc(dataset, fractions)\n",
    "    \n",
    "    return experiment_logs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "importance_sampling_constants = [0.01, 0.1, 1, 10, 100, 1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Current experiment:  0\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 7.8392     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.9074     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.6842     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.6567     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.0350     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.8915     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.2741     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.6842     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.2582     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 8.0190     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 7.6364     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 7.9871     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 7.7161     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 74s - loss: 0.7700    \n",
      "Current experiment:  1\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 3.2383     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 1.0725     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.8256     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7593     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7367     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7339     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7346     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7225     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7246     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.6996     - ETA\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7121     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.6924     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7014     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 78s - loss: 0.7321    \n",
      "Current experiment:  2\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 3s - loss: 6.5105     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 1.1590     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7790     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7275     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7558     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7142     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7227     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7160     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7086     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7113     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6937     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.6838     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6901     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 73s - loss: 7.9261    \n",
      "Current experiment:  0\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 1.9502     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.8135     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7537     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7370     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7240     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7105     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7064     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7041     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7093     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7043     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6992     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7073     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6972     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 76s - loss: 0.7486    \n",
      "Current experiment:  1\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 2.0790     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.9351     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7961     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7383     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7261     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7028     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7217     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7057     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7133     - ETA: 0s - loss: 0\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7051     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7031     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7105     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7023     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 74s - loss: 7.9064    \n",
      "Current experiment:  2\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 3.3306     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0224     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.8327     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7523     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7295     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7015     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7147     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7041     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7016     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7057     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7013     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7001     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.6942     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 73s - loss: 0.7726    \n",
      "Current experiment:  0\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 2.2550     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.9830     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.8132     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7557     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7259     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7157     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7345     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7117     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.6991     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.6966     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7088     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.7090     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.6928     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 71s - loss: 0.7734    \n",
      "Current experiment:  1\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 7.3301     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 4.3037     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.9570     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7654     \n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 1s - loss: 0.7062     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7154     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7048     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7044     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6972     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6876     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6970     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6998     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7005     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 61s - loss: 0.7575    \n",
      "Current experiment:  2\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 2.2978     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.8654     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7573     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7300     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7075     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7247     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6958     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7001     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6930     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7046     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6925     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6935     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6969     - ETA:\n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 61s - loss: 0.8095    \n",
      "Current experiment:  0\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 5.0286     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0956     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.8182     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7463     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7403     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7323     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7080     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7103     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7003     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7062     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7099     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6917     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7048     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 59s - loss: 0.7378    \n",
      "Current experiment:  1\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 1.8415     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.8216     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7563     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7260     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7236     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7163     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7050     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7136     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7081     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7068     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6955     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7065     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7088     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 57s - loss: 0.7743    \n",
      "Current experiment:  2\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 2.0455     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7974     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7367     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7137     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7050     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7021     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7064     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7011     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6972     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6994     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6910     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6933     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7021     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 57s - loss: 7.9209    \n",
      "Current experiment:  0\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 2.1541     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.9104     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7695     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7392     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7096     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7127     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7076     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6954     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7055     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6948     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7028     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.6895     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 0.7013     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 58s - loss: 0.7288    \n",
      "Current experiment:  1\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 7.8637     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.7044     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.8334     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.9785     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.8495     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.5265     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.6883     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.9462     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.1235     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.8334     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.2525     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.8979     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.9462     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 59s - loss: 0.7443    \n",
      "Current experiment:  2\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 7.5605     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.2900     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.9712     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.5089     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.7321     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.1784     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.0350     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.1147     \n",
      "Epoch 1/1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000/1000 [==============================] - 1s - loss: 7.2857     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.7958     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 8.1944     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.5089     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 7.7639     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 56s - loss: 0.7380    \n",
      "Current experiment:  0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:30: RuntimeWarning: overflow encountered in exp\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:30: RuntimeWarning: invalid value encountered in true_divide\n",
      "/usr/local/lib/python3.5/dist-packages/ipykernel_launcher.py:33: RuntimeWarning: invalid value encountered in less\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 2s - loss: 0.0139     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.1921e-07     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 57s - loss: 0.7390    \n",
      "Current experiment:  1\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 3s - loss: 4.2744e-04     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 1.0000e-07     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 57s - loss: 0.8365    \n",
      "Current experiment:  2\n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 3s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "1000/1000 [==============================] - 1s - loss: 16.1181     \n",
      "Epoch 1/1\n",
      "37488/37488 [==============================] - 58s - loss: 0.7222    \n"
     ]
    }
   ],
   "source": [
    "logs = []\n",
    "for importance_sampling_constant in importance_sampling_constants:\n",
    "    for i in range(3):\n",
    "        print('Current experiment: ', i)\n",
    "        logs.append(\n",
    "            experiment([0], [30, 450, 31000, 31000], importance_sampling_constant)\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.589474604287 0.775100024694 0.772281581571\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for log in logs:\n",
    "    print(log['logreg rocauc'],log['cnn_aux rocauc'],log['cnn_pure rocauc'])\n",
    "    # print(log['logreg rocauc'],log['cnn_aux rocauc plus epoch'],log['cnn_pure rocauc plus epoch'])\n",
    "    print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>C0 values</th>\n",
       "      <th>alpha</th>\n",
       "      <th>beta</th>\n",
       "      <th>cnn_aux rocauc</th>\n",
       "      <th>cnn_aux time</th>\n",
       "      <th>cnn_pure rocauc</th>\n",
       "      <th>cnn_pure time</th>\n",
       "      <th>importance_sampling_constant</th>\n",
       "      <th>logreg rocauc</th>\n",
       "      <th>logreg time</th>\n",
       "      <th>number of C0 in D0</th>\n",
       "      <th>number of C0 in D1</th>\n",
       "      <th>number of C1 in D0</th>\n",
       "      <th>number of C1 in D1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>27.934686</td>\n",
       "      <td>0.545440</td>\n",
       "      <td>75.385020</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.387129</td>\n",
       "      <td>1.428216</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.418178</td>\n",
       "      <td>30.856762</td>\n",
       "      <td>0.436780</td>\n",
       "      <td>78.983442</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.564905</td>\n",
       "      <td>1.486955</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.652204</td>\n",
       "      <td>29.428717</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>73.759881</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.693464</td>\n",
       "      <td>1.620913</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.616173</td>\n",
       "      <td>28.164245</td>\n",
       "      <td>0.896379</td>\n",
       "      <td>76.843398</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.615444</td>\n",
       "      <td>1.714485</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.539078</td>\n",
       "      <td>31.313852</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>75.415642</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.612049</td>\n",
       "      <td>1.794774</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.652513</td>\n",
       "      <td>28.520525</td>\n",
       "      <td>0.463652</td>\n",
       "      <td>74.071143</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.674259</td>\n",
       "      <td>1.894963</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.624387</td>\n",
       "      <td>30.699365</td>\n",
       "      <td>0.841932</td>\n",
       "      <td>72.122681</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.388551</td>\n",
       "      <td>2.003900</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.746312</td>\n",
       "      <td>24.829697</td>\n",
       "      <td>0.686297</td>\n",
       "      <td>62.254979</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.381280</td>\n",
       "      <td>2.142799</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.685559</td>\n",
       "      <td>24.958680</td>\n",
       "      <td>0.697531</td>\n",
       "      <td>61.758000</td>\n",
       "      <td>1.00</td>\n",
       "      <td>0.565675</td>\n",
       "      <td>2.239764</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.687537</td>\n",
       "      <td>25.302016</td>\n",
       "      <td>0.710150</td>\n",
       "      <td>60.057472</td>\n",
       "      <td>10.00</td>\n",
       "      <td>0.512336</td>\n",
       "      <td>2.357135</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.658306</td>\n",
       "      <td>23.266249</td>\n",
       "      <td>0.699665</td>\n",
       "      <td>57.678090</td>\n",
       "      <td>10.00</td>\n",
       "      <td>0.592292</td>\n",
       "      <td>2.501479</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.797496</td>\n",
       "      <td>23.655939</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>57.583004</td>\n",
       "      <td>10.00</td>\n",
       "      <td>0.697266</td>\n",
       "      <td>3.094244</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.613290</td>\n",
       "      <td>24.443915</td>\n",
       "      <td>0.829909</td>\n",
       "      <td>59.395902</td>\n",
       "      <td>100.00</td>\n",
       "      <td>0.666156</td>\n",
       "      <td>2.684584</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>24.856266</td>\n",
       "      <td>0.807025</td>\n",
       "      <td>59.510944</td>\n",
       "      <td>100.00</td>\n",
       "      <td>0.478109</td>\n",
       "      <td>2.833612</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>24.909344</td>\n",
       "      <td>0.481117</td>\n",
       "      <td>56.453596</td>\n",
       "      <td>100.00</td>\n",
       "      <td>0.492989</td>\n",
       "      <td>2.940324</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>24.260801</td>\n",
       "      <td>0.729688</td>\n",
       "      <td>58.265461</td>\n",
       "      <td>1000.00</td>\n",
       "      <td>0.702876</td>\n",
       "      <td>3.063643</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>24.629208</td>\n",
       "      <td>0.533533</td>\n",
       "      <td>58.090938</td>\n",
       "      <td>1000.00</td>\n",
       "      <td>0.577873</td>\n",
       "      <td>3.156959</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>[0]</td>\n",
       "      <td>0.000967</td>\n",
       "      <td>0.014308</td>\n",
       "      <td>0.499980</td>\n",
       "      <td>25.403611</td>\n",
       "      <td>0.578188</td>\n",
       "      <td>58.663348</td>\n",
       "      <td>1000.00</td>\n",
       "      <td>0.707536</td>\n",
       "      <td>3.268312</td>\n",
       "      <td>30</td>\n",
       "      <td>450</td>\n",
       "      <td>31000</td>\n",
       "      <td>31000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   C0 values     alpha      beta  cnn_aux rocauc  cnn_aux time  \\\n",
       "0        [0]  0.000967  0.014308        0.500000     27.934686   \n",
       "1        [0]  0.000967  0.014308        0.418178     30.856762   \n",
       "2        [0]  0.000967  0.014308        0.652204     29.428717   \n",
       "3        [0]  0.000967  0.014308        0.616173     28.164245   \n",
       "4        [0]  0.000967  0.014308        0.539078     31.313852   \n",
       "5        [0]  0.000967  0.014308        0.652513     28.520525   \n",
       "6        [0]  0.000967  0.014308        0.624387     30.699365   \n",
       "7        [0]  0.000967  0.014308        0.746312     24.829697   \n",
       "8        [0]  0.000967  0.014308        0.685559     24.958680   \n",
       "9        [0]  0.000967  0.014308        0.687537     25.302016   \n",
       "10       [0]  0.000967  0.014308        0.658306     23.266249   \n",
       "11       [0]  0.000967  0.014308        0.797496     23.655939   \n",
       "12       [0]  0.000967  0.014308        0.613290     24.443915   \n",
       "13       [0]  0.000967  0.014308        0.500000     24.856266   \n",
       "14       [0]  0.000967  0.014308        0.500000     24.909344   \n",
       "15       [0]  0.000967  0.014308        0.500000     24.260801   \n",
       "16       [0]  0.000967  0.014308        0.500000     24.629208   \n",
       "17       [0]  0.000967  0.014308        0.499980     25.403611   \n",
       "\n",
       "    cnn_pure rocauc  cnn_pure time  importance_sampling_constant  \\\n",
       "0          0.545440      75.385020                          0.01   \n",
       "1          0.436780      78.983442                          0.01   \n",
       "2          0.500000      73.759881                          0.01   \n",
       "3          0.896379      76.843398                          0.10   \n",
       "4          0.500000      75.415642                          0.10   \n",
       "5          0.463652      74.071143                          0.10   \n",
       "6          0.841932      72.122681                          1.00   \n",
       "7          0.686297      62.254979                          1.00   \n",
       "8          0.697531      61.758000                          1.00   \n",
       "9          0.710150      60.057472                         10.00   \n",
       "10         0.699665      57.678090                         10.00   \n",
       "11         0.500000      57.583004                         10.00   \n",
       "12         0.829909      59.395902                        100.00   \n",
       "13         0.807025      59.510944                        100.00   \n",
       "14         0.481117      56.453596                        100.00   \n",
       "15         0.729688      58.265461                       1000.00   \n",
       "16         0.533533      58.090938                       1000.00   \n",
       "17         0.578188      58.663348                       1000.00   \n",
       "\n",
       "    logreg rocauc  logreg time  number of C0 in D0  number of C0 in D1  \\\n",
       "0        0.387129     1.428216                  30                 450   \n",
       "1        0.564905     1.486955                  30                 450   \n",
       "2        0.693464     1.620913                  30                 450   \n",
       "3        0.615444     1.714485                  30                 450   \n",
       "4        0.612049     1.794774                  30                 450   \n",
       "5        0.674259     1.894963                  30                 450   \n",
       "6        0.388551     2.003900                  30                 450   \n",
       "7        0.381280     2.142799                  30                 450   \n",
       "8        0.565675     2.239764                  30                 450   \n",
       "9        0.512336     2.357135                  30                 450   \n",
       "10       0.592292     2.501479                  30                 450   \n",
       "11       0.697266     3.094244                  30                 450   \n",
       "12       0.666156     2.684584                  30                 450   \n",
       "13       0.478109     2.833612                  30                 450   \n",
       "14       0.492989     2.940324                  30                 450   \n",
       "15       0.702876     3.063643                  30                 450   \n",
       "16       0.577873     3.156959                  30                 450   \n",
       "17       0.707536     3.268312                  30                 450   \n",
       "\n",
       "    number of C1 in D0  number of C1 in D1  \n",
       "0                31000               31000  \n",
       "1                31000               31000  \n",
       "2                31000               31000  \n",
       "3                31000               31000  \n",
       "4                31000               31000  \n",
       "5                31000               31000  \n",
       "6                31000               31000  \n",
       "7                31000               31000  \n",
       "8                31000               31000  \n",
       "9                31000               31000  \n",
       "10               31000               31000  \n",
       "11               31000               31000  \n",
       "12               31000               31000  \n",
       "13               31000               31000  \n",
       "14               31000               31000  \n",
       "15               31000               31000  \n",
       "16               31000               31000  \n",
       "17               31000               31000  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.DataFrame(logs).to_csv('logs_i_s', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}