Nested-CV-workflow.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-02T17:36:28.434881Z",
     "start_time": "2019-05-02T17:36:24.803118Z"
    }
   },
   "outputs": [],
   "source": [
    "import os,sys\n",
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(\"/projects/cc/kxsw653/ExCAPE-ML_v5/IT4I-data/qsar_models/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-02T17:36:29.752167Z",
     "start_time": "2019-05-02T17:36:29.745710Z"
    }
   },
   "outputs": [],
   "source": [
    "# HP SELECTION - INNER LOOP\n",
    "# find out the best HP per fold\n",
    "def selectHP(metric, perf_inner):\n",
    "    data1 = []\n",
    "    data2 = []\n",
    "    data3 = []\n",
    "    for target in perf_inner['target'].unique():\n",
    "        # extract results for the current target\n",
    "        sub = perf_inner.loc[perf_inner['target']==target]\n",
    "    \n",
    "        # get AUC performance averaged for each given fold i.e. averaged over the two inner folds\n",
    "        subgrouped3 = sub.groupby(['ofold', 'hp']).mean().reset_index().sort_values('auc', ascending=False)    \n",
    "        \n",
    "        \n",
    "        # extract best AUC for each of the three folds\n",
    "        sub1 = subgrouped3.loc[subgrouped3['ofold']==1].sort_values(metric, ascending=False).drop(['ofold','ifold','auc','recall','precision','kappa'],axis=1).head(1)\n",
    "        sub2 = subgrouped3.loc[subgrouped3['ofold']==2].sort_values(metric, ascending=False).drop(['ofold','ifold','auc','recall','precision','kappa'],axis=1).head(1)\n",
    "        sub3 = subgrouped3.loc[subgrouped3['ofold']==3].sort_values(metric, ascending=False).drop(['ofold','ifold','auc','recall','precision','kappa'],axis=1).head(1)\n",
    "\n",
    "        data1.append(sub1.values[0])\n",
    "        data2.append(sub2.values[0])\n",
    "        data3.append(sub3.values[0])\n",
    "        \n",
    "    hp_fold1 = pd.DataFrame(data1, columns=[ 'hp_fold1','target'])\n",
    "    hp_fold2 = pd.DataFrame(data2, columns=[ 'hp_fold2','target'])\n",
    "    hp_fold3 = pd.DataFrame(data3, columns=[ 'hp_fold3','target'])\n",
    "    \n",
    "    return hp_fold1.merge(hp_fold2, on='target').merge(hp_fold3, on='target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-02T17:36:30.949276Z",
     "start_time": "2019-05-02T17:36:30.943016Z"
    }
   },
   "outputs": [],
   "source": [
    "# Find out the performance - OUTER LOOP\n",
    "# using the HP selected in inner loop\n",
    "def getOuterLoopPerformance(perf_outer, hp_select):    \n",
    "    \n",
    "    data1,data2,data3=[],[],[]\n",
    "    for i in range(526):\n",
    "        \n",
    "        hp_fold1 = hp_selection.loc[hp_selection['target']==i]['hp_fold1'].values[0]\n",
    "        hp_fold2 = hp_selection.loc[hp_selection['target']==i]['hp_fold2'].values[0]\n",
    "        hp_fold3 = hp_selection.loc[hp_selection['target']==i]['hp_fold3'].values[0]\n",
    "        \n",
    "        perf_fold1 = perf_outer.loc[(perf_outer['target']==i)&(perf_outer['hp']==hp_fold1)&(perf_outer['ofold']==1)]\n",
    "        perf_fold2 = perf_outer.loc[(perf_outer['target']==i)&(perf_outer['hp']==hp_fold2)&(perf_outer['ofold']==2)]\n",
    "        perf_fold3 = perf_outer.loc[(perf_outer['target']==i)&(perf_outer['hp']==hp_fold3)&(perf_outer['ofold']==3)]\n",
    "\n",
    "        ####  one target missing for SVM and RF\n",
    "        if perf_fold1.shape[0]==0 or perf_fold2.shape[0]==0 or perf_fold3.shape[0]==0:continue\n",
    "        ####\n",
    "                \n",
    "        data1.append(perf_fold1.values[0])\n",
    "        data2.append(perf_fold2.values[0])\n",
    "        data3.append(perf_fold3.values[0])\n",
    "        \n",
    "        \n",
    "    all_perf_fold1 = pd.DataFrame(data1, columns=perf_fold1.columns.values)\n",
    "    all_perf_fold2 = pd.DataFrame(data2, columns=perf_fold2.columns.values)\n",
    "    all_perf_fold3 = pd.DataFrame(data3, columns=perf_fold3.columns.values)\n",
    "    \n",
    "    return all_perf_fold1,all_perf_fold2,all_perf_fold3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# XGboost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Inner fold - HP search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T20:47:11.509456Z",
     "start_time": "2019-04-04T20:47:11.086749Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "xgboost_df = pd.read_csv('results_v5_xgboost_ecfp/all-nested-cv-perf-scores.csv', header=0, sep=\"\\t\")\n",
    "\n",
    "# separate inner and out loop results\n",
    "xgb_inner = xgboost_df.loc[xgboost_df['ifold'].isin([1,2])].copy(deep=True).drop(['al','sample'],axis=1)\n",
    "xgb_outer = xgboost_df.loc[xgboost_df['ifold'].isin([0])].copy(deep=True).drop(['al','sample'],axis=1)\n",
    "del xgboost_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:55:41.809216Z",
     "start_time": "2019-04-03T08:55:41.777460Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ofold</th>\n",
       "      <th>ifold</th>\n",
       "      <th>target</th>\n",
       "      <th>hp</th>\n",
       "      <th>auc</th>\n",
       "      <th>recall</th>\n",
       "      <th>precision</th>\n",
       "      <th>kappa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9468</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>0.808374</td>\n",
       "      <td>0.702326</td>\n",
       "      <td>0.731826</td>\n",
       "      <td>0.507079</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9469</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>0.804593</td>\n",
       "      <td>0.857364</td>\n",
       "      <td>0.603053</td>\n",
       "      <td>0.404244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9470</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>0.792723</td>\n",
       "      <td>0.913178</td>\n",
       "      <td>0.523556</td>\n",
       "      <td>0.253211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9471</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>0.810966</td>\n",
       "      <td>0.699225</td>\n",
       "      <td>0.722756</td>\n",
       "      <td>0.495144</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9472</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>0.810930</td>\n",
       "      <td>0.865116</td>\n",
       "      <td>0.605863</td>\n",
       "      <td>0.412486</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      ofold  ifold  target                                                 hp  \\\n",
       "9468      1      1       0  booster_gbtree-learning_rate_0.05-max_depth_5-...   \n",
       "9469      1      1       0  booster_gbtree-learning_rate_0.05-max_depth_5-...   \n",
       "9470      1      1       0  booster_gbtree-learning_rate_0.05-max_depth_5-...   \n",
       "9471      1      1       0  booster_gbtree-learning_rate_0.05-max_depth_5-...   \n",
       "9472      1      1       0  booster_gbtree-learning_rate_0.05-max_depth_5-...   \n",
       "\n",
       "           auc    recall  precision     kappa  \n",
       "9468  0.808374  0.702326   0.731826  0.507079  \n",
       "9469  0.804593  0.857364   0.603053  0.404244  \n",
       "9470  0.792723  0.913178   0.523556  0.253211  \n",
       "9471  0.810966  0.699225   0.722756  0.495144  \n",
       "9472  0.810930  0.865116   0.605863  0.412486  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_inner.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:55:58.332406Z",
     "start_time": "2019-04-03T08:55:42.906246Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# select best HP per fold\n",
    "hp_selection = selectHP('auc', xgb_inner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:57:32.307866Z",
     "start_time": "2019-04-03T08:57:32.297133Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hp_fold1</th>\n",
       "      <th>target</th>\n",
       "      <th>hp_fold2</th>\n",
       "      <th>hp_fold3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_10...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_10...</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>100.0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_10...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>101.0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_10...</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>102.0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_10...</td>\n",
       "      <td>103.0</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_5-...</td>\n",
       "      <td>booster_gbtree-learning_rate_0.05-max_depth_10...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            hp_fold1  target  \\\n",
       "0  booster_gbtree-learning_rate_0.05-max_depth_10...     0.0   \n",
       "1  booster_gbtree-learning_rate_0.05-max_depth_5-...   100.0   \n",
       "2  booster_gbtree-learning_rate_0.05-max_depth_5-...   101.0   \n",
       "3  booster_gbtree-learning_rate_0.05-max_depth_5-...   102.0   \n",
       "4  booster_gbtree-learning_rate_0.05-max_depth_10...   103.0   \n",
       "\n",
       "                                            hp_fold2  \\\n",
       "0  booster_gbtree-learning_rate_0.05-max_depth_10...   \n",
       "1  booster_gbtree-learning_rate_0.05-max_depth_5-...   \n",
       "2  booster_gbtree-learning_rate_0.05-max_depth_10...   \n",
       "3  booster_gbtree-learning_rate_0.05-max_depth_5-...   \n",
       "4  booster_gbtree-learning_rate_0.05-max_depth_5-...   \n",
       "\n",
       "                                            hp_fold3  \n",
       "0  booster_gbtree-learning_rate_0.05-max_depth_10...  \n",
       "1  booster_gbtree-learning_rate_0.05-max_depth_10...  \n",
       "2  booster_gbtree-learning_rate_0.05-max_depth_5-...  \n",
       "3  booster_gbtree-learning_rate_0.05-max_depth_5-...  \n",
       "4  booster_gbtree-learning_rate_0.05-max_depth_10...  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hp_selection.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Outer folds - Models evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:57:46.018285Z",
     "start_time": "2019-04-03T08:57:37.844218Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# use the selected HP to find out the performance in outer loop\n",
    "perf_fold1, perf_fold2, perf_fold3 = getOuterLoopPerformance(xgb_outer, hp_selection)\n",
    "perf_fold1.to_csv(\"results_v5_xgboost_ecfp/perf_fold1.csv\", sep=\"\\t\", index=None)\n",
    "perf_fold2.to_csv(\"results_v5_xgboost_ecfp/perf_fold2.csv\", sep=\"\\t\", index=None)\n",
    "perf_fold3.to_csv(\"results_v5_xgboost_ecfp/perf_fold3.csv\", sep=\"\\t\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:56:02.680071Z",
     "start_time": "2019-04-03T08:56:02.666906Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           fold1           fold2           fold3\n",
      "AUC      =(0.8084+/-0.1212,0.8067+/-0.1224,0.8106+/-0.1177)\n",
      "PRECISION=(0.6579+/-0.2562,0.6549+/-0.2602,0.6677+/-0.2410)\n",
      "RECALL   =(0.5599+/-0.3462,0.5524+/-0.3426,0.5577+/-0.3394)\n",
      "KAPPA    =(0.3626+/-0.2411,0.3619+/-0.2339,0.3649+/-0.2359)\n"
     ]
    }
   ],
   "source": [
    "# hp selection based on AUC\n",
    "print(\"           fold1           fold2           fold3\")\n",
    "print(\"AUC      =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['auc'].mean(),\n",
    "                                                                           perf_fold1['auc'].std(),\n",
    "                                                                           perf_fold2['auc'].mean(),\n",
    "                                                                           perf_fold2['auc'].std(),\n",
    "                                                                           perf_fold3['auc'].mean(),\n",
    "                                                                           perf_fold3['auc'].std()))\n",
    "\n",
    "print(\"PRECISION=({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['precision'].mean(),\n",
    "                                                                           perf_fold1['precision'].std(),\n",
    "                                                                           perf_fold2['precision'].mean(),\n",
    "                                                                           perf_fold2['precision'].std(),\n",
    "                                                                           perf_fold3['precision'].mean(),\n",
    "                                                                           perf_fold3['precision'].std()))\n",
    "\n",
    "print(\"RECALL   =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['recall'].mean(),\n",
    "                                                                           perf_fold1['recall'].std(),\n",
    "                                                                           perf_fold2['recall'].mean(),\n",
    "                                                                           perf_fold2['recall'].std(),\n",
    "                                                                           perf_fold3['recall'].mean(),\n",
    "                                                                           perf_fold3['recall'].std()))\n",
    "\n",
    "print(\"KAPPA    =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['kappa'].mean(),\n",
    "                                                                           perf_fold1['kappa'].std(),\n",
    "                                                                           perf_fold2['kappa'].mean(),\n",
    "                                                                           perf_fold2['kappa'].std(),\n",
    "                                                                           perf_fold3['kappa'].mean(),\n",
    "                                                                           perf_fold3['kappa'].std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## find out best HP out of three outer folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:56:48.153913Z",
     "start_time": "2019-04-03T08:56:47.474283Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# get the best HP out of the three HP selected in inner loop based on outer loop performances\n",
    "best_hp_for_full_scale = []\n",
    "for i in range(526):\n",
    "    \n",
    "    best_fold = np.argmax(np.array([perf_fold1.iloc[i]['auc'],\n",
    "                                    perf_fold2.iloc[i]['auc'],\n",
    "                                    perf_fold3.iloc[i]['auc']]))\n",
    "    \n",
    "    \n",
    "    best_hp = np.array([perf_fold1.iloc[i]['hp'],\n",
    "                        perf_fold2.iloc[i]['hp'],\n",
    "                        perf_fold3.iloc[i]['hp']])[best_fold]\n",
    "    \n",
    "    \n",
    "    best_hp_for_full_scale.append(best_hp)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# select one best HP per target based on full outer loop perf\n",
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T20:58:06.500028Z",
     "start_time": "2019-04-04T20:58:06.482721Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "best_hp_for_full_scale = xgb_outer.sort_values('auc',ascending=False).groupby(['target']).first().reset_index()['hp'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T20:58:08.356417Z",
     "start_time": "2019-04-04T20:58:08.338502Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb0 python xgboost_model.py 0 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb1 python xgboost_model.py 1 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb2 python xgboost_model.py 2 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb3 python xgboost_model.py 3 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb4 python xgboost_model.py 4 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb5 python xgboost_model.py 5 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb6 python xgboost_model.py 6 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb7 python xgboost_model.py 7 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb8 python xgboost_model.py 8 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb9 python xgboost_model.py 9 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb10 python xgboost_model.py 10 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb11 python xgboost_model.py 11 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb12 python xgboost_model.py 12 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb13 python xgboost_model.py 13 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb14 python xgboost_model.py 14 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb15 python xgboost_model.py 15 binary:logistic 0.05 10 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb16 python xgboost_model.py 16 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb17 python xgboost_model.py 17 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb18 python xgboost_model.py 18 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb19 python xgboost_model.py 19 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb20 python xgboost_model.py 20 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb21 python xgboost_model.py 21 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb22 python xgboost_model.py 22 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb23 python xgboost_model.py 23 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb24 python xgboost_model.py 24 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb25 python xgboost_model.py 25 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb26 python xgboost_model.py 26 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb27 python xgboost_model.py 27 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb28 python xgboost_model.py 28 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb29 python xgboost_model.py 29 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb30 python xgboost_model.py 30 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb31 python xgboost_model.py 31 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb32 python xgboost_model.py 32 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb33 python xgboost_model.py 33 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb34 python xgboost_model.py 34 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb35 python xgboost_model.py 35 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb36 python xgboost_model.py 36 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb37 python xgboost_model.py 37 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb38 python xgboost_model.py 38 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb39 python xgboost_model.py 39 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb40 python xgboost_model.py 40 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb41 python xgboost_model.py 41 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb42 python xgboost_model.py 42 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb43 python xgboost_model.py 43 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb44 python xgboost_model.py 44 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb45 python xgboost_model.py 45 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb46 python xgboost_model.py 46 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb47 python xgboost_model.py 47 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb48 python xgboost_model.py 48 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb49 python xgboost_model.py 49 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb50 python xgboost_model.py 50 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb51 python xgboost_model.py 51 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb52 python xgboost_model.py 52 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb53 python xgboost_model.py 53 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb54 python xgboost_model.py 54 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb55 python xgboost_model.py 55 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb56 python xgboost_model.py 56 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb57 python xgboost_model.py 57 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb58 python xgboost_model.py 58 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb59 python xgboost_model.py 59 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb60 python xgboost_model.py 60 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb61 python xgboost_model.py 61 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb62 python xgboost_model.py 62 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb63 python xgboost_model.py 63 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb64 python xgboost_model.py 64 binary:logistic 0.05 5 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb65 python xgboost_model.py 65 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb66 python xgboost_model.py 66 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb67 python xgboost_model.py 67 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb68 python xgboost_model.py 68 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb69 python xgboost_model.py 69 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb70 python xgboost_model.py 70 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb71 python xgboost_model.py 71 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb72 python xgboost_model.py 72 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb73 python xgboost_model.py 73 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb74 python xgboost_model.py 74 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb75 python xgboost_model.py 75 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb76 python xgboost_model.py 76 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb77 python xgboost_model.py 77 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb78 python xgboost_model.py 78 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb79 python xgboost_model.py 79 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb80 python xgboost_model.py 80 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb81 python xgboost_model.py 81 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb82 python xgboost_model.py 82 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb83 python xgboost_model.py 83 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb84 python xgboost_model.py 84 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb85 python xgboost_model.py 85 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb86 python xgboost_model.py 86 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb87 python xgboost_model.py 87 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb88 python xgboost_model.py 88 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb89 python xgboost_model.py 89 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb90 python xgboost_model.py 90 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb91 python xgboost_model.py 91 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb92 python xgboost_model.py 92 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb93 python xgboost_model.py 93 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb94 python xgboost_model.py 94 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb95 python xgboost_model.py 95 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb96 python xgboost_model.py 96 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb97 python xgboost_model.py 97 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb98 python xgboost_model.py 98 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb99 python xgboost_model.py 99 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb100 python xgboost_model.py 100 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb101 python xgboost_model.py 101 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb102 python xgboost_model.py 102 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb103 python xgboost_model.py 103 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb104 python xgboost_model.py 104 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb105 python xgboost_model.py 105 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb106 python xgboost_model.py 106 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb107 python xgboost_model.py 107 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb108 python xgboost_model.py 108 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb109 python xgboost_model.py 109 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb110 python xgboost_model.py 110 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb111 python xgboost_model.py 111 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb112 python xgboost_model.py 112 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb113 python xgboost_model.py 113 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb114 python xgboost_model.py 114 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb115 python xgboost_model.py 115 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb116 python xgboost_model.py 116 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb117 python xgboost_model.py 117 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb118 python xgboost_model.py 118 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb119 python xgboost_model.py 119 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb120 python xgboost_model.py 120 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb121 python xgboost_model.py 121 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb122 python xgboost_model.py 122 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb123 python xgboost_model.py 123 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb124 python xgboost_model.py 124 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb125 python xgboost_model.py 125 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb126 python xgboost_model.py 126 binary:logistic 0.05 10 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb127 python xgboost_model.py 127 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb128 python xgboost_model.py 128 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb129 python xgboost_model.py 129 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb130 python xgboost_model.py 130 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb131 python xgboost_model.py 131 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb132 python xgboost_model.py 132 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb133 python xgboost_model.py 133 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb134 python xgboost_model.py 134 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb135 python xgboost_model.py 135 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb136 python xgboost_model.py 136 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb137 python xgboost_model.py 137 binary:logistic 0.05 10 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb138 python xgboost_model.py 138 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb139 python xgboost_model.py 139 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb140 python xgboost_model.py 140 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb141 python xgboost_model.py 141 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb142 python xgboost_model.py 142 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb143 python xgboost_model.py 143 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb144 python xgboost_model.py 144 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb145 python xgboost_model.py 145 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb146 python xgboost_model.py 146 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb147 python xgboost_model.py 147 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb148 python xgboost_model.py 148 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb149 python xgboost_model.py 149 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb150 python xgboost_model.py 150 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb151 python xgboost_model.py 151 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb152 python xgboost_model.py 152 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb153 python xgboost_model.py 153 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb154 python xgboost_model.py 154 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb155 python xgboost_model.py 155 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb156 python xgboost_model.py 156 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb157 python xgboost_model.py 157 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb158 python xgboost_model.py 158 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb159 python xgboost_model.py 159 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb160 python xgboost_model.py 160 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb161 python xgboost_model.py 161 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb162 python xgboost_model.py 162 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb163 python xgboost_model.py 163 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb164 python xgboost_model.py 164 binary:logistic 0.05 10 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb165 python xgboost_model.py 165 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb166 python xgboost_model.py 166 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb167 python xgboost_model.py 167 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb168 python xgboost_model.py 168 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb169 python xgboost_model.py 169 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb170 python xgboost_model.py 170 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb171 python xgboost_model.py 171 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb172 python xgboost_model.py 172 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb173 python xgboost_model.py 173 binary:logistic 0.05 5 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb174 python xgboost_model.py 174 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb175 python xgboost_model.py 175 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb176 python xgboost_model.py 176 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb177 python xgboost_model.py 177 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb178 python xgboost_model.py 178 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb179 python xgboost_model.py 179 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb180 python xgboost_model.py 180 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb181 python xgboost_model.py 181 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb182 python xgboost_model.py 182 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb183 python xgboost_model.py 183 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb184 python xgboost_model.py 184 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb185 python xgboost_model.py 185 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb186 python xgboost_model.py 186 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb187 python xgboost_model.py 187 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb188 python xgboost_model.py 188 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb189 python xgboost_model.py 189 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb190 python xgboost_model.py 190 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb191 python xgboost_model.py 191 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb192 python xgboost_model.py 192 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb193 python xgboost_model.py 193 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb194 python xgboost_model.py 194 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb195 python xgboost_model.py 195 binary:logistic 0.05 10 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb196 python xgboost_model.py 196 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb197 python xgboost_model.py 197 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb198 python xgboost_model.py 198 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb199 python xgboost_model.py 199 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb200 python xgboost_model.py 200 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb201 python xgboost_model.py 201 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb202 python xgboost_model.py 202 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb203 python xgboost_model.py 203 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb204 python xgboost_model.py 204 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb205 python xgboost_model.py 205 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb206 python xgboost_model.py 206 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb207 python xgboost_model.py 207 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb208 python xgboost_model.py 208 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb209 python xgboost_model.py 209 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb210 python xgboost_model.py 210 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb211 python xgboost_model.py 211 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb212 python xgboost_model.py 212 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb213 python xgboost_model.py 213 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb214 python xgboost_model.py 214 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb215 python xgboost_model.py 215 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb216 python xgboost_model.py 216 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb217 python xgboost_model.py 217 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb218 python xgboost_model.py 218 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb219 python xgboost_model.py 219 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb220 python xgboost_model.py 220 binary:logistic 0.05 5 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb221 python xgboost_model.py 221 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb222 python xgboost_model.py 222 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb223 python xgboost_model.py 223 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb224 python xgboost_model.py 224 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb225 python xgboost_model.py 225 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb226 python xgboost_model.py 226 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb227 python xgboost_model.py 227 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb228 python xgboost_model.py 228 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb229 python xgboost_model.py 229 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb230 python xgboost_model.py 230 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb231 python xgboost_model.py 231 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb232 python xgboost_model.py 232 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb233 python xgboost_model.py 233 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb234 python xgboost_model.py 234 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb235 python xgboost_model.py 235 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb236 python xgboost_model.py 236 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb237 python xgboost_model.py 237 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb238 python xgboost_model.py 238 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb239 python xgboost_model.py 239 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb240 python xgboost_model.py 240 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb241 python xgboost_model.py 241 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb242 python xgboost_model.py 242 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb243 python xgboost_model.py 243 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb244 python xgboost_model.py 244 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb245 python xgboost_model.py 245 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb246 python xgboost_model.py 246 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb247 python xgboost_model.py 247 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb248 python xgboost_model.py 248 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb249 python xgboost_model.py 249 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb250 python xgboost_model.py 250 binary:logistic 0.05 10 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb251 python xgboost_model.py 251 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb252 python xgboost_model.py 252 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb253 python xgboost_model.py 253 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb254 python xgboost_model.py 254 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb255 python xgboost_model.py 255 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb256 python xgboost_model.py 256 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb257 python xgboost_model.py 257 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb258 python xgboost_model.py 258 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb259 python xgboost_model.py 259 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb260 python xgboost_model.py 260 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb261 python xgboost_model.py 261 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb262 python xgboost_model.py 262 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb263 python xgboost_model.py 263 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb264 python xgboost_model.py 264 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb265 python xgboost_model.py 265 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb266 python xgboost_model.py 266 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb267 python xgboost_model.py 267 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb268 python xgboost_model.py 268 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb269 python xgboost_model.py 269 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb270 python xgboost_model.py 270 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb271 python xgboost_model.py 271 binary:logistic 0.05 10 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb272 python xgboost_model.py 272 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb273 python xgboost_model.py 273 binary:logistic 0.05 10 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb274 python xgboost_model.py 274 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb275 python xgboost_model.py 275 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb276 python xgboost_model.py 276 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb277 python xgboost_model.py 277 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb278 python xgboost_model.py 278 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb279 python xgboost_model.py 279 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb280 python xgboost_model.py 280 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb281 python xgboost_model.py 281 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb282 python xgboost_model.py 282 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb283 python xgboost_model.py 283 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb284 python xgboost_model.py 284 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb285 python xgboost_model.py 285 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb286 python xgboost_model.py 286 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb287 python xgboost_model.py 287 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb288 python xgboost_model.py 288 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb289 python xgboost_model.py 289 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb290 python xgboost_model.py 290 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb291 python xgboost_model.py 291 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb292 python xgboost_model.py 292 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb293 python xgboost_model.py 293 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb294 python xgboost_model.py 294 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb295 python xgboost_model.py 295 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb296 python xgboost_model.py 296 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb297 python xgboost_model.py 297 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb298 python xgboost_model.py 298 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb299 python xgboost_model.py 299 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb300 python xgboost_model.py 300 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb301 python xgboost_model.py 301 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb302 python xgboost_model.py 302 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb303 python xgboost_model.py 303 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb304 python xgboost_model.py 304 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb305 python xgboost_model.py 305 binary:logistic 0.05 10 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb306 python xgboost_model.py 306 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb307 python xgboost_model.py 307 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb308 python xgboost_model.py 308 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb309 python xgboost_model.py 309 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb310 python xgboost_model.py 310 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb311 python xgboost_model.py 311 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb312 python xgboost_model.py 312 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb313 python xgboost_model.py 313 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb314 python xgboost_model.py 314 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb315 python xgboost_model.py 315 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb316 python xgboost_model.py 316 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb317 python xgboost_model.py 317 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb318 python xgboost_model.py 318 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb319 python xgboost_model.py 319 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb320 python xgboost_model.py 320 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb321 python xgboost_model.py 321 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb322 python xgboost_model.py 322 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb323 python xgboost_model.py 323 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb324 python xgboost_model.py 324 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb325 python xgboost_model.py 325 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb326 python xgboost_model.py 326 binary:logistic 0.05 5 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb327 python xgboost_model.py 327 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb328 python xgboost_model.py 328 binary:logistic 0.05 10 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb329 python xgboost_model.py 329 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb330 python xgboost_model.py 330 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb331 python xgboost_model.py 331 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb332 python xgboost_model.py 332 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb333 python xgboost_model.py 333 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb334 python xgboost_model.py 334 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb335 python xgboost_model.py 335 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb336 python xgboost_model.py 336 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb337 python xgboost_model.py 337 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb338 python xgboost_model.py 338 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb339 python xgboost_model.py 339 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb340 python xgboost_model.py 340 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb341 python xgboost_model.py 341 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb342 python xgboost_model.py 342 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb343 python xgboost_model.py 343 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb344 python xgboost_model.py 344 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb345 python xgboost_model.py 345 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb346 python xgboost_model.py 346 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb347 python xgboost_model.py 347 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb348 python xgboost_model.py 348 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb349 python xgboost_model.py 349 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb350 python xgboost_model.py 350 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb351 python xgboost_model.py 351 binary:logistic 0.05 10 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb352 python xgboost_model.py 352 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb353 python xgboost_model.py 353 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb354 python xgboost_model.py 354 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb355 python xgboost_model.py 355 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb356 python xgboost_model.py 356 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb357 python xgboost_model.py 357 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb358 python xgboost_model.py 358 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb359 python xgboost_model.py 359 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb360 python xgboost_model.py 360 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb361 python xgboost_model.py 361 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb362 python xgboost_model.py 362 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb363 python xgboost_model.py 363 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb364 python xgboost_model.py 364 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb365 python xgboost_model.py 365 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb366 python xgboost_model.py 366 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb367 python xgboost_model.py 367 binary:logistic 0.05 5 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb368 python xgboost_model.py 368 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb369 python xgboost_model.py 369 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb370 python xgboost_model.py 370 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb371 python xgboost_model.py 371 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb372 python xgboost_model.py 372 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb373 python xgboost_model.py 373 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb374 python xgboost_model.py 374 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb375 python xgboost_model.py 375 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb376 python xgboost_model.py 376 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb377 python xgboost_model.py 377 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb378 python xgboost_model.py 378 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb379 python xgboost_model.py 379 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb380 python xgboost_model.py 380 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb381 python xgboost_model.py 381 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb382 python xgboost_model.py 382 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb383 python xgboost_model.py 383 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb384 python xgboost_model.py 384 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb385 python xgboost_model.py 385 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb386 python xgboost_model.py 386 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb387 python xgboost_model.py 387 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb388 python xgboost_model.py 388 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb389 python xgboost_model.py 389 binary:logistic 0.05 10 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb390 python xgboost_model.py 390 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb391 python xgboost_model.py 391 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb392 python xgboost_model.py 392 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb393 python xgboost_model.py 393 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb394 python xgboost_model.py 394 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb395 python xgboost_model.py 395 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb396 python xgboost_model.py 396 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb397 python xgboost_model.py 397 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb398 python xgboost_model.py 398 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb399 python xgboost_model.py 399 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb400 python xgboost_model.py 400 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb401 python xgboost_model.py 401 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb402 python xgboost_model.py 402 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb403 python xgboost_model.py 403 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb404 python xgboost_model.py 404 binary:logistic 0.05 10 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb405 python xgboost_model.py 405 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb406 python xgboost_model.py 406 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb407 python xgboost_model.py 407 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb408 python xgboost_model.py 408 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb409 python xgboost_model.py 409 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb410 python xgboost_model.py 410 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb411 python xgboost_model.py 411 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb412 python xgboost_model.py 412 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb413 python xgboost_model.py 413 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb414 python xgboost_model.py 414 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb415 python xgboost_model.py 415 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb416 python xgboost_model.py 416 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb417 python xgboost_model.py 417 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb418 python xgboost_model.py 418 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb419 python xgboost_model.py 419 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb420 python xgboost_model.py 420 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb421 python xgboost_model.py 421 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb422 python xgboost_model.py 422 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb423 python xgboost_model.py 423 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb424 python xgboost_model.py 424 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb425 python xgboost_model.py 425 binary:logistic 0.05 10 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb426 python xgboost_model.py 426 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb427 python xgboost_model.py 427 binary:logistic 0.05 10 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb428 python xgboost_model.py 428 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb429 python xgboost_model.py 429 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb430 python xgboost_model.py 430 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb431 python xgboost_model.py 431 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb432 python xgboost_model.py 432 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb433 python xgboost_model.py 433 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb434 python xgboost_model.py 434 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb435 python xgboost_model.py 435 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb436 python xgboost_model.py 436 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb437 python xgboost_model.py 437 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb438 python xgboost_model.py 438 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb439 python xgboost_model.py 439 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb440 python xgboost_model.py 440 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb441 python xgboost_model.py 441 binary:logistic 0.05 5 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb442 python xgboost_model.py 442 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb443 python xgboost_model.py 443 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb444 python xgboost_model.py 444 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb445 python xgboost_model.py 445 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb446 python xgboost_model.py 446 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb447 python xgboost_model.py 447 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb448 python xgboost_model.py 448 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb449 python xgboost_model.py 449 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb450 python xgboost_model.py 450 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb451 python xgboost_model.py 451 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb452 python xgboost_model.py 452 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb453 python xgboost_model.py 453 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb454 python xgboost_model.py 454 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb455 python xgboost_model.py 455 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb456 python xgboost_model.py 456 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb457 python xgboost_model.py 457 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb458 python xgboost_model.py 458 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb459 python xgboost_model.py 459 binary:logistic 0.05 10 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb460 python xgboost_model.py 460 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb461 python xgboost_model.py 461 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb462 python xgboost_model.py 462 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb463 python xgboost_model.py 463 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb464 python xgboost_model.py 464 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb465 python xgboost_model.py 465 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb466 python xgboost_model.py 466 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb467 python xgboost_model.py 467 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb468 python xgboost_model.py 468 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb469 python xgboost_model.py 469 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb470 python xgboost_model.py 470 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb471 python xgboost_model.py 471 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb472 python xgboost_model.py 472 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb473 python xgboost_model.py 473 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb474 python xgboost_model.py 474 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb475 python xgboost_model.py 475 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb476 python xgboost_model.py 476 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb477 python xgboost_model.py 477 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb478 python xgboost_model.py 478 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb479 python xgboost_model.py 479 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb480 python xgboost_model.py 480 binary:logistic 0.05 10 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb481 python xgboost_model.py 481 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb482 python xgboost_model.py 482 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb483 python xgboost_model.py 483 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb484 python xgboost_model.py 484 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb485 python xgboost_model.py 485 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb486 python xgboost_model.py 486 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb487 python xgboost_model.py 487 binary:logistic 0.05 5 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb488 python xgboost_model.py 488 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb489 python xgboost_model.py 489 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb490 python xgboost_model.py 490 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb491 python xgboost_model.py 491 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb492 python xgboost_model.py 492 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb493 python xgboost_model.py 493 binary:logistic 0.05 1 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb494 python xgboost_model.py 494 binary:logistic 0.05 5 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb495 python xgboost_model.py 495 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb496 python xgboost_model.py 496 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb497 python xgboost_model.py 497 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb498 python xgboost_model.py 498 binary:logistic 0.05 5 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb499 python xgboost_model.py 499 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb500 python xgboost_model.py 500 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb501 python xgboost_model.py 501 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb502 python xgboost_model.py 502 binary:logistic 0.05 10 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb503 python xgboost_model.py 503 binary:logistic 0.05 1 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb504 python xgboost_model.py 504 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb505 python xgboost_model.py 505 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb506 python xgboost_model.py 506 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb507 python xgboost_model.py 507 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb508 python xgboost_model.py 508 binary:logistic 0.05 5 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb509 python xgboost_model.py 509 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb510 python xgboost_model.py 510 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb511 python xgboost_model.py 511 binary:logistic 0.05 1 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb512 python xgboost_model.py 512 binary:logistic 0.05 10 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb513 python xgboost_model.py 513 binary:logistic 0.05 10 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb514 python xgboost_model.py 514 binary:logistic 0.05 10 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb515 python xgboost_model.py 515 binary:logistic 0.05 1 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb516 python xgboost_model.py 516 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb517 python xgboost_model.py 517 binary:logistic 0.05 5 200 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb518 python xgboost_model.py 518 binary:logistic 0.05 5 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb519 python xgboost_model.py 519 binary:logistic 0.05 1 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb520 python xgboost_model.py 520 binary:logistic 0.05 10 50 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb521 python xgboost_model.py 521 binary:logistic 0.05 10 50 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb522 python xgboost_model.py 522 binary:logistic 0.05 1 100 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb523 python xgboost_model.py 523 binary:logistic 0.05 5 200 10 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb524 python xgboost_model.py 524 binary:logistic 0.05 10 100 5 & \n",
      "srun --partition=core -c 12 --time=3-00:00:00 -J xgb525 python xgboost_model.py 525 binary:logistic 0.05 10 200 5 & \n"
     ]
    }
   ],
   "source": [
    "# generate commands for full-scale models to run \n",
    "#\n",
    "# Usage: python xgboost_model.py [target_idx] [obj] [lr] [scale_pos_w] [n_Estim] [max_depth]\n",
    "srun_cmd = 'srun --partition=core -c 12 --time=3-00:00:00 -J xgb'\n",
    "\n",
    "for i in range(526):\n",
    "    target  = i\n",
    "    hp      = best_hp_for_full_scale[i].split('-')\n",
    "    booster = hp[0].replace('booster_', '')\n",
    "    lr      = hp[1].replace('learning_rate_', '')\n",
    "    max_d   = hp[2].replace('max_depth_', '')\n",
    "    n_estim = hp[3].replace('n_estimators_', '')\n",
    "    obj     = hp[4].replace('objective_', '')\n",
    "    scale_pw= hp[5].replace('scale_pos_weight_', '')\n",
    "    print(\"{}{} python xgboost_model.py {} {} {} {} {} {} & \".format(srun_cmd, i, target, obj, lr, scale_pw, n_estim, max_d))\n",
    "\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Random Forest"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true,
    "hidden": true
   },
   "source": [
    "## Inner fold - find HP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:58:51.981546Z",
     "start_time": "2019-04-03T08:58:51.906492Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample</th>\n",
       "      <th>ofold</th>\n",
       "      <th>ifold</th>\n",
       "      <th>al</th>\n",
       "      <th>target</th>\n",
       "      <th>hp</th>\n",
       "      <th>auc</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>kappa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>class_weight_balanced-max_features_auto-n_esti...</td>\n",
       "      <td>0.775111</td>\n",
       "      <td>0.763749</td>\n",
       "      <td>0.816901</td>\n",
       "      <td>0.545830</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>class_weight_balanced-max_features_auto-n_esti...</td>\n",
       "      <td>0.768052</td>\n",
       "      <td>0.758327</td>\n",
       "      <td>0.809760</td>\n",
       "      <td>0.531926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>class_weight_balanced-max_features_auto-n_esti...</td>\n",
       "      <td>0.773379</td>\n",
       "      <td>0.769946</td>\n",
       "      <td>0.811429</td>\n",
       "      <td>0.543322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>100</td>\n",
       "      <td>class_weight_balanced-max_features_auto-n_esti...</td>\n",
       "      <td>0.503381</td>\n",
       "      <td>0.006873</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.013154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>100</td>\n",
       "      <td>class_weight_balanced-max_features_auto-n_esti...</td>\n",
       "      <td>0.503381</td>\n",
       "      <td>0.006873</td>\n",
       "      <td>0.363636</td>\n",
       "      <td>0.013154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample  ofold  ifold  al  target  \\\n",
       "0       1      1      0   6       0   \n",
       "1       1      1      0   6       0   \n",
       "2       1      1      0   6       0   \n",
       "3       1      1      0   6     100   \n",
       "4       1      1      0   6     100   \n",
       "\n",
       "                                                  hp       auc  precision  \\\n",
       "0  class_weight_balanced-max_features_auto-n_esti...  0.775111   0.763749   \n",
       "1  class_weight_balanced-max_features_auto-n_esti...  0.768052   0.758327   \n",
       "2  class_weight_balanced-max_features_auto-n_esti...  0.773379   0.769946   \n",
       "3  class_weight_balanced-max_features_auto-n_esti...  0.503381   0.006873   \n",
       "4  class_weight_balanced-max_features_auto-n_esti...  0.503381   0.006873   \n",
       "\n",
       "     recall     kappa  \n",
       "0  0.816901  0.545830  \n",
       "1  0.809760  0.531926  \n",
       "2  0.811429  0.543322  \n",
       "3  0.363636  0.013154  \n",
       "4  0.363636  0.013154  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf_df = pd.read_csv('results_v5_rf_ecfp/all-nested-cv-perf-scores.csv', header=0, sep=\"\\t\")\n",
    "rf_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:58:56.220304Z",
     "start_time": "2019-04-03T08:58:56.211466Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# separate inner and out loop results\n",
    "rf_inner = rf_df.loc[rf_df['ifold'].isin([1,2])].copy(deep=True).drop(['al','sample'],axis=1)\n",
    "rf_outer = rf_df.loc[rf_df['ifold'].isin([0])].copy(deep=True).drop(['al','sample'],axis=1)\n",
    "del rf_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:59:05.133836Z",
     "start_time": "2019-04-03T08:58:58.513950Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# select best HP per fold\n",
    "hp_selection = selectHP('auc',rf_inner)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:59:07.404698Z",
     "start_time": "2019-04-03T08:59:07.401020Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(526, 4)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hp_selection.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "hidden": true
   },
   "source": [
    "## Outer folds - model evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:59:24.781570Z",
     "start_time": "2019-04-03T08:59:20.245261Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# use the selected HP to find out the performance in outer loop\n",
    "perf_fold1, perf_fold2, perf_fold3 = getOuterLoopPerformance(rf_outer, hp_selection)\n",
    "\n",
    "perf_fold1.to_csv(\"results_v5_rf_ecfp/perf_fold1.csv\", sep=\"\\t\", index=None)\n",
    "perf_fold2.to_csv(\"results_v5_rf_ecfp/perf_fold2.csv\", sep=\"\\t\", index=None)\n",
    "perf_fold3.to_csv(\"results_v5_rf_ecfp/perf_fold3.csv\", sep=\"\\t\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T08:59:27.251692Z",
     "start_time": "2019-04-03T08:59:27.237929Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           fold1           fold2           fold3\n",
      "AUC      =(0.6444+/-0.1137,0.6409+/-0.1123,0.6379+/-0.1123)\n",
      "PRECISION=(0.4646+/-0.3641,0.4581+/-0.3664,0.4505+/-0.3637)\n",
      "RECALL   =(0.7294+/-0.2732,0.7112+/-0.2736,0.7141+/-0.2791)\n",
      "KAPPA    =(0.3270+/-0.2347,0.3185+/-0.2304,0.3151+/-0.2341)\n"
     ]
    }
   ],
   "source": [
    "# hp selection based on AUC\n",
    "print(\"           fold1           fold2           fold3\")\n",
    "print(\"AUC      =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['auc'].mean(),\n",
    "                                                                           perf_fold1['auc'].std(),\n",
    "                                                                           perf_fold2['auc'].mean(),\n",
    "                                                                           perf_fold2['auc'].std(),\n",
    "                                                                           perf_fold3['auc'].mean(),\n",
    "                                                                           perf_fold3['auc'].std()))\n",
    "\n",
    "print(\"PRECISION=({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['precision'].mean(),\n",
    "                                                                           perf_fold1['precision'].std(),\n",
    "                                                                           perf_fold2['precision'].mean(),\n",
    "                                                                           perf_fold2['precision'].std(),\n",
    "                                                                           perf_fold3['precision'].mean(),\n",
    "                                                                           perf_fold3['precision'].std()))\n",
    "\n",
    "print(\"RECALL   =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['recall'].mean(),\n",
    "                                                                           perf_fold1['recall'].std(),\n",
    "                                                                           perf_fold2['recall'].mean(),\n",
    "                                                                           perf_fold2['recall'].std(),\n",
    "                                                                           perf_fold3['recall'].mean(),\n",
    "                                                                           perf_fold3['recall'].std()))\n",
    "\n",
    "print(\"KAPPA    =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['kappa'].mean(),\n",
    "                                                                           perf_fold1['kappa'].std(),\n",
    "                                                                           perf_fold2['kappa'].mean(),\n",
    "                                                                           perf_fold2['kappa'].std(),\n",
    "                                                                           perf_fold3['kappa'].mean(),\n",
    "                                                                           perf_fold3['kappa'].std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SVM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inner fold - find HP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-02T17:36:37.663878Z",
     "start_time": "2019-05-02T17:36:37.315872Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample</th>\n",
       "      <th>ofold</th>\n",
       "      <th>ifold</th>\n",
       "      <th>al</th>\n",
       "      <th>target</th>\n",
       "      <th>hp</th>\n",
       "      <th>auc</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>kappa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>C_300-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.666746</td>\n",
       "      <td>0.620449</td>\n",
       "      <td>0.729508</td>\n",
       "      <td>0.327439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>C_100-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.670803</td>\n",
       "      <td>0.618900</td>\n",
       "      <td>0.735727</td>\n",
       "      <td>0.335026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>C_30-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.685857</td>\n",
       "      <td>0.692486</td>\n",
       "      <td>0.729201</td>\n",
       "      <td>0.369413</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>C_10-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.690788</td>\n",
       "      <td>0.695585</td>\n",
       "      <td>0.734260</td>\n",
       "      <td>0.379106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>C_1-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.727089</td>\n",
       "      <td>0.761425</td>\n",
       "      <td>0.755573</td>\n",
       "      <td>0.454614</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample  ofold  ifold  al  target  \\\n",
       "0       1      1      0   6       0   \n",
       "1       1      1      0   6       0   \n",
       "2       1      1      0   6       0   \n",
       "3       1      1      0   6       0   \n",
       "4       1      1      0   6       0   \n",
       "\n",
       "                                              hp       auc  precision  \\\n",
       "0  C_300-dual_True-loss_squared_hinge-penalty_l2  0.666746   0.620449   \n",
       "1  C_100-dual_True-loss_squared_hinge-penalty_l2  0.670803   0.618900   \n",
       "2   C_30-dual_True-loss_squared_hinge-penalty_l2  0.685857   0.692486   \n",
       "3   C_10-dual_True-loss_squared_hinge-penalty_l2  0.690788   0.695585   \n",
       "4    C_1-dual_True-loss_squared_hinge-penalty_l2  0.727089   0.761425   \n",
       "\n",
       "     recall     kappa  \n",
       "0  0.729508  0.327439  \n",
       "1  0.735727  0.335026  \n",
       "2  0.729201  0.369413  \n",
       "3  0.734260  0.379106  \n",
       "4  0.755573  0.454614  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm_df = pd.read_csv('results_v5_svm_ecfp/all-nested-cv-perf-scores.tsv', header=0, sep=\"\\t\")\n",
    "svm_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-02T17:36:41.908325Z",
     "start_time": "2019-05-02T17:36:41.679315Z"
    }
   },
   "outputs": [],
   "source": [
    "# separate inner and out loop results\n",
    "svm_inner = svm_df.loc[svm_df['ifold'].isin([1,2])].copy(deep=True).drop(['al','sample'],axis=1)\n",
    "svm_outer = svm_df.loc[svm_df['ifold'].isin([0])].copy(deep=True).drop(['al','sample'],axis=1)\n",
    "del svm_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-02T17:36:53.661931Z",
     "start_time": "2019-05-02T17:36:46.048244Z"
    }
   },
   "outputs": [],
   "source": [
    "# select best HP per fold\n",
    "hp_selection = selectHP('auc',svm_inner)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outer fold - model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-03T11:08:38.994825Z",
     "start_time": "2019-05-03T11:08:38.723747Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/projects/cc/kxsw653/ExCAPE-ML_v5/IT4I-data/qsar_models\r\n"
     ]
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-02T17:37:07.335977Z",
     "start_time": "2019-05-02T17:36:53.703823Z"
    }
   },
   "outputs": [],
   "source": [
    "# use the selected HP to find out the performance in outer loop\n",
    "perf_fold1, perf_fold2, perf_fold3 = getOuterLoopPerformance(svm_outer, hp_selection)\n",
    "\n",
    "perf_fold1.to_csv(\"results_v5_svm_ecfp/perf_fold1.csv\", sep=\"\\t\", index=None)\n",
    "perf_fold2.to_csv(\"results_v5_svm_ecfp/perf_fold2.csv\", sep=\"\\t\", index=None)\n",
    "perf_fold3.to_csv(\"results_v5_svm_ecfp/perf_fold3.csv\", sep=\"\\t\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-05-02T17:37:07.446843Z",
     "start_time": "2019-05-02T17:37:07.414059Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ofold</th>\n",
       "      <th>ifold</th>\n",
       "      <th>target</th>\n",
       "      <th>hp</th>\n",
       "      <th>auc</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>kappa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>C_0.1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.764828</td>\n",
       "      <td>0.815647</td>\n",
       "      <td>0.780578</td>\n",
       "      <td>0.532616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>C_0.01-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.795744</td>\n",
       "      <td>0.780059</td>\n",
       "      <td>0.801205</td>\n",
       "      <td>0.591688</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>C_100-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.515563</td>\n",
       "      <td>0.033141</td>\n",
       "      <td>0.140244</td>\n",
       "      <td>0.050047</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>C_100-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.512383</td>\n",
       "      <td>0.025000</td>\n",
       "      <td>0.088235</td>\n",
       "      <td>0.038578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>C_300-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.504152</td>\n",
       "      <td>0.009724</td>\n",
       "      <td>0.071006</td>\n",
       "      <td>0.014484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>C_100-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.499914</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000312</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>C_300-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.514861</td>\n",
       "      <td>0.046445</td>\n",
       "      <td>0.148624</td>\n",
       "      <td>0.043847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>C_300-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.550642</td>\n",
       "      <td>0.102848</td>\n",
       "      <td>0.383481</td>\n",
       "      <td>0.158863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>C_300-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.504614</td>\n",
       "      <td>0.015766</td>\n",
       "      <td>0.119701</td>\n",
       "      <td>0.015632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>C_300-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.500971</td>\n",
       "      <td>0.004515</td>\n",
       "      <td>0.039867</td>\n",
       "      <td>0.003423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>C_30-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.569938</td>\n",
       "      <td>0.140625</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.196537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>C_100-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.501319</td>\n",
       "      <td>0.003795</td>\n",
       "      <td>0.023529</td>\n",
       "      <td>0.004519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>C_30-dual_False-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.524515</td>\n",
       "      <td>0.049645</td>\n",
       "      <td>0.122807</td>\n",
       "      <td>0.069780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>C_300-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.509835</td>\n",
       "      <td>0.027586</td>\n",
       "      <td>0.038835</td>\n",
       "      <td>0.022956</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>14</td>\n",
       "      <td>C_300-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.504573</td>\n",
       "      <td>0.012165</td>\n",
       "      <td>0.053381</td>\n",
       "      <td>0.014767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>15</td>\n",
       "      <td>C_300-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.513505</td>\n",
       "      <td>0.029162</td>\n",
       "      <td>0.142857</td>\n",
       "      <td>0.044500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>C_300-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.509672</td>\n",
       "      <td>0.020737</td>\n",
       "      <td>0.110429</td>\n",
       "      <td>0.032388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>17</td>\n",
       "      <td>C_10-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.606351</td>\n",
       "      <td>0.212766</td>\n",
       "      <td>0.555556</td>\n",
       "      <td>0.307549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "      <td>C_300-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.510409</td>\n",
       "      <td>0.022502</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>0.035025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>19</td>\n",
       "      <td>C_300-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.505293</td>\n",
       "      <td>0.012708</td>\n",
       "      <td>0.109244</td>\n",
       "      <td>0.018665</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "      <td>C_300-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.502665</td>\n",
       "      <td>0.007330</td>\n",
       "      <td>0.043636</td>\n",
       "      <td>0.009046</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>21</td>\n",
       "      <td>C_100-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.523205</td>\n",
       "      <td>0.093644</td>\n",
       "      <td>0.278716</td>\n",
       "      <td>0.063340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>22</td>\n",
       "      <td>C_300-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.501571</td>\n",
       "      <td>0.004098</td>\n",
       "      <td>0.038168</td>\n",
       "      <td>0.005632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>C_30-dual_False-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.692763</td>\n",
       "      <td>0.386364</td>\n",
       "      <td>0.390805</td>\n",
       "      <td>0.387726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>24</td>\n",
       "      <td>C_100-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.705816</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.538462</td>\n",
       "      <td>0.466492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "      <td>C_100-dual_False-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.717170</td>\n",
       "      <td>0.434783</td>\n",
       "      <td>0.263158</td>\n",
       "      <td>0.327565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>26</td>\n",
       "      <td>C_30-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.788167</td>\n",
       "      <td>0.577528</td>\n",
       "      <td>0.647355</td>\n",
       "      <td>0.609058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>27</td>\n",
       "      <td>C_100-dual_False-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.899996</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.960000</td>\n",
       "      <td>0.872698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>28</td>\n",
       "      <td>C_10-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.916653</td>\n",
       "      <td>0.833663</td>\n",
       "      <td>0.907328</td>\n",
       "      <td>0.868410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>29</td>\n",
       "      <td>C_30-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.753408</td>\n",
       "      <td>0.508671</td>\n",
       "      <td>0.454780</td>\n",
       "      <td>0.478548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>496</td>\n",
       "      <td>C_0.1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.722838</td>\n",
       "      <td>0.642105</td>\n",
       "      <td>0.734940</td>\n",
       "      <td>0.449991</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>497</td>\n",
       "      <td>C_0.001-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.678912</td>\n",
       "      <td>0.557823</td>\n",
       "      <td>0.854167</td>\n",
       "      <td>0.300486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>498</td>\n",
       "      <td>C_0.05-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.511074</td>\n",
       "      <td>0.096774</td>\n",
       "      <td>0.375000</td>\n",
       "      <td>0.027661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>499</td>\n",
       "      <td>C_1-dual_False-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.731771</td>\n",
       "      <td>0.854167</td>\n",
       "      <td>0.621212</td>\n",
       "      <td>0.442786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>500</td>\n",
       "      <td>C_1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.592980</td>\n",
       "      <td>0.543103</td>\n",
       "      <td>0.715909</td>\n",
       "      <td>0.172296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>501</td>\n",
       "      <td>C_0.1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.583528</td>\n",
       "      <td>0.977401</td>\n",
       "      <td>0.786364</td>\n",
       "      <td>0.222511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>502</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>502</td>\n",
       "      <td>C_1-dual_False-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.577261</td>\n",
       "      <td>0.948640</td>\n",
       "      <td>0.853261</td>\n",
       "      <td>0.197138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>503</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>503</td>\n",
       "      <td>C_0.05-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.685897</td>\n",
       "      <td>0.564103</td>\n",
       "      <td>0.814815</td>\n",
       "      <td>0.345238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>504</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>504</td>\n",
       "      <td>C_0.1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.644762</td>\n",
       "      <td>0.809524</td>\n",
       "      <td>0.566667</td>\n",
       "      <td>0.279926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>505</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>505</td>\n",
       "      <td>C_1-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.731693</td>\n",
       "      <td>0.816327</td>\n",
       "      <td>0.769231</td>\n",
       "      <td>0.469729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>506</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>506</td>\n",
       "      <td>C_1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.677003</td>\n",
       "      <td>0.465116</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.384346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>507</td>\n",
       "      <td>C_0.1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.578125</td>\n",
       "      <td>0.656250</td>\n",
       "      <td>0.777778</td>\n",
       "      <td>0.138249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>508</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>508</td>\n",
       "      <td>C_0.1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.909699</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>0.884615</td>\n",
       "      <td>0.819398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>509</td>\n",
       "      <td>C_1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.756352</td>\n",
       "      <td>0.736842</td>\n",
       "      <td>0.518519</td>\n",
       "      <td>0.449126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>510</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>510</td>\n",
       "      <td>C_1-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.662608</td>\n",
       "      <td>0.407407</td>\n",
       "      <td>0.647059</td>\n",
       "      <td>0.368179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>511</td>\n",
       "      <td>C_1-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.659905</td>\n",
       "      <td>0.895349</td>\n",
       "      <td>0.658120</td>\n",
       "      <td>0.334105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>512</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>512</td>\n",
       "      <td>C_0.01-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.705882</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.464286</td>\n",
       "      <td>0.320988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>513</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>513</td>\n",
       "      <td>C_1-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.760274</td>\n",
       "      <td>0.767123</td>\n",
       "      <td>0.756757</td>\n",
       "      <td>0.520548</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>514</td>\n",
       "      <td>C_300-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.586022</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.662162</td>\n",
       "      <td>0.174276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>515</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>515</td>\n",
       "      <td>C_0.01-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.834158</td>\n",
       "      <td>0.926380</td>\n",
       "      <td>0.825137</td>\n",
       "      <td>0.681465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>516</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>516</td>\n",
       "      <td>C_1-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.514610</td>\n",
       "      <td>0.068182</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.040816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>517</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>517</td>\n",
       "      <td>C_0.05-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.834652</td>\n",
       "      <td>0.989305</td>\n",
       "      <td>0.885167</td>\n",
       "      <td>0.733761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>518</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>518</td>\n",
       "      <td>C_0.1-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.820366</td>\n",
       "      <td>0.956522</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>0.679612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>519</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>519</td>\n",
       "      <td>C_0.05-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.738517</td>\n",
       "      <td>0.742857</td>\n",
       "      <td>0.832000</td>\n",
       "      <td>0.458086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>520</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>520</td>\n",
       "      <td>C_1-dual_False-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.523941</td>\n",
       "      <td>0.675000</td>\n",
       "      <td>0.593407</td>\n",
       "      <td>0.049082</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>521</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>521</td>\n",
       "      <td>C_0.01-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.802701</td>\n",
       "      <td>0.949153</td>\n",
       "      <td>0.835821</td>\n",
       "      <td>0.642135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>522</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>522</td>\n",
       "      <td>C_0.05-dual_True-loss_hinge-penalty_l2</td>\n",
       "      <td>0.677521</td>\n",
       "      <td>0.926471</td>\n",
       "      <td>0.887324</td>\n",
       "      <td>0.388060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>523</td>\n",
       "      <td>C_300-dual_True-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.722134</td>\n",
       "      <td>0.849673</td>\n",
       "      <td>0.896552</td>\n",
       "      <td>0.410612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>524</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>524</td>\n",
       "      <td>C_10-dual_False-loss_squared_hinge-penalty_l2</td>\n",
       "      <td>0.695186</td>\n",
       "      <td>0.877551</td>\n",
       "      <td>0.819048</td>\n",
       "      <td>0.412668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>525</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>525</td>\n",
       "      <td>C_0.1-dual_False-loss_squared_hinge-penalty_l1</td>\n",
       "      <td>0.722929</td>\n",
       "      <td>0.804348</td>\n",
       "      <td>0.795699</td>\n",
       "      <td>0.447647</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>526 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     ofold  ifold  target                                               hp  \\\n",
       "0        1      0       0            C_0.1-dual_True-loss_hinge-penalty_l2   \n",
       "1        1      0       1           C_0.01-dual_True-loss_hinge-penalty_l2   \n",
       "2        1      0       2            C_100-dual_True-loss_hinge-penalty_l2   \n",
       "3        1      0       3   C_100-dual_False-loss_squared_hinge-penalty_l1   \n",
       "4        1      0       4            C_300-dual_True-loss_hinge-penalty_l2   \n",
       "5        1      0       5            C_100-dual_True-loss_hinge-penalty_l2   \n",
       "6        1      0       6    C_300-dual_True-loss_squared_hinge-penalty_l2   \n",
       "7        1      0       7    C_300-dual_True-loss_squared_hinge-penalty_l2   \n",
       "8        1      0       8    C_300-dual_True-loss_squared_hinge-penalty_l2   \n",
       "9        1      0       9   C_300-dual_False-loss_squared_hinge-penalty_l1   \n",
       "10       1      0      10             C_30-dual_True-loss_hinge-penalty_l2   \n",
       "11       1      0      11   C_100-dual_False-loss_squared_hinge-penalty_l1   \n",
       "12       1      0      12    C_30-dual_False-loss_squared_hinge-penalty_l2   \n",
       "13       1      0      13            C_300-dual_True-loss_hinge-penalty_l2   \n",
       "14       1      0      14            C_300-dual_True-loss_hinge-penalty_l2   \n",
       "15       1      0      15            C_300-dual_True-loss_hinge-penalty_l2   \n",
       "16       1      0      16    C_300-dual_True-loss_squared_hinge-penalty_l2   \n",
       "17       1      0      17             C_10-dual_True-loss_hinge-penalty_l2   \n",
       "18       1      0      18    C_300-dual_True-loss_squared_hinge-penalty_l2   \n",
       "19       1      0      19    C_300-dual_True-loss_squared_hinge-penalty_l2   \n",
       "20       1      0      20            C_300-dual_True-loss_hinge-penalty_l2   \n",
       "21       1      0      21    C_100-dual_True-loss_squared_hinge-penalty_l2   \n",
       "22       1      0      22            C_300-dual_True-loss_hinge-penalty_l2   \n",
       "23       1      0      23    C_30-dual_False-loss_squared_hinge-penalty_l2   \n",
       "24       1      0      24   C_100-dual_False-loss_squared_hinge-penalty_l1   \n",
       "25       1      0      25   C_100-dual_False-loss_squared_hinge-penalty_l2   \n",
       "26       1      0      26             C_30-dual_True-loss_hinge-penalty_l2   \n",
       "27       1      0      27   C_100-dual_False-loss_squared_hinge-penalty_l2   \n",
       "28       1      0      28             C_10-dual_True-loss_hinge-penalty_l2   \n",
       "29       1      0      29             C_30-dual_True-loss_hinge-penalty_l2   \n",
       "..     ...    ...     ...                                              ...   \n",
       "496      1      0     496            C_0.1-dual_True-loss_hinge-penalty_l2   \n",
       "497      1      0     497  C_0.001-dual_True-loss_squared_hinge-penalty_l2   \n",
       "498      1      0     498           C_0.05-dual_True-loss_hinge-penalty_l2   \n",
       "499      1      0     499     C_1-dual_False-loss_squared_hinge-penalty_l2   \n",
       "500      1      0     500              C_1-dual_True-loss_hinge-penalty_l2   \n",
       "501      1      0     501            C_0.1-dual_True-loss_hinge-penalty_l2   \n",
       "502      1      0     502     C_1-dual_False-loss_squared_hinge-penalty_l2   \n",
       "503      1      0     503  C_0.05-dual_False-loss_squared_hinge-penalty_l1   \n",
       "504      1      0     504            C_0.1-dual_True-loss_hinge-penalty_l2   \n",
       "505      1      0     505     C_1-dual_False-loss_squared_hinge-penalty_l1   \n",
       "506      1      0     506              C_1-dual_True-loss_hinge-penalty_l2   \n",
       "507      1      0     507            C_0.1-dual_True-loss_hinge-penalty_l2   \n",
       "508      1      0     508            C_0.1-dual_True-loss_hinge-penalty_l2   \n",
       "509      1      0     509              C_1-dual_True-loss_hinge-penalty_l2   \n",
       "510      1      0     510     C_1-dual_False-loss_squared_hinge-penalty_l1   \n",
       "511      1      0     511     C_1-dual_False-loss_squared_hinge-penalty_l1   \n",
       "512      1      0     512           C_0.01-dual_True-loss_hinge-penalty_l2   \n",
       "513      1      0     513     C_1-dual_False-loss_squared_hinge-penalty_l1   \n",
       "514      1      0     514            C_300-dual_True-loss_hinge-penalty_l2   \n",
       "515      1      0     515   C_0.01-dual_True-loss_squared_hinge-penalty_l2   \n",
       "516      1      0     516     C_1-dual_False-loss_squared_hinge-penalty_l1   \n",
       "517      1      0     517           C_0.05-dual_True-loss_hinge-penalty_l2   \n",
       "518      1      0     518            C_0.1-dual_True-loss_hinge-penalty_l2   \n",
       "519      1      0     519   C_0.05-dual_True-loss_squared_hinge-penalty_l2   \n",
       "520      1      0     520     C_1-dual_False-loss_squared_hinge-penalty_l2   \n",
       "521      1      0     521           C_0.01-dual_True-loss_hinge-penalty_l2   \n",
       "522      1      0     522           C_0.05-dual_True-loss_hinge-penalty_l2   \n",
       "523      1      0     523    C_300-dual_True-loss_squared_hinge-penalty_l2   \n",
       "524      1      0     524    C_10-dual_False-loss_squared_hinge-penalty_l2   \n",
       "525      1      0     525   C_0.1-dual_False-loss_squared_hinge-penalty_l1   \n",
       "\n",
       "          auc  precision    recall     kappa  \n",
       "0    0.764828   0.815647  0.780578  0.532616  \n",
       "1    0.795744   0.780059  0.801205  0.591688  \n",
       "2    0.515563   0.033141  0.140244  0.050047  \n",
       "3    0.512383   0.025000  0.088235  0.038578  \n",
       "4    0.504152   0.009724  0.071006  0.014484  \n",
       "5    0.499914   0.000000  0.000000 -0.000312  \n",
       "6    0.514861   0.046445  0.148624  0.043847  \n",
       "7    0.550642   0.102848  0.383481  0.158863  \n",
       "8    0.504614   0.015766  0.119701  0.015632  \n",
       "9    0.500971   0.004515  0.039867  0.003423  \n",
       "10   0.569938   0.140625  0.333333  0.196537  \n",
       "11   0.501319   0.003795  0.023529  0.004519  \n",
       "12   0.524515   0.049645  0.122807  0.069780  \n",
       "13   0.509835   0.027586  0.038835  0.022956  \n",
       "14   0.504573   0.012165  0.053381  0.014767  \n",
       "15   0.513505   0.029162  0.142857  0.044500  \n",
       "16   0.509672   0.020737  0.110429  0.032388  \n",
       "17   0.606351   0.212766  0.555556  0.307549  \n",
       "18   0.510409   0.022502  0.125000  0.035025  \n",
       "19   0.505293   0.012708  0.109244  0.018665  \n",
       "20   0.502665   0.007330  0.043636  0.009046  \n",
       "21   0.523205   0.093644  0.278716  0.063340  \n",
       "22   0.501571   0.004098  0.038168  0.005632  \n",
       "23   0.692763   0.386364  0.390805  0.387726  \n",
       "24   0.705816   0.411765  0.538462  0.466492  \n",
       "25   0.717170   0.434783  0.263158  0.327565  \n",
       "26   0.788167   0.577528  0.647355  0.609058  \n",
       "27   0.899996   0.800000  0.960000  0.872698  \n",
       "28   0.916653   0.833663  0.907328  0.868410  \n",
       "29   0.753408   0.508671  0.454780  0.478548  \n",
       "..        ...        ...       ...       ...  \n",
       "496  0.722838   0.642105  0.734940  0.449991  \n",
       "497  0.678912   0.557823  0.854167  0.300486  \n",
       "498  0.511074   0.096774  0.375000  0.027661  \n",
       "499  0.731771   0.854167  0.621212  0.442786  \n",
       "500  0.592980   0.543103  0.715909  0.172296  \n",
       "501  0.583528   0.977401  0.786364  0.222511  \n",
       "502  0.577261   0.948640  0.853261  0.197138  \n",
       "503  0.685897   0.564103  0.814815  0.345238  \n",
       "504  0.644762   0.809524  0.566667  0.279926  \n",
       "505  0.731693   0.816327  0.769231  0.469729  \n",
       "506  0.677003   0.465116  0.666667  0.384346  \n",
       "507  0.578125   0.656250  0.777778  0.138249  \n",
       "508  0.909699   0.884615  0.884615  0.819398  \n",
       "509  0.756352   0.736842  0.518519  0.449126  \n",
       "510  0.662608   0.407407  0.647059  0.368179  \n",
       "511  0.659905   0.895349  0.658120  0.334105  \n",
       "512  0.705882   1.000000  0.464286  0.320988  \n",
       "513  0.760274   0.767123  0.756757  0.520548  \n",
       "514  0.586022   0.333333  0.662162  0.174276  \n",
       "515  0.834158   0.926380  0.825137  0.681465  \n",
       "516  0.514610   0.068182  0.333333  0.040816  \n",
       "517  0.834652   0.989305  0.885167  0.733761  \n",
       "518  0.820366   0.956522  0.916667  0.679612  \n",
       "519  0.738517   0.742857  0.832000  0.458086  \n",
       "520  0.523941   0.675000  0.593407  0.049082  \n",
       "521  0.802701   0.949153  0.835821  0.642135  \n",
       "522  0.677521   0.926471  0.887324  0.388060  \n",
       "523  0.722134   0.849673  0.896552  0.410612  \n",
       "524  0.695186   0.877551  0.819048  0.412668  \n",
       "525  0.722929   0.804348  0.795699  0.447647  \n",
       "\n",
       "[526 rows x 8 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "perf_fold1.sort_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:00:44.335837Z",
     "start_time": "2019-04-03T09:00:44.323156Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "           fold1           fold2           fold3\n",
      "AUC      =(0.6700+/-0.1168,0.6710+/-0.1128,0.6691+/-0.1138)\n",
      "PRECISION=(0.5254+/-0.3149,0.5292+/-0.3114,0.5257+/-0.3107)\n",
      "RECALL   =(0.5758+/-0.2999,0.5680+/-0.2979,0.5763+/-0.2901)\n",
      "KAPPA    =(0.3430+/-0.2323,0.3454+/-0.2266,0.3482+/-0.2309)\n"
     ]
    }
   ],
   "source": [
    "# hp selection based on AUC\n",
    "print(\"           fold1           fold2           fold3\")\n",
    "print(\"AUC      =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['auc'].mean(),\n",
    "                                                                           perf_fold1['auc'].std(),\n",
    "                                                                           perf_fold2['auc'].mean(),\n",
    "                                                                           perf_fold2['auc'].std(),\n",
    "                                                                           perf_fold3['auc'].mean(),\n",
    "                                                                           perf_fold3['auc'].std()))\n",
    "\n",
    "print(\"PRECISION=({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['precision'].mean(),\n",
    "                                                                           perf_fold1['precision'].std(),\n",
    "                                                                           perf_fold2['precision'].mean(),\n",
    "                                                                           perf_fold2['precision'].std(),\n",
    "                                                                           perf_fold3['precision'].mean(),\n",
    "                                                                           perf_fold3['precision'].std()))\n",
    "\n",
    "print(\"RECALL   =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['recall'].mean(),\n",
    "                                                                           perf_fold1['recall'].std(),\n",
    "                                                                           perf_fold2['recall'].mean(),\n",
    "                                                                           perf_fold2['recall'].std(),\n",
    "                                                                           perf_fold3['recall'].mean(),\n",
    "                                                                           perf_fold3['recall'].std()))\n",
    "\n",
    "print(\"KAPPA    =({:.4f}+/-{:.4f},{:.4f}+/-{:.4f},{:.4f}+/-{:.4f})\".format(perf_fold1['kappa'].mean(),\n",
    "                                                                           perf_fold1['kappa'].std(),\n",
    "                                                                           perf_fold2['kappa'].mean(),\n",
    "                                                                           perf_fold2['kappa'].std(),\n",
    "                                                                           perf_fold3['kappa'].mean(),\n",
    "                                                                           perf_fold3['kappa'].std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ExNET "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Inner Loop - HP search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:01:18.575969Z",
     "start_time": "2019-04-03T09:01:18.557438Z"
    }
   },
   "outputs": [],
   "source": [
    "target_map = pd.read_csv('../../DATA/bioactivities/MAP_TARGETS_ExCAPE-ml.csv',header=None, sep=',', index_col=1, squeeze=True)\n",
    "header=['sample','ofold','ifold','al','hp', 'step']\n",
    "useless = [header.append(x) for x in target_map.values]\n",
    "del useless"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:01:22.839753Z",
     "start_time": "2019-04-03T09:01:22.004404Z"
    }
   },
   "outputs": [],
   "source": [
    "# one columns per target +     hyperparameters\n",
    "#exnet_inner = pd.read_csv(\"/projects/cc/kxsw653/ExCAPE-ML_v5/IT4I-data/qsar_models/results_v5_exnet/exnet_inner_metrics_lite.tsv\", header = 0, sep=\"\\t\")\n",
    "exnet_inner = pd.read_csv(\"/projects/cc/kxsw653/ExCAPE-ML_v5/IT4I-data/qsar_models/results_v5_exnet/target_wise/exnet_inner_fullpack-target-wise.tsv\", \n",
    "                          header = None, sep=\"\\t\", names=header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:01:26.404514Z",
     "start_time": "2019-04-03T09:01:26.379761Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample</th>\n",
       "      <th>ofold</th>\n",
       "      <th>ifold</th>\n",
       "      <th>al</th>\n",
       "      <th>hp</th>\n",
       "      <th>step</th>\n",
       "      <th>EGFR</th>\n",
       "      <th>ERBB2</th>\n",
       "      <th>ALDH1A1</th>\n",
       "      <th>APOBEC3G</th>\n",
       "      <th>...</th>\n",
       "      <th>SLC12A5</th>\n",
       "      <th>ADK</th>\n",
       "      <th>S1PR5</th>\n",
       "      <th>FBP1</th>\n",
       "      <th>C5AR1</th>\n",
       "      <th>THRA</th>\n",
       "      <th>F7</th>\n",
       "      <th>LTA4H</th>\n",
       "      <th>CMA1</th>\n",
       "      <th>HCAR2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-0_8-0_0</td>\n",
       "      <td>130000</td>\n",
       "      <td>0.639472</td>\n",
       "      <td>0.363534</td>\n",
       "      <td>0.519393</td>\n",
       "      <td>0.608180</td>\n",
       "      <td>...</td>\n",
       "      <td>0.385377</td>\n",
       "      <td>0.166154</td>\n",
       "      <td>0.211647</td>\n",
       "      <td>0.577778</td>\n",
       "      <td>0.398931</td>\n",
       "      <td>0.493761</td>\n",
       "      <td>0.562999</td>\n",
       "      <td>0.392628</td>\n",
       "      <td>0.447002</td>\n",
       "      <td>0.462589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-0_8-0_0</td>\n",
       "      <td>170000</td>\n",
       "      <td>0.649232</td>\n",
       "      <td>0.388696</td>\n",
       "      <td>0.522374</td>\n",
       "      <td>0.627334</td>\n",
       "      <td>...</td>\n",
       "      <td>0.380154</td>\n",
       "      <td>0.138462</td>\n",
       "      <td>0.192369</td>\n",
       "      <td>0.573333</td>\n",
       "      <td>0.435860</td>\n",
       "      <td>0.677514</td>\n",
       "      <td>0.369463</td>\n",
       "      <td>0.370994</td>\n",
       "      <td>0.449206</td>\n",
       "      <td>0.465514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-0_8-0_0</td>\n",
       "      <td>210000</td>\n",
       "      <td>0.658029</td>\n",
       "      <td>0.399156</td>\n",
       "      <td>0.521756</td>\n",
       "      <td>0.623755</td>\n",
       "      <td>...</td>\n",
       "      <td>0.370533</td>\n",
       "      <td>0.126154</td>\n",
       "      <td>0.169076</td>\n",
       "      <td>0.580000</td>\n",
       "      <td>0.430029</td>\n",
       "      <td>0.738194</td>\n",
       "      <td>0.280501</td>\n",
       "      <td>0.358173</td>\n",
       "      <td>0.438536</td>\n",
       "      <td>0.482181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-0_8-0_0</td>\n",
       "      <td>260000</td>\n",
       "      <td>0.652803</td>\n",
       "      <td>0.535138</td>\n",
       "      <td>0.520949</td>\n",
       "      <td>0.621915</td>\n",
       "      <td>...</td>\n",
       "      <td>0.390874</td>\n",
       "      <td>0.123077</td>\n",
       "      <td>0.167470</td>\n",
       "      <td>0.584444</td>\n",
       "      <td>0.417396</td>\n",
       "      <td>0.780034</td>\n",
       "      <td>0.322077</td>\n",
       "      <td>0.346154</td>\n",
       "      <td>0.443122</td>\n",
       "      <td>0.555496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-0_8-0_0</td>\n",
       "      <td>300000</td>\n",
       "      <td>0.654717</td>\n",
       "      <td>0.707544</td>\n",
       "      <td>0.523126</td>\n",
       "      <td>0.611750</td>\n",
       "      <td>...</td>\n",
       "      <td>0.411765</td>\n",
       "      <td>0.121538</td>\n",
       "      <td>0.153815</td>\n",
       "      <td>0.571111</td>\n",
       "      <td>0.375121</td>\n",
       "      <td>0.794715</td>\n",
       "      <td>0.345861</td>\n",
       "      <td>0.331731</td>\n",
       "      <td>0.436067</td>\n",
       "      <td>0.670745</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 532 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample  ofold  ifold  al                               hp    step  \\\n",
       "0       1      1      1   6  1024_1024_1024-0_01-0_5-0_8-0_0  130000   \n",
       "1       1      1      1   6  1024_1024_1024-0_01-0_5-0_8-0_0  170000   \n",
       "2       1      1      1   6  1024_1024_1024-0_01-0_5-0_8-0_0  210000   \n",
       "3       1      1      1   6  1024_1024_1024-0_01-0_5-0_8-0_0  260000   \n",
       "4       1      1      1   6  1024_1024_1024-0_01-0_5-0_8-0_0  300000   \n",
       "\n",
       "       EGFR     ERBB2   ALDH1A1  APOBEC3G    ...      SLC12A5       ADK  \\\n",
       "0  0.639472  0.363534  0.519393  0.608180    ...     0.385377  0.166154   \n",
       "1  0.649232  0.388696  0.522374  0.627334    ...     0.380154  0.138462   \n",
       "2  0.658029  0.399156  0.521756  0.623755    ...     0.370533  0.126154   \n",
       "3  0.652803  0.535138  0.520949  0.621915    ...     0.390874  0.123077   \n",
       "4  0.654717  0.707544  0.523126  0.611750    ...     0.411765  0.121538   \n",
       "\n",
       "      S1PR5      FBP1     C5AR1      THRA        F7     LTA4H      CMA1  \\\n",
       "0  0.211647  0.577778  0.398931  0.493761  0.562999  0.392628  0.447002   \n",
       "1  0.192369  0.573333  0.435860  0.677514  0.369463  0.370994  0.449206   \n",
       "2  0.169076  0.580000  0.430029  0.738194  0.280501  0.358173  0.438536   \n",
       "3  0.167470  0.584444  0.417396  0.780034  0.322077  0.346154  0.443122   \n",
       "4  0.153815  0.571111  0.375121  0.794715  0.345861  0.331731  0.436067   \n",
       "\n",
       "      HCAR2  \n",
       "0  0.462589  \n",
       "1  0.465514  \n",
       "2  0.482181  \n",
       "3  0.555496  \n",
       "4  0.670745  \n",
       "\n",
       "[5 rows x 532 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exnet_inner.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:01:30.283873Z",
     "start_time": "2019-04-03T09:01:30.255213Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ofold</th>\n",
       "      <th>ifold</th>\n",
       "      <th>hp</th>\n",
       "      <th>step</th>\n",
       "      <th>EGFR</th>\n",
       "      <th>ERBB2</th>\n",
       "      <th>ALDH1A1</th>\n",
       "      <th>APOBEC3G</th>\n",
       "      <th>ATAD5</th>\n",
       "      <th>BRCA1</th>\n",
       "      <th>...</th>\n",
       "      <th>SLC12A5</th>\n",
       "      <th>ADK</th>\n",
       "      <th>S1PR5</th>\n",
       "      <th>FBP1</th>\n",
       "      <th>C5AR1</th>\n",
       "      <th>THRA</th>\n",
       "      <th>F7</th>\n",
       "      <th>LTA4H</th>\n",
       "      <th>CMA1</th>\n",
       "      <th>HCAR2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-0_8-0_0</td>\n",
       "      <td>480000</td>\n",
       "      <td>0.646505</td>\n",
       "      <td>0.723674</td>\n",
       "      <td>0.538322</td>\n",
       "      <td>0.626419</td>\n",
       "      <td>0.595862</td>\n",
       "      <td>0.652656</td>\n",
       "      <td>...</td>\n",
       "      <td>0.450797</td>\n",
       "      <td>0.133846</td>\n",
       "      <td>0.129719</td>\n",
       "      <td>0.584444</td>\n",
       "      <td>0.495141</td>\n",
       "      <td>0.801321</td>\n",
       "      <td>0.338235</td>\n",
       "      <td>0.323718</td>\n",
       "      <td>0.453263</td>\n",
       "      <td>0.660461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-0_8-0_4</td>\n",
       "      <td>480000</td>\n",
       "      <td>0.655590</td>\n",
       "      <td>0.720136</td>\n",
       "      <td>0.555043</td>\n",
       "      <td>0.644933</td>\n",
       "      <td>0.602999</td>\n",
       "      <td>0.672664</td>\n",
       "      <td>...</td>\n",
       "      <td>0.493678</td>\n",
       "      <td>0.141538</td>\n",
       "      <td>0.162651</td>\n",
       "      <td>0.544444</td>\n",
       "      <td>0.568513</td>\n",
       "      <td>0.796428</td>\n",
       "      <td>0.323529</td>\n",
       "      <td>0.312500</td>\n",
       "      <td>0.511199</td>\n",
       "      <td>0.633333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-1_0-0_0</td>\n",
       "      <td>480000</td>\n",
       "      <td>0.659623</td>\n",
       "      <td>0.734540</td>\n",
       "      <td>0.559255</td>\n",
       "      <td>0.662940</td>\n",
       "      <td>0.604420</td>\n",
       "      <td>0.670085</td>\n",
       "      <td>...</td>\n",
       "      <td>0.510720</td>\n",
       "      <td>0.176923</td>\n",
       "      <td>0.851004</td>\n",
       "      <td>0.542222</td>\n",
       "      <td>0.664237</td>\n",
       "      <td>0.802055</td>\n",
       "      <td>0.344953</td>\n",
       "      <td>0.296474</td>\n",
       "      <td>0.523192</td>\n",
       "      <td>0.585018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1024_1024_1024-0_01-0_5-1_0-0_4</td>\n",
       "      <td>480000</td>\n",
       "      <td>0.667653</td>\n",
       "      <td>0.735606</td>\n",
       "      <td>0.566943</td>\n",
       "      <td>0.664524</td>\n",
       "      <td>0.606412</td>\n",
       "      <td>0.678587</td>\n",
       "      <td>...</td>\n",
       "      <td>0.413964</td>\n",
       "      <td>0.244615</td>\n",
       "      <td>0.886345</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.692420</td>\n",
       "      <td>0.805725</td>\n",
       "      <td>0.377905</td>\n",
       "      <td>0.310096</td>\n",
       "      <td>0.485891</td>\n",
       "      <td>0.620656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1024_1024_1024-0_1-0_5-0_8-0_0</td>\n",
       "      <td>480000</td>\n",
       "      <td>0.684571</td>\n",
       "      <td>0.748714</td>\n",
       "      <td>0.560915</td>\n",
       "      <td>0.755989</td>\n",
       "      <td>0.653678</td>\n",
       "      <td>0.845998</td>\n",
       "      <td>...</td>\n",
       "      <td>0.411215</td>\n",
       "      <td>0.798462</td>\n",
       "      <td>0.895984</td>\n",
       "      <td>0.566667</td>\n",
       "      <td>0.752187</td>\n",
       "      <td>0.770981</td>\n",
       "      <td>0.464415</td>\n",
       "      <td>0.710737</td>\n",
       "      <td>0.492945</td>\n",
       "      <td>0.553546</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 530 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    ofold  ifold                               hp    step      EGFR     ERBB2  \\\n",
       "10      1      1  1024_1024_1024-0_01-0_5-0_8-0_0  480000  0.646505  0.723674   \n",
       "25      1      1  1024_1024_1024-0_01-0_5-0_8-0_4  480000  0.655590  0.720136   \n",
       "40      1      1  1024_1024_1024-0_01-0_5-1_0-0_0  480000  0.659623  0.734540   \n",
       "55      1      1  1024_1024_1024-0_01-0_5-1_0-0_4  480000  0.667653  0.735606   \n",
       "70      1      1   1024_1024_1024-0_1-0_5-0_8-0_0  480000  0.684571  0.748714   \n",
       "\n",
       "     ALDH1A1  APOBEC3G     ATAD5     BRCA1    ...      SLC12A5       ADK  \\\n",
       "10  0.538322  0.626419  0.595862  0.652656    ...     0.450797  0.133846   \n",
       "25  0.555043  0.644933  0.602999  0.672664    ...     0.493678  0.141538   \n",
       "40  0.559255  0.662940  0.604420  0.670085    ...     0.510720  0.176923   \n",
       "55  0.566943  0.664524  0.606412  0.678587    ...     0.413964  0.244615   \n",
       "70  0.560915  0.755989  0.653678  0.845998    ...     0.411215  0.798462   \n",
       "\n",
       "       S1PR5      FBP1     C5AR1      THRA        F7     LTA4H      CMA1  \\\n",
       "10  0.129719  0.584444  0.495141  0.801321  0.338235  0.323718  0.453263   \n",
       "25  0.162651  0.544444  0.568513  0.796428  0.323529  0.312500  0.511199   \n",
       "40  0.851004  0.542222  0.664237  0.802055  0.344953  0.296474  0.523192   \n",
       "55  0.886345  0.600000  0.692420  0.805725  0.377905  0.310096  0.485891   \n",
       "70  0.895984  0.566667  0.752187  0.770981  0.464415  0.710737  0.492945   \n",
       "\n",
       "       HCAR2  \n",
       "10  0.660461  \n",
       "25  0.633333  \n",
       "40  0.585018  \n",
       "55  0.620656  \n",
       "70  0.553546  \n",
       "\n",
       "[5 rows x 530 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exnet_inner_final = exnet_inner.loc[exnet_inner['step']==480000].drop(['sample', 'al'],axis=1)\n",
    "exnet_inner_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:01:34.689029Z",
     "start_time": "2019-04-03T09:01:34.668661Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold-1: hp\n",
      "2048_2048-0_1-0_5-1_0-0_4    0.759769\n",
      "2048_2048-0_1-0_5-0_8-0_4    0.754120\n",
      "dtype: float64\n",
      "fold-2: hp\n",
      "2048_2048-0_1-0_5-1_0-0_4    0.764706\n",
      "2048_2048-0_1-0_5-0_8-0_4    0.756711\n",
      "dtype: float64\n",
      "fold-3: hp\n",
      "2048_2048-0_1-0_5-1_0-0_4    0.757835\n",
      "2048_2048-0_1-0_5-0_8-0_4    0.749826\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "# get the mean across inner folds for each HP\n",
    "mean_perf_per_ifold = exnet_inner_final.groupby(['ofold','hp']).mean().reset_index()\n",
    "\n",
    "# get the mean across targets\n",
    "perf = mean_perf_per_ifold.drop(['step','ifold'],axis=1).set_index(['ofold','hp']).T\n",
    "\n",
    "# only the first one for each fold should be selected\n",
    "print(\"fold-1:\",perf[1].mean().sort_values(ascending=False).head(2))\n",
    "print(\"fold-2:\",perf[2].mean().sort_values(ascending=False).head(2))\n",
    "print(\"fold-3:\",perf[3].mean().sort_values(ascending=False).head(2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Outer Loop - Model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T10:14:16.213759Z",
     "start_time": "2019-04-03T10:14:13.591876Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>ofold</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>EGFR</th>\n",
       "      <td>0.836050</td>\n",
       "      <td>0.872380</td>\n",
       "      <td>0.832454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERBB2</th>\n",
       "      <td>0.900838</td>\n",
       "      <td>0.917334</td>\n",
       "      <td>0.924384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ALDH1A1</th>\n",
       "      <td>0.710809</td>\n",
       "      <td>0.695130</td>\n",
       "      <td>0.687927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>APOBEC3G</th>\n",
       "      <td>0.857099</td>\n",
       "      <td>0.853981</td>\n",
       "      <td>0.804838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATAD5</th>\n",
       "      <td>0.689444</td>\n",
       "      <td>0.707411</td>\n",
       "      <td>0.680767</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "ofold            1         2         3\n",
       "EGFR      0.836050  0.872380  0.832454\n",
       "ERBB2     0.900838  0.917334  0.924384\n",
       "ALDH1A1   0.710809  0.695130  0.687927\n",
       "APOBEC3G  0.857099  0.853981  0.804838\n",
       "ATAD5     0.689444  0.707411  0.680767"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# there is one columns per target\n",
    "ex_outer = pd.read_csv('results_v5_exnet/target_wise/exnet_outer_fullpack-target-wise.tsv', header=None, sep=\"\\t\", names=header)\n",
    "                      #dtype={'ofold':'category', \n",
    "                      #       'ifold':'category', \n",
    "                      #       'network':'category', \n",
    "                      #       'param':'category'})\n",
    "\n",
    "# select the final checkpoints for each model, keep only checkpoints saved at same time in each outer loop\n",
    "ex_outer_final_steps = ex_outer.loc[ex_outer['step']==1940000]\n",
    "\n",
    "# get performance of selected HP from inner fold\n",
    "selected_hp = '2048_2048-0_1-0_5-1_0-0_4'\n",
    "perf = ex_outer_final_steps.loc[ex_outer_final_steps['hp']==selected_hp].drop(['sample','al','step','hp', 'ifold'],axis=1).set_index('ofold').T\n",
    "perf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T10:10:35.060598Z",
     "start_time": "2019-04-03T10:10:35.054850Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(96, 532)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ex_outer_final_steps.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T10:08:27.664935Z",
     "start_time": "2019-04-03T10:08:27.639334Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>522</th>\n",
       "      <th>523</th>\n",
       "      <th>524</th>\n",
       "      <th>525</th>\n",
       "      <th>526</th>\n",
       "      <th>527</th>\n",
       "      <th>528</th>\n",
       "      <th>529</th>\n",
       "      <th>530</th>\n",
       "      <th>531</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3030</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>1940000</td>\n",
       "      <td>0.836050</td>\n",
       "      <td>0.900838</td>\n",
       "      <td>0.710809</td>\n",
       "      <td>0.857099</td>\n",
       "      <td>...</td>\n",
       "      <td>0.518743</td>\n",
       "      <td>0.669519</td>\n",
       "      <td>0.951564</td>\n",
       "      <td>0.690326</td>\n",
       "      <td>0.850742</td>\n",
       "      <td>0.869703</td>\n",
       "      <td>0.676733</td>\n",
       "      <td>0.799682</td>\n",
       "      <td>0.787284</td>\n",
       "      <td>0.795324</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9360</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>1940000</td>\n",
       "      <td>0.872380</td>\n",
       "      <td>0.917334</td>\n",
       "      <td>0.695130</td>\n",
       "      <td>0.853981</td>\n",
       "      <td>...</td>\n",
       "      <td>0.530819</td>\n",
       "      <td>0.884633</td>\n",
       "      <td>0.891255</td>\n",
       "      <td>0.713287</td>\n",
       "      <td>0.702344</td>\n",
       "      <td>0.884457</td>\n",
       "      <td>0.674712</td>\n",
       "      <td>0.827938</td>\n",
       "      <td>0.726667</td>\n",
       "      <td>0.822149</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15822</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>1940000</td>\n",
       "      <td>0.832454</td>\n",
       "      <td>0.924384</td>\n",
       "      <td>0.687927</td>\n",
       "      <td>0.804838</td>\n",
       "      <td>...</td>\n",
       "      <td>0.669599</td>\n",
       "      <td>0.890769</td>\n",
       "      <td>0.881526</td>\n",
       "      <td>0.931111</td>\n",
       "      <td>0.788630</td>\n",
       "      <td>0.804991</td>\n",
       "      <td>0.673566</td>\n",
       "      <td>0.893429</td>\n",
       "      <td>0.552822</td>\n",
       "      <td>0.683954</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 532 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       0    1    2    3                          4        5         6    \\\n",
       "3030     1    1    0    6  2048_2048-0_1-0_5-1_0-0_4  1940000  0.836050   \n",
       "9360     1    2    0    6  2048_2048-0_1-0_5-1_0-0_4  1940000  0.872380   \n",
       "15822    1    3    0    6  2048_2048-0_1-0_5-1_0-0_4  1940000  0.832454   \n",
       "\n",
       "            7         8         9      ...          522       523       524  \\\n",
       "3030   0.900838  0.710809  0.857099    ...     0.518743  0.669519  0.951564   \n",
       "9360   0.917334  0.695130  0.853981    ...     0.530819  0.884633  0.891255   \n",
       "15822  0.924384  0.687927  0.804838    ...     0.669599  0.890769  0.881526   \n",
       "\n",
       "            525       526       527       528       529       530       531  \n",
       "3030   0.690326  0.850742  0.869703  0.676733  0.799682  0.787284  0.795324  \n",
       "9360   0.713287  0.702344  0.884457  0.674712  0.827938  0.726667  0.822149  \n",
       "15822  0.931111  0.788630  0.804991  0.673566  0.893429  0.552822  0.683954  \n",
       "\n",
       "[3 rows x 532 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ex_outer_final_steps.loc[ex_outer_final_steps[4]==selected_hp]#.drop(['sample','al','step','hp', 'ifold'],axis=1)#.set_index('ofold')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:02:35.622595Z",
     "start_time": "2019-04-03T09:02:35.599197Z"
    }
   },
   "outputs": [],
   "source": [
    "perf.to_csv(\"results_v5_exnet/auc_perf_ofolds.csv\", sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get confusion matrix based metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T10:17:40.286801Z",
     "start_time": "2019-04-03T10:17:40.258933Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>target</th>\n",
       "      <th>auc</th>\n",
       "      <th>recall</th>\n",
       "      <th>precision</th>\n",
       "      <th>f1</th>\n",
       "      <th>cohen_kappa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>EGFR</td>\n",
       "      <td>0.768526</td>\n",
       "      <td>0.886135</td>\n",
       "      <td>0.665503</td>\n",
       "      <td>0.760133</td>\n",
       "      <td>0.412870</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>ERBB2</td>\n",
       "      <td>0.650448</td>\n",
       "      <td>0.727273</td>\n",
       "      <td>0.589074</td>\n",
       "      <td>0.650919</td>\n",
       "      <td>0.232286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>ALDH1A1</td>\n",
       "      <td>0.672300</td>\n",
       "      <td>0.080692</td>\n",
       "      <td>0.129032</td>\n",
       "      <td>0.099291</td>\n",
       "      <td>0.092433</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>APOBEC3G</td>\n",
       "      <td>0.703815</td>\n",
       "      <td>0.016667</td>\n",
       "      <td>0.013333</td>\n",
       "      <td>0.014815</td>\n",
       "      <td>0.013825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>ATAD5</td>\n",
       "      <td>0.638408</td>\n",
       "      <td>0.081848</td>\n",
       "      <td>0.096743</td>\n",
       "      <td>0.088674</td>\n",
       "      <td>0.079353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>BRCA1</td>\n",
       "      <td>0.869893</td>\n",
       "      <td>0.084821</td>\n",
       "      <td>0.022196</td>\n",
       "      <td>0.035185</td>\n",
       "      <td>0.032392</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4</td>\n",
       "      <td>EHMT2</td>\n",
       "      <td>0.584642</td>\n",
       "      <td>0.176606</td>\n",
       "      <td>0.100065</td>\n",
       "      <td>0.127748</td>\n",
       "      <td>0.046793</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        filename    target       auc  \\\n",
       "0  predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4      EGFR  0.768526   \n",
       "1  predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4     ERBB2  0.650448   \n",
       "2  predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4   ALDH1A1  0.672300   \n",
       "3  predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4  APOBEC3G  0.703815   \n",
       "4  predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4     ATAD5  0.638408   \n",
       "5  predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4     BRCA1  0.869893   \n",
       "6  predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4     EHMT2  0.584642   \n",
       "\n",
       "     recall  precision        f1  cohen_kappa  \n",
       "0  0.886135   0.665503  0.760133     0.412870  \n",
       "1  0.727273   0.589074  0.650919     0.232286  \n",
       "2  0.080692   0.129032  0.099291     0.092433  \n",
       "3  0.016667   0.013333  0.014815     0.013825  \n",
       "4  0.081848   0.096743  0.088674     0.079353  \n",
       "5  0.084821   0.022196  0.035185     0.032392  \n",
       "6  0.176606   0.100065  0.127748     0.046793  "
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fold1 = pd.read_csv(\"results_v5_exnet/nested-cv-predictions/predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4.testmetrics\", header=0, sep=\"\\t\")\n",
    "fold1.head(7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Why the f*** is AUC different in fold1 and in perf ??????????"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SMURFF -- WRONG look and smruff HP selection.ipynb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Inner folds - HP selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:53:24.422301Z",
     "start_time": "2019-04-03T09:53:02.372423Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "smurff_inner = pd.read_csv('results_v5_smurff/inner_folds_final.csv.gz', usecols=[1,8,12,16,20,24,44,45,46,47,48,49], compression='gzip', header=0, sep='\\t')# ,error_bad_lines=False\n",
    "smurff_inner = smurff_inner.dropna(axis=0, how=\"all\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:53:31.816252Z",
     "start_time": "2019-04-03T09:53:31.590464Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "s1_c2_i1    327172\n",
       "s1_c3_i2    324542\n",
       "s1_c3_i1    323490\n",
       "s1_c1_i1    322964\n",
       "s1_c1_i2    322964\n",
       "s1_c2_i2    280884\n",
       "Name: cv_id, dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the number of inner fold records\n",
    "\n",
    "smurff_inner['cv_id'].value_counts()\n",
    "# --> there are some missing data ... "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:54:02.773316Z",
     "start_time": "2019-04-03T09:53:38.976297Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# format the data in a way that works: create one HP columns with all hyperparam\n",
    "smurff_inner['ofold'] = smurff_inner['cv_id'].str.split('_',expand=True)[1]\n",
    "smurff_inner['ifold'] = smurff_inner['cv_id'].str.split('_',expand=True)[2]\n",
    "smurff_inner['hp'] = smurff_inner['samples'].astype(str).str.cat(smurff_inner['nl'].astype(str), sep=\"_nl\").str.cat(smurff_inner['bp'].astype(str), sep=\"_bp\").str.cat(smurff_inner['tp'].astype(str), sep=\"_tp\")\n",
    "smurff_inner = smurff_inner.drop(['cv_id','nl','bp','tp'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:54:10.384720Z",
     "start_time": "2019-04-03T09:54:10.369506Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_id</th>\n",
       "      <th>auc_l6</th>\n",
       "      <th>prec_l6</th>\n",
       "      <th>rec_l6</th>\n",
       "      <th>f1_l6</th>\n",
       "      <th>kappa_l6</th>\n",
       "      <th>burnins</th>\n",
       "      <th>samples</th>\n",
       "      <th>ofold</th>\n",
       "      <th>ifold</th>\n",
       "      <th>hp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.754203</td>\n",
       "      <td>0.585284</td>\n",
       "      <td>0.813953</td>\n",
       "      <td>0.680934</td>\n",
       "      <td>0.115081</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>c1</td>\n",
       "      <td>i1</td>\n",
       "      <td>1000_nl16_bp10_tp10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.819002</td>\n",
       "      <td>0.592697</td>\n",
       "      <td>0.864754</td>\n",
       "      <td>0.703333</td>\n",
       "      <td>0.260595</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>c1</td>\n",
       "      <td>i1</td>\n",
       "      <td>1000_nl16_bp10_tp10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.661777</td>\n",
       "      <td>0.148936</td>\n",
       "      <td>0.010574</td>\n",
       "      <td>0.019746</td>\n",
       "      <td>0.029248</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>c1</td>\n",
       "      <td>i1</td>\n",
       "      <td>1000_nl16_bp10_tp10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.550285</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000510</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>c1</td>\n",
       "      <td>i1</td>\n",
       "      <td>1000_nl16_bp10_tp10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.629571</td>\n",
       "      <td>0.076087</td>\n",
       "      <td>0.005121</td>\n",
       "      <td>0.009596</td>\n",
       "      <td>0.046246</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>c1</td>\n",
       "      <td>i1</td>\n",
       "      <td>1000_nl16_bp10_tp10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   col_id    auc_l6   prec_l6    rec_l6     f1_l6  kappa_l6  burnins  samples  \\\n",
       "0       0  0.754203  0.585284  0.813953  0.680934  0.115081      200     1000   \n",
       "1       1  0.819002  0.592697  0.864754  0.703333  0.260595      200     1000   \n",
       "2       2  0.661777  0.148936  0.010574  0.019746  0.029248      200     1000   \n",
       "3       3  0.550285  0.000000  0.000000  0.000000  0.000510      200     1000   \n",
       "4       4  0.629571  0.076087  0.005121  0.009596  0.046246      200     1000   \n",
       "\n",
       "  ofold ifold                   hp  \n",
       "0    c1    i1  1000_nl16_bp10_tp10  \n",
       "1    c1    i1  1000_nl16_bp10_tp10  \n",
       "2    c1    i1  1000_nl16_bp10_tp10  \n",
       "3    c1    i1  1000_nl16_bp10_tp10  \n",
       "4    c1    i1  1000_nl16_bp10_tp10  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smurff_inner.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:54:18.139587Z",
     "start_time": "2019-04-03T09:54:17.697476Z"
    },
    "hidden": true,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>hp</th>\n",
       "      <th>col_id</th>\n",
       "      <th>auc_l6</th>\n",
       "      <th>prec_l6</th>\n",
       "      <th>rec_l6</th>\n",
       "      <th>f1_l6</th>\n",
       "      <th>kappa_l6</th>\n",
       "      <th>burnins</th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>342</th>\n",
       "      <td>1800_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.760919</td>\n",
       "      <td>0.475949</td>\n",
       "      <td>0.453340</td>\n",
       "      <td>0.426336</td>\n",
       "      <td>0.125686</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>1200_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.758250</td>\n",
       "      <td>0.470010</td>\n",
       "      <td>0.443106</td>\n",
       "      <td>0.415792</td>\n",
       "      <td>0.112477</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>1300_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.758203</td>\n",
       "      <td>0.469519</td>\n",
       "      <td>0.442859</td>\n",
       "      <td>0.415460</td>\n",
       "      <td>0.112217</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>1100_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.758076</td>\n",
       "      <td>0.469749</td>\n",
       "      <td>0.442975</td>\n",
       "      <td>0.415863</td>\n",
       "      <td>0.111988</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1000_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.757817</td>\n",
       "      <td>0.470004</td>\n",
       "      <td>0.442700</td>\n",
       "      <td>0.415685</td>\n",
       "      <td>0.111682</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>1400_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.757741</td>\n",
       "      <td>0.467570</td>\n",
       "      <td>0.441540</td>\n",
       "      <td>0.411996</td>\n",
       "      <td>0.108204</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>237</th>\n",
       "      <td>1500_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.757560</td>\n",
       "      <td>0.467026</td>\n",
       "      <td>0.441219</td>\n",
       "      <td>0.411652</td>\n",
       "      <td>0.107899</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>631</th>\n",
       "      <td>900_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.757338</td>\n",
       "      <td>0.470834</td>\n",
       "      <td>0.442255</td>\n",
       "      <td>0.415361</td>\n",
       "      <td>0.110624</td>\n",
       "      <td>200.0</td>\n",
       "      <td>900.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>273</th>\n",
       "      <td>1600_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.757233</td>\n",
       "      <td>0.467038</td>\n",
       "      <td>0.441305</td>\n",
       "      <td>0.411375</td>\n",
       "      <td>0.107717</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>1400_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756999</td>\n",
       "      <td>0.462895</td>\n",
       "      <td>0.445471</td>\n",
       "      <td>0.415012</td>\n",
       "      <td>0.113179</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>1200_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756981</td>\n",
       "      <td>0.464514</td>\n",
       "      <td>0.445646</td>\n",
       "      <td>0.415743</td>\n",
       "      <td>0.114748</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>309</th>\n",
       "      <td>1700_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756971</td>\n",
       "      <td>0.465622</td>\n",
       "      <td>0.441113</td>\n",
       "      <td>0.410809</td>\n",
       "      <td>0.106882</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>1300_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756945</td>\n",
       "      <td>0.464138</td>\n",
       "      <td>0.445524</td>\n",
       "      <td>0.415277</td>\n",
       "      <td>0.113968</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>234</th>\n",
       "      <td>1500_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756929</td>\n",
       "      <td>0.461918</td>\n",
       "      <td>0.445490</td>\n",
       "      <td>0.414655</td>\n",
       "      <td>0.112310</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>1100_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756922</td>\n",
       "      <td>0.464054</td>\n",
       "      <td>0.445569</td>\n",
       "      <td>0.415800</td>\n",
       "      <td>0.115154</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>800_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756805</td>\n",
       "      <td>0.461880</td>\n",
       "      <td>0.445382</td>\n",
       "      <td>0.414201</td>\n",
       "      <td>0.112990</td>\n",
       "      <td>200.0</td>\n",
       "      <td>800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>270</th>\n",
       "      <td>1600_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756779</td>\n",
       "      <td>0.461105</td>\n",
       "      <td>0.445760</td>\n",
       "      <td>0.414395</td>\n",
       "      <td>0.111844</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1000_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756730</td>\n",
       "      <td>0.464309</td>\n",
       "      <td>0.445389</td>\n",
       "      <td>0.415820</td>\n",
       "      <td>0.114951</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>306</th>\n",
       "      <td>1700_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756631</td>\n",
       "      <td>0.459301</td>\n",
       "      <td>0.445832</td>\n",
       "      <td>0.413905</td>\n",
       "      <td>0.111296</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>628</th>\n",
       "      <td>900_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756517</td>\n",
       "      <td>0.463882</td>\n",
       "      <td>0.445304</td>\n",
       "      <td>0.415723</td>\n",
       "      <td>0.114995</td>\n",
       "      <td>200.0</td>\n",
       "      <td>900.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>556</th>\n",
       "      <td>700_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756494</td>\n",
       "      <td>0.461926</td>\n",
       "      <td>0.445420</td>\n",
       "      <td>0.414583</td>\n",
       "      <td>0.113863</td>\n",
       "      <td>200.0</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>520</th>\n",
       "      <td>600_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755858</td>\n",
       "      <td>0.464208</td>\n",
       "      <td>0.445510</td>\n",
       "      <td>0.415872</td>\n",
       "      <td>0.114408</td>\n",
       "      <td>200.0</td>\n",
       "      <td>600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>1200_nl64_bp5_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755780</td>\n",
       "      <td>0.456860</td>\n",
       "      <td>0.439222</td>\n",
       "      <td>0.408322</td>\n",
       "      <td>0.108126</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>595</th>\n",
       "      <td>800_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755751</td>\n",
       "      <td>0.470179</td>\n",
       "      <td>0.441418</td>\n",
       "      <td>0.415273</td>\n",
       "      <td>0.110980</td>\n",
       "      <td>200.0</td>\n",
       "      <td>800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>1300_nl64_bp5_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755750</td>\n",
       "      <td>0.456743</td>\n",
       "      <td>0.439555</td>\n",
       "      <td>0.408652</td>\n",
       "      <td>0.108139</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>204</th>\n",
       "      <td>1400_nl64_bp5_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755729</td>\n",
       "      <td>0.455253</td>\n",
       "      <td>0.439727</td>\n",
       "      <td>0.408566</td>\n",
       "      <td>0.108425</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>1100_nl64_bp5_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755641</td>\n",
       "      <td>0.457640</td>\n",
       "      <td>0.439274</td>\n",
       "      <td>0.408418</td>\n",
       "      <td>0.107861</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>1500_nl64_bp5_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755617</td>\n",
       "      <td>0.455359</td>\n",
       "      <td>0.439690</td>\n",
       "      <td>0.408387</td>\n",
       "      <td>0.108143</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>484</th>\n",
       "      <td>500_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755556</td>\n",
       "      <td>0.465717</td>\n",
       "      <td>0.445629</td>\n",
       "      <td>0.416891</td>\n",
       "      <td>0.117209</td>\n",
       "      <td>200.0</td>\n",
       "      <td>500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>559</th>\n",
       "      <td>700_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.755548</td>\n",
       "      <td>0.472394</td>\n",
       "      <td>0.441424</td>\n",
       "      <td>0.415897</td>\n",
       "      <td>0.112344</td>\n",
       "      <td>200.0</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>644</th>\n",
       "      <td>900_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.686368</td>\n",
       "      <td>0.427890</td>\n",
       "      <td>0.451250</td>\n",
       "      <td>0.403178</td>\n",
       "      <td>0.098433</td>\n",
       "      <td>200.0</td>\n",
       "      <td>900.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>536</th>\n",
       "      <td>600_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.686319</td>\n",
       "      <td>0.428676</td>\n",
       "      <td>0.451117</td>\n",
       "      <td>0.403307</td>\n",
       "      <td>0.098826</td>\n",
       "      <td>200.0</td>\n",
       "      <td>600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>572</th>\n",
       "      <td>700_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.686274</td>\n",
       "      <td>0.427762</td>\n",
       "      <td>0.451088</td>\n",
       "      <td>0.403257</td>\n",
       "      <td>0.098581</td>\n",
       "      <td>200.0</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>608</th>\n",
       "      <td>800_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.686248</td>\n",
       "      <td>0.427426</td>\n",
       "      <td>0.451126</td>\n",
       "      <td>0.403116</td>\n",
       "      <td>0.098110</td>\n",
       "      <td>200.0</td>\n",
       "      <td>800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>100_nl8_bp5_tp5</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.686233</td>\n",
       "      <td>0.429786</td>\n",
       "      <td>0.444065</td>\n",
       "      <td>0.401794</td>\n",
       "      <td>0.100672</td>\n",
       "      <td>200.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>500</th>\n",
       "      <td>500_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.686125</td>\n",
       "      <td>0.427879</td>\n",
       "      <td>0.450956</td>\n",
       "      <td>0.403067</td>\n",
       "      <td>0.098522</td>\n",
       "      <td>200.0</td>\n",
       "      <td>500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>464</th>\n",
       "      <td>400_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.685611</td>\n",
       "      <td>0.428598</td>\n",
       "      <td>0.451179</td>\n",
       "      <td>0.403210</td>\n",
       "      <td>0.098772</td>\n",
       "      <td>200.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>100_nl8_bp1_tp5</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.685393</td>\n",
       "      <td>0.431242</td>\n",
       "      <td>0.446264</td>\n",
       "      <td>0.403191</td>\n",
       "      <td>0.101379</td>\n",
       "      <td>200.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>100_nl8_bp10_tp5</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.684928</td>\n",
       "      <td>0.432923</td>\n",
       "      <td>0.444649</td>\n",
       "      <td>0.404124</td>\n",
       "      <td>0.100177</td>\n",
       "      <td>200.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>428</th>\n",
       "      <td>300_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.684751</td>\n",
       "      <td>0.427448</td>\n",
       "      <td>0.450439</td>\n",
       "      <td>0.402475</td>\n",
       "      <td>0.098466</td>\n",
       "      <td>200.0</td>\n",
       "      <td>300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>1700_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.684300</td>\n",
       "      <td>0.429397</td>\n",
       "      <td>0.453289</td>\n",
       "      <td>0.405352</td>\n",
       "      <td>0.097529</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>283</th>\n",
       "      <td>1600_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.684003</td>\n",
       "      <td>0.429155</td>\n",
       "      <td>0.453424</td>\n",
       "      <td>0.405299</td>\n",
       "      <td>0.097807</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>1500_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.683745</td>\n",
       "      <td>0.429096</td>\n",
       "      <td>0.453473</td>\n",
       "      <td>0.405330</td>\n",
       "      <td>0.097778</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>1400_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.683455</td>\n",
       "      <td>0.428971</td>\n",
       "      <td>0.453433</td>\n",
       "      <td>0.405265</td>\n",
       "      <td>0.097924</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>1300_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.683208</td>\n",
       "      <td>0.429055</td>\n",
       "      <td>0.453394</td>\n",
       "      <td>0.405170</td>\n",
       "      <td>0.098099</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>392</th>\n",
       "      <td>200_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.683097</td>\n",
       "      <td>0.426798</td>\n",
       "      <td>0.450060</td>\n",
       "      <td>0.402233</td>\n",
       "      <td>0.097941</td>\n",
       "      <td>200.0</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>1200_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.682927</td>\n",
       "      <td>0.428796</td>\n",
       "      <td>0.453210</td>\n",
       "      <td>0.404956</td>\n",
       "      <td>0.098064</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>1100_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.682585</td>\n",
       "      <td>0.428699</td>\n",
       "      <td>0.453036</td>\n",
       "      <td>0.404796</td>\n",
       "      <td>0.098069</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1000_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.682277</td>\n",
       "      <td>0.428515</td>\n",
       "      <td>0.452790</td>\n",
       "      <td>0.404639</td>\n",
       "      <td>0.097950</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>500_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.682251</td>\n",
       "      <td>0.428804</td>\n",
       "      <td>0.453657</td>\n",
       "      <td>0.405367</td>\n",
       "      <td>0.099355</td>\n",
       "      <td>200.0</td>\n",
       "      <td>500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>533</th>\n",
       "      <td>600_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.682170</td>\n",
       "      <td>0.429407</td>\n",
       "      <td>0.453393</td>\n",
       "      <td>0.405311</td>\n",
       "      <td>0.099146</td>\n",
       "      <td>200.0</td>\n",
       "      <td>600.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>461</th>\n",
       "      <td>400_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.682091</td>\n",
       "      <td>0.428402</td>\n",
       "      <td>0.453698</td>\n",
       "      <td>0.405357</td>\n",
       "      <td>0.099208</td>\n",
       "      <td>200.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>641</th>\n",
       "      <td>900_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.682087</td>\n",
       "      <td>0.428171</td>\n",
       "      <td>0.452789</td>\n",
       "      <td>0.404429</td>\n",
       "      <td>0.097808</td>\n",
       "      <td>200.0</td>\n",
       "      <td>900.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>569</th>\n",
       "      <td>700_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.681990</td>\n",
       "      <td>0.428407</td>\n",
       "      <td>0.453188</td>\n",
       "      <td>0.405027</td>\n",
       "      <td>0.098682</td>\n",
       "      <td>200.0</td>\n",
       "      <td>700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>605</th>\n",
       "      <td>800_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.681976</td>\n",
       "      <td>0.428084</td>\n",
       "      <td>0.452829</td>\n",
       "      <td>0.404465</td>\n",
       "      <td>0.098039</td>\n",
       "      <td>200.0</td>\n",
       "      <td>800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>353</th>\n",
       "      <td>1800_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.681604</td>\n",
       "      <td>0.428103</td>\n",
       "      <td>0.449139</td>\n",
       "      <td>0.402598</td>\n",
       "      <td>0.095492</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1800.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>425</th>\n",
       "      <td>300_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.681398</td>\n",
       "      <td>0.427874</td>\n",
       "      <td>0.453319</td>\n",
       "      <td>0.404902</td>\n",
       "      <td>0.098912</td>\n",
       "      <td>200.0</td>\n",
       "      <td>300.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>100_nl8_bp5_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.680397</td>\n",
       "      <td>0.423794</td>\n",
       "      <td>0.449650</td>\n",
       "      <td>0.401572</td>\n",
       "      <td>0.097871</td>\n",
       "      <td>200.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>389</th>\n",
       "      <td>200_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.680008</td>\n",
       "      <td>0.427224</td>\n",
       "      <td>0.452671</td>\n",
       "      <td>0.404613</td>\n",
       "      <td>0.098808</td>\n",
       "      <td>200.0</td>\n",
       "      <td>200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>100_nl8_bp1_tp10</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.675817</td>\n",
       "      <td>0.423713</td>\n",
       "      <td>0.450255</td>\n",
       "      <td>0.402256</td>\n",
       "      <td>0.096958</td>\n",
       "      <td>200.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>646 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     hp  col_id    auc_l6   prec_l6    rec_l6     f1_l6  \\\n",
       "342  1800_nl64_bp10_tp1   262.5  0.760919  0.475949  0.453340  0.426336   \n",
       "129   1200_nl64_bp1_tp1   262.5  0.758250  0.470010  0.443106  0.415792   \n",
       "165   1300_nl64_bp1_tp1   262.5  0.758203  0.469519  0.442859  0.415460   \n",
       "93    1100_nl64_bp1_tp1   262.5  0.758076  0.469749  0.442975  0.415863   \n",
       "21    1000_nl64_bp1_tp1   262.5  0.757817  0.470004  0.442700  0.415685   \n",
       "201   1400_nl64_bp1_tp1   262.5  0.757741  0.467570  0.441540  0.411996   \n",
       "237   1500_nl64_bp1_tp1   262.5  0.757560  0.467026  0.441219  0.411652   \n",
       "631    900_nl64_bp1_tp1   262.5  0.757338  0.470834  0.442255  0.415361   \n",
       "273   1600_nl64_bp1_tp1   262.5  0.757233  0.467038  0.441305  0.411375   \n",
       "198  1400_nl64_bp10_tp1   262.5  0.756999  0.462895  0.445471  0.415012   \n",
       "126  1200_nl64_bp10_tp1   262.5  0.756981  0.464514  0.445646  0.415743   \n",
       "309   1700_nl64_bp1_tp1   262.5  0.756971  0.465622  0.441113  0.410809   \n",
       "162  1300_nl64_bp10_tp1   262.5  0.756945  0.464138  0.445524  0.415277   \n",
       "234  1500_nl64_bp10_tp1   262.5  0.756929  0.461918  0.445490  0.414655   \n",
       "90   1100_nl64_bp10_tp1   262.5  0.756922  0.464054  0.445569  0.415800   \n",
       "592   800_nl64_bp10_tp1   262.5  0.756805  0.461880  0.445382  0.414201   \n",
       "270  1600_nl64_bp10_tp1   262.5  0.756779  0.461105  0.445760  0.414395   \n",
       "18   1000_nl64_bp10_tp1   262.5  0.756730  0.464309  0.445389  0.415820   \n",
       "306  1700_nl64_bp10_tp1   262.5  0.756631  0.459301  0.445832  0.413905   \n",
       "628   900_nl64_bp10_tp1   262.5  0.756517  0.463882  0.445304  0.415723   \n",
       "556   700_nl64_bp10_tp1   262.5  0.756494  0.461926  0.445420  0.414583   \n",
       "520   600_nl64_bp10_tp1   262.5  0.755858  0.464208  0.445510  0.415872   \n",
       "132   1200_nl64_bp5_tp1   262.5  0.755780  0.456860  0.439222  0.408322   \n",
       "595    800_nl64_bp1_tp1   262.5  0.755751  0.470179  0.441418  0.415273   \n",
       "168   1300_nl64_bp5_tp1   262.5  0.755750  0.456743  0.439555  0.408652   \n",
       "204   1400_nl64_bp5_tp1   262.5  0.755729  0.455253  0.439727  0.408566   \n",
       "96    1100_nl64_bp5_tp1   262.5  0.755641  0.457640  0.439274  0.408418   \n",
       "240   1500_nl64_bp5_tp1   262.5  0.755617  0.455359  0.439690  0.408387   \n",
       "484   500_nl64_bp10_tp1   262.5  0.755556  0.465717  0.445629  0.416891   \n",
       "559    700_nl64_bp1_tp1   262.5  0.755548  0.472394  0.441424  0.415897   \n",
       "..                  ...     ...       ...       ...       ...       ...   \n",
       "644    900_nl8_bp5_tp10   262.5  0.686368  0.427890  0.451250  0.403178   \n",
       "536    600_nl8_bp5_tp10   262.5  0.686319  0.428676  0.451117  0.403307   \n",
       "572    700_nl8_bp5_tp10   262.5  0.686274  0.427762  0.451088  0.403257   \n",
       "608    800_nl8_bp5_tp10   262.5  0.686248  0.427426  0.451126  0.403116   \n",
       "71      100_nl8_bp5_tp5   262.5  0.686233  0.429786  0.444065  0.401794   \n",
       "500    500_nl8_bp5_tp10   262.5  0.686125  0.427879  0.450956  0.403067   \n",
       "464    400_nl8_bp5_tp10   262.5  0.685611  0.428598  0.451179  0.403210   \n",
       "68      100_nl8_bp1_tp5   262.5  0.685393  0.431242  0.446264  0.403191   \n",
       "65     100_nl8_bp10_tp5   262.5  0.684928  0.432923  0.444649  0.404124   \n",
       "428    300_nl8_bp5_tp10   262.5  0.684751  0.427448  0.450439  0.402475   \n",
       "319   1700_nl8_bp1_tp10   262.5  0.684300  0.429397  0.453289  0.405352   \n",
       "283   1600_nl8_bp1_tp10   262.5  0.684003  0.429155  0.453424  0.405299   \n",
       "247   1500_nl8_bp1_tp10   262.5  0.683745  0.429096  0.453473  0.405330   \n",
       "211   1400_nl8_bp1_tp10   262.5  0.683455  0.428971  0.453433  0.405265   \n",
       "175   1300_nl8_bp1_tp10   262.5  0.683208  0.429055  0.453394  0.405170   \n",
       "392    200_nl8_bp5_tp10   262.5  0.683097  0.426798  0.450060  0.402233   \n",
       "139   1200_nl8_bp1_tp10   262.5  0.682927  0.428796  0.453210  0.404956   \n",
       "103   1100_nl8_bp1_tp10   262.5  0.682585  0.428699  0.453036  0.404796   \n",
       "31    1000_nl8_bp1_tp10   262.5  0.682277  0.428515  0.452790  0.404639   \n",
       "497    500_nl8_bp1_tp10   262.5  0.682251  0.428804  0.453657  0.405367   \n",
       "533    600_nl8_bp1_tp10   262.5  0.682170  0.429407  0.453393  0.405311   \n",
       "461    400_nl8_bp1_tp10   262.5  0.682091  0.428402  0.453698  0.405357   \n",
       "641    900_nl8_bp1_tp10   262.5  0.682087  0.428171  0.452789  0.404429   \n",
       "569    700_nl8_bp1_tp10   262.5  0.681990  0.428407  0.453188  0.405027   \n",
       "605    800_nl8_bp1_tp10   262.5  0.681976  0.428084  0.452829  0.404465   \n",
       "353   1800_nl8_bp1_tp10   262.5  0.681604  0.428103  0.449139  0.402598   \n",
       "425    300_nl8_bp1_tp10   262.5  0.681398  0.427874  0.453319  0.404902   \n",
       "70     100_nl8_bp5_tp10   262.5  0.680397  0.423794  0.449650  0.401572   \n",
       "389    200_nl8_bp1_tp10   262.5  0.680008  0.427224  0.452671  0.404613   \n",
       "67     100_nl8_bp1_tp10   262.5  0.675817  0.423713  0.450255  0.402256   \n",
       "\n",
       "     kappa_l6  burnins  samples  \n",
       "342  0.125686    200.0   1800.0  \n",
       "129  0.112477    200.0   1200.0  \n",
       "165  0.112217    200.0   1300.0  \n",
       "93   0.111988    200.0   1100.0  \n",
       "21   0.111682    200.0   1000.0  \n",
       "201  0.108204    200.0   1400.0  \n",
       "237  0.107899    200.0   1500.0  \n",
       "631  0.110624    200.0    900.0  \n",
       "273  0.107717    200.0   1600.0  \n",
       "198  0.113179    200.0   1400.0  \n",
       "126  0.114748    200.0   1200.0  \n",
       "309  0.106882    200.0   1700.0  \n",
       "162  0.113968    200.0   1300.0  \n",
       "234  0.112310    200.0   1500.0  \n",
       "90   0.115154    200.0   1100.0  \n",
       "592  0.112990    200.0    800.0  \n",
       "270  0.111844    200.0   1600.0  \n",
       "18   0.114951    200.0   1000.0  \n",
       "306  0.111296    200.0   1700.0  \n",
       "628  0.114995    200.0    900.0  \n",
       "556  0.113863    200.0    700.0  \n",
       "520  0.114408    200.0    600.0  \n",
       "132  0.108126    200.0   1200.0  \n",
       "595  0.110980    200.0    800.0  \n",
       "168  0.108139    200.0   1300.0  \n",
       "204  0.108425    200.0   1400.0  \n",
       "96   0.107861    200.0   1100.0  \n",
       "240  0.108143    200.0   1500.0  \n",
       "484  0.117209    200.0    500.0  \n",
       "559  0.112344    200.0    700.0  \n",
       "..        ...      ...      ...  \n",
       "644  0.098433    200.0    900.0  \n",
       "536  0.098826    200.0    600.0  \n",
       "572  0.098581    200.0    700.0  \n",
       "608  0.098110    200.0    800.0  \n",
       "71   0.100672    200.0    100.0  \n",
       "500  0.098522    200.0    500.0  \n",
       "464  0.098772    200.0    400.0  \n",
       "68   0.101379    200.0    100.0  \n",
       "65   0.100177    200.0    100.0  \n",
       "428  0.098466    200.0    300.0  \n",
       "319  0.097529    200.0   1700.0  \n",
       "283  0.097807    200.0   1600.0  \n",
       "247  0.097778    200.0   1500.0  \n",
       "211  0.097924    200.0   1400.0  \n",
       "175  0.098099    200.0   1300.0  \n",
       "392  0.097941    200.0    200.0  \n",
       "139  0.098064    200.0   1200.0  \n",
       "103  0.098069    200.0   1100.0  \n",
       "31   0.097950    200.0   1000.0  \n",
       "497  0.099355    200.0    500.0  \n",
       "533  0.099146    200.0    600.0  \n",
       "461  0.099208    200.0    400.0  \n",
       "641  0.097808    200.0    900.0  \n",
       "569  0.098682    200.0    700.0  \n",
       "605  0.098039    200.0    800.0  \n",
       "353  0.095492    200.0   1800.0  \n",
       "425  0.098912    200.0    300.0  \n",
       "70   0.097871    200.0    100.0  \n",
       "389  0.098808    200.0    200.0  \n",
       "67   0.096958    200.0    100.0  \n",
       "\n",
       "[646 rows x 9 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# average inner fold results over the six inner fold runs\n",
    "smurff_inner.groupby(['hp']).mean().reset_index().sort_values('auc_l6', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:54:26.594069Z",
     "start_time": "2019-04-03T09:54:25.888506Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ofold</th>\n",
       "      <th>hp</th>\n",
       "      <th>col_id</th>\n",
       "      <th>auc_l6</th>\n",
       "      <th>prec_l6</th>\n",
       "      <th>rec_l6</th>\n",
       "      <th>f1_l6</th>\n",
       "      <th>kappa_l6</th>\n",
       "      <th>burnins</th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c1</td>\n",
       "      <td>1200_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.757327</td>\n",
       "      <td>0.469492</td>\n",
       "      <td>0.442140</td>\n",
       "      <td>0.413736</td>\n",
       "      <td>0.108458</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1200.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c2</td>\n",
       "      <td>1500_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.759611</td>\n",
       "      <td>0.464144</td>\n",
       "      <td>0.448272</td>\n",
       "      <td>0.411971</td>\n",
       "      <td>0.108071</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c3</td>\n",
       "      <td>1800_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.760919</td>\n",
       "      <td>0.475949</td>\n",
       "      <td>0.453340</td>\n",
       "      <td>0.426336</td>\n",
       "      <td>0.125686</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1800.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ofold                  hp  col_id    auc_l6   prec_l6    rec_l6     f1_l6  \\\n",
       "0    c1  1200_nl64_bp10_tp1   262.5  0.757327  0.469492  0.442140  0.413736   \n",
       "1    c2   1500_nl64_bp1_tp1   262.5  0.759611  0.464144  0.448272  0.411971   \n",
       "2    c3  1800_nl64_bp10_tp1   262.5  0.760919  0.475949  0.453340  0.426336   \n",
       "\n",
       "   kappa_l6  burnins  samples  \n",
       "0  0.108458    200.0   1200.0  \n",
       "1  0.108071    200.0   1500.0  \n",
       "2  0.125686    200.0   1800.0  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find out the best HPs for each fold by averaging over the two inner folds in each fold \n",
    "# no restrinction made on the number of samples\n",
    "mean=smurff_inner.groupby(['hp','ofold']).mean().reset_index().sort_values('auc_l6', ascending=False)\n",
    "mean.groupby(['ofold']).first().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# => fold 1:  samples = 1200, nl=64, bp=10, tp=1\n",
    "# => fold 2:  samples = 1500, nl=64, bp=1 , tp=1\n",
    "# => fold 3:  samples = 1800, nl=64, bp=10, tp=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:48:22.568117Z",
     "start_time": "2019-04-03T12:48:22.207253Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ofold</th>\n",
       "      <th>hp</th>\n",
       "      <th>col_id</th>\n",
       "      <th>auc_l6</th>\n",
       "      <th>prec_l6</th>\n",
       "      <th>rec_l6</th>\n",
       "      <th>f1_l6</th>\n",
       "      <th>kappa_l6</th>\n",
       "      <th>burnins</th>\n",
       "      <th>samples</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c1</td>\n",
       "      <td>1700_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756770</td>\n",
       "      <td>0.461765</td>\n",
       "      <td>0.442699</td>\n",
       "      <td>0.412033</td>\n",
       "      <td>0.106115</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c2</td>\n",
       "      <td>1700_nl64_bp1_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.758963</td>\n",
       "      <td>0.464874</td>\n",
       "      <td>0.448420</td>\n",
       "      <td>0.411268</td>\n",
       "      <td>0.107306</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1700.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c3</td>\n",
       "      <td>1700_nl64_bp10_tp1</td>\n",
       "      <td>262.5</td>\n",
       "      <td>0.756491</td>\n",
       "      <td>0.456837</td>\n",
       "      <td>0.448964</td>\n",
       "      <td>0.415776</td>\n",
       "      <td>0.116478</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1700.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  ofold                  hp  col_id    auc_l6   prec_l6    rec_l6     f1_l6  \\\n",
       "0    c1  1700_nl64_bp10_tp1   262.5  0.756770  0.461765  0.442699  0.412033   \n",
       "1    c2   1700_nl64_bp1_tp1   262.5  0.758963  0.464874  0.448420  0.411268   \n",
       "2    c3  1700_nl64_bp10_tp1   262.5  0.756491  0.456837  0.448964  0.415776   \n",
       "\n",
       "   kappa_l6  burnins  samples  \n",
       "0  0.106115    200.0   1700.0  \n",
       "1  0.107306    200.0   1700.0  \n",
       "2  0.116478    200.0   1700.0  "
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# alternative \n",
    "# Select the best HPs at sample=1700\n",
    "smurff_inner1700 = smurff_inner.loc[smurff_inner['samples']==1700].copy(deep=True)\n",
    "smurff_inner1700.groupby(['hp','ofold']).mean().reset_index().sort_values('auc_l6', ascending=False).groupby(['ofold']).first().reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-01T14:37:12.039087Z",
     "start_time": "2019-04-01T14:37:12.001311Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "smurff_inner1700['ofold'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "## Outer folds - Model evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T20:28:05.879732Z",
     "start_time": "2019-04-04T20:27:54.677401Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "smurff_outer = pd.read_csv('results_v5_smurff/outer_folds_final.csv.gz', usecols=[1,8,12,16,20,24,44,45,46,47,48,49], compression='gzip', header=0, sep='\\t')# ,error_bad_lines=False\n",
    "smurff_outer = smurff_outer.dropna(axis=0, how='all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-04T20:28:05.983462Z",
     "start_time": "2019-04-04T20:28:05.929646Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_id</th>\n",
       "      <th>auc_l6</th>\n",
       "      <th>prec_l6</th>\n",
       "      <th>rec_l6</th>\n",
       "      <th>f1_l6</th>\n",
       "      <th>kappa_l6</th>\n",
       "      <th>cv_id</th>\n",
       "      <th>burnins</th>\n",
       "      <th>samples</th>\n",
       "      <th>nl</th>\n",
       "      <th>bp</th>\n",
       "      <th>tp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.803692</td>\n",
       "      <td>0.715830</td>\n",
       "      <td>0.872192</td>\n",
       "      <td>0.786313</td>\n",
       "      <td>0.125994</td>\n",
       "      <td>s1_c1</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.851680</td>\n",
       "      <td>0.715618</td>\n",
       "      <td>0.900293</td>\n",
       "      <td>0.797403</td>\n",
       "      <td>0.099906</td>\n",
       "      <td>s1_c1</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.652197</td>\n",
       "      <td>0.022222</td>\n",
       "      <td>0.001441</td>\n",
       "      <td>0.002706</td>\n",
       "      <td>0.022962</td>\n",
       "      <td>s1_c1</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.608130</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000817</td>\n",
       "      <td>s1_c1</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.638063</td>\n",
       "      <td>0.061856</td>\n",
       "      <td>0.004862</td>\n",
       "      <td>0.009016</td>\n",
       "      <td>0.039437</td>\n",
       "      <td>s1_c1</td>\n",
       "      <td>200</td>\n",
       "      <td>1000</td>\n",
       "      <td>16</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   col_id    auc_l6   prec_l6    rec_l6     f1_l6  kappa_l6  cv_id  burnins  \\\n",
       "0       0  0.803692  0.715830  0.872192  0.786313  0.125994  s1_c1      200   \n",
       "1       1  0.851680  0.715618  0.900293  0.797403  0.099906  s1_c1      200   \n",
       "2       2  0.652197  0.022222  0.001441  0.002706  0.022962  s1_c1      200   \n",
       "3       3  0.608130  0.000000  0.000000  0.000000  0.000817  s1_c1      200   \n",
       "4       4  0.638063  0.061856  0.004862  0.009016  0.039437  s1_c1      200   \n",
       "\n",
       "   samples  nl  bp  tp  \n",
       "0     1000  16  10  10  \n",
       "1     1000  16  10  10  \n",
       "2     1000  16  10  10  \n",
       "3     1000  16  10  10  \n",
       "4     1000  16  10  10  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smurff_outer.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:54:53.661291Z",
     "start_time": "2019-04-03T09:54:53.540598Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "s1_c3    321912\n",
       "s1_c2    321912\n",
       "s1_c1    303502\n",
       "Name: cv_id, dtype: int64"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# count the number of records per fold\n",
    "smurff_outer['cv_id'].value_counts()\n",
    "\n",
    "# missing data as well"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T09:55:09.553201Z",
     "start_time": "2019-04-03T09:55:01.530852Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "# format the data in a way that works: create one HP columns with all hyperparam\n",
    "smurff_outer['ofold'] = smurff_outer['cv_id'].str.split('_',expand=True)[1]\n",
    "smurff_outer['hp'] = smurff_outer['samples'].astype(str).str.cat(smurff_outer['nl'].astype(str), sep=\"_nl\").str.cat(smurff_outer['bp'].astype(str), sep=\"_bp\").str.cat(smurff_outer['tp'].astype(str), sep=\"_tp\")\n",
    "smurff_outer = smurff_outer.drop(['cv_id','nl','bp','tp'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:49:06.969372Z",
     "start_time": "2019-04-03T12:49:06.365199Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "hp_c1 = \"1700_nl64_bp10_tp1\"\n",
    "hp_c2 = \"1700_nl64_bp1_tp1\"\n",
    "hp_c3 = \"1700_nl64_bp10_tp1\"\n",
    "\n",
    "\n",
    "smurff_outer_c1 = smurff_outer.loc[(smurff_outer['ofold']=='c1')&(smurff_outer['hp']==hp_c1)]\n",
    "smurff_outer_c2 = smurff_outer.loc[(smurff_outer['ofold']=='c2')&(smurff_outer['hp']==hp_c2)]\n",
    "smurff_outer_c3 = smurff_outer.loc[(smurff_outer['ofold']=='c3')&(smurff_outer['hp']==hp_c3)]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:49:22.410753Z",
     "start_time": "2019-04-03T12:49:22.395610Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_id</th>\n",
       "      <th>auc_l6</th>\n",
       "      <th>prec_l6</th>\n",
       "      <th>rec_l6</th>\n",
       "      <th>f1_l6</th>\n",
       "      <th>kappa_l6</th>\n",
       "      <th>burnins</th>\n",
       "      <th>samples</th>\n",
       "      <th>ofold</th>\n",
       "      <th>hp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>799520</th>\n",
       "      <td>0</td>\n",
       "      <td>0.810862</td>\n",
       "      <td>0.685915</td>\n",
       "      <td>0.755039</td>\n",
       "      <td>0.718819</td>\n",
       "      <td>0.300324</td>\n",
       "      <td>200</td>\n",
       "      <td>1700</td>\n",
       "      <td>c3</td>\n",
       "      <td>1700_nl64_bp10_tp1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799521</th>\n",
       "      <td>1</td>\n",
       "      <td>0.911781</td>\n",
       "      <td>0.685015</td>\n",
       "      <td>0.918033</td>\n",
       "      <td>0.784588</td>\n",
       "      <td>0.257552</td>\n",
       "      <td>200</td>\n",
       "      <td>1700</td>\n",
       "      <td>c3</td>\n",
       "      <td>1700_nl64_bp10_tp1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799522</th>\n",
       "      <td>2</td>\n",
       "      <td>0.683782</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004332</td>\n",
       "      <td>200</td>\n",
       "      <td>1700</td>\n",
       "      <td>c3</td>\n",
       "      <td>1700_nl64_bp10_tp1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799523</th>\n",
       "      <td>3</td>\n",
       "      <td>0.803055</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000015</td>\n",
       "      <td>200</td>\n",
       "      <td>1700</td>\n",
       "      <td>c3</td>\n",
       "      <td>1700_nl64_bp10_tp1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>799524</th>\n",
       "      <td>4</td>\n",
       "      <td>0.657538</td>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.001463</td>\n",
       "      <td>0.002909</td>\n",
       "      <td>0.011847</td>\n",
       "      <td>200</td>\n",
       "      <td>1700</td>\n",
       "      <td>c3</td>\n",
       "      <td>1700_nl64_bp10_tp1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        col_id    auc_l6   prec_l6    rec_l6     f1_l6  kappa_l6  burnins  \\\n",
       "799520       0  0.810862  0.685915  0.755039  0.718819  0.300324      200   \n",
       "799521       1  0.911781  0.685015  0.918033  0.784588  0.257552      200   \n",
       "799522       2  0.683782  0.000000  0.000000  0.000000  0.004332      200   \n",
       "799523       3  0.803055  0.000000  0.000000  0.000000 -0.000015      200   \n",
       "799524       4  0.657538  0.250000  0.001463  0.002909  0.011847      200   \n",
       "\n",
       "        samples ofold                  hp  \n",
       "799520     1700    c3  1700_nl64_bp10_tp1  \n",
       "799521     1700    c3  1700_nl64_bp10_tp1  \n",
       "799522     1700    c3  1700_nl64_bp10_tp1  \n",
       "799523     1700    c3  1700_nl64_bp10_tp1  \n",
       "799524     1700    c3  1700_nl64_bp10_tp1  "
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "smurff_outer_c3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:53:40.380079Z",
     "start_time": "2019-04-03T12:53:40.342496Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "smurff_outer_c1.to_csv('results_v5_smurff/perf_ofold1.csv', sep=\"\\t\", index=None)\n",
    "smurff_outer_c2.to_csv('results_v5_smurff/perf_ofold2.csv', sep=\"\\t\", index=None)\n",
    "smurff_outer_c3.to_csv('results_v5_smurff/perf_ofold3.csv', sep=\"\\t\", index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:52:44.809615Z",
     "start_time": "2019-04-03T12:52:44.804201Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "auc:0.7838133771642667 +/-0.12118727211881054\n",
      "auc:0.7793024205155963 +/-0.12706133717521323\n",
      "auc:0.7833483080113223 +/-0.12398470563634535\n"
     ]
    }
   ],
   "source": [
    "print(\"auc:{} +/-{}\".format(smurff_outer_c1['auc_l6'].mean(), smurff_outer_c1['auc_l6'].std()))\n",
    "print(\"auc:{} +/-{}\".format(smurff_outer_c2['auc_l6'].mean(), smurff_outer_c2['auc_l6'].std()))\n",
    "print(\"auc:{} +/-{}\".format(smurff_outer_c3['auc_l6'].mean(), smurff_outer_c3['auc_l6'].std()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "heading_collapsed": true
   },
   "source": [
    "# Stich everything together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T14:39:06.910119Z",
     "start_time": "2019-04-09T14:39:06.817342Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "xgb1 = pd.read_csv('results_v5_xgboost_ecfp/perf_fold1.csv', header=0, sep=\"\\t\")\n",
    "xgb2 = pd.read_csv('results_v5_xgboost_ecfp/perf_fold2.csv', header=0, sep=\"\\t\")\n",
    "xgb3 = pd.read_csv('results_v5_xgboost_ecfp/perf_fold3.csv', header=0, sep=\"\\t\")\n",
    "\n",
    "x1 = xgb1.drop(['ofold','ifold','hp'],axis=1)\n",
    "x2 = xgb2.drop(['ofold','ifold','hp'],axis=1)\n",
    "x3 = xgb3.drop(['ofold','ifold','hp'],axis=1)\n",
    "x1.columns = ['target','1_xgb_auc', '1_xgb_recall', '1_xgb_precision', '1_xgb_kappa']\n",
    "x2.columns = ['target','2_xgb_auc', '2_xgb_recall', '2_xgb_precision', '2_xgb_kappa']\n",
    "x3.columns = ['target','3_xgb_auc', '3_xgb_recall', '3_xgb_precision', '3_xgb_kappa']\n",
    "\n",
    "all_xgb = x1.merge(x2, on='target').merge(x3, on='target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T10:33:14.568984Z",
     "start_time": "2019-04-03T10:33:14.527072Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T14:39:08.710518Z",
     "start_time": "2019-04-09T14:39:08.661414Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "svm1 = pd.read_csv('results_v5_svm_ecfp/perf_fold1.csv', header=0, sep=\"\\t\")\n",
    "svm2 = pd.read_csv('results_v5_svm_ecfp/perf_fold2.csv', header=0, sep=\"\\t\")\n",
    "svm3 = pd.read_csv('results_v5_svm_ecfp/perf_fold3.csv', header=0, sep=\"\\t\")\n",
    "\n",
    "s1 = svm1.drop(['ofold','ifold','hp'],axis=1)\n",
    "s2 = svm2.drop(['ofold','ifold','hp'],axis=1)\n",
    "s3 = svm3.drop(['ofold','ifold','hp'],axis=1)\n",
    "s1.columns = ['target','1_svm_auc', '1_svm_precision', '1_svm_recall',  '1_svm_kappa']\n",
    "s2.columns = ['target','2_svm_auc', '2_svm_precision', '2_svm_recall',  '2_svm_kappa']\n",
    "s3.columns = ['target','3_svm_auc', '3_svm_precision', '3_svm_recall',  '3_svm_kappa']\n",
    "\n",
    "all_svm = s1.merge(s2, on='target').merge(s3, on='target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T14:39:11.404869Z",
     "start_time": "2019-04-09T14:39:11.352169Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "rf1 = pd.read_csv('results_v5_rf_ecfp/perf_fold1.csv', header=0, sep=\"\\t\")\n",
    "rf2 = pd.read_csv('results_v5_rf_ecfp/perf_fold2.csv', header=0, sep=\"\\t\")\n",
    "rf3 = pd.read_csv('results_v5_rf_ecfp/perf_fold3.csv', header=0, sep=\"\\t\")\n",
    "\n",
    "r1 = rf1.drop(['ofold','ifold','hp'],axis=1)\n",
    "r2 = rf2.drop(['ofold','ifold','hp'],axis=1)\n",
    "r3 = rf3.drop(['ofold','ifold','hp'],axis=1)\n",
    "\n",
    "r1.columns = ['target','1_rf_auc', '1_rf_precision', '1_rf_recall',  '1_rf_kappa']\n",
    "r2.columns = ['target','2_rf_auc', '2_rf_precision', '2_rf_recall',  '2_rf_kappa']\n",
    "r3.columns = ['target','3_rf_auc', '3_rf_precision', '3_rf_recall',  '3_rf_kappa']\n",
    "\n",
    "all_rf = r1.merge(r2, on='target').merge(r3, on='target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T11:19:23.702149Z",
     "start_time": "2019-04-03T11:19:23.683712Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>target</th>\n",
       "      <th>1_auc</th>\n",
       "      <th>1_precision</th>\n",
       "      <th>1_recall</th>\n",
       "      <th>1_kappa</th>\n",
       "      <th>2_auc</th>\n",
       "      <th>2_precision</th>\n",
       "      <th>2_recall</th>\n",
       "      <th>2_kappa</th>\n",
       "      <th>3_auc</th>\n",
       "      <th>3_precision</th>\n",
       "      <th>3_recall</th>\n",
       "      <th>3_kappa</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0.773379</td>\n",
       "      <td>0.769946</td>\n",
       "      <td>0.811429</td>\n",
       "      <td>0.543322</td>\n",
       "      <td>0.813441</td>\n",
       "      <td>0.769542</td>\n",
       "      <td>0.847181</td>\n",
       "      <td>0.626088</td>\n",
       "      <td>0.774280</td>\n",
       "      <td>0.727132</td>\n",
       "      <td>0.757674</td>\n",
       "      <td>0.551139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0.794876</td>\n",
       "      <td>0.712610</td>\n",
       "      <td>0.849650</td>\n",
       "      <td>0.590978</td>\n",
       "      <td>0.883388</td>\n",
       "      <td>0.888660</td>\n",
       "      <td>0.907368</td>\n",
       "      <td>0.764070</td>\n",
       "      <td>0.877033</td>\n",
       "      <td>0.823770</td>\n",
       "      <td>0.885463</td>\n",
       "      <td>0.763261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0.505728</td>\n",
       "      <td>0.011527</td>\n",
       "      <td>0.615385</td>\n",
       "      <td>0.022278</td>\n",
       "      <td>0.507672</td>\n",
       "      <td>0.015385</td>\n",
       "      <td>0.785714</td>\n",
       "      <td>0.029822</td>\n",
       "      <td>0.506035</td>\n",
       "      <td>0.012085</td>\n",
       "      <td>0.888889</td>\n",
       "      <td>0.023603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0.504159</td>\n",
       "      <td>0.008333</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.016217</td>\n",
       "      <td>0.499996</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000014</td>\n",
       "      <td>0.499985</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0.501981</td>\n",
       "      <td>0.004052</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>0.007743</td>\n",
       "      <td>0.505276</td>\n",
       "      <td>0.010638</td>\n",
       "      <td>0.600000</td>\n",
       "      <td>0.020492</td>\n",
       "      <td>0.503969</td>\n",
       "      <td>0.008047</td>\n",
       "      <td>0.478261</td>\n",
       "      <td>0.015430</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   target     1_auc  1_precision  1_recall   1_kappa     2_auc  2_precision  \\\n",
       "0       0  0.773379     0.769946  0.811429  0.543322  0.813441     0.769542   \n",
       "1       1  0.794876     0.712610  0.849650  0.590978  0.883388     0.888660   \n",
       "2       2  0.505728     0.011527  0.615385  0.022278  0.507672     0.015385   \n",
       "3       3  0.504159     0.008333  0.333333  0.016217  0.499996     0.000000   \n",
       "4       4  0.501981     0.004052  0.333333  0.007743  0.505276     0.010638   \n",
       "\n",
       "   2_recall   2_kappa     3_auc  3_precision  3_recall   3_kappa  \n",
       "0  0.847181  0.626088  0.774280     0.727132  0.757674  0.551139  \n",
       "1  0.907368  0.764070  0.877033     0.823770  0.885463  0.763261  \n",
       "2  0.785714  0.029822  0.506035     0.012085  0.888889  0.023603  \n",
       "3  0.000000 -0.000014  0.499985     0.000000  0.000000 -0.000058  \n",
       "4  0.600000  0.020492  0.503969     0.008047  0.478261  0.015430  "
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_rf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T13:04:59.094218Z",
     "start_time": "2019-04-03T13:04:59.084491Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21552019945540535"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prk1 = pd.read_csv(\"results_v5_exnet/nested-cv-predictions/\", header=0, sep=\"\\t\")\n",
    "prk1['cohen_kappa'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T11:44:18.889369Z",
     "start_time": "2019-04-03T11:44:18.851520Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "exnet = pd.read_csv(\"results_v5_exnet/auc_perf_ofolds.csv\", header=0, sep=\"\\t\", index_col=0).reset_index()\n",
    "exnet.columns = ['target', '1_exnet_auc', '2_exnet_auc', '3_exnet_auc']\n",
    "\n",
    "prk1 = pd.read_csv(\"results_v5_exnet/nested-cv-predictions/predictions-1-1-0-6-2048_2048-0_1-0_5-1_0-0_4.testmetrics\", header=0, sep=\"\\t\")\n",
    "prk2 = pd.read_csv(\"results_v5_exnet/nested-cv-predictions/predictions-1-2-0-6-2048_2048-0_1-0_5-1_0-0_4.testmetrics\", header=0, sep=\"\\t\")\n",
    "prk3 = pd.read_csv(\"results_v5_exnet/nested-cv-predictions/predictions-1-3-0-6-2048_2048-0_1-0_5-1_0-0_4.testmetrics\", header=0, sep=\"\\t\")\n",
    "\n",
    "prk1 = prk1.drop(['filename','auc'],axis=1)\n",
    "prk2 = prk2.drop(['filename','auc'],axis=1)\n",
    "prk3 = prk3.drop(['filename','auc'],axis=1)\n",
    "\n",
    "prk1.columns=['target','1_exnet_recall', '1_exnet_precision', '1_exnet_f1', '1_exnet_kappa']\n",
    "prk2.columns=['target','2_exnet_recall', '2_exnet_precision', '2_exnet_f1', '2_exnet_kappa']\n",
    "prk3.columns=['target','3_exnet_recall', '3_exnet_precision', '3_exnet_f1', '3_exnet_kappa']\n",
    "\n",
    "all_exnet = exnet.merge(prk1, on='target').merge(prk2, on='target').merge(prk3, on='target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:55:20.526092Z",
     "start_time": "2019-04-03T12:55:20.482958Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "smu1 = pd.read_csv(\"results_v5_smurff/nested-cv-evaluations/perf_smurff_ofold1.csv\", header=0, sep=\"\\t\")\n",
    "smu2 = pd.read_csv(\"results_v5_smurff/nested-cv-evaluations/perf_smurff_ofold2.csv\", header=0, sep=\"\\t\")\n",
    "smu3 = pd.read_csv(\"results_v5_smurff/nested-cv-evaluations/perf_smurff_ofold3.csv\", header=0, sep=\"\\t\")\n",
    "\n",
    "#smu1 = smu1.drop(['burnins', 'samples', 'ofold', 'hp'],axis=1)\n",
    "#smu2 = smu2.drop(['burnins', 'samples', 'ofold', 'hp'],axis=1)\n",
    "#smu3 = smu3.drop(['burnins', 'samples', 'ofold', 'hp'],axis=1)\n",
    "#\n",
    "#\n",
    "#smu1.columns=['target', '1_smurff_auc', '1_smurff_precision', '1_smurff_recall', '1_smurff_f1', '1_smurff_kappa']\n",
    "#smu2.columns=['target', '2_smurff_auc', '2_smurff_precision', '2_smurff_recall', '2_smurff_f1', '2_smurff_kappa']\n",
    "#smu3.columns=['target', '3_smurff_auc', '3_smurff_precision', '3_smurff_recall', '3_smurff_f1', '3_smurff_kappa']\n",
    "#\n",
    "#all_smurff = smu1.merge(smu2, on='target').merge(smu3, on='target')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:55:39.811713Z",
     "start_time": "2019-04-03T12:55:39.807356Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['target', '1_smurff_auc', '1_smurff_precision', '1_smurff_recall',\n",
       "       '1_smurff_f1', '1_smurff_kappa', '2_smurff_auc', '2_smurff_precision',\n",
       "       '2_smurff_recall', '2_smurff_f1', '2_smurff_kappa', '3_smurff_auc',\n",
       "       '3_smurff_precision', '3_smurff_recall', '3_smurff_f1',\n",
       "       '3_smurff_kappa'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_smurff.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:53:55.825536Z",
     "start_time": "2019-04-03T12:53:55.796282Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "all_perf = all_exnet.merge(all_xgb, left_index=True, right_on='target').merge(all_smurff, on='target').merge(all_svm, on='target').merge(all_rf, on='target')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-03T12:54:01.213699Z",
     "start_time": "2019-04-03T12:54:01.145864Z"
    },
    "hidden": true
   },
   "outputs": [],
   "source": [
    "all_perf.to_csv('ALL_PERF.csv', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-04-09T14:54:04.017145Z",
     "start_time": "2019-04-09T14:54:04.004843Z"
    },
    "hidden": true
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'all_perf' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-44-b4c61090c80a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mall_perf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'all_perf' is not defined"
     ]
    }
   ],
   "source": [
    "all_perf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hidden": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}