CoPheeKSA_XGBoost.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "650ac471",
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    import google.colab\n",
    "    IN_COLAB = True\n",
    "except ImportError:\n",
    "    IN_COLAB = False\n",
    "\n",
    "if IN_COLAB:\n",
    "    %pip uninstall -y plotnine\n",
    "    !git clone https://github.com/bzhanglab/CoPheeMap.git\n",
    "    %cd CoPheeMap\n",
    "    %pip install -q -r requirements.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "258e3518",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import re\n",
    "import numpy as np\n",
    "import random\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import scipy.stats as sts\n",
    "import math\n",
    "from sklearn import metrics\n",
    "from statannot import add_stat_annotation\n",
    "from collections import Counter\n",
    "import pickle\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import average_precision_score\n",
    "from os import listdir\n",
    "from os.path import isfile\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import networkx as nx\n",
    "import pickle\n",
    "import joblib\n",
    "import xgboost as xgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f96011d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "Cancer=['BRCA','CCRCC','COAD','GBM','HCC','HNSCC','LSCC','LUAD','OV','PDAC','UCEC'] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c4a2d3eb",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Kinase name, family, group information\n",
    "kinases_id=pd.read_excel('KSA/kinase_table.xlsx')\n",
    "kinases_id.index=kinases_id.Name\n",
    "\n",
    "#gold standard KSAs\n",
    "K_S_GPS=pd.read_csv('KSA/KSA_gold_standard.csv',index_col=0)\n",
    "ks=list(set(K_S_GPS['Gene name']))\n",
    "\n",
    "#All sites identified in the CPTAC PanCan datasets\n",
    "a_file = open(\"PanCan/all_sites_id_1cancer.pkl\", \"rb\")\n",
    "All_sites = pickle.load(a_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "81612a0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Classify kinases into under-studied and other kinases according to # of known substrates\n",
    "poor_kinase=[]\n",
    "other_kinase=[]\n",
    "poor_num=[]\n",
    "other_num=[]\n",
    "for i in ks:\n",
    "    if 'family' not in i:\n",
    "        if 'Isoform' not in i:\n",
    "            tmp=K_S_GPS[K_S_GPS['Gene name']==i]\n",
    "            if len(set(tmp['seq_15']))<=10:\n",
    "                poor_kinase.append(i)\n",
    "                poor_num.append(len(set(tmp['seq_15'])))\n",
    "            if len(set(tmp['seq_15']))>10:\n",
    "                other_kinase.append(i)\n",
    "                other_num.append(len(set(tmp['seq_15'])))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "05bffc41",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load CoPheeMap\n",
    "network = pd.read_csv('Supplementary_table/Table_S2_CoPheeMap.tsv.zip', compression='zip', sep='\\t')\n",
    "\n",
    "seq1=[i.split('|')[3] for i in network.site1.tolist()]\n",
    "seq2=[i.split('|')[3] for i in network.site2.tolist()]\n",
    "network['seq1']=np.array(seq1)\n",
    "network['seq2']=np.array(seq2)\n",
    "sites=list(set(network.site1.tolist()+network.site2.tolist()))\n",
    "G = nx.Graph()\n",
    "elist=list(zip(network.site1.tolist(),network.site2.tolist()))\n",
    "G.add_edges_from(elist)\n",
    "\n",
    "idx_ST=[]\n",
    "idx_Y=[]\n",
    "for i in range(len(network)):\n",
    "    if network.seq1.iloc[i][7] in ['S','T']:\n",
    "        idx_ST.append(i)\n",
    "    if network.seq1.iloc[i][7] in ['Y']:\n",
    "        idx_Y.append(i)\n",
    "network_ST=network.iloc[idx_ST,:]\n",
    "network_Y=network.iloc[idx_Y,:]\n",
    "lst_ST=list(set(network_ST.site1.tolist()+network_ST.site2.tolist()))\n",
    "H=G.subgraph(lst_ST)\n",
    "nodes_ST=max(nx.connected_components(H))\n",
    "G_ST=G.subgraph(nodes_ST)\n",
    "network_Y=network.iloc[idx_Y,:]\n",
    "lst_Y=list(set(network_Y.site1.tolist()+network_Y.site2.tolist()))\n",
    "H=G.subgraph(lst_Y)\n",
    "nodes_Y=max(nx.connected_components(H))\n",
    "G_Y=G.subgraph(nodes_Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "59ee84ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Load embeddings of KMap and CoPheeMap\n",
    "kk_emb=pd.read_csv('CoPheeKSA/kinase_network/n2v_KK_PPI_PCC.csv',index_col=0)\n",
    "cophee_emb=pd.read_csv('CoPheeMap/data_construction/n2v_networkST.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9da7b837",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Supervised model trained with one static feature: PSSM scores\n",
    "auc=[]\n",
    "label=[]\n",
    "for i in range(10):\n",
    "    data_train=pd.read_csv('CoPheeKSA/static_features/test/test_'+str(i)+'.csv',index_col=0)\n",
    "    data_train=data_train.drop_duplicates()\n",
    "    data_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    data_train=data_train.dropna()\n",
    "    data_neg=pd.read_csv('Supplementary_table/Table_S3_negative_KSA_w_features.tsv.zip', compression='zip', sep='\\t')\n",
    "    data_neg=data_neg.drop_duplicates()\n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_train))\n",
    "    data_neg=data_neg.iloc[idx]\n",
    "    \n",
    "    y=np.concatenate((np.repeat(1,len(data_train)),np.repeat(0,len(data_neg))))\n",
    "    y_pred=np.concatenate((data_train.PSSM_score,data_neg.PSSM))\n",
    "    fpr, tpr, _ = metrics.roc_curve(y,  y_pred)\n",
    "    auc.append(metrics.auc(fpr, tpr))\n",
    "    label.append('PSSM')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "cb29a80b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.87407\teval-auc:0.85568\n",
      "[1]\ttrain-auc:0.87682\teval-auc:0.85912\n",
      "[2]\ttrain-auc:0.89918\teval-auc:0.86608\n",
      "[3]\ttrain-auc:0.90070\teval-auc:0.86680\n",
      "[4]\ttrain-auc:0.90049\teval-auc:0.86704\n",
      "[5]\ttrain-auc:0.90116\teval-auc:0.86632\n",
      "[6]\ttrain-auc:0.90215\teval-auc:0.86424\n",
      "[7]\ttrain-auc:0.90706\teval-auc:0.86000\n",
      "[8]\ttrain-auc:0.90772\teval-auc:0.86120\n",
      "[9]\ttrain-auc:0.90814\teval-auc:0.86232\n",
      "[10]\ttrain-auc:0.90895\teval-auc:0.86184\n",
      "[11]\ttrain-auc:0.90899\teval-auc:0.86088\n",
      "[12]\ttrain-auc:0.91029\teval-auc:0.86024\n",
      "[13]\ttrain-auc:0.91137\teval-auc:0.85944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.86400\teval-auc:0.89880\n",
      "[1]\ttrain-auc:0.87976\teval-auc:0.91512\n",
      "[2]\ttrain-auc:0.90379\teval-auc:0.94208\n",
      "[3]\ttrain-auc:0.90515\teval-auc:0.94512\n",
      "[4]\ttrain-auc:0.90554\teval-auc:0.94496\n",
      "[5]\ttrain-auc:0.90582\teval-auc:0.94472\n",
      "[6]\ttrain-auc:0.90614\teval-auc:0.94568\n",
      "[7]\ttrain-auc:0.90666\teval-auc:0.94552\n",
      "[8]\ttrain-auc:0.90704\teval-auc:0.94544\n",
      "[9]\ttrain-auc:0.90979\teval-auc:0.94584\n",
      "[10]\ttrain-auc:0.91018\teval-auc:0.94968\n",
      "[11]\ttrain-auc:0.91250\teval-auc:0.95168\n",
      "[12]\ttrain-auc:0.91449\teval-auc:0.95096\n",
      "[13]\ttrain-auc:0.91633\teval-auc:0.95096\n",
      "[14]\ttrain-auc:0.91775\teval-auc:0.95080\n",
      "[15]\ttrain-auc:0.91841\teval-auc:0.95128\n",
      "[16]\ttrain-auc:0.92069\teval-auc:0.95096\n",
      "[17]\ttrain-auc:0.92268\teval-auc:0.95096\n",
      "[18]\ttrain-auc:0.92343\teval-auc:0.95048\n",
      "[19]\ttrain-auc:0.92428\teval-auc:0.95064\n",
      "[20]\ttrain-auc:0.92627\teval-auc:0.95112\n",
      "[21]\ttrain-auc:0.92769\teval-auc:0.95352\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[22]\ttrain-auc:0.92868\teval-auc:0.95296\n",
      "[23]\ttrain-auc:0.92900\teval-auc:0.95248\n",
      "[24]\ttrain-auc:0.93035\teval-auc:0.95232\n",
      "[25]\ttrain-auc:0.93102\teval-auc:0.95296\n",
      "[26]\ttrain-auc:0.93170\teval-auc:0.95472\n",
      "[27]\ttrain-auc:0.93299\teval-auc:0.95768\n",
      "[28]\ttrain-auc:0.93382\teval-auc:0.95776\n",
      "[29]\ttrain-auc:0.93418\teval-auc:0.95872\n",
      "[30]\ttrain-auc:0.93494\teval-auc:0.95920\n",
      "[31]\ttrain-auc:0.93559\teval-auc:0.95968\n",
      "[32]\ttrain-auc:0.93616\teval-auc:0.96000\n",
      "[33]\ttrain-auc:0.93664\teval-auc:0.96080\n",
      "[34]\ttrain-auc:0.93683\teval-auc:0.96176\n",
      "[35]\ttrain-auc:0.93699\teval-auc:0.96128\n",
      "[36]\ttrain-auc:0.93724\teval-auc:0.96048\n",
      "[37]\ttrain-auc:0.93761\teval-auc:0.96032\n",
      "[38]\ttrain-auc:0.93832\teval-auc:0.96000\n",
      "[39]\ttrain-auc:0.93870\teval-auc:0.96096\n",
      "[40]\ttrain-auc:0.93917\teval-auc:0.96160\n",
      "[41]\ttrain-auc:0.93969\teval-auc:0.96160\n",
      "[42]\ttrain-auc:0.94024\teval-auc:0.96256\n",
      "[43]\ttrain-auc:0.94043\teval-auc:0.96240\n",
      "[44]\ttrain-auc:0.94096\teval-auc:0.96336\n",
      "[45]\ttrain-auc:0.94134\teval-auc:0.96352\n",
      "[46]\ttrain-auc:0.94147\teval-auc:0.96368\n",
      "[47]\ttrain-auc:0.94195\teval-auc:0.96368\n",
      "[48]\ttrain-auc:0.94235\teval-auc:0.96400\n",
      "[49]\ttrain-auc:0.94289\teval-auc:0.96464\n",
      "[50]\ttrain-auc:0.94347\teval-auc:0.96560\n",
      "[51]\ttrain-auc:0.94364\teval-auc:0.96560\n",
      "[52]\ttrain-auc:0.94377\teval-auc:0.96608\n",
      "[53]\ttrain-auc:0.94422\teval-auc:0.96496\n",
      "[54]\ttrain-auc:0.94440\teval-auc:0.96448\n",
      "[55]\ttrain-auc:0.94481\teval-auc:0.96528\n",
      "[56]\ttrain-auc:0.94522\teval-auc:0.96576\n",
      "[57]\ttrain-auc:0.94562\teval-auc:0.96592\n",
      "[58]\ttrain-auc:0.94595\teval-auc:0.96624\n",
      "[59]\ttrain-auc:0.94606\teval-auc:0.96672\n",
      "[60]\ttrain-auc:0.94660\teval-auc:0.96544\n",
      "[61]\ttrain-auc:0.94668\teval-auc:0.96592\n",
      "[62]\ttrain-auc:0.94685\teval-auc:0.96624\n",
      "[63]\ttrain-auc:0.94723\teval-auc:0.96608\n",
      "[64]\ttrain-auc:0.94757\teval-auc:0.96720\n",
      "[65]\ttrain-auc:0.94780\teval-auc:0.96784\n",
      "[66]\ttrain-auc:0.94818\teval-auc:0.96800\n",
      "[67]\ttrain-auc:0.94846\teval-auc:0.96800\n",
      "[68]\ttrain-auc:0.94861\teval-auc:0.96800\n",
      "[69]\ttrain-auc:0.94880\teval-auc:0.96816\n",
      "[70]\ttrain-auc:0.94898\teval-auc:0.96784\n",
      "[71]\ttrain-auc:0.94940\teval-auc:0.96848\n",
      "[72]\ttrain-auc:0.94961\teval-auc:0.96784\n",
      "[73]\ttrain-auc:0.95001\teval-auc:0.96768\n",
      "[74]\ttrain-auc:0.95017\teval-auc:0.96912\n",
      "[75]\ttrain-auc:0.95044\teval-auc:0.96896\n",
      "[76]\ttrain-auc:0.95075\teval-auc:0.96864\n",
      "[77]\ttrain-auc:0.95086\teval-auc:0.96816\n",
      "[78]\ttrain-auc:0.95107\teval-auc:0.96784\n",
      "[79]\ttrain-auc:0.95133\teval-auc:0.96816\n",
      "[80]\ttrain-auc:0.95182\teval-auc:0.96816\n",
      "[81]\ttrain-auc:0.95199\teval-auc:0.96816\n",
      "[82]\ttrain-auc:0.95217\teval-auc:0.96816\n",
      "[83]\ttrain-auc:0.95230\teval-auc:0.96816\n",
      "[84]\ttrain-auc:0.95240\teval-auc:0.96848\n",
      "[0]\ttrain-auc:0.87795\teval-auc:0.89952\n",
      "[1]\ttrain-auc:0.88169\teval-auc:0.90400\n",
      "[2]\ttrain-auc:0.90142\teval-auc:0.91504\n",
      "[3]\ttrain-auc:0.90295\teval-auc:0.91576\n",
      "[4]\ttrain-auc:0.90544\teval-auc:0.91432\n",
      "[5]\ttrain-auc:0.90599\teval-auc:0.91280\n",
      "[6]\ttrain-auc:0.90642\teval-auc:0.91248\n",
      "[7]\ttrain-auc:0.90730\teval-auc:0.92728\n",
      "[8]\ttrain-auc:0.90857\teval-auc:0.92704\n",
      "[9]\ttrain-auc:0.91349\teval-auc:0.92424\n",
      "[10]\ttrain-auc:0.91422\teval-auc:0.92440\n",
      "[11]\ttrain-auc:0.91690\teval-auc:0.92480\n",
      "[12]\ttrain-auc:0.91727\teval-auc:0.92568\n",
      "[13]\ttrain-auc:0.91926\teval-auc:0.92368\n",
      "[14]\ttrain-auc:0.91946\teval-auc:0.92400\n",
      "[15]\ttrain-auc:0.92098\teval-auc:0.92336\n",
      "[16]\ttrain-auc:0.92202\teval-auc:0.92344\n",
      "[17]\ttrain-auc:0.92224\teval-auc:0.92472\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.87092\teval-auc:0.84816\n",
      "[1]\ttrain-auc:0.89626\teval-auc:0.86872\n",
      "[2]\ttrain-auc:0.90033\teval-auc:0.88080\n",
      "[3]\ttrain-auc:0.90184\teval-auc:0.88280\n",
      "[4]\ttrain-auc:0.90187\teval-auc:0.88280\n",
      "[5]\ttrain-auc:0.90288\teval-auc:0.88792\n",
      "[6]\ttrain-auc:0.90538\teval-auc:0.88712\n",
      "[7]\ttrain-auc:0.90597\teval-auc:0.88744\n",
      "[8]\ttrain-auc:0.90665\teval-auc:0.89152\n",
      "[9]\ttrain-auc:0.90758\teval-auc:0.90288\n",
      "[10]\ttrain-auc:0.90809\teval-auc:0.90344\n",
      "[11]\ttrain-auc:0.90834\teval-auc:0.90448\n",
      "[12]\ttrain-auc:0.90980\teval-auc:0.90408\n",
      "[13]\ttrain-auc:0.91222\teval-auc:0.90416\n",
      "[14]\ttrain-auc:0.91544\teval-auc:0.90216\n",
      "[15]\ttrain-auc:0.91617\teval-auc:0.90776\n",
      "[16]\ttrain-auc:0.91874\teval-auc:0.91448\n",
      "[17]\ttrain-auc:0.91980\teval-auc:0.91768\n",
      "[18]\ttrain-auc:0.92066\teval-auc:0.91680\n",
      "[19]\ttrain-auc:0.92352\teval-auc:0.92256\n",
      "[20]\ttrain-auc:0.92411\teval-auc:0.92744\n",
      "[21]\ttrain-auc:0.92464\teval-auc:0.92856\n",
      "[22]\ttrain-auc:0.92625\teval-auc:0.93128\n",
      "[23]\ttrain-auc:0.92714\teval-auc:0.93200\n",
      "[24]\ttrain-auc:0.92825\teval-auc:0.93360\n",
      "[25]\ttrain-auc:0.92877\teval-auc:0.93456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[26]\ttrain-auc:0.92993\teval-auc:0.93472\n",
      "[27]\ttrain-auc:0.93083\teval-auc:0.93760\n",
      "[28]\ttrain-auc:0.93109\teval-auc:0.93696\n",
      "[29]\ttrain-auc:0.93132\teval-auc:0.93760\n",
      "[30]\ttrain-auc:0.93204\teval-auc:0.93872\n",
      "[31]\ttrain-auc:0.93267\teval-auc:0.93968\n",
      "[32]\ttrain-auc:0.93379\teval-auc:0.94288\n",
      "[33]\ttrain-auc:0.93437\teval-auc:0.94400\n",
      "[34]\ttrain-auc:0.93484\teval-auc:0.94480\n",
      "[35]\ttrain-auc:0.93575\teval-auc:0.94480\n",
      "[36]\ttrain-auc:0.93598\teval-auc:0.94432\n",
      "[37]\ttrain-auc:0.93640\teval-auc:0.94560\n",
      "[38]\ttrain-auc:0.93651\teval-auc:0.94400\n",
      "[39]\ttrain-auc:0.93799\teval-auc:0.94944\n",
      "[40]\ttrain-auc:0.93833\teval-auc:0.94992\n",
      "[41]\ttrain-auc:0.93905\teval-auc:0.94752\n",
      "[42]\ttrain-auc:0.93957\teval-auc:0.94928\n",
      "[43]\ttrain-auc:0.93975\teval-auc:0.94912\n",
      "[44]\ttrain-auc:0.94041\teval-auc:0.94928\n",
      "[45]\ttrain-auc:0.94040\teval-auc:0.95280\n",
      "[46]\ttrain-auc:0.94084\teval-auc:0.95168\n",
      "[47]\ttrain-auc:0.94133\teval-auc:0.95200\n",
      "[48]\ttrain-auc:0.94167\teval-auc:0.95280\n",
      "[49]\ttrain-auc:0.94220\teval-auc:0.95248\n",
      "[50]\ttrain-auc:0.94235\teval-auc:0.95280\n",
      "[51]\ttrain-auc:0.94275\teval-auc:0.95232\n",
      "[52]\ttrain-auc:0.94314\teval-auc:0.95472\n",
      "[53]\ttrain-auc:0.94363\teval-auc:0.95424\n",
      "[54]\ttrain-auc:0.94395\teval-auc:0.95488\n",
      "[55]\ttrain-auc:0.94470\teval-auc:0.95760\n",
      "[56]\ttrain-auc:0.94478\teval-auc:0.95664\n",
      "[57]\ttrain-auc:0.94537\teval-auc:0.95728\n",
      "[58]\ttrain-auc:0.94576\teval-auc:0.95808\n",
      "[59]\ttrain-auc:0.94590\teval-auc:0.95776\n",
      "[60]\ttrain-auc:0.94627\teval-auc:0.95728\n",
      "[61]\ttrain-auc:0.94649\teval-auc:0.95776\n",
      "[62]\ttrain-auc:0.94705\teval-auc:0.95712\n",
      "[63]\ttrain-auc:0.94733\teval-auc:0.95712\n",
      "[64]\ttrain-auc:0.94765\teval-auc:0.95856\n",
      "[65]\ttrain-auc:0.94792\teval-auc:0.95904\n",
      "[66]\ttrain-auc:0.94809\teval-auc:0.96016\n",
      "[67]\ttrain-auc:0.94845\teval-auc:0.96240\n",
      "[68]\ttrain-auc:0.94883\teval-auc:0.96208\n",
      "[69]\ttrain-auc:0.94918\teval-auc:0.96112\n",
      "[70]\ttrain-auc:0.94928\teval-auc:0.96080\n",
      "[71]\ttrain-auc:0.94940\teval-auc:0.96096\n",
      "[72]\ttrain-auc:0.94957\teval-auc:0.96096\n",
      "[73]\ttrain-auc:0.94986\teval-auc:0.96208\n",
      "[74]\ttrain-auc:0.95016\teval-auc:0.96240\n",
      "[75]\ttrain-auc:0.95062\teval-auc:0.96272\n",
      "[76]\ttrain-auc:0.95069\teval-auc:0.96304\n",
      "[77]\ttrain-auc:0.95075\teval-auc:0.96400\n",
      "[78]\ttrain-auc:0.95106\teval-auc:0.96432\n",
      "[79]\ttrain-auc:0.95138\teval-auc:0.96384\n",
      "[80]\ttrain-auc:0.95161\teval-auc:0.96416\n",
      "[81]\ttrain-auc:0.95162\teval-auc:0.96384\n",
      "[82]\ttrain-auc:0.95186\teval-auc:0.96416\n",
      "[83]\ttrain-auc:0.95196\teval-auc:0.96432\n",
      "[84]\ttrain-auc:0.95232\teval-auc:0.96432\n",
      "[85]\ttrain-auc:0.95229\teval-auc:0.96400\n",
      "[86]\ttrain-auc:0.95251\teval-auc:0.96384\n",
      "[87]\ttrain-auc:0.95276\teval-auc:0.96560\n",
      "[88]\ttrain-auc:0.95307\teval-auc:0.96576\n",
      "[89]\ttrain-auc:0.95351\teval-auc:0.96528\n",
      "[90]\ttrain-auc:0.95367\teval-auc:0.96528\n",
      "[91]\ttrain-auc:0.95373\teval-auc:0.96512\n",
      "[92]\ttrain-auc:0.95388\teval-auc:0.96560\n",
      "[93]\ttrain-auc:0.95427\teval-auc:0.96640\n",
      "[94]\ttrain-auc:0.95455\teval-auc:0.96656\n",
      "[95]\ttrain-auc:0.95476\teval-auc:0.96704\n",
      "[96]\ttrain-auc:0.95487\teval-auc:0.96672\n",
      "[97]\ttrain-auc:0.95495\teval-auc:0.96672\n",
      "[98]\ttrain-auc:0.95513\teval-auc:0.96752\n",
      "[99]\ttrain-auc:0.95522\teval-auc:0.96720\n",
      "[100]\ttrain-auc:0.95533\teval-auc:0.96752\n",
      "[101]\ttrain-auc:0.95541\teval-auc:0.96720\n",
      "[102]\ttrain-auc:0.95563\teval-auc:0.96784\n",
      "[103]\ttrain-auc:0.95569\teval-auc:0.96784\n",
      "[104]\ttrain-auc:0.95590\teval-auc:0.96752\n",
      "[105]\ttrain-auc:0.95602\teval-auc:0.96816\n",
      "[106]\ttrain-auc:0.95623\teval-auc:0.96880\n",
      "[107]\ttrain-auc:0.95630\teval-auc:0.96912\n",
      "[108]\ttrain-auc:0.95641\teval-auc:0.96928\n",
      "[109]\ttrain-auc:0.95661\teval-auc:0.96992\n",
      "[110]\ttrain-auc:0.95684\teval-auc:0.96960\n",
      "[111]\ttrain-auc:0.95693\teval-auc:0.96976\n",
      "[112]\ttrain-auc:0.95707\teval-auc:0.96992\n",
      "[113]\ttrain-auc:0.95722\teval-auc:0.96960\n",
      "[114]\ttrain-auc:0.95731\teval-auc:0.97024\n",
      "[115]\ttrain-auc:0.95756\teval-auc:0.96976\n",
      "[116]\ttrain-auc:0.95771\teval-auc:0.96992\n",
      "[117]\ttrain-auc:0.95784\teval-auc:0.97056\n",
      "[118]\ttrain-auc:0.95799\teval-auc:0.97056\n",
      "[119]\ttrain-auc:0.95823\teval-auc:0.97104\n",
      "[120]\ttrain-auc:0.95852\teval-auc:0.97136\n",
      "[121]\ttrain-auc:0.95869\teval-auc:0.97184\n",
      "[122]\ttrain-auc:0.95873\teval-auc:0.97184\n",
      "[123]\ttrain-auc:0.95882\teval-auc:0.97184\n",
      "[124]\ttrain-auc:0.95893\teval-auc:0.97136\n",
      "[125]\ttrain-auc:0.95902\teval-auc:0.97232\n",
      "[126]\ttrain-auc:0.95920\teval-auc:0.97200\n",
      "[127]\ttrain-auc:0.95934\teval-auc:0.97232\n",
      "[128]\ttrain-auc:0.95940\teval-auc:0.97232\n",
      "[129]\ttrain-auc:0.95951\teval-auc:0.97232\n",
      "[130]\ttrain-auc:0.95959\teval-auc:0.97296\n",
      "[131]\ttrain-auc:0.95957\teval-auc:0.97296\n",
      "[132]\ttrain-auc:0.95972\teval-auc:0.97296\n",
      "[133]\ttrain-auc:0.95995\teval-auc:0.97296\n",
      "[134]\ttrain-auc:0.96007\teval-auc:0.97296\n",
      "[135]\ttrain-auc:0.96012\teval-auc:0.97264\n",
      "[136]\ttrain-auc:0.96006\teval-auc:0.97328\n",
      "[137]\ttrain-auc:0.96025\teval-auc:0.97312\n",
      "[138]\ttrain-auc:0.96042\teval-auc:0.97328\n",
      "[139]\ttrain-auc:0.96053\teval-auc:0.97344\n",
      "[140]\ttrain-auc:0.96067\teval-auc:0.97408\n",
      "[141]\ttrain-auc:0.96084\teval-auc:0.97392\n",
      "[142]\ttrain-auc:0.96091\teval-auc:0.97360\n",
      "[143]\ttrain-auc:0.96088\teval-auc:0.97392\n",
      "[144]\ttrain-auc:0.96094\teval-auc:0.97408\n",
      "[145]\ttrain-auc:0.96101\teval-auc:0.97408\n",
      "[146]\ttrain-auc:0.96108\teval-auc:0.97392\n",
      "[147]\ttrain-auc:0.96140\teval-auc:0.97376\n",
      "[148]\ttrain-auc:0.96148\teval-auc:0.97392\n",
      "[149]\ttrain-auc:0.96158\teval-auc:0.97424\n",
      "[150]\ttrain-auc:0.96177\teval-auc:0.97440\n",
      "[151]\ttrain-auc:0.96190\teval-auc:0.97424\n",
      "[152]\ttrain-auc:0.96208\teval-auc:0.97600\n",
      "[153]\ttrain-auc:0.96223\teval-auc:0.97616\n",
      "[154]\ttrain-auc:0.96234\teval-auc:0.97600\n",
      "[155]\ttrain-auc:0.96243\teval-auc:0.97584\n",
      "[156]\ttrain-auc:0.96251\teval-auc:0.97584\n",
      "[157]\ttrain-auc:0.96258\teval-auc:0.97616\n",
      "[158]\ttrain-auc:0.96264\teval-auc:0.97616\n",
      "[159]\ttrain-auc:0.96260\teval-auc:0.97568\n",
      "[160]\ttrain-auc:0.96272\teval-auc:0.97552\n",
      "[161]\ttrain-auc:0.96280\teval-auc:0.97568\n",
      "[162]\ttrain-auc:0.96297\teval-auc:0.97632\n",
      "[163]\ttrain-auc:0.96316\teval-auc:0.97712\n",
      "[164]\ttrain-auc:0.96320\teval-auc:0.97744\n",
      "[165]\ttrain-auc:0.96328\teval-auc:0.97712\n",
      "[166]\ttrain-auc:0.96338\teval-auc:0.97712\n",
      "[167]\ttrain-auc:0.96353\teval-auc:0.97712\n",
      "[168]\ttrain-auc:0.96368\teval-auc:0.97760\n",
      "[169]\ttrain-auc:0.96385\teval-auc:0.97760\n",
      "[170]\ttrain-auc:0.96398\teval-auc:0.97744\n",
      "[171]\ttrain-auc:0.96411\teval-auc:0.97744\n",
      "[172]\ttrain-auc:0.96416\teval-auc:0.97744\n",
      "[173]\ttrain-auc:0.96422\teval-auc:0.97712\n",
      "[174]\ttrain-auc:0.96433\teval-auc:0.97760\n",
      "[175]\ttrain-auc:0.96443\teval-auc:0.97760\n",
      "[176]\ttrain-auc:0.96446\teval-auc:0.97744\n",
      "[177]\ttrain-auc:0.96455\teval-auc:0.97808\n",
      "[178]\ttrain-auc:0.96459\teval-auc:0.97808\n",
      "[179]\ttrain-auc:0.96470\teval-auc:0.97776\n",
      "[180]\ttrain-auc:0.96473\teval-auc:0.97760\n",
      "[181]\ttrain-auc:0.96491\teval-auc:0.97776\n",
      "[182]\ttrain-auc:0.96512\teval-auc:0.97760\n",
      "[183]\ttrain-auc:0.96514\teval-auc:0.97776\n",
      "[184]\ttrain-auc:0.96517\teval-auc:0.97776\n",
      "[185]\ttrain-auc:0.96538\teval-auc:0.97808\n",
      "[186]\ttrain-auc:0.96551\teval-auc:0.97856\n",
      "[187]\ttrain-auc:0.96564\teval-auc:0.97888\n",
      "[188]\ttrain-auc:0.96568\teval-auc:0.97856\n",
      "[189]\ttrain-auc:0.96580\teval-auc:0.97856\n",
      "[190]\ttrain-auc:0.96590\teval-auc:0.97888\n",
      "[191]\ttrain-auc:0.96593\teval-auc:0.97888\n",
      "[192]\ttrain-auc:0.96593\teval-auc:0.97904\n",
      "[193]\ttrain-auc:0.96604\teval-auc:0.97968\n",
      "[194]\ttrain-auc:0.96617\teval-auc:0.97968\n",
      "[195]\ttrain-auc:0.96628\teval-auc:0.97952\n",
      "[196]\ttrain-auc:0.96641\teval-auc:0.97968\n",
      "[197]\ttrain-auc:0.96655\teval-auc:0.97968\n",
      "[198]\ttrain-auc:0.96660\teval-auc:0.97968\n",
      "[199]\ttrain-auc:0.96664\teval-auc:0.97984\n",
      "[200]\ttrain-auc:0.96679\teval-auc:0.98016\n",
      "[201]\ttrain-auc:0.96697\teval-auc:0.98064\n",
      "[202]\ttrain-auc:0.96705\teval-auc:0.98064\n",
      "[203]\ttrain-auc:0.96706\teval-auc:0.98080\n",
      "[204]\ttrain-auc:0.96714\teval-auc:0.98080\n",
      "[205]\ttrain-auc:0.96721\teval-auc:0.98064\n",
      "[206]\ttrain-auc:0.96735\teval-auc:0.98080\n",
      "[207]\ttrain-auc:0.96745\teval-auc:0.98080\n",
      "[208]\ttrain-auc:0.96751\teval-auc:0.98080\n",
      "[209]\ttrain-auc:0.96754\teval-auc:0.98080\n",
      "[210]\ttrain-auc:0.96759\teval-auc:0.98080\n",
      "[211]\ttrain-auc:0.96767\teval-auc:0.98080\n",
      "[212]\ttrain-auc:0.96776\teval-auc:0.98080\n",
      "[213]\ttrain-auc:0.96791\teval-auc:0.98048\n",
      "[0]\ttrain-auc:0.86184\teval-auc:0.88360\n",
      "[1]\ttrain-auc:0.89160\teval-auc:0.89440\n",
      "[2]\ttrain-auc:0.90107\teval-auc:0.89312\n",
      "[3]\ttrain-auc:0.90232\teval-auc:0.89440\n",
      "[4]\ttrain-auc:0.90233\teval-auc:0.89440\n",
      "[5]\ttrain-auc:0.90286\teval-auc:0.89304\n",
      "[6]\ttrain-auc:0.90409\teval-auc:0.89304\n",
      "[7]\ttrain-auc:0.90452\teval-auc:0.89368\n",
      "[8]\ttrain-auc:0.90455\teval-auc:0.89376\n",
      "[9]\ttrain-auc:0.91180\teval-auc:0.91472\n",
      "[10]\ttrain-auc:0.91245\teval-auc:0.91552\n",
      "[11]\ttrain-auc:0.91477\teval-auc:0.91728\n",
      "[12]\ttrain-auc:0.91577\teval-auc:0.91680\n",
      "[13]\ttrain-auc:0.91701\teval-auc:0.91696\n",
      "[14]\ttrain-auc:0.91882\teval-auc:0.91720\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[15]\ttrain-auc:0.91899\teval-auc:0.91656\n",
      "[16]\ttrain-auc:0.91988\teval-auc:0.91904\n",
      "[17]\ttrain-auc:0.92114\teval-auc:0.91904\n",
      "[18]\ttrain-auc:0.92168\teval-auc:0.91856\n",
      "[19]\ttrain-auc:0.92360\teval-auc:0.91816\n",
      "[20]\ttrain-auc:0.92542\teval-auc:0.92496\n",
      "[21]\ttrain-auc:0.92568\teval-auc:0.92592\n",
      "[22]\ttrain-auc:0.92639\teval-auc:0.92528\n",
      "[23]\ttrain-auc:0.92693\teval-auc:0.92568\n",
      "[24]\ttrain-auc:0.92777\teval-auc:0.92624\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\ttrain-auc:0.92861\teval-auc:0.92688\n",
      "[26]\ttrain-auc:0.93000\teval-auc:0.92992\n",
      "[27]\ttrain-auc:0.93054\teval-auc:0.92832\n",
      "[28]\ttrain-auc:0.93096\teval-auc:0.93136\n",
      "[29]\ttrain-auc:0.93118\teval-auc:0.93408\n",
      "[30]\ttrain-auc:0.93157\teval-auc:0.93424\n",
      "[31]\ttrain-auc:0.93262\teval-auc:0.93520\n",
      "[32]\ttrain-auc:0.93287\teval-auc:0.93472\n",
      "[33]\ttrain-auc:0.93341\teval-auc:0.93200\n",
      "[34]\ttrain-auc:0.93366\teval-auc:0.93232\n",
      "[35]\ttrain-auc:0.93419\teval-auc:0.93344\n",
      "[36]\ttrain-auc:0.93463\teval-auc:0.93472\n",
      "[37]\ttrain-auc:0.93501\teval-auc:0.93152\n",
      "[38]\ttrain-auc:0.93568\teval-auc:0.93280\n",
      "[39]\ttrain-auc:0.93614\teval-auc:0.93360\n",
      "[40]\ttrain-auc:0.93663\teval-auc:0.93504\n",
      "[0]\ttrain-auc:0.87596\teval-auc:0.89104\n",
      "[1]\ttrain-auc:0.87671\teval-auc:0.89088\n",
      "[2]\ttrain-auc:0.89987\teval-auc:0.88312\n",
      "[3]\ttrain-auc:0.90039\teval-auc:0.88344\n",
      "[4]\ttrain-auc:0.90084\teval-auc:0.88328\n",
      "[5]\ttrain-auc:0.90420\teval-auc:0.92312\n",
      "[6]\ttrain-auc:0.90783\teval-auc:0.92248\n",
      "[7]\ttrain-auc:0.90845\teval-auc:0.92336\n",
      "[8]\ttrain-auc:0.90919\teval-auc:0.92592\n",
      "[9]\ttrain-auc:0.90911\teval-auc:0.92592\n",
      "[10]\ttrain-auc:0.91388\teval-auc:0.92824\n",
      "[11]\ttrain-auc:0.91502\teval-auc:0.92776\n",
      "[12]\ttrain-auc:0.91568\teval-auc:0.92872\n",
      "[13]\ttrain-auc:0.91609\teval-auc:0.92824\n",
      "[14]\ttrain-auc:0.91658\teval-auc:0.92960\n",
      "[15]\ttrain-auc:0.91760\teval-auc:0.93096\n",
      "[16]\ttrain-auc:0.91817\teval-auc:0.93296\n",
      "[17]\ttrain-auc:0.92135\teval-auc:0.93344\n",
      "[18]\ttrain-auc:0.92174\teval-auc:0.93536\n",
      "[19]\ttrain-auc:0.92321\teval-auc:0.93648\n",
      "[20]\ttrain-auc:0.92356\teval-auc:0.93664\n",
      "[21]\ttrain-auc:0.92434\teval-auc:0.93680\n",
      "[22]\ttrain-auc:0.92624\teval-auc:0.93712\n",
      "[23]\ttrain-auc:0.92783\teval-auc:0.94128\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[24]\ttrain-auc:0.92929\teval-auc:0.94160\n",
      "[25]\ttrain-auc:0.93012\teval-auc:0.94352\n",
      "[26]\ttrain-auc:0.93043\teval-auc:0.94352\n",
      "[27]\ttrain-auc:0.93101\teval-auc:0.94432\n",
      "[28]\ttrain-auc:0.93218\teval-auc:0.94560\n",
      "[29]\ttrain-auc:0.93260\teval-auc:0.94432\n",
      "[30]\ttrain-auc:0.93359\teval-auc:0.94496\n",
      "[31]\ttrain-auc:0.93411\teval-auc:0.94512\n",
      "[32]\ttrain-auc:0.93505\teval-auc:0.94656\n",
      "[33]\ttrain-auc:0.93529\teval-auc:0.94608\n",
      "[34]\ttrain-auc:0.93628\teval-auc:0.94656\n",
      "[35]\ttrain-auc:0.93673\teval-auc:0.94832\n",
      "[36]\ttrain-auc:0.93706\teval-auc:0.94912\n",
      "[37]\ttrain-auc:0.93768\teval-auc:0.94944\n",
      "[38]\ttrain-auc:0.93798\teval-auc:0.94928\n",
      "[39]\ttrain-auc:0.93850\teval-auc:0.94928\n",
      "[40]\ttrain-auc:0.93924\teval-auc:0.95200\n",
      "[41]\ttrain-auc:0.93977\teval-auc:0.95280\n",
      "[42]\ttrain-auc:0.94039\teval-auc:0.95360\n",
      "[43]\ttrain-auc:0.94056\teval-auc:0.95360\n",
      "[44]\ttrain-auc:0.94085\teval-auc:0.95440\n",
      "[45]\ttrain-auc:0.94124\teval-auc:0.95488\n",
      "[46]\ttrain-auc:0.94135\teval-auc:0.95520\n",
      "[47]\ttrain-auc:0.94181\teval-auc:0.95488\n",
      "[48]\ttrain-auc:0.94230\teval-auc:0.95488\n",
      "[49]\ttrain-auc:0.94268\teval-auc:0.95600\n",
      "[50]\ttrain-auc:0.94305\teval-auc:0.95648\n",
      "[51]\ttrain-auc:0.94321\teval-auc:0.95696\n",
      "[52]\ttrain-auc:0.94389\teval-auc:0.95648\n",
      "[53]\ttrain-auc:0.94413\teval-auc:0.95664\n",
      "[54]\ttrain-auc:0.94452\teval-auc:0.95648\n",
      "[55]\ttrain-auc:0.94514\teval-auc:0.95616\n",
      "[56]\ttrain-auc:0.94557\teval-auc:0.95696\n",
      "[57]\ttrain-auc:0.94567\teval-auc:0.95728\n",
      "[58]\ttrain-auc:0.94614\teval-auc:0.95712\n",
      "[59]\ttrain-auc:0.94641\teval-auc:0.95792\n",
      "[60]\ttrain-auc:0.94674\teval-auc:0.95888\n",
      "[61]\ttrain-auc:0.94709\teval-auc:0.95904\n",
      "[62]\ttrain-auc:0.94739\teval-auc:0.95920\n",
      "[63]\ttrain-auc:0.94747\teval-auc:0.95872\n",
      "[64]\ttrain-auc:0.94767\teval-auc:0.95888\n",
      "[65]\ttrain-auc:0.94793\teval-auc:0.95936\n",
      "[66]\ttrain-auc:0.94798\teval-auc:0.95920\n",
      "[67]\ttrain-auc:0.94840\teval-auc:0.95952\n",
      "[68]\ttrain-auc:0.94865\teval-auc:0.95952\n",
      "[69]\ttrain-auc:0.94893\teval-auc:0.95984\n",
      "[70]\ttrain-auc:0.94914\teval-auc:0.96064\n",
      "[71]\ttrain-auc:0.94942\teval-auc:0.96208\n",
      "[72]\ttrain-auc:0.94970\teval-auc:0.96224\n",
      "[73]\ttrain-auc:0.94990\teval-auc:0.96256\n",
      "[74]\ttrain-auc:0.94997\teval-auc:0.96192\n",
      "[75]\ttrain-auc:0.95011\teval-auc:0.96208\n",
      "[76]\ttrain-auc:0.95017\teval-auc:0.96160\n",
      "[77]\ttrain-auc:0.95049\teval-auc:0.96096\n",
      "[78]\ttrain-auc:0.95086\teval-auc:0.96128\n",
      "[79]\ttrain-auc:0.95116\teval-auc:0.96192\n",
      "[80]\ttrain-auc:0.95137\teval-auc:0.96192\n",
      "[81]\ttrain-auc:0.95166\teval-auc:0.96208\n",
      "[82]\ttrain-auc:0.95188\teval-auc:0.96192\n",
      "[83]\ttrain-auc:0.95201\teval-auc:0.96272\n",
      "[84]\ttrain-auc:0.95229\teval-auc:0.96240\n",
      "[85]\ttrain-auc:0.95258\teval-auc:0.96304\n",
      "[86]\ttrain-auc:0.95269\teval-auc:0.96288\n",
      "[87]\ttrain-auc:0.95275\teval-auc:0.96288\n",
      "[88]\ttrain-auc:0.95285\teval-auc:0.96320\n",
      "[89]\ttrain-auc:0.95320\teval-auc:0.96384\n",
      "[90]\ttrain-auc:0.95333\teval-auc:0.96352\n",
      "[91]\ttrain-auc:0.95352\teval-auc:0.96320\n",
      "[92]\ttrain-auc:0.95370\teval-auc:0.96320\n",
      "[93]\ttrain-auc:0.95389\teval-auc:0.96384\n",
      "[94]\ttrain-auc:0.95419\teval-auc:0.96400\n",
      "[95]\ttrain-auc:0.95425\teval-auc:0.96336\n",
      "[96]\ttrain-auc:0.95450\teval-auc:0.96336\n",
      "[97]\ttrain-auc:0.95477\teval-auc:0.96400\n",
      "[98]\ttrain-auc:0.95497\teval-auc:0.96352\n",
      "[99]\ttrain-auc:0.95519\teval-auc:0.96320\n",
      "[100]\ttrain-auc:0.95566\teval-auc:0.96320\n",
      "[101]\ttrain-auc:0.95579\teval-auc:0.96304\n",
      "[102]\ttrain-auc:0.95587\teval-auc:0.96288\n",
      "[103]\ttrain-auc:0.95613\teval-auc:0.96304\n",
      "[0]\ttrain-auc:0.87369\teval-auc:0.90184\n",
      "[1]\ttrain-auc:0.87631\teval-auc:0.91032\n",
      "[2]\ttrain-auc:0.87979\teval-auc:0.91000\n",
      "[3]\ttrain-auc:0.89517\teval-auc:0.90976\n",
      "[4]\ttrain-auc:0.90236\teval-auc:0.90672\n",
      "[5]\ttrain-auc:0.90232\teval-auc:0.90640\n",
      "[6]\ttrain-auc:0.90292\teval-auc:0.90608\n",
      "[7]\ttrain-auc:0.90515\teval-auc:0.90320\n",
      "[8]\ttrain-auc:0.90583\teval-auc:0.90280\n",
      "[9]\ttrain-auc:0.90577\teval-auc:0.90280\n",
      "[10]\ttrain-auc:0.91171\teval-auc:0.90792\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.87720\teval-auc:0.88216\n",
      "[1]\ttrain-auc:0.88235\teval-auc:0.89560\n",
      "[2]\ttrain-auc:0.89221\teval-auc:0.89688\n",
      "[3]\ttrain-auc:0.89886\teval-auc:0.89600\n",
      "[4]\ttrain-auc:0.89964\teval-auc:0.89912\n",
      "[5]\ttrain-auc:0.90006\teval-auc:0.89952\n",
      "[6]\ttrain-auc:0.90210\teval-auc:0.89840\n",
      "[7]\ttrain-auc:0.90246\teval-auc:0.89968\n",
      "[8]\ttrain-auc:0.90331\teval-auc:0.89888\n",
      "[9]\ttrain-auc:0.90443\teval-auc:0.89952\n",
      "[10]\ttrain-auc:0.90484\teval-auc:0.90016\n",
      "[11]\ttrain-auc:0.90914\teval-auc:0.89744\n",
      "[12]\ttrain-auc:0.90958\teval-auc:0.89752\n",
      "[13]\ttrain-auc:0.91096\teval-auc:0.89880\n",
      "[14]\ttrain-auc:0.91291\teval-auc:0.89920\n",
      "[15]\ttrain-auc:0.91451\teval-auc:0.91320\n",
      "[16]\ttrain-auc:0.91513\teval-auc:0.91296\n",
      "[17]\ttrain-auc:0.91779\teval-auc:0.92344\n",
      "[18]\ttrain-auc:0.91855\teval-auc:0.92160\n",
      "[19]\ttrain-auc:0.92122\teval-auc:0.92432\n",
      "[20]\ttrain-auc:0.92194\teval-auc:0.92512\n",
      "[21]\ttrain-auc:0.92378\teval-auc:0.92144\n",
      "[22]\ttrain-auc:0.92446\teval-auc:0.92376\n",
      "[23]\ttrain-auc:0.92541\teval-auc:0.92424\n",
      "[24]\ttrain-auc:0.92551\teval-auc:0.92312\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\ttrain-auc:0.92724\teval-auc:0.92456\n",
      "[26]\ttrain-auc:0.92754\teval-auc:0.92352\n",
      "[27]\ttrain-auc:0.92804\teval-auc:0.92576\n",
      "[28]\ttrain-auc:0.92886\teval-auc:0.92912\n",
      "[29]\ttrain-auc:0.92978\teval-auc:0.93040\n",
      "[30]\ttrain-auc:0.93030\teval-auc:0.93024\n",
      "[31]\ttrain-auc:0.93077\teval-auc:0.92864\n",
      "[32]\ttrain-auc:0.93133\teval-auc:0.92880\n",
      "[33]\ttrain-auc:0.93249\teval-auc:0.92912\n",
      "[34]\ttrain-auc:0.93282\teval-auc:0.92848\n",
      "[35]\ttrain-auc:0.93344\teval-auc:0.92816\n",
      "[36]\ttrain-auc:0.93383\teval-auc:0.92848\n",
      "[37]\ttrain-auc:0.93444\teval-auc:0.93248\n",
      "[38]\ttrain-auc:0.93494\teval-auc:0.93328\n",
      "[39]\ttrain-auc:0.93549\teval-auc:0.93376\n",
      "[40]\ttrain-auc:0.93608\teval-auc:0.93392\n",
      "[41]\ttrain-auc:0.93629\teval-auc:0.93424\n",
      "[42]\ttrain-auc:0.93718\teval-auc:0.93360\n",
      "[43]\ttrain-auc:0.93762\teval-auc:0.93472\n",
      "[44]\ttrain-auc:0.93812\teval-auc:0.93456\n",
      "[45]\ttrain-auc:0.93856\teval-auc:0.93472\n",
      "[46]\ttrain-auc:0.93871\teval-auc:0.93488\n",
      "[47]\ttrain-auc:0.93931\teval-auc:0.93616\n",
      "[48]\ttrain-auc:0.93959\teval-auc:0.93568\n",
      "[49]\ttrain-auc:0.94014\teval-auc:0.93824\n",
      "[50]\ttrain-auc:0.94056\teval-auc:0.93840\n",
      "[51]\ttrain-auc:0.94091\teval-auc:0.93856\n",
      "[52]\ttrain-auc:0.94115\teval-auc:0.93792\n",
      "[53]\ttrain-auc:0.94129\teval-auc:0.93840\n",
      "[54]\ttrain-auc:0.94193\teval-auc:0.93856\n",
      "[55]\ttrain-auc:0.94204\teval-auc:0.94032\n",
      "[56]\ttrain-auc:0.94285\teval-auc:0.94032\n",
      "[57]\ttrain-auc:0.94331\teval-auc:0.94096\n",
      "[58]\ttrain-auc:0.94331\teval-auc:0.94144\n",
      "[59]\ttrain-auc:0.94359\teval-auc:0.94160\n",
      "[60]\ttrain-auc:0.94389\teval-auc:0.94272\n",
      "[61]\ttrain-auc:0.94430\teval-auc:0.94464\n",
      "[62]\ttrain-auc:0.94460\teval-auc:0.94448\n",
      "[63]\ttrain-auc:0.94479\teval-auc:0.94512\n",
      "[64]\ttrain-auc:0.94522\teval-auc:0.94512\n",
      "[65]\ttrain-auc:0.94525\teval-auc:0.94448\n",
      "[66]\ttrain-auc:0.94571\teval-auc:0.94512\n",
      "[67]\ttrain-auc:0.94603\teval-auc:0.94512\n",
      "[68]\ttrain-auc:0.94653\teval-auc:0.94560\n",
      "[69]\ttrain-auc:0.94692\teval-auc:0.94720\n",
      "[70]\ttrain-auc:0.94703\teval-auc:0.94720\n",
      "[71]\ttrain-auc:0.94735\teval-auc:0.94688\n",
      "[72]\ttrain-auc:0.94754\teval-auc:0.94640\n",
      "[73]\ttrain-auc:0.94791\teval-auc:0.94672\n",
      "[74]\ttrain-auc:0.94810\teval-auc:0.94528\n",
      "[75]\ttrain-auc:0.94840\teval-auc:0.94624\n",
      "[76]\ttrain-auc:0.94855\teval-auc:0.94736\n",
      "[77]\ttrain-auc:0.94872\teval-auc:0.94736\n",
      "[78]\ttrain-auc:0.94898\teval-auc:0.94688\n",
      "[79]\ttrain-auc:0.94913\teval-auc:0.94704\n",
      "[80]\ttrain-auc:0.94944\teval-auc:0.94656\n",
      "[81]\ttrain-auc:0.94961\teval-auc:0.94656\n",
      "[82]\ttrain-auc:0.95004\teval-auc:0.94784\n",
      "[83]\ttrain-auc:0.95012\teval-auc:0.94816\n",
      "[84]\ttrain-auc:0.95032\teval-auc:0.94848\n",
      "[85]\ttrain-auc:0.95050\teval-auc:0.94944\n",
      "[86]\ttrain-auc:0.95069\teval-auc:0.94928\n",
      "[87]\ttrain-auc:0.95087\teval-auc:0.94944\n",
      "[88]\ttrain-auc:0.95098\teval-auc:0.94944\n",
      "[89]\ttrain-auc:0.95114\teval-auc:0.95024\n",
      "[90]\ttrain-auc:0.95129\teval-auc:0.95056\n",
      "[91]\ttrain-auc:0.95145\teval-auc:0.95072\n",
      "[92]\ttrain-auc:0.95160\teval-auc:0.95104\n",
      "[93]\ttrain-auc:0.95185\teval-auc:0.95088\n",
      "[94]\ttrain-auc:0.95206\teval-auc:0.95152\n",
      "[95]\ttrain-auc:0.95219\teval-auc:0.95136\n",
      "[96]\ttrain-auc:0.95247\teval-auc:0.95248\n",
      "[97]\ttrain-auc:0.95268\teval-auc:0.95280\n",
      "[98]\ttrain-auc:0.95289\teval-auc:0.95296\n",
      "[99]\ttrain-auc:0.95295\teval-auc:0.95248\n",
      "[100]\ttrain-auc:0.95306\teval-auc:0.95248\n",
      "[101]\ttrain-auc:0.95329\teval-auc:0.95280\n",
      "[102]\ttrain-auc:0.95338\teval-auc:0.95280\n",
      "[103]\ttrain-auc:0.95352\teval-auc:0.95280\n",
      "[104]\ttrain-auc:0.95367\teval-auc:0.95280\n",
      "[105]\ttrain-auc:0.95376\teval-auc:0.95296\n",
      "[106]\ttrain-auc:0.95395\teval-auc:0.95424\n",
      "[107]\ttrain-auc:0.95405\teval-auc:0.95440\n",
      "[108]\ttrain-auc:0.95428\teval-auc:0.95344\n",
      "[109]\ttrain-auc:0.95453\teval-auc:0.95392\n",
      "[110]\ttrain-auc:0.95487\teval-auc:0.95408\n",
      "[111]\ttrain-auc:0.95486\teval-auc:0.95424\n",
      "[112]\ttrain-auc:0.95504\teval-auc:0.95424\n",
      "[113]\ttrain-auc:0.95517\teval-auc:0.95456\n",
      "[114]\ttrain-auc:0.95554\teval-auc:0.95504\n",
      "[115]\ttrain-auc:0.95585\teval-auc:0.95456\n",
      "[116]\ttrain-auc:0.95617\teval-auc:0.95424\n",
      "[117]\ttrain-auc:0.95641\teval-auc:0.95440\n",
      "[118]\ttrain-auc:0.95647\teval-auc:0.95472\n",
      "[119]\ttrain-auc:0.95664\teval-auc:0.95488\n",
      "[120]\ttrain-auc:0.95683\teval-auc:0.95520\n",
      "[121]\ttrain-auc:0.95687\teval-auc:0.95536\n",
      "[122]\ttrain-auc:0.95717\teval-auc:0.95616\n",
      "[123]\ttrain-auc:0.95736\teval-auc:0.95664\n",
      "[124]\ttrain-auc:0.95750\teval-auc:0.95664\n",
      "[125]\ttrain-auc:0.95755\teval-auc:0.95664\n",
      "[126]\ttrain-auc:0.95781\teval-auc:0.95680\n",
      "[127]\ttrain-auc:0.95793\teval-auc:0.95584\n",
      "[128]\ttrain-auc:0.95820\teval-auc:0.95568\n",
      "[129]\ttrain-auc:0.95836\teval-auc:0.95568\n",
      "[130]\ttrain-auc:0.95842\teval-auc:0.95568\n",
      "[131]\ttrain-auc:0.95851\teval-auc:0.95568\n",
      "[132]\ttrain-auc:0.95862\teval-auc:0.95568\n",
      "[133]\ttrain-auc:0.95868\teval-auc:0.95616\n",
      "[134]\ttrain-auc:0.95882\teval-auc:0.95616\n",
      "[135]\ttrain-auc:0.95899\teval-auc:0.95696\n",
      "[136]\ttrain-auc:0.95919\teval-auc:0.95712\n",
      "[137]\ttrain-auc:0.95919\teval-auc:0.95584\n",
      "[138]\ttrain-auc:0.95936\teval-auc:0.95536\n",
      "[139]\ttrain-auc:0.95946\teval-auc:0.95584\n",
      "[140]\ttrain-auc:0.95949\teval-auc:0.95568\n",
      "[141]\ttrain-auc:0.95974\teval-auc:0.95552\n",
      "[142]\ttrain-auc:0.95990\teval-auc:0.95568\n",
      "[143]\ttrain-auc:0.96001\teval-auc:0.95568\n",
      "[144]\ttrain-auc:0.96018\teval-auc:0.95520\n",
      "[145]\ttrain-auc:0.96029\teval-auc:0.95552\n",
      "[0]\ttrain-auc:0.85308\teval-auc:0.86728\n",
      "[1]\ttrain-auc:0.88292\teval-auc:0.86216\n",
      "[2]\ttrain-auc:0.88633\teval-auc:0.86144\n",
      "[3]\ttrain-auc:0.90554\teval-auc:0.86136\n",
      "[4]\ttrain-auc:0.90569\teval-auc:0.86144\n",
      "[5]\ttrain-auc:0.90646\teval-auc:0.86184\n",
      "[6]\ttrain-auc:0.90652\teval-auc:0.86224\n",
      "[7]\ttrain-auc:0.90740\teval-auc:0.86432\n",
      "[8]\ttrain-auc:0.90983\teval-auc:0.86232\n",
      "[9]\ttrain-auc:0.91058\teval-auc:0.86200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.87518\teval-auc:0.93776\n",
      "[1]\ttrain-auc:0.87838\teval-auc:0.94672\n",
      "[2]\ttrain-auc:0.89853\teval-auc:0.96144\n",
      "[3]\ttrain-auc:0.90027\teval-auc:0.96264\n",
      "[4]\ttrain-auc:0.90230\teval-auc:0.96280\n",
      "[5]\ttrain-auc:0.90304\teval-auc:0.96344\n",
      "[6]\ttrain-auc:0.90308\teval-auc:0.96312\n",
      "[7]\ttrain-auc:0.90444\teval-auc:0.96392\n",
      "[8]\ttrain-auc:0.90993\teval-auc:0.96368\n",
      "[9]\ttrain-auc:0.91070\teval-auc:0.96472\n",
      "[10]\ttrain-auc:0.91140\teval-auc:0.96520\n",
      "[11]\ttrain-auc:0.91145\teval-auc:0.96520\n",
      "[12]\ttrain-auc:0.91481\teval-auc:0.96768\n",
      "[13]\ttrain-auc:0.91582\teval-auc:0.96688\n",
      "[14]\ttrain-auc:0.91741\teval-auc:0.96816\n",
      "[15]\ttrain-auc:0.91776\teval-auc:0.96800\n",
      "[16]\ttrain-auc:0.91832\teval-auc:0.96928\n",
      "[17]\ttrain-auc:0.91913\teval-auc:0.96896\n",
      "[18]\ttrain-auc:0.92140\teval-auc:0.97016\n",
      "[19]\ttrain-auc:0.92160\teval-auc:0.97032\n",
      "[20]\ttrain-auc:0.92233\teval-auc:0.97064\n",
      "[21]\ttrain-auc:0.92384\teval-auc:0.97064\n",
      "[22]\ttrain-auc:0.92440\teval-auc:0.96968\n",
      "[23]\ttrain-auc:0.92593\teval-auc:0.97048\n",
      "[24]\ttrain-auc:0.92672\teval-auc:0.97048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\ttrain-auc:0.92710\teval-auc:0.97112\n",
      "[26]\ttrain-auc:0.92846\teval-auc:0.97208\n",
      "[27]\ttrain-auc:0.92956\teval-auc:0.97288\n",
      "[28]\ttrain-auc:0.93014\teval-auc:0.97272\n",
      "[29]\ttrain-auc:0.93114\teval-auc:0.97448\n",
      "[30]\ttrain-auc:0.93183\teval-auc:0.97536\n",
      "[31]\ttrain-auc:0.93244\teval-auc:0.97552\n",
      "[32]\ttrain-auc:0.93293\teval-auc:0.97600\n",
      "[33]\ttrain-auc:0.93369\teval-auc:0.97504\n",
      "[34]\ttrain-auc:0.93468\teval-auc:0.97600\n",
      "[35]\ttrain-auc:0.93528\teval-auc:0.97632\n",
      "[36]\ttrain-auc:0.93559\teval-auc:0.97680\n",
      "[37]\ttrain-auc:0.93628\teval-auc:0.97712\n",
      "[38]\ttrain-auc:0.93696\teval-auc:0.97616\n",
      "[39]\ttrain-auc:0.93733\teval-auc:0.97792\n",
      "[40]\ttrain-auc:0.93792\teval-auc:0.97808\n",
      "[41]\ttrain-auc:0.93828\teval-auc:0.97888\n",
      "[42]\ttrain-auc:0.93882\teval-auc:0.97968\n",
      "[43]\ttrain-auc:0.93923\teval-auc:0.97968\n",
      "[44]\ttrain-auc:0.93971\teval-auc:0.97984\n",
      "[45]\ttrain-auc:0.93996\teval-auc:0.98000\n",
      "[46]\ttrain-auc:0.94043\teval-auc:0.97984\n",
      "[47]\ttrain-auc:0.94076\teval-auc:0.98016\n",
      "[48]\ttrain-auc:0.94106\teval-auc:0.98016\n",
      "[49]\ttrain-auc:0.94120\teval-auc:0.98048\n",
      "[50]\ttrain-auc:0.94164\teval-auc:0.98064\n",
      "[51]\ttrain-auc:0.94205\teval-auc:0.98064\n",
      "[52]\ttrain-auc:0.94242\teval-auc:0.98032\n",
      "[53]\ttrain-auc:0.94248\teval-auc:0.98016\n",
      "[54]\ttrain-auc:0.94300\teval-auc:0.97920\n",
      "[55]\ttrain-auc:0.94345\teval-auc:0.97936\n",
      "[56]\ttrain-auc:0.94390\teval-auc:0.97856\n",
      "[57]\ttrain-auc:0.94414\teval-auc:0.97952\n",
      "[58]\ttrain-auc:0.94446\teval-auc:0.98032\n",
      "[59]\ttrain-auc:0.94452\teval-auc:0.97968\n"
     ]
    }
   ],
   "source": [
    "#Supervised model trained with dynamic features + PSSM scores + CoPheeMap info + KMap info\n",
    "Cancer=['BRCA','CCRCC','COAD','GBM','HCC','HNSCC','LSCC','LUAD','OV','PDAC','UCEC'] \n",
    "col_idx=['cophee_emb0', 'cophee_emb1', 'cophee_emb2', 'cophee_emb3', 'cophee_emb4', \n",
    "         'cophee_emb5', 'cophee_emb6', 'cophee_emb7', 'cophee_emb8', 'cophee_emb9', \n",
    "         'cophee_emb10','cophee_emb11', 'cophee_emb12', 'cophee_emb13', 'cophee_emb14', \n",
    "         'cophee_emb15', 'k_emb0', 'k_emb1', 'k_emb2', 'k_emb3','k_emb4', 'k_emb5', \n",
    "         'k_emb6', 'k_emb7', 'k_emb8', 'k_emb9', 'k_emb10',\n",
    "         'k_emb11', 'k_emb12', 'k_emb13', 'k_emb14', 'k_emb15','BRCA_spc_k_s', \n",
    "         'CCRCC_spc_k_s', 'COAD_spc_k_s', 'GBM_spc_k_s',\n",
    "         'HCC_spc_k_s', 'HNSCC_spc_k_s', 'LSCC_spc_k_s', 'LUAD_spc_k_s',\n",
    "         'OV_spc_k_s', 'PDAC_spc_k_s', 'UCEC_spc_k_s', 'BRCA_spc_kactivity_s',\n",
    "         'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "         'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "         'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "         'UCEC_spc_kactivity_s','PSSM']\n",
    "for i in range(10):\n",
    "    data_train=pd.read_csv('CoPheeKSA/train/w_feature/train_'+str(i)+'.csv',index_col=0)\n",
    "    data_train=data_train.drop_duplicates()\n",
    "    data_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    data_train=data_train[~data_train['PSSM'].isna()]\n",
    "    \n",
    "    data_neg=pd.read_csv('Supplementary_table/Table_S3_negative_KSA_w_features.tsv.zip', compression='zip', sep='\\t')\n",
    "    data_neg=data_neg.drop_duplicates()\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    kinases=list(set(data_train.kinase))\n",
    "    data_train_neg=pd.DataFrame()\n",
    "    for k in kinases:\n",
    "        tmp=data_train[data_train.kinase==k]\n",
    "        tmp2=data_neg[data_neg.kinase==k]\n",
    "        if len(tmp2) > 10*len(tmp):\n",
    "            idx=random.sample(range(len(tmp2)),10*len(tmp))\n",
    "            tmp2=tmp2.iloc[idx]\n",
    "        \n",
    "        c=tmp[['BRCA_spc_kactivity_s',\n",
    "             'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "             'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "             'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "             'UCEC_spc_kactivity_s']].isna().sum()\n",
    "        for idx_col in ['BRCA_spc_kactivity_s',\n",
    "             'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "             'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "             'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "             'UCEC_spc_kactivity_s']:\n",
    "            num=int(c.loc[idx_col]/len(tmp)*len(tmp2))\n",
    "            idx=random.sample(tmp2.index.tolist(),num)\n",
    "            \n",
    "            for z in idx:\n",
    "                tmp2.loc[z, idx_col]=np.nan\n",
    "            \n",
    "        data_train_neg=pd.concat([data_train_neg,tmp2])\n",
    "    idx=random.sample(range(len(data_neg)),2*len(data_train))\n",
    "    data_train_neg=pd.concat([data_train_neg,data_neg.iloc[idx]])\n",
    "    data_train_neg=data_train_neg.drop_duplicates()\n",
    "    \n",
    "    idx_removed=data_train_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    data_test=pd.read_csv('CoPheeKSA/test/w_feature/test_'+str(i)+'.csv',index_col=0)\n",
    "    data_test=data_test.drop_duplicates()\n",
    "    data_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    data_test=data_test[~data_test['PSSM'].isna()]\n",
    "    \n",
    "    idx=random.sample(range(len(data_test)),25)\n",
    "    data_eval=data_test.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_test)) if i not in idx]\n",
    "    data_test=data_test.iloc[idx2,:]\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_eval))\n",
    "    data_eval_neg=data_neg.iloc[idx]\n",
    "    idx_removed=data_eval_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_test))\n",
    "    data_test_neg=data_neg.iloc[idx]\n",
    "    idx_removed=data_test_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    \n",
    "    data_train=data_train[col_idx]\n",
    "    data_eval=data_eval[col_idx]\n",
    "    data_test=data_test[col_idx]\n",
    "    data_neg=data_neg[col_idx]\n",
    "    data_train_neg=data_train_neg[col_idx]\n",
    "    data_eval_neg=data_eval_neg[col_idx]\n",
    "    \n",
    "    X_train=pd.concat([data_train,data_train_neg])\n",
    "    y_train=np.concatenate((np.repeat(1,len(data_train)),np.repeat(0,len(data_train_neg))))\n",
    "    X_train=X_train[col_idx]\n",
    "    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "    \n",
    "    X_eval=pd.concat([data_eval,data_eval_neg])\n",
    "    y_eval=np.concatenate((np.repeat(1,len(data_eval)),np.repeat(0,len(data_eval_neg))))\n",
    "    X_eval=X_eval[col_idx]\n",
    "    X_eval.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    deval = xgb.DMatrix(X_eval,label=y_eval)\n",
    "    param = {'max_depth': 2, 'eta': 0.2, 'objective': 'binary:logistic','verbosity':0}\n",
    "    param['nthread'] = 4\n",
    "    param['eval_metric'] = 'auc'\n",
    "    evallist = [(dtrain, 'train'),(deval, 'eval')]\n",
    "    num_round = 300\n",
    "    bst = xgb.train(param, dtrain, num_round, evallist, early_stopping_rounds=10)\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_test))\n",
    "    data_neg2=data_neg.iloc[idx,:]\n",
    "    X_test=pd.concat([data_test,data_neg2])\n",
    "    X_test=X_test[col_idx]\n",
    "    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    y_test=np.concatenate((np.repeat(1,len(data_test)),np.repeat(0,len(data_neg2))))\n",
    "    dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "    y_pred = bst.predict(dtest)\n",
    "\n",
    "    fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred)\n",
    "    auc_tmp=metrics.auc(fpr, tpr)\n",
    "    \n",
    "    auc.append(auc_tmp)\n",
    "    label.append('CoPheeKSA')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "db77f4b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.56125\teval-auc:0.50464\n",
      "[1]\ttrain-auc:0.56783\teval-auc:0.48216\n",
      "[2]\ttrain-auc:0.60394\teval-auc:0.49512\n",
      "[3]\ttrain-auc:0.61880\teval-auc:0.53456\n",
      "[4]\ttrain-auc:0.61995\teval-auc:0.53392\n",
      "[5]\ttrain-auc:0.62599\teval-auc:0.53144\n",
      "[6]\ttrain-auc:0.62503\teval-auc:0.52800\n",
      "[7]\ttrain-auc:0.63474\teval-auc:0.52968\n",
      "[8]\ttrain-auc:0.63583\teval-auc:0.53816\n",
      "[9]\ttrain-auc:0.63860\teval-auc:0.54616\n",
      "[10]\ttrain-auc:0.63602\teval-auc:0.55584\n",
      "[11]\ttrain-auc:0.65070\teval-auc:0.58376\n",
      "[12]\ttrain-auc:0.65613\teval-auc:0.61424\n",
      "[13]\ttrain-auc:0.66211\teval-auc:0.60280\n",
      "[14]\ttrain-auc:0.66701\teval-auc:0.63776\n",
      "[15]\ttrain-auc:0.67095\teval-auc:0.65152\n",
      "[16]\ttrain-auc:0.67235\teval-auc:0.69792\n",
      "[17]\ttrain-auc:0.67705\teval-auc:0.70176\n",
      "[18]\ttrain-auc:0.67839\teval-auc:0.70768\n",
      "[19]\ttrain-auc:0.67686\teval-auc:0.70520\n",
      "[20]\ttrain-auc:0.68046\teval-auc:0.70688\n",
      "[21]\ttrain-auc:0.68100\teval-auc:0.71344\n",
      "[22]\ttrain-auc:0.68417\teval-auc:0.70512\n",
      "[23]\ttrain-auc:0.68724\teval-auc:0.71312\n",
      "[24]\ttrain-auc:0.69302\teval-auc:0.71216\n",
      "[25]\ttrain-auc:0.69500\teval-auc:0.71168\n",
      "[26]\ttrain-auc:0.69526\teval-auc:0.71808\n",
      "[27]\ttrain-auc:0.69493\teval-auc:0.71568\n",
      "[28]\ttrain-auc:0.69605\teval-auc:0.71008\n",
      "[29]\ttrain-auc:0.69874\teval-auc:0.72432\n",
      "[30]\ttrain-auc:0.69929\teval-auc:0.73056\n",
      "[31]\ttrain-auc:0.70030\teval-auc:0.72512\n",
      "[32]\ttrain-auc:0.70051\teval-auc:0.72448\n",
      "[33]\ttrain-auc:0.70262\teval-auc:0.73264\n",
      "[34]\ttrain-auc:0.70332\teval-auc:0.73584\n",
      "[35]\ttrain-auc:0.70546\teval-auc:0.73520\n",
      "[36]\ttrain-auc:0.70640\teval-auc:0.73552\n",
      "[37]\ttrain-auc:0.70841\teval-auc:0.74896\n",
      "[38]\ttrain-auc:0.71047\teval-auc:0.75392\n",
      "[39]\ttrain-auc:0.71673\teval-auc:0.73904\n",
      "[40]\ttrain-auc:0.71768\teval-auc:0.74096\n",
      "[41]\ttrain-auc:0.72070\teval-auc:0.74544\n",
      "[42]\ttrain-auc:0.72161\teval-auc:0.74144\n",
      "[43]\ttrain-auc:0.72317\teval-auc:0.75280\n",
      "[44]\ttrain-auc:0.72338\teval-auc:0.74768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[45]\ttrain-auc:0.72355\teval-auc:0.74704\n",
      "[46]\ttrain-auc:0.72566\teval-auc:0.74272\n",
      "[47]\ttrain-auc:0.72877\teval-auc:0.74400\n",
      "[48]\ttrain-auc:0.73035\teval-auc:0.74096\n",
      "0.8227706122448981\n",
      "[0]\ttrain-auc:0.54016\teval-auc:0.52848\n",
      "[1]\ttrain-auc:0.60837\teval-auc:0.64280\n",
      "[2]\ttrain-auc:0.62143\teval-auc:0.65272\n",
      "[3]\ttrain-auc:0.63716\teval-auc:0.66112\n",
      "[4]\ttrain-auc:0.63609\teval-auc:0.66336\n",
      "[5]\ttrain-auc:0.63577\teval-auc:0.66096\n",
      "[6]\ttrain-auc:0.64181\teval-auc:0.65936\n",
      "[7]\ttrain-auc:0.64009\teval-auc:0.65920\n",
      "[8]\ttrain-auc:0.63992\teval-auc:0.64944\n",
      "[9]\ttrain-auc:0.64834\teval-auc:0.66112\n",
      "[10]\ttrain-auc:0.65056\teval-auc:0.66704\n",
      "[11]\ttrain-auc:0.65045\teval-auc:0.68592\n",
      "[12]\ttrain-auc:0.65758\teval-auc:0.70824\n",
      "[13]\ttrain-auc:0.66025\teval-auc:0.71928\n",
      "[14]\ttrain-auc:0.66489\teval-auc:0.75272\n",
      "[15]\ttrain-auc:0.67018\teval-auc:0.76096\n",
      "[16]\ttrain-auc:0.67652\teval-auc:0.78048\n",
      "[17]\ttrain-auc:0.67684\teval-auc:0.79072\n",
      "[18]\ttrain-auc:0.67999\teval-auc:0.80144\n",
      "[19]\ttrain-auc:0.67940\teval-auc:0.80432\n",
      "[20]\ttrain-auc:0.67923\teval-auc:0.80240\n",
      "[21]\ttrain-auc:0.67933\teval-auc:0.80880\n",
      "[22]\ttrain-auc:0.68047\teval-auc:0.82912\n",
      "[23]\ttrain-auc:0.68118\teval-auc:0.83056\n",
      "[24]\ttrain-auc:0.68540\teval-auc:0.83120\n",
      "[25]\ttrain-auc:0.69239\teval-auc:0.84264\n",
      "[26]\ttrain-auc:0.69718\teval-auc:0.85384\n",
      "[27]\ttrain-auc:0.69860\teval-auc:0.85768\n",
      "[28]\ttrain-auc:0.70107\teval-auc:0.85224\n",
      "[29]\ttrain-auc:0.70480\teval-auc:0.86680\n",
      "[30]\ttrain-auc:0.70535\teval-auc:0.87896\n",
      "[31]\ttrain-auc:0.70753\teval-auc:0.87784\n",
      "[32]\ttrain-auc:0.70957\teval-auc:0.86560\n",
      "[33]\ttrain-auc:0.71050\teval-auc:0.86032\n",
      "[34]\ttrain-auc:0.71208\teval-auc:0.85312\n",
      "[35]\ttrain-auc:0.71325\teval-auc:0.85504\n",
      "[36]\ttrain-auc:0.71502\teval-auc:0.85904\n",
      "[37]\ttrain-auc:0.71518\teval-auc:0.86352\n",
      "[38]\ttrain-auc:0.71594\teval-auc:0.86928\n",
      "[39]\ttrain-auc:0.71645\teval-auc:0.86976\n",
      "[40]\ttrain-auc:0.71977\teval-auc:0.86768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.772575542355372\n",
      "[0]\ttrain-auc:0.54062\teval-auc:0.51792\n",
      "[1]\ttrain-auc:0.56267\teval-auc:0.54504\n",
      "[2]\ttrain-auc:0.60862\teval-auc:0.55448\n",
      "[3]\ttrain-auc:0.62090\teval-auc:0.60816\n",
      "[4]\ttrain-auc:0.63407\teval-auc:0.57832\n",
      "[5]\ttrain-auc:0.63340\teval-auc:0.60112\n",
      "[6]\ttrain-auc:0.63664\teval-auc:0.60032\n",
      "[7]\ttrain-auc:0.64241\teval-auc:0.64288\n",
      "[8]\ttrain-auc:0.63948\teval-auc:0.61360\n",
      "[9]\ttrain-auc:0.65179\teval-auc:0.63456\n",
      "[10]\ttrain-auc:0.66101\teval-auc:0.68736\n",
      "[11]\ttrain-auc:0.65816\teval-auc:0.69624\n",
      "[12]\ttrain-auc:0.66547\teval-auc:0.73040\n",
      "[13]\ttrain-auc:0.66656\teval-auc:0.71528\n",
      "[14]\ttrain-auc:0.66914\teval-auc:0.72216\n",
      "[15]\ttrain-auc:0.67020\teval-auc:0.72488\n",
      "[16]\ttrain-auc:0.68009\teval-auc:0.71424\n",
      "[17]\ttrain-auc:0.68455\teval-auc:0.70152\n",
      "[18]\ttrain-auc:0.68391\teval-auc:0.69920\n",
      "[19]\ttrain-auc:0.68680\teval-auc:0.70528\n",
      "[20]\ttrain-auc:0.68840\teval-auc:0.71504\n",
      "[21]\ttrain-auc:0.68805\teval-auc:0.71760\n",
      "[22]\ttrain-auc:0.69286\teval-auc:0.71888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7416638686242789\n",
      "[0]\ttrain-auc:0.54136\teval-auc:0.49600\n",
      "[1]\ttrain-auc:0.54381\teval-auc:0.49600\n",
      "[2]\ttrain-auc:0.56853\teval-auc:0.52160\n",
      "[3]\ttrain-auc:0.62049\teval-auc:0.51888\n",
      "[4]\ttrain-auc:0.63317\teval-auc:0.56544\n",
      "[5]\ttrain-auc:0.63294\teval-auc:0.55896\n",
      "[6]\ttrain-auc:0.63001\teval-auc:0.56952\n",
      "[7]\ttrain-auc:0.63491\teval-auc:0.55848\n",
      "[8]\ttrain-auc:0.62870\teval-auc:0.56056\n",
      "[9]\ttrain-auc:0.63459\teval-auc:0.59896\n",
      "[10]\ttrain-auc:0.65184\teval-auc:0.62568\n",
      "[11]\ttrain-auc:0.66224\teval-auc:0.62760\n",
      "[12]\ttrain-auc:0.66822\teval-auc:0.63600\n",
      "[13]\ttrain-auc:0.66774\teval-auc:0.64000\n",
      "[14]\ttrain-auc:0.67066\teval-auc:0.64832\n",
      "[15]\ttrain-auc:0.67186\teval-auc:0.65240\n",
      "[16]\ttrain-auc:0.67257\teval-auc:0.66504\n",
      "[17]\ttrain-auc:0.67257\teval-auc:0.67336\n",
      "[18]\ttrain-auc:0.67661\teval-auc:0.66760\n",
      "[19]\ttrain-auc:0.67614\teval-auc:0.68072\n",
      "[20]\ttrain-auc:0.67739\teval-auc:0.69392\n",
      "[21]\ttrain-auc:0.68077\teval-auc:0.69056\n",
      "[22]\ttrain-auc:0.68349\teval-auc:0.71224\n",
      "[23]\ttrain-auc:0.68317\teval-auc:0.71952\n",
      "[24]\ttrain-auc:0.68670\teval-auc:0.72688\n",
      "[25]\ttrain-auc:0.68580\teval-auc:0.73216\n",
      "[26]\ttrain-auc:0.68841\teval-auc:0.72400\n",
      "[27]\ttrain-auc:0.69020\teval-auc:0.72808\n",
      "[28]\ttrain-auc:0.69507\teval-auc:0.73104\n",
      "[29]\ttrain-auc:0.69558\teval-auc:0.73936\n",
      "[30]\ttrain-auc:0.69470\teval-auc:0.74336\n",
      "[31]\ttrain-auc:0.70160\teval-auc:0.76288\n",
      "[32]\ttrain-auc:0.70262\teval-auc:0.76624\n",
      "[33]\ttrain-auc:0.70312\teval-auc:0.76304\n",
      "[34]\ttrain-auc:0.70278\teval-auc:0.76768\n",
      "[35]\ttrain-auc:0.70625\teval-auc:0.76224\n",
      "[36]\ttrain-auc:0.71139\teval-auc:0.76864\n",
      "[37]\ttrain-auc:0.71204\teval-auc:0.76368\n",
      "[38]\ttrain-auc:0.71300\teval-auc:0.76720\n",
      "[39]\ttrain-auc:0.71369\teval-auc:0.76288\n",
      "[40]\ttrain-auc:0.71479\teval-auc:0.76064\n",
      "[41]\ttrain-auc:0.71890\teval-auc:0.77232\n",
      "[42]\ttrain-auc:0.72315\teval-auc:0.78496\n",
      "[43]\ttrain-auc:0.72312\teval-auc:0.78432\n",
      "[44]\ttrain-auc:0.72588\teval-auc:0.78720\n",
      "[45]\ttrain-auc:0.72755\teval-auc:0.78864\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[46]\ttrain-auc:0.73032\teval-auc:0.80560\n",
      "[47]\ttrain-auc:0.73109\teval-auc:0.80384\n",
      "[48]\ttrain-auc:0.73280\teval-auc:0.80288\n",
      "[49]\ttrain-auc:0.73330\teval-auc:0.80160\n",
      "[50]\ttrain-auc:0.73457\teval-auc:0.80464\n",
      "[51]\ttrain-auc:0.73611\teval-auc:0.80768\n",
      "[52]\ttrain-auc:0.73763\teval-auc:0.80752\n",
      "[53]\ttrain-auc:0.73966\teval-auc:0.81232\n",
      "[54]\ttrain-auc:0.73899\teval-auc:0.80464\n",
      "[55]\ttrain-auc:0.73933\teval-auc:0.80288\n",
      "[56]\ttrain-auc:0.73978\teval-auc:0.80224\n",
      "[57]\ttrain-auc:0.74123\teval-auc:0.80352\n",
      "[58]\ttrain-auc:0.74211\teval-auc:0.80640\n",
      "[59]\ttrain-auc:0.74279\teval-auc:0.80800\n",
      "[60]\ttrain-auc:0.74252\teval-auc:0.80784\n",
      "[61]\ttrain-auc:0.74397\teval-auc:0.81312\n",
      "[62]\ttrain-auc:0.74390\teval-auc:0.81296\n",
      "[63]\ttrain-auc:0.74583\teval-auc:0.81136\n",
      "[64]\ttrain-auc:0.74643\teval-auc:0.81392\n",
      "[65]\ttrain-auc:0.74771\teval-auc:0.81504\n",
      "[66]\ttrain-auc:0.74834\teval-auc:0.81168\n",
      "[67]\ttrain-auc:0.74976\teval-auc:0.81472\n",
      "[68]\ttrain-auc:0.75272\teval-auc:0.81632\n",
      "[69]\ttrain-auc:0.75549\teval-auc:0.80512\n",
      "[70]\ttrain-auc:0.75740\teval-auc:0.80816\n",
      "[71]\ttrain-auc:0.75859\teval-auc:0.81088\n",
      "[72]\ttrain-auc:0.76006\teval-auc:0.82096\n",
      "[73]\ttrain-auc:0.76108\teval-auc:0.82304\n",
      "[74]\ttrain-auc:0.76230\teval-auc:0.81824\n",
      "[75]\ttrain-auc:0.76305\teval-auc:0.82048\n",
      "[76]\ttrain-auc:0.76376\teval-auc:0.82000\n",
      "[77]\ttrain-auc:0.76403\teval-auc:0.81600\n",
      "[78]\ttrain-auc:0.76458\teval-auc:0.82256\n",
      "[79]\ttrain-auc:0.76678\teval-auc:0.82448\n",
      "[80]\ttrain-auc:0.76667\teval-auc:0.81984\n",
      "[81]\ttrain-auc:0.76894\teval-auc:0.82016\n",
      "[82]\ttrain-auc:0.76952\teval-auc:0.82208\n",
      "[83]\ttrain-auc:0.76935\teval-auc:0.82176\n",
      "[84]\ttrain-auc:0.76995\teval-auc:0.82592\n",
      "[85]\ttrain-auc:0.77174\teval-auc:0.83264\n",
      "[86]\ttrain-auc:0.77198\teval-auc:0.83072\n",
      "[87]\ttrain-auc:0.77361\teval-auc:0.83520\n",
      "[88]\ttrain-auc:0.77399\teval-auc:0.83536\n",
      "[89]\ttrain-auc:0.77540\teval-auc:0.83520\n",
      "[90]\ttrain-auc:0.77639\teval-auc:0.83456\n",
      "[91]\ttrain-auc:0.77657\teval-auc:0.83168\n",
      "[92]\ttrain-auc:0.77724\teval-auc:0.82944\n",
      "[93]\ttrain-auc:0.77880\teval-auc:0.82656\n",
      "[94]\ttrain-auc:0.77898\teval-auc:0.82864\n",
      "[95]\ttrain-auc:0.77934\teval-auc:0.83072\n",
      "[96]\ttrain-auc:0.78059\teval-auc:0.83088\n",
      "[97]\ttrain-auc:0.78317\teval-auc:0.83536\n",
      "[98]\ttrain-auc:0.78451\teval-auc:0.84032\n",
      "[99]\ttrain-auc:0.78564\teval-auc:0.83760\n",
      "[100]\ttrain-auc:0.78584\teval-auc:0.83712\n",
      "[101]\ttrain-auc:0.78764\teval-auc:0.84048\n",
      "[102]\ttrain-auc:0.78754\teval-auc:0.84176\n",
      "[103]\ttrain-auc:0.78773\teval-auc:0.84080\n",
      "[104]\ttrain-auc:0.78916\teval-auc:0.84000\n",
      "[105]\ttrain-auc:0.79029\teval-auc:0.83904\n",
      "[106]\ttrain-auc:0.79049\teval-auc:0.83904\n",
      "[107]\ttrain-auc:0.79040\teval-auc:0.83920\n",
      "[108]\ttrain-auc:0.79242\teval-auc:0.84064\n",
      "[109]\ttrain-auc:0.79372\teval-auc:0.83920\n",
      "[110]\ttrain-auc:0.79413\teval-auc:0.83936\n",
      "[111]\ttrain-auc:0.79451\teval-auc:0.83904\n",
      "[112]\ttrain-auc:0.79460\teval-auc:0.83872\n",
      "0.8168130612244897\n",
      "[0]\ttrain-auc:0.56236\teval-auc:0.60256\n",
      "[1]\ttrain-auc:0.56413\teval-auc:0.60688\n",
      "[2]\ttrain-auc:0.56574\teval-auc:0.60464\n",
      "[3]\ttrain-auc:0.60923\teval-auc:0.64112\n",
      "[4]\ttrain-auc:0.61114\teval-auc:0.63904\n",
      "[5]\ttrain-auc:0.62254\teval-auc:0.67960\n",
      "[6]\ttrain-auc:0.62259\teval-auc:0.66352\n",
      "[7]\ttrain-auc:0.62298\teval-auc:0.66240\n",
      "[8]\ttrain-auc:0.63245\teval-auc:0.68104\n",
      "[9]\ttrain-auc:0.64557\teval-auc:0.70472\n",
      "[10]\ttrain-auc:0.64770\teval-auc:0.70392\n",
      "[11]\ttrain-auc:0.64642\teval-auc:0.70088\n",
      "[12]\ttrain-auc:0.65140\teval-auc:0.68184\n",
      "[13]\ttrain-auc:0.65994\teval-auc:0.69544\n",
      "[14]\ttrain-auc:0.66275\teval-auc:0.70248\n",
      "[15]\ttrain-auc:0.67073\teval-auc:0.70784\n",
      "[16]\ttrain-auc:0.66899\teval-auc:0.72224\n",
      "[17]\ttrain-auc:0.67278\teval-auc:0.71456\n",
      "[18]\ttrain-auc:0.67379\teval-auc:0.72904\n",
      "[19]\ttrain-auc:0.67756\teval-auc:0.74104\n",
      "[20]\ttrain-auc:0.67751\teval-auc:0.74928\n",
      "[21]\ttrain-auc:0.68074\teval-auc:0.75752\n",
      "[22]\ttrain-auc:0.68121\teval-auc:0.76784\n",
      "[23]\ttrain-auc:0.68818\teval-auc:0.76960\n",
      "[24]\ttrain-auc:0.68964\teval-auc:0.77184\n",
      "[25]\ttrain-auc:0.69222\teval-auc:0.77696\n",
      "[26]\ttrain-auc:0.69322\teval-auc:0.77792\n",
      "[27]\ttrain-auc:0.69382\teval-auc:0.77312\n",
      "[28]\ttrain-auc:0.69543\teval-auc:0.77040\n",
      "[29]\ttrain-auc:0.69562\teval-auc:0.76656\n",
      "[30]\ttrain-auc:0.69609\teval-auc:0.76752\n",
      "[31]\ttrain-auc:0.69860\teval-auc:0.77392\n",
      "[32]\ttrain-auc:0.69955\teval-auc:0.77120\n",
      "[33]\ttrain-auc:0.69919\teval-auc:0.76640\n",
      "[34]\ttrain-auc:0.70157\teval-auc:0.75552\n",
      "[35]\ttrain-auc:0.70436\teval-auc:0.76160\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7760570987654322\n",
      "[0]\ttrain-auc:0.56225\teval-auc:0.52528\n",
      "[1]\ttrain-auc:0.56424\teval-auc:0.52464\n",
      "[2]\ttrain-auc:0.60568\teval-auc:0.56112\n",
      "[3]\ttrain-auc:0.60785\teval-auc:0.56376\n",
      "[4]\ttrain-auc:0.60741\teval-auc:0.50352\n",
      "[5]\ttrain-auc:0.60174\teval-auc:0.55840\n",
      "[6]\ttrain-auc:0.61294\teval-auc:0.59424\n",
      "[7]\ttrain-auc:0.62002\teval-auc:0.63192\n",
      "[8]\ttrain-auc:0.62960\teval-auc:0.64472\n",
      "[9]\ttrain-auc:0.63620\teval-auc:0.65216\n",
      "[10]\ttrain-auc:0.64328\teval-auc:0.64024\n",
      "[11]\ttrain-auc:0.65844\teval-auc:0.64424\n",
      "[12]\ttrain-auc:0.66178\teval-auc:0.65600\n",
      "[13]\ttrain-auc:0.66325\teval-auc:0.68984\n",
      "[14]\ttrain-auc:0.66366\teval-auc:0.69920\n",
      "[15]\ttrain-auc:0.66902\teval-auc:0.71064\n",
      "[16]\ttrain-auc:0.66911\teval-auc:0.71552\n",
      "[17]\ttrain-auc:0.67046\teval-auc:0.72496\n",
      "[18]\ttrain-auc:0.66984\teval-auc:0.73176\n",
      "[19]\ttrain-auc:0.67650\teval-auc:0.75112\n",
      "[20]\ttrain-auc:0.68079\teval-auc:0.76568\n",
      "[21]\ttrain-auc:0.68840\teval-auc:0.78024\n",
      "[22]\ttrain-auc:0.69060\teval-auc:0.77456\n",
      "[23]\ttrain-auc:0.69054\teval-auc:0.75808\n",
      "[24]\ttrain-auc:0.68955\teval-auc:0.75968\n",
      "[25]\ttrain-auc:0.69066\teval-auc:0.76704\n",
      "[26]\ttrain-auc:0.69317\teval-auc:0.76880\n",
      "[27]\ttrain-auc:0.69561\teval-auc:0.76416\n",
      "[28]\ttrain-auc:0.69955\teval-auc:0.77184\n",
      "[29]\ttrain-auc:0.70040\teval-auc:0.77040\n",
      "[30]\ttrain-auc:0.70294\teval-auc:0.76768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7884604209710744\n",
      "[0]\ttrain-auc:0.53939\teval-auc:0.59448\n",
      "[1]\ttrain-auc:0.54181\teval-auc:0.59472\n",
      "[2]\ttrain-auc:0.60541\teval-auc:0.67656\n",
      "[3]\ttrain-auc:0.62514\teval-auc:0.71416\n",
      "[4]\ttrain-auc:0.62494\teval-auc:0.71336\n",
      "[5]\ttrain-auc:0.63721\teval-auc:0.70888\n",
      "[6]\ttrain-auc:0.63647\teval-auc:0.70816\n",
      "[7]\ttrain-auc:0.63550\teval-auc:0.70816\n",
      "[8]\ttrain-auc:0.63603\teval-auc:0.69776\n",
      "[9]\ttrain-auc:0.64617\teval-auc:0.72016\n",
      "[10]\ttrain-auc:0.65727\teval-auc:0.74920\n",
      "[11]\ttrain-auc:0.66522\teval-auc:0.75032\n",
      "[12]\ttrain-auc:0.66293\teval-auc:0.76208\n",
      "[13]\ttrain-auc:0.66898\teval-auc:0.76720\n",
      "[14]\ttrain-auc:0.67144\teval-auc:0.76624\n",
      "[15]\ttrain-auc:0.67162\teval-auc:0.77136\n",
      "[16]\ttrain-auc:0.67469\teval-auc:0.76920\n",
      "[17]\ttrain-auc:0.67569\teval-auc:0.75672\n",
      "[18]\ttrain-auc:0.67437\teval-auc:0.76376\n",
      "[19]\ttrain-auc:0.68050\teval-auc:0.76080\n",
      "[20]\ttrain-auc:0.68056\teval-auc:0.75256\n",
      "[21]\ttrain-auc:0.68482\teval-auc:0.76360\n",
      "[22]\ttrain-auc:0.68485\teval-auc:0.76360\n",
      "[23]\ttrain-auc:0.68627\teval-auc:0.77160\n",
      "[24]\ttrain-auc:0.68925\teval-auc:0.77464\n",
      "[25]\ttrain-auc:0.69319\teval-auc:0.78096\n",
      "[26]\ttrain-auc:0.69893\teval-auc:0.79328\n",
      "[27]\ttrain-auc:0.70125\teval-auc:0.79696\n",
      "[28]\ttrain-auc:0.70436\teval-auc:0.79168\n",
      "[29]\ttrain-auc:0.70713\teval-auc:0.78080\n",
      "[30]\ttrain-auc:0.70741\teval-auc:0.78160\n",
      "[31]\ttrain-auc:0.70762\teval-auc:0.78048\n",
      "[32]\ttrain-auc:0.70934\teval-auc:0.78016\n",
      "[33]\ttrain-auc:0.71144\teval-auc:0.77888\n",
      "[34]\ttrain-auc:0.71210\teval-auc:0.77168\n",
      "[35]\ttrain-auc:0.71340\teval-auc:0.77184\n",
      "[36]\ttrain-auc:0.71489\teval-auc:0.75984\n",
      "[37]\ttrain-auc:0.71606\teval-auc:0.75744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7796918351169242\n",
      "[0]\ttrain-auc:0.54076\teval-auc:0.54600\n",
      "[1]\ttrain-auc:0.56235\teval-auc:0.61712\n",
      "[2]\ttrain-auc:0.57499\teval-auc:0.65232\n",
      "[3]\ttrain-auc:0.61870\teval-auc:0.70448\n",
      "[4]\ttrain-auc:0.63002\teval-auc:0.72200\n",
      "[5]\ttrain-auc:0.63021\teval-auc:0.70456\n",
      "[6]\ttrain-auc:0.63563\teval-auc:0.73320\n",
      "[7]\ttrain-auc:0.63106\teval-auc:0.73544\n",
      "[8]\ttrain-auc:0.63233\teval-auc:0.71976\n",
      "[9]\ttrain-auc:0.64186\teval-auc:0.73448\n",
      "[10]\ttrain-auc:0.64867\teval-auc:0.75760\n",
      "[11]\ttrain-auc:0.64886\teval-auc:0.75584\n",
      "[12]\ttrain-auc:0.65753\teval-auc:0.78424\n",
      "[13]\ttrain-auc:0.65709\teval-auc:0.77496\n",
      "[14]\ttrain-auc:0.65982\teval-auc:0.77616\n",
      "[15]\ttrain-auc:0.66246\teval-auc:0.77360\n",
      "[16]\ttrain-auc:0.66121\teval-auc:0.79088\n",
      "[17]\ttrain-auc:0.66899\teval-auc:0.79312\n",
      "[18]\ttrain-auc:0.67067\teval-auc:0.80960\n",
      "[19]\ttrain-auc:0.67917\teval-auc:0.81408\n",
      "[20]\ttrain-auc:0.68027\teval-auc:0.80736\n",
      "[21]\ttrain-auc:0.68098\teval-auc:0.81456\n",
      "[22]\ttrain-auc:0.68436\teval-auc:0.81408\n",
      "[23]\ttrain-auc:0.69010\teval-auc:0.81504\n",
      "[24]\ttrain-auc:0.69067\teval-auc:0.81600\n",
      "[25]\ttrain-auc:0.69360\teval-auc:0.81680\n",
      "[26]\ttrain-auc:0.69548\teval-auc:0.81856\n",
      "[27]\ttrain-auc:0.69735\teval-auc:0.81648\n",
      "[28]\ttrain-auc:0.69835\teval-auc:0.81968\n",
      "[29]\ttrain-auc:0.69891\teval-auc:0.82352\n",
      "[30]\ttrain-auc:0.69996\teval-auc:0.81856\n",
      "[31]\ttrain-auc:0.70246\teval-auc:0.83392\n",
      "[32]\ttrain-auc:0.70318\teval-auc:0.84336\n",
      "[33]\ttrain-auc:0.70740\teval-auc:0.85184\n",
      "[34]\ttrain-auc:0.70873\teval-auc:0.85136\n",
      "[35]\ttrain-auc:0.71265\teval-auc:0.85200\n",
      "[36]\ttrain-auc:0.71749\teval-auc:0.85984\n",
      "[37]\ttrain-auc:0.71890\teval-auc:0.85312\n",
      "[38]\ttrain-auc:0.71966\teval-auc:0.85408\n",
      "[39]\ttrain-auc:0.71940\teval-auc:0.84928\n",
      "[40]\ttrain-auc:0.72179\teval-auc:0.85632\n",
      "[41]\ttrain-auc:0.72264\teval-auc:0.86112\n",
      "[42]\ttrain-auc:0.72279\teval-auc:0.85936\n",
      "[43]\ttrain-auc:0.72646\teval-auc:0.86288\n",
      "[44]\ttrain-auc:0.72870\teval-auc:0.86544\n",
      "[45]\ttrain-auc:0.73068\teval-auc:0.87168\n",
      "[46]\ttrain-auc:0.73314\teval-auc:0.87536\n",
      "[47]\ttrain-auc:0.73543\teval-auc:0.87376\n",
      "[48]\ttrain-auc:0.73546\teval-auc:0.87104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[49]\ttrain-auc:0.73580\teval-auc:0.87456\n",
      "[50]\ttrain-auc:0.73689\teval-auc:0.87536\n",
      "[51]\ttrain-auc:0.73752\teval-auc:0.88000\n",
      "[52]\ttrain-auc:0.74053\teval-auc:0.87936\n",
      "[53]\ttrain-auc:0.74212\teval-auc:0.87520\n",
      "[54]\ttrain-auc:0.74217\teval-auc:0.87696\n",
      "[55]\ttrain-auc:0.74299\teval-auc:0.87888\n",
      "[56]\ttrain-auc:0.74389\teval-auc:0.88048\n",
      "[57]\ttrain-auc:0.74632\teval-auc:0.87872\n",
      "[58]\ttrain-auc:0.74692\teval-auc:0.87952\n",
      "[59]\ttrain-auc:0.74807\teval-auc:0.88272\n",
      "[60]\ttrain-auc:0.74847\teval-auc:0.88272\n",
      "[61]\ttrain-auc:0.74995\teval-auc:0.88320\n",
      "[62]\ttrain-auc:0.74961\teval-auc:0.88592\n",
      "[63]\ttrain-auc:0.75081\teval-auc:0.88896\n",
      "[64]\ttrain-auc:0.75095\teval-auc:0.88144\n",
      "[65]\ttrain-auc:0.75114\teval-auc:0.88080\n",
      "[66]\ttrain-auc:0.75124\teval-auc:0.87760\n",
      "[67]\ttrain-auc:0.75137\teval-auc:0.87680\n",
      "[68]\ttrain-auc:0.75244\teval-auc:0.87616\n",
      "[69]\ttrain-auc:0.75300\teval-auc:0.87792\n",
      "[70]\ttrain-auc:0.75406\teval-auc:0.86960\n",
      "[71]\ttrain-auc:0.75374\teval-auc:0.87072\n",
      "[72]\ttrain-auc:0.75428\teval-auc:0.87040\n",
      "[73]\ttrain-auc:0.75425\teval-auc:0.87264\n",
      "0.8037064282480761\n",
      "[0]\ttrain-auc:0.53778\teval-auc:0.51416\n",
      "[1]\ttrain-auc:0.60614\teval-auc:0.46384\n",
      "[2]\ttrain-auc:0.60607\teval-auc:0.46384\n",
      "[3]\ttrain-auc:0.62649\teval-auc:0.51432\n",
      "[4]\ttrain-auc:0.63202\teval-auc:0.54072\n",
      "[5]\ttrain-auc:0.63444\teval-auc:0.54416\n",
      "[6]\ttrain-auc:0.63865\teval-auc:0.55000\n",
      "[7]\ttrain-auc:0.64010\teval-auc:0.55304\n",
      "[8]\ttrain-auc:0.64083\teval-auc:0.55512\n",
      "[9]\ttrain-auc:0.64959\teval-auc:0.55952\n",
      "[10]\ttrain-auc:0.65904\teval-auc:0.57144\n",
      "[11]\ttrain-auc:0.66430\teval-auc:0.60392\n",
      "[12]\ttrain-auc:0.66575\teval-auc:0.60512\n",
      "[13]\ttrain-auc:0.67128\teval-auc:0.64328\n",
      "[14]\ttrain-auc:0.67540\teval-auc:0.63232\n",
      "[15]\ttrain-auc:0.67976\teval-auc:0.64144\n",
      "[16]\ttrain-auc:0.68016\teval-auc:0.63616\n",
      "[17]\ttrain-auc:0.68037\teval-auc:0.64416\n",
      "[18]\ttrain-auc:0.68576\teval-auc:0.66640\n",
      "[19]\ttrain-auc:0.68619\teval-auc:0.68168\n",
      "[20]\ttrain-auc:0.69050\teval-auc:0.69656\n",
      "[21]\ttrain-auc:0.69165\teval-auc:0.69000\n",
      "[22]\ttrain-auc:0.69150\teval-auc:0.68120\n",
      "[23]\ttrain-auc:0.69903\teval-auc:0.68224\n",
      "[24]\ttrain-auc:0.70049\teval-auc:0.67920\n",
      "[25]\ttrain-auc:0.70183\teval-auc:0.69344\n",
      "[26]\ttrain-auc:0.70210\teval-auc:0.71328\n",
      "[27]\ttrain-auc:0.70379\teval-auc:0.71216\n",
      "[28]\ttrain-auc:0.70558\teval-auc:0.69872\n",
      "[29]\ttrain-auc:0.70694\teval-auc:0.69632\n",
      "[30]\ttrain-auc:0.70801\teval-auc:0.69760\n",
      "[31]\ttrain-auc:0.70942\teval-auc:0.69808\n",
      "[32]\ttrain-auc:0.71065\teval-auc:0.69424\n",
      "[33]\ttrain-auc:0.71397\teval-auc:0.69424\n",
      "[34]\ttrain-auc:0.71582\teval-auc:0.68960\n",
      "[35]\ttrain-auc:0.71865\teval-auc:0.69680\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.790280864197531\n",
      "[0]\ttrain-auc:0.54100\teval-auc:0.53384\n",
      "[1]\ttrain-auc:0.56252\teval-auc:0.57424\n",
      "[2]\ttrain-auc:0.56471\teval-auc:0.57456\n",
      "[3]\ttrain-auc:0.61123\teval-auc:0.65576\n",
      "[4]\ttrain-auc:0.62408\teval-auc:0.67448\n",
      "[5]\ttrain-auc:0.63555\teval-auc:0.68712\n",
      "[6]\ttrain-auc:0.63736\teval-auc:0.70184\n",
      "[7]\ttrain-auc:0.64128\teval-auc:0.69832\n",
      "[8]\ttrain-auc:0.63938\teval-auc:0.71544\n",
      "[9]\ttrain-auc:0.64542\teval-auc:0.76272\n",
      "[10]\ttrain-auc:0.64483\teval-auc:0.78480\n",
      "[11]\ttrain-auc:0.65437\teval-auc:0.83816\n",
      "[12]\ttrain-auc:0.65659\teval-auc:0.85424\n",
      "[13]\ttrain-auc:0.65842\teval-auc:0.86896\n",
      "[14]\ttrain-auc:0.65754\teval-auc:0.86744\n",
      "[15]\ttrain-auc:0.66622\teval-auc:0.86368\n",
      "[16]\ttrain-auc:0.66861\teval-auc:0.88056\n",
      "[17]\ttrain-auc:0.67411\teval-auc:0.86288\n",
      "[18]\ttrain-auc:0.67519\teval-auc:0.87040\n",
      "[19]\ttrain-auc:0.68466\teval-auc:0.87152\n",
      "[20]\ttrain-auc:0.68681\teval-auc:0.85536\n",
      "[21]\ttrain-auc:0.68860\teval-auc:0.86272\n",
      "[22]\ttrain-auc:0.69067\teval-auc:0.85712\n",
      "[23]\ttrain-auc:0.69090\teval-auc:0.86384\n",
      "[24]\ttrain-auc:0.68974\teval-auc:0.86288\n",
      "[25]\ttrain-auc:0.69307\teval-auc:0.86288\n",
      "0.7191945417220142\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "#Supervised model trained with CoPheeMap info + KMap info\n",
    "Cancer=['BRCA','CCRCC','COAD','GBM','HCC','HNSCC','LSCC','LUAD','OV','PDAC','UCEC'] \n",
    "col_idx=['cophee_emb0', 'cophee_emb1', 'cophee_emb2', 'cophee_emb3', 'cophee_emb4', \n",
    "         'cophee_emb5', 'cophee_emb6', 'cophee_emb7', 'cophee_emb8', 'cophee_emb9', \n",
    "         'cophee_emb10','cophee_emb11', 'cophee_emb12', 'cophee_emb13', 'cophee_emb14', \n",
    "         'cophee_emb15', 'k_emb0', 'k_emb1', 'k_emb2', 'k_emb3','k_emb4', 'k_emb5', \n",
    "         'k_emb6', 'k_emb7', 'k_emb8', 'k_emb9', 'k_emb10',\n",
    "         'k_emb11', 'k_emb12', 'k_emb13', 'k_emb14', 'k_emb15']\n",
    "for i in range(10):\n",
    "    data_neg=pd.read_csv('Supplementary_table/Table_S3_negative_KSA_w_features.tsv.zip', compression='zip', sep='\\t')\n",
    "    data_neg=data_neg.drop_duplicates()\n",
    "    data_neg=data_neg.replace(0,np.nan)\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    #data_neg=data_neg[~data_neg['PSSM'].isna()]\n",
    "    \n",
    "    data_train=pd.read_csv('CoPheeKSA/train/w_feature/train_'+str(i)+'.csv',index_col=0)\n",
    "    data_train=data_train.drop_duplicates()\n",
    "    data_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    #data_train=data_train[~data_train['PSSM'].isna()]\n",
    "    \n",
    "    kinases=list(set(data_train.kinase))\n",
    "    data_train_neg=pd.DataFrame()\n",
    "    for k in kinases:\n",
    "        tmp=data_train[data_train.kinase==k]\n",
    "        tmp2=data_neg[data_neg.kinase==k]\n",
    "        if len(tmp2) > 10*len(tmp):\n",
    "            idx=random.sample(range(len(tmp2)),10*len(tmp))\n",
    "            tmp2=tmp2.iloc[idx]\n",
    "            \n",
    "        data_train_neg=pd.concat([data_train_neg,tmp2])\n",
    "    idx=random.sample(range(len(data_neg)),2*len(data_train))\n",
    "    data_train_neg=pd.concat([data_train_neg,data_neg.iloc[idx]])\n",
    "    data_train_neg=data_train_neg.drop_duplicates()\n",
    "    \n",
    "    idx_removed=data_train_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    data_test=pd.read_csv('CoPheeKSA/test/w_feature/test_'+str(i)+'.csv',index_col=0)\n",
    "    data_test=data_test.drop_duplicates()\n",
    "    data_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    #data_test=data_test[~data_test['PSSM'].isna()]\n",
    "    \n",
    "    idx=random.sample(range(len(data_test)),25)\n",
    "    data_eval=data_test.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_test)) if i not in idx]\n",
    "    data_test=data_test.iloc[idx2,:]\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_eval))\n",
    "    data_neg_eval=data_neg.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_neg)) if i not in idx]\n",
    "    data_neg=data_neg.iloc[idx2,:]\n",
    "    \n",
    "    \n",
    "    data_train=data_train[col_idx]\n",
    "    data_eval=data_eval[col_idx]\n",
    "    data_test=data_test[col_idx]\n",
    "    data_neg=data_neg[col_idx]\n",
    "    data_train_neg=data_train_neg[col_idx]\n",
    "    data_neg_eval=data_neg_eval[col_idx]\n",
    "    \n",
    "    X_train=pd.concat([data_train,data_train_neg])\n",
    "    y_train=np.concatenate((np.repeat(1,len(data_train)),np.repeat(0,len(data_train_neg))))\n",
    "    X_train=X_train[col_idx]\n",
    "    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    #X_train=X_train.replace(np.nan,0)\n",
    "    dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "    \n",
    "    X_eval=pd.concat([data_eval,data_neg_eval])\n",
    "    y_eval=np.concatenate((np.repeat(1,len(data_eval)),np.repeat(0,len(data_neg_eval))))\n",
    "    X_eval=X_eval[col_idx]\n",
    "    X_eval.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    deval = xgb.DMatrix(X_eval,label=y_eval)\n",
    "    param = {'max_depth': 2, 'eta': 0.2, 'objective': 'binary:logistic','verbosity':0}\n",
    "    param['nthread'] = 4\n",
    "    param['eval_metric'] = 'auc'\n",
    "    evallist = [(dtrain, 'train'),(deval, 'eval')]\n",
    "    num_round = 300\n",
    "    bst = xgb.train(param, dtrain, num_round, evallist, early_stopping_rounds=10)\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_test))\n",
    "    data_neg2=data_neg.iloc[idx,:]\n",
    "    X_test=pd.concat([data_test,data_neg2])\n",
    "    X_test=X_test[col_idx]\n",
    "    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    y_test=np.concatenate((np.repeat(1,len(data_test)),np.repeat(0,len(data_neg2))))\n",
    "    dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "    y_pred = bst.predict(dtest)\n",
    "\n",
    "    fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred)\n",
    "    auc_tmp=metrics.auc(fpr, tpr)\n",
    "    print(auc_tmp)\n",
    "    \n",
    "    auc.append(auc_tmp)\n",
    "    label.append('CoPheeMap+KMap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "f0e6052e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.87492\teval-auc:0.86424\n",
      "[1]\ttrain-auc:0.87692\teval-auc:0.86592\n",
      "[2]\ttrain-auc:0.89887\teval-auc:0.87640\n",
      "[3]\ttrain-auc:0.90033\teval-auc:0.87720\n",
      "[4]\ttrain-auc:0.90041\teval-auc:0.87696\n",
      "[5]\ttrain-auc:0.90146\teval-auc:0.87576\n",
      "[6]\ttrain-auc:0.90167\teval-auc:0.87472\n",
      "[7]\ttrain-auc:0.90661\teval-auc:0.89000\n",
      "[8]\ttrain-auc:0.90658\teval-auc:0.88944\n",
      "[9]\ttrain-auc:0.90685\teval-auc:0.88896\n",
      "[10]\ttrain-auc:0.90746\teval-auc:0.89120\n",
      "[11]\ttrain-auc:0.91081\teval-auc:0.88720\n",
      "[12]\ttrain-auc:0.91216\teval-auc:0.89816\n",
      "[13]\ttrain-auc:0.91248\teval-auc:0.89840\n",
      "[14]\ttrain-auc:0.91463\teval-auc:0.89816\n",
      "[15]\ttrain-auc:0.91579\teval-auc:0.91336\n",
      "[16]\ttrain-auc:0.91653\teval-auc:0.91224\n",
      "[17]\ttrain-auc:0.91672\teval-auc:0.91120\n",
      "[18]\ttrain-auc:0.91713\teval-auc:0.90768\n",
      "[19]\ttrain-auc:0.91777\teval-auc:0.91080\n",
      "[20]\ttrain-auc:0.91824\teval-auc:0.90504\n",
      "[21]\ttrain-auc:0.91920\teval-auc:0.90560\n",
      "[22]\ttrain-auc:0.92029\teval-auc:0.90608\n",
      "[23]\ttrain-auc:0.92084\teval-auc:0.90592\n",
      "[24]\ttrain-auc:0.92170\teval-auc:0.90512\n",
      "[25]\ttrain-auc:0.92211\teval-auc:0.90704\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9350285714285714\n",
      "[0]\ttrain-auc:0.87745\teval-auc:0.91120\n",
      "[1]\ttrain-auc:0.88018\teval-auc:0.91552\n",
      "[2]\ttrain-auc:0.90536\teval-auc:0.92664\n",
      "[3]\ttrain-auc:0.90619\teval-auc:0.92832\n",
      "[4]\ttrain-auc:0.90622\teval-auc:0.92816\n",
      "[5]\ttrain-auc:0.90659\teval-auc:0.92832\n",
      "[6]\ttrain-auc:0.90667\teval-auc:0.92896\n",
      "[7]\ttrain-auc:0.90675\teval-auc:0.92848\n",
      "[8]\ttrain-auc:0.90711\teval-auc:0.92832\n",
      "[9]\ttrain-auc:0.90970\teval-auc:0.92704\n",
      "[10]\ttrain-auc:0.90988\teval-auc:0.92672\n",
      "[11]\ttrain-auc:0.91362\teval-auc:0.92128\n",
      "[12]\ttrain-auc:0.91638\teval-auc:0.91784\n",
      "[13]\ttrain-auc:0.91692\teval-auc:0.91800\n",
      "[14]\ttrain-auc:0.91786\teval-auc:0.93400\n",
      "[15]\ttrain-auc:0.91808\teval-auc:0.92920\n",
      "[16]\ttrain-auc:0.91916\teval-auc:0.92864\n",
      "[17]\ttrain-auc:0.91977\teval-auc:0.93504\n",
      "[18]\ttrain-auc:0.92080\teval-auc:0.93440\n",
      "[19]\ttrain-auc:0.92093\teval-auc:0.93040\n",
      "[20]\ttrain-auc:0.92200\teval-auc:0.92888\n",
      "[21]\ttrain-auc:0.92246\teval-auc:0.93400\n",
      "[22]\ttrain-auc:0.92301\teval-auc:0.93584\n",
      "[23]\ttrain-auc:0.92356\teval-auc:0.93496\n",
      "[24]\ttrain-auc:0.92414\teval-auc:0.93512\n",
      "[25]\ttrain-auc:0.92463\teval-auc:0.93504\n",
      "[26]\ttrain-auc:0.92574\teval-auc:0.93584\n",
      "[27]\ttrain-auc:0.92599\teval-auc:0.93600\n",
      "[28]\ttrain-auc:0.92614\teval-auc:0.93536\n",
      "[29]\ttrain-auc:0.92696\teval-auc:0.93832\n",
      "[30]\ttrain-auc:0.92863\teval-auc:0.93920\n",
      "[31]\ttrain-auc:0.92921\teval-auc:0.93904\n",
      "[32]\ttrain-auc:0.92946\teval-auc:0.93952\n",
      "[33]\ttrain-auc:0.92963\teval-auc:0.93760\n",
      "[34]\ttrain-auc:0.93022\teval-auc:0.93776\n",
      "[35]\ttrain-auc:0.93054\teval-auc:0.93728\n",
      "[36]\ttrain-auc:0.93064\teval-auc:0.93808\n",
      "[37]\ttrain-auc:0.93124\teval-auc:0.93808\n",
      "[38]\ttrain-auc:0.93168\teval-auc:0.93792\n",
      "[39]\ttrain-auc:0.93230\teval-auc:0.93776\n",
      "[40]\ttrain-auc:0.93274\teval-auc:0.93888\n",
      "[41]\ttrain-auc:0.93318\teval-auc:0.94032\n",
      "[42]\ttrain-auc:0.93362\teval-auc:0.94176\n",
      "[43]\ttrain-auc:0.93404\teval-auc:0.94224\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[44]\ttrain-auc:0.93414\teval-auc:0.94224\n",
      "[45]\ttrain-auc:0.93481\teval-auc:0.94208\n",
      "[46]\ttrain-auc:0.93508\teval-auc:0.94176\n",
      "[47]\ttrain-auc:0.93543\teval-auc:0.94288\n",
      "[48]\ttrain-auc:0.93564\teval-auc:0.94368\n",
      "[49]\ttrain-auc:0.93650\teval-auc:0.94576\n",
      "[50]\ttrain-auc:0.93658\teval-auc:0.94672\n",
      "[51]\ttrain-auc:0.93698\teval-auc:0.94688\n",
      "[52]\ttrain-auc:0.93730\teval-auc:0.94800\n",
      "[53]\ttrain-auc:0.93735\teval-auc:0.94800\n",
      "[54]\ttrain-auc:0.93776\teval-auc:0.94784\n",
      "[55]\ttrain-auc:0.93824\teval-auc:0.94816\n",
      "[56]\ttrain-auc:0.93826\teval-auc:0.94832\n",
      "[57]\ttrain-auc:0.93849\teval-auc:0.94880\n",
      "[58]\ttrain-auc:0.93888\teval-auc:0.94832\n",
      "[59]\ttrain-auc:0.93915\teval-auc:0.94848\n",
      "[60]\ttrain-auc:0.93952\teval-auc:0.94832\n",
      "[61]\ttrain-auc:0.93960\teval-auc:0.94816\n",
      "[62]\ttrain-auc:0.93979\teval-auc:0.94800\n",
      "[63]\ttrain-auc:0.94025\teval-auc:0.94784\n",
      "[64]\ttrain-auc:0.94033\teval-auc:0.94832\n",
      "[65]\ttrain-auc:0.94048\teval-auc:0.94960\n",
      "[66]\ttrain-auc:0.94082\teval-auc:0.95040\n",
      "[67]\ttrain-auc:0.94112\teval-auc:0.95056\n",
      "[68]\ttrain-auc:0.94127\teval-auc:0.95120\n",
      "[69]\ttrain-auc:0.94146\teval-auc:0.95168\n",
      "[70]\ttrain-auc:0.94163\teval-auc:0.95168\n",
      "[71]\ttrain-auc:0.94181\teval-auc:0.95168\n",
      "[72]\ttrain-auc:0.94206\teval-auc:0.95152\n",
      "[73]\ttrain-auc:0.94233\teval-auc:0.95200\n",
      "[74]\ttrain-auc:0.94251\teval-auc:0.95104\n",
      "[75]\ttrain-auc:0.94265\teval-auc:0.95152\n",
      "[76]\ttrain-auc:0.94276\teval-auc:0.95184\n",
      "[77]\ttrain-auc:0.94293\teval-auc:0.95216\n",
      "[78]\ttrain-auc:0.94300\teval-auc:0.95232\n",
      "[79]\ttrain-auc:0.94304\teval-auc:0.95264\n",
      "[80]\ttrain-auc:0.94356\teval-auc:0.95232\n",
      "[81]\ttrain-auc:0.94364\teval-auc:0.95232\n",
      "[82]\ttrain-auc:0.94376\teval-auc:0.95248\n",
      "[83]\ttrain-auc:0.94404\teval-auc:0.95264\n",
      "[84]\ttrain-auc:0.94416\teval-auc:0.95312\n",
      "[85]\ttrain-auc:0.94440\teval-auc:0.95328\n",
      "[86]\ttrain-auc:0.94460\teval-auc:0.95088\n",
      "[87]\ttrain-auc:0.94458\teval-auc:0.95072\n",
      "[88]\ttrain-auc:0.94491\teval-auc:0.95104\n",
      "[89]\ttrain-auc:0.94501\teval-auc:0.95184\n",
      "[90]\ttrain-auc:0.94534\teval-auc:0.95328\n",
      "[91]\ttrain-auc:0.94546\teval-auc:0.95312\n",
      "[92]\ttrain-auc:0.94554\teval-auc:0.95328\n",
      "[93]\ttrain-auc:0.94559\teval-auc:0.95344\n",
      "[94]\ttrain-auc:0.94584\teval-auc:0.95376\n",
      "[95]\ttrain-auc:0.94610\teval-auc:0.95504\n",
      "[96]\ttrain-auc:0.94634\teval-auc:0.95536\n",
      "[97]\ttrain-auc:0.94639\teval-auc:0.95536\n",
      "[98]\ttrain-auc:0.94644\teval-auc:0.95536\n",
      "[99]\ttrain-auc:0.94663\teval-auc:0.95568\n",
      "[100]\ttrain-auc:0.94669\teval-auc:0.95568\n",
      "[101]\ttrain-auc:0.94679\teval-auc:0.95568\n",
      "[102]\ttrain-auc:0.94697\teval-auc:0.95552\n",
      "[103]\ttrain-auc:0.94705\teval-auc:0.95568\n",
      "[104]\ttrain-auc:0.94718\teval-auc:0.95536\n",
      "[105]\ttrain-auc:0.94730\teval-auc:0.95520\n",
      "[106]\ttrain-auc:0.94746\teval-auc:0.95504\n",
      "[107]\ttrain-auc:0.94756\teval-auc:0.95568\n",
      "[108]\ttrain-auc:0.94765\teval-auc:0.95568\n",
      "[109]\ttrain-auc:0.94792\teval-auc:0.95616\n",
      "[110]\ttrain-auc:0.94813\teval-auc:0.95632\n",
      "[111]\ttrain-auc:0.94825\teval-auc:0.95648\n",
      "[112]\ttrain-auc:0.94862\teval-auc:0.95776\n",
      "[113]\ttrain-auc:0.94869\teval-auc:0.95712\n",
      "[114]\ttrain-auc:0.94878\teval-auc:0.95696\n",
      "[115]\ttrain-auc:0.94895\teval-auc:0.95824\n",
      "[116]\ttrain-auc:0.94911\teval-auc:0.95888\n",
      "[117]\ttrain-auc:0.94919\teval-auc:0.95920\n",
      "[118]\ttrain-auc:0.94923\teval-auc:0.95936\n",
      "[119]\ttrain-auc:0.94932\teval-auc:0.95936\n",
      "[120]\ttrain-auc:0.94952\teval-auc:0.95936\n",
      "[121]\ttrain-auc:0.94959\teval-auc:0.96000\n",
      "[122]\ttrain-auc:0.94970\teval-auc:0.96000\n",
      "[123]\ttrain-auc:0.94985\teval-auc:0.96000\n",
      "[124]\ttrain-auc:0.95028\teval-auc:0.96192\n",
      "[125]\ttrain-auc:0.95033\teval-auc:0.96208\n",
      "[126]\ttrain-auc:0.95051\teval-auc:0.96208\n",
      "[127]\ttrain-auc:0.95074\teval-auc:0.96208\n",
      "[128]\ttrain-auc:0.95081\teval-auc:0.96208\n",
      "[129]\ttrain-auc:0.95091\teval-auc:0.96208\n",
      "[130]\ttrain-auc:0.95098\teval-auc:0.96192\n",
      "[131]\ttrain-auc:0.95104\teval-auc:0.96208\n",
      "[132]\ttrain-auc:0.95117\teval-auc:0.96304\n",
      "[133]\ttrain-auc:0.95125\teval-auc:0.96336\n",
      "[134]\ttrain-auc:0.95145\teval-auc:0.96336\n",
      "[135]\ttrain-auc:0.95155\teval-auc:0.96336\n",
      "[136]\ttrain-auc:0.95158\teval-auc:0.96336\n",
      "[137]\ttrain-auc:0.95173\teval-auc:0.96288\n",
      "[138]\ttrain-auc:0.95177\teval-auc:0.96288\n",
      "[139]\ttrain-auc:0.95186\teval-auc:0.96304\n",
      "[140]\ttrain-auc:0.95199\teval-auc:0.96304\n",
      "[141]\ttrain-auc:0.95222\teval-auc:0.96256\n",
      "[142]\ttrain-auc:0.95229\teval-auc:0.96304\n",
      "0.9513881714876032\n",
      "[0]\ttrain-auc:0.87768\teval-auc:0.85456\n",
      "[1]\ttrain-auc:0.88057\teval-auc:0.86304\n",
      "[2]\ttrain-auc:0.90101\teval-auc:0.87008\n",
      "[3]\ttrain-auc:0.90240\teval-auc:0.86864\n",
      "[4]\ttrain-auc:0.90315\teval-auc:0.86648\n",
      "[5]\ttrain-auc:0.90454\teval-auc:0.86416\n",
      "[6]\ttrain-auc:0.90447\teval-auc:0.86416\n",
      "[7]\ttrain-auc:0.90503\teval-auc:0.86488\n",
      "[8]\ttrain-auc:0.90569\teval-auc:0.86464\n",
      "[9]\ttrain-auc:0.91148\teval-auc:0.86896\n",
      "[10]\ttrain-auc:0.91145\teval-auc:0.86880\n",
      "[11]\ttrain-auc:0.91289\teval-auc:0.86888\n",
      "[12]\ttrain-auc:0.91366\teval-auc:0.86760\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9052180946857544\n",
      "[0]\ttrain-auc:0.87102\teval-auc:0.88168\n",
      "[1]\ttrain-auc:0.87818\teval-auc:0.91208\n",
      "[2]\ttrain-auc:0.90088\teval-auc:0.90432\n",
      "[3]\ttrain-auc:0.90173\teval-auc:0.90264\n",
      "[4]\ttrain-auc:0.90185\teval-auc:0.90264\n",
      "[5]\ttrain-auc:0.90275\teval-auc:0.90440\n",
      "[6]\ttrain-auc:0.90285\teval-auc:0.90416\n",
      "[7]\ttrain-auc:0.90353\teval-auc:0.90528\n",
      "[8]\ttrain-auc:0.90567\teval-auc:0.90432\n",
      "[9]\ttrain-auc:0.90685\teval-auc:0.90520\n",
      "[10]\ttrain-auc:0.90682\teval-auc:0.90504\n",
      "[11]\ttrain-auc:0.91237\teval-auc:0.91928\n",
      "[12]\ttrain-auc:0.91295\teval-auc:0.91912\n",
      "[13]\ttrain-auc:0.91368\teval-auc:0.91664\n",
      "[14]\ttrain-auc:0.91368\teval-auc:0.91600\n",
      "[15]\ttrain-auc:0.91452\teval-auc:0.91568\n",
      "[16]\ttrain-auc:0.91593\teval-auc:0.92352\n",
      "[17]\ttrain-auc:0.91704\teval-auc:0.92232\n",
      "[18]\ttrain-auc:0.91703\teval-auc:0.92168\n",
      "[19]\ttrain-auc:0.91756\teval-auc:0.91960\n",
      "[20]\ttrain-auc:0.91873\teval-auc:0.91904\n",
      "[21]\ttrain-auc:0.91957\teval-auc:0.92184\n",
      "[22]\ttrain-auc:0.92041\teval-auc:0.92080\n",
      "[23]\ttrain-auc:0.92120\teval-auc:0.92152\n",
      "[24]\ttrain-auc:0.92158\teval-auc:0.92208\n",
      "[25]\ttrain-auc:0.92194\teval-auc:0.92160\n",
      "[26]\ttrain-auc:0.92244\teval-auc:0.92320\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9239738775510203\n",
      "[0]\ttrain-auc:0.86148\teval-auc:0.87616\n",
      "[1]\ttrain-auc:0.89111\teval-auc:0.91352\n",
      "[2]\ttrain-auc:0.90014\teval-auc:0.91040\n",
      "[3]\ttrain-auc:0.90168\teval-auc:0.90848\n",
      "[4]\ttrain-auc:0.90184\teval-auc:0.91008\n",
      "[5]\ttrain-auc:0.90201\teval-auc:0.90968\n",
      "[6]\ttrain-auc:0.90385\teval-auc:0.91056\n",
      "[7]\ttrain-auc:0.90376\teval-auc:0.91080\n",
      "[8]\ttrain-auc:0.90401\teval-auc:0.91096\n",
      "[9]\ttrain-auc:0.91051\teval-auc:0.92416\n",
      "[10]\ttrain-auc:0.91074\teval-auc:0.92904\n",
      "[11]\ttrain-auc:0.91073\teval-auc:0.92904\n",
      "[12]\ttrain-auc:0.91089\teval-auc:0.92904\n",
      "[13]\ttrain-auc:0.91201\teval-auc:0.92792\n",
      "[14]\ttrain-auc:0.91366\teval-auc:0.92696\n",
      "[15]\ttrain-auc:0.91448\teval-auc:0.92936\n",
      "[16]\ttrain-auc:0.91499\teval-auc:0.92904\n",
      "[17]\ttrain-auc:0.91607\teval-auc:0.93368\n",
      "[18]\ttrain-auc:0.91655\teval-auc:0.93496\n",
      "[19]\ttrain-auc:0.91784\teval-auc:0.93496\n",
      "[20]\ttrain-auc:0.91839\teval-auc:0.93464\n",
      "[21]\ttrain-auc:0.91869\teval-auc:0.93512\n",
      "[22]\ttrain-auc:0.91975\teval-auc:0.93560\n",
      "[23]\ttrain-auc:0.92092\teval-auc:0.93512\n",
      "[24]\ttrain-auc:0.92151\teval-auc:0.93736\n",
      "[25]\ttrain-auc:0.92227\teval-auc:0.93752\n",
      "[26]\ttrain-auc:0.92240\teval-auc:0.93848\n",
      "[27]\ttrain-auc:0.92322\teval-auc:0.93848\n",
      "[28]\ttrain-auc:0.92340\teval-auc:0.93848\n",
      "[29]\ttrain-auc:0.92457\teval-auc:0.93904\n",
      "[30]\ttrain-auc:0.92485\teval-auc:0.93920\n",
      "[31]\ttrain-auc:0.92555\teval-auc:0.93936\n",
      "[32]\ttrain-auc:0.92614\teval-auc:0.94032\n",
      "[33]\ttrain-auc:0.92640\teval-auc:0.94016\n",
      "[34]\ttrain-auc:0.92697\teval-auc:0.94240\n",
      "[35]\ttrain-auc:0.92740\teval-auc:0.94224\n",
      "[36]\ttrain-auc:0.92825\teval-auc:0.94560\n",
      "[37]\ttrain-auc:0.92847\teval-auc:0.94592\n",
      "[38]\ttrain-auc:0.92902\teval-auc:0.94864\n",
      "[39]\ttrain-auc:0.92957\teval-auc:0.94864\n",
      "[40]\ttrain-auc:0.92989\teval-auc:0.94832\n",
      "[41]\ttrain-auc:0.93017\teval-auc:0.95008\n",
      "[42]\ttrain-auc:0.93067\teval-auc:0.94928\n",
      "[43]\ttrain-auc:0.93128\teval-auc:0.94928\n",
      "[44]\ttrain-auc:0.93154\teval-auc:0.94928\n",
      "[45]\ttrain-auc:0.93178\teval-auc:0.94944\n",
      "[46]\ttrain-auc:0.93240\teval-auc:0.95056\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[47]\ttrain-auc:0.93250\teval-auc:0.95072\n",
      "[48]\ttrain-auc:0.93331\teval-auc:0.95040\n",
      "[49]\ttrain-auc:0.93387\teval-auc:0.95168\n",
      "[50]\ttrain-auc:0.93399\teval-auc:0.95168\n",
      "[51]\ttrain-auc:0.93480\teval-auc:0.95280\n",
      "[52]\ttrain-auc:0.93504\teval-auc:0.95264\n",
      "[53]\ttrain-auc:0.93519\teval-auc:0.95440\n",
      "[54]\ttrain-auc:0.93570\teval-auc:0.95376\n",
      "[55]\ttrain-auc:0.93599\teval-auc:0.95408\n",
      "[56]\ttrain-auc:0.93604\teval-auc:0.95344\n",
      "[57]\ttrain-auc:0.93620\teval-auc:0.95376\n",
      "[58]\ttrain-auc:0.93657\teval-auc:0.95360\n",
      "[59]\ttrain-auc:0.93688\teval-auc:0.95376\n",
      "[60]\ttrain-auc:0.93701\teval-auc:0.95344\n",
      "[61]\ttrain-auc:0.93720\teval-auc:0.95392\n",
      "[62]\ttrain-auc:0.93743\teval-auc:0.95360\n",
      "[63]\ttrain-auc:0.93766\teval-auc:0.95344\n",
      "0.9314320987654321\n",
      "[0]\ttrain-auc:0.87503\teval-auc:0.88568\n",
      "[1]\ttrain-auc:0.87849\teval-auc:0.88544\n",
      "[2]\ttrain-auc:0.90012\teval-auc:0.89992\n",
      "[3]\ttrain-auc:0.90102\teval-auc:0.90048\n",
      "[4]\ttrain-auc:0.90111\teval-auc:0.90064\n",
      "[5]\ttrain-auc:0.90464\teval-auc:0.89952\n",
      "[6]\ttrain-auc:0.90528\teval-auc:0.90304\n",
      "[7]\ttrain-auc:0.90798\teval-auc:0.90176\n",
      "[8]\ttrain-auc:0.90779\teval-auc:0.90160\n",
      "[9]\ttrain-auc:0.90830\teval-auc:0.90216\n",
      "[10]\ttrain-auc:0.90862\teval-auc:0.90472\n",
      "[11]\ttrain-auc:0.91204\teval-auc:0.89888\n",
      "[12]\ttrain-auc:0.91306\teval-auc:0.89840\n",
      "[13]\ttrain-auc:0.91503\teval-auc:0.90376\n",
      "[14]\ttrain-auc:0.91524\teval-auc:0.90296\n",
      "[15]\ttrain-auc:0.91628\teval-auc:0.90480\n",
      "[16]\ttrain-auc:0.91707\teval-auc:0.90392\n",
      "[17]\ttrain-auc:0.91856\teval-auc:0.91200\n",
      "[18]\ttrain-auc:0.91917\teval-auc:0.91352\n",
      "[19]\ttrain-auc:0.91956\teval-auc:0.91296\n",
      "[20]\ttrain-auc:0.91997\teval-auc:0.91424\n",
      "[21]\ttrain-auc:0.92111\teval-auc:0.91272\n",
      "[22]\ttrain-auc:0.92170\teval-auc:0.91448\n",
      "[23]\ttrain-auc:0.92225\teval-auc:0.91464\n",
      "[24]\ttrain-auc:0.92343\teval-auc:0.91672\n",
      "[25]\ttrain-auc:0.92346\teval-auc:0.91992\n",
      "[26]\ttrain-auc:0.92476\teval-auc:0.91736\n",
      "[27]\ttrain-auc:0.92541\teval-auc:0.91608\n",
      "[28]\ttrain-auc:0.92608\teval-auc:0.91704\n",
      "[29]\ttrain-auc:0.92686\teval-auc:0.92072\n",
      "[30]\ttrain-auc:0.92717\teval-auc:0.92104\n",
      "[31]\ttrain-auc:0.92737\teval-auc:0.92168\n",
      "[32]\ttrain-auc:0.92804\teval-auc:0.92384\n",
      "[33]\ttrain-auc:0.92845\teval-auc:0.92360\n",
      "[34]\ttrain-auc:0.92857\teval-auc:0.92384\n",
      "[35]\ttrain-auc:0.92882\teval-auc:0.92384\n",
      "[36]\ttrain-auc:0.92934\teval-auc:0.92544\n",
      "[37]\ttrain-auc:0.93003\teval-auc:0.92592\n",
      "[38]\ttrain-auc:0.93058\teval-auc:0.92608\n",
      "[39]\ttrain-auc:0.93140\teval-auc:0.92528\n",
      "[40]\ttrain-auc:0.93162\teval-auc:0.92496\n",
      "[41]\ttrain-auc:0.93201\teval-auc:0.92560\n",
      "[42]\ttrain-auc:0.93226\teval-auc:0.92496\n",
      "[43]\ttrain-auc:0.93277\teval-auc:0.92128\n",
      "[44]\ttrain-auc:0.93304\teval-auc:0.92144\n",
      "[45]\ttrain-auc:0.93359\teval-auc:0.92128\n",
      "[46]\ttrain-auc:0.93368\teval-auc:0.92192\n",
      "[47]\ttrain-auc:0.93395\teval-auc:0.92176\n",
      "[48]\ttrain-auc:0.93415\teval-auc:0.92192\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9225739282024793\n",
      "[0]\ttrain-auc:0.87365\teval-auc:0.86072\n",
      "[1]\ttrain-auc:0.87778\teval-auc:0.86184\n",
      "[2]\ttrain-auc:0.89202\teval-auc:0.86896\n",
      "[3]\ttrain-auc:0.89956\teval-auc:0.86936\n",
      "[4]\ttrain-auc:0.90064\teval-auc:0.86856\n",
      "[5]\ttrain-auc:0.90055\teval-auc:0.86856\n",
      "[6]\ttrain-auc:0.90129\teval-auc:0.86968\n",
      "[7]\ttrain-auc:0.90362\teval-auc:0.87032\n",
      "[8]\ttrain-auc:0.90446\teval-auc:0.86824\n",
      "[9]\ttrain-auc:0.90692\teval-auc:0.86672\n",
      "[10]\ttrain-auc:0.90730\teval-auc:0.86720\n",
      "[11]\ttrain-auc:0.90728\teval-auc:0.86704\n",
      "[12]\ttrain-auc:0.90725\teval-auc:0.86704\n",
      "[13]\ttrain-auc:0.90778\teval-auc:0.86960\n",
      "[14]\ttrain-auc:0.90890\teval-auc:0.86944\n",
      "[15]\ttrain-auc:0.91095\teval-auc:0.88440\n",
      "[16]\ttrain-auc:0.91104\teval-auc:0.88368\n",
      "[17]\ttrain-auc:0.91427\teval-auc:0.87736\n",
      "[18]\ttrain-auc:0.91470\teval-auc:0.87720\n",
      "[19]\ttrain-auc:0.91669\teval-auc:0.87256\n",
      "[20]\ttrain-auc:0.91857\teval-auc:0.87760\n",
      "[21]\ttrain-auc:0.91974\teval-auc:0.87288\n",
      "[22]\ttrain-auc:0.91996\teval-auc:0.87456\n",
      "[23]\ttrain-auc:0.92058\teval-auc:0.87360\n",
      "[24]\ttrain-auc:0.92104\teval-auc:0.87480\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9240305852820718\n",
      "[0]\ttrain-auc:0.87858\teval-auc:0.86112\n",
      "[1]\ttrain-auc:0.88232\teval-auc:0.87384\n",
      "[2]\ttrain-auc:0.89218\teval-auc:0.87240\n",
      "[3]\ttrain-auc:0.90191\teval-auc:0.91184\n",
      "[4]\ttrain-auc:0.90203\teval-auc:0.91232\n",
      "[5]\ttrain-auc:0.90232\teval-auc:0.91312\n",
      "[6]\ttrain-auc:0.90253\teval-auc:0.91296\n",
      "[7]\ttrain-auc:0.90265\teval-auc:0.91232\n",
      "[8]\ttrain-auc:0.90457\teval-auc:0.91184\n",
      "[9]\ttrain-auc:0.90518\teval-auc:0.91368\n",
      "[10]\ttrain-auc:0.90528\teval-auc:0.91352\n",
      "[11]\ttrain-auc:0.90555\teval-auc:0.91464\n",
      "[12]\ttrain-auc:0.90546\teval-auc:0.91432\n",
      "[13]\ttrain-auc:0.90581\teval-auc:0.91432\n",
      "[14]\ttrain-auc:0.90736\teval-auc:0.91560\n",
      "[15]\ttrain-auc:0.91035\teval-auc:0.91688\n",
      "[16]\ttrain-auc:0.91233\teval-auc:0.91800\n",
      "[17]\ttrain-auc:0.91466\teval-auc:0.91568\n",
      "[18]\ttrain-auc:0.91493\teval-auc:0.91728\n",
      "[19]\ttrain-auc:0.91607\teval-auc:0.91632\n",
      "[20]\ttrain-auc:0.91758\teval-auc:0.91480\n",
      "[21]\ttrain-auc:0.91861\teval-auc:0.91352\n",
      "[22]\ttrain-auc:0.91971\teval-auc:0.91704\n",
      "[23]\ttrain-auc:0.92052\teval-auc:0.91536\n",
      "[24]\ttrain-auc:0.92151\teval-auc:0.91832\n",
      "[25]\ttrain-auc:0.92242\teval-auc:0.91704\n",
      "[26]\ttrain-auc:0.92470\teval-auc:0.91720\n",
      "[27]\ttrain-auc:0.92507\teval-auc:0.91672\n",
      "[28]\ttrain-auc:0.92526\teval-auc:0.91592\n",
      "[29]\ttrain-auc:0.92582\teval-auc:0.91560\n",
      "[30]\ttrain-auc:0.92659\teval-auc:0.91936\n",
      "[31]\ttrain-auc:0.92689\teval-auc:0.91952\n",
      "[32]\ttrain-auc:0.92751\teval-auc:0.91984\n",
      "[33]\ttrain-auc:0.92791\teval-auc:0.92080\n",
      "[34]\ttrain-auc:0.92803\teval-auc:0.91968\n",
      "[35]\ttrain-auc:0.92845\teval-auc:0.92000\n",
      "[36]\ttrain-auc:0.92863\teval-auc:0.92032\n",
      "[37]\ttrain-auc:0.92894\teval-auc:0.92112\n",
      "[38]\ttrain-auc:0.92907\teval-auc:0.92112\n",
      "[39]\ttrain-auc:0.92970\teval-auc:0.92232\n",
      "[40]\ttrain-auc:0.93004\teval-auc:0.92216\n",
      "[41]\ttrain-auc:0.93026\teval-auc:0.92256\n",
      "[42]\ttrain-auc:0.93078\teval-auc:0.92304\n",
      "[43]\ttrain-auc:0.93110\teval-auc:0.92352\n",
      "[44]\ttrain-auc:0.93132\teval-auc:0.92384\n",
      "[45]\ttrain-auc:0.93155\teval-auc:0.92368\n",
      "[46]\ttrain-auc:0.93197\teval-auc:0.92432\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[47]\ttrain-auc:0.93235\teval-auc:0.92480\n",
      "[48]\ttrain-auc:0.93270\teval-auc:0.92576\n",
      "[49]\ttrain-auc:0.93285\teval-auc:0.92560\n",
      "[50]\ttrain-auc:0.93299\teval-auc:0.92592\n",
      "[51]\ttrain-auc:0.93388\teval-auc:0.92944\n",
      "[52]\ttrain-auc:0.93399\teval-auc:0.92944\n",
      "[53]\ttrain-auc:0.93404\teval-auc:0.92944\n",
      "[54]\ttrain-auc:0.93409\teval-auc:0.92896\n",
      "[55]\ttrain-auc:0.93428\teval-auc:0.92944\n",
      "[56]\ttrain-auc:0.93468\teval-auc:0.93024\n",
      "[57]\ttrain-auc:0.93507\teval-auc:0.92976\n",
      "[58]\ttrain-auc:0.93563\teval-auc:0.93136\n",
      "[59]\ttrain-auc:0.93582\teval-auc:0.93152\n",
      "[60]\ttrain-auc:0.93612\teval-auc:0.93264\n",
      "[61]\ttrain-auc:0.93659\teval-auc:0.93344\n",
      "[62]\ttrain-auc:0.93686\teval-auc:0.93392\n",
      "[63]\ttrain-auc:0.93706\teval-auc:0.93344\n",
      "[64]\ttrain-auc:0.93730\teval-auc:0.93328\n",
      "[65]\ttrain-auc:0.93767\teval-auc:0.93328\n",
      "[66]\ttrain-auc:0.93779\teval-auc:0.93232\n",
      "[67]\ttrain-auc:0.93781\teval-auc:0.93200\n",
      "[68]\ttrain-auc:0.93799\teval-auc:0.93184\n",
      "[69]\ttrain-auc:0.93823\teval-auc:0.93296\n",
      "[70]\ttrain-auc:0.93845\teval-auc:0.93408\n",
      "[71]\ttrain-auc:0.93868\teval-auc:0.93376\n",
      "[72]\ttrain-auc:0.93874\teval-auc:0.93328\n",
      "[73]\ttrain-auc:0.93927\teval-auc:0.93248\n",
      "[74]\ttrain-auc:0.93944\teval-auc:0.93312\n",
      "[75]\ttrain-auc:0.93951\teval-auc:0.93328\n",
      "[76]\ttrain-auc:0.93995\teval-auc:0.93280\n",
      "[77]\ttrain-auc:0.94006\teval-auc:0.93280\n",
      "[78]\ttrain-auc:0.94027\teval-auc:0.93408\n",
      "[79]\ttrain-auc:0.94038\teval-auc:0.93424\n",
      "[80]\ttrain-auc:0.94044\teval-auc:0.93408\n",
      "[81]\ttrain-auc:0.94054\teval-auc:0.93408\n",
      "[82]\ttrain-auc:0.94074\teval-auc:0.93568\n",
      "[83]\ttrain-auc:0.94092\teval-auc:0.93568\n",
      "[84]\ttrain-auc:0.94127\teval-auc:0.93728\n",
      "[85]\ttrain-auc:0.94172\teval-auc:0.93920\n",
      "[86]\ttrain-auc:0.94184\teval-auc:0.93984\n",
      "[87]\ttrain-auc:0.94191\teval-auc:0.93968\n",
      "[88]\ttrain-auc:0.94205\teval-auc:0.93792\n",
      "[89]\ttrain-auc:0.94243\teval-auc:0.93808\n",
      "[90]\ttrain-auc:0.94257\teval-auc:0.93760\n",
      "[91]\ttrain-auc:0.94277\teval-auc:0.93760\n",
      "[92]\ttrain-auc:0.94284\teval-auc:0.93744\n",
      "[93]\ttrain-auc:0.94302\teval-auc:0.93728\n",
      "[94]\ttrain-auc:0.94325\teval-auc:0.93792\n",
      "[95]\ttrain-auc:0.94332\teval-auc:0.93680\n",
      "[96]\ttrain-auc:0.94355\teval-auc:0.93712\n",
      "0.95263128112268\n",
      "[0]\ttrain-auc:0.87861\teval-auc:0.75688\n",
      "[1]\ttrain-auc:0.88248\teval-auc:0.75848\n",
      "[2]\ttrain-auc:0.88370\teval-auc:0.75984\n",
      "[3]\ttrain-auc:0.90580\teval-auc:0.79560\n",
      "[4]\ttrain-auc:0.90595\teval-auc:0.79912\n",
      "[5]\ttrain-auc:0.90632\teval-auc:0.79896\n",
      "[6]\ttrain-auc:0.90623\teval-auc:0.79904\n",
      "[7]\ttrain-auc:0.90899\teval-auc:0.81000\n",
      "[8]\ttrain-auc:0.90910\teval-auc:0.80976\n",
      "[9]\ttrain-auc:0.90955\teval-auc:0.81064\n",
      "[10]\ttrain-auc:0.91013\teval-auc:0.81296\n",
      "[11]\ttrain-auc:0.91223\teval-auc:0.82072\n",
      "[12]\ttrain-auc:0.91444\teval-auc:0.82536\n",
      "[13]\ttrain-auc:0.91502\teval-auc:0.82576\n",
      "[14]\ttrain-auc:0.91665\teval-auc:0.82656\n",
      "[15]\ttrain-auc:0.91856\teval-auc:0.83152\n",
      "[16]\ttrain-auc:0.91914\teval-auc:0.83120\n",
      "[17]\ttrain-auc:0.91989\teval-auc:0.83000\n",
      "[18]\ttrain-auc:0.92016\teval-auc:0.83096\n",
      "[19]\ttrain-auc:0.92091\teval-auc:0.83648\n",
      "[20]\ttrain-auc:0.92141\teval-auc:0.83568\n",
      "[21]\ttrain-auc:0.92223\teval-auc:0.83528\n",
      "[22]\ttrain-auc:0.92388\teval-auc:0.84072\n",
      "[23]\ttrain-auc:0.92458\teval-auc:0.83872\n",
      "[24]\ttrain-auc:0.92521\teval-auc:0.84080\n",
      "[25]\ttrain-auc:0.92542\teval-auc:0.84304\n",
      "[26]\ttrain-auc:0.92612\teval-auc:0.84384\n",
      "[27]\ttrain-auc:0.92672\teval-auc:0.84248\n",
      "[28]\ttrain-auc:0.92702\teval-auc:0.84712\n",
      "[29]\ttrain-auc:0.92741\teval-auc:0.84776\n",
      "[30]\ttrain-auc:0.92772\teval-auc:0.84776\n",
      "[31]\ttrain-auc:0.92830\teval-auc:0.85112\n",
      "[32]\ttrain-auc:0.92892\teval-auc:0.85272\n",
      "[33]\ttrain-auc:0.92938\teval-auc:0.85160\n",
      "[34]\ttrain-auc:0.93007\teval-auc:0.85184\n",
      "[35]\ttrain-auc:0.93055\teval-auc:0.85104\n",
      "[36]\ttrain-auc:0.93086\teval-auc:0.84768\n",
      "[37]\ttrain-auc:0.93123\teval-auc:0.85168\n",
      "[38]\ttrain-auc:0.93153\teval-auc:0.85472\n",
      "[39]\ttrain-auc:0.93175\teval-auc:0.85616\n",
      "[40]\ttrain-auc:0.93235\teval-auc:0.85536\n",
      "[41]\ttrain-auc:0.93259\teval-auc:0.85488\n",
      "[42]\ttrain-auc:0.93306\teval-auc:0.85408\n",
      "[43]\ttrain-auc:0.93342\teval-auc:0.85456\n",
      "[44]\ttrain-auc:0.93385\teval-auc:0.85520\n",
      "[45]\ttrain-auc:0.93430\teval-auc:0.85632\n",
      "[46]\ttrain-auc:0.93433\teval-auc:0.85680\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[47]\ttrain-auc:0.93460\teval-auc:0.85376\n",
      "[48]\ttrain-auc:0.93503\teval-auc:0.85520\n",
      "[49]\ttrain-auc:0.93567\teval-auc:0.85632\n",
      "[50]\ttrain-auc:0.93578\teval-auc:0.85648\n",
      "[51]\ttrain-auc:0.93663\teval-auc:0.85888\n",
      "[52]\ttrain-auc:0.93693\teval-auc:0.85808\n",
      "[53]\ttrain-auc:0.93709\teval-auc:0.85504\n",
      "[54]\ttrain-auc:0.93750\teval-auc:0.85728\n",
      "[55]\ttrain-auc:0.93787\teval-auc:0.85664\n",
      "[56]\ttrain-auc:0.93780\teval-auc:0.85664\n",
      "[57]\ttrain-auc:0.93812\teval-auc:0.85968\n",
      "[58]\ttrain-auc:0.93824\teval-auc:0.85984\n",
      "[59]\ttrain-auc:0.93856\teval-auc:0.85936\n",
      "[60]\ttrain-auc:0.93865\teval-auc:0.85760\n",
      "[61]\ttrain-auc:0.93909\teval-auc:0.85952\n",
      "[62]\ttrain-auc:0.93951\teval-auc:0.85936\n",
      "[63]\ttrain-auc:0.93990\teval-auc:0.86144\n",
      "[64]\ttrain-auc:0.94016\teval-auc:0.86208\n",
      "[65]\ttrain-auc:0.94035\teval-auc:0.86176\n",
      "[66]\ttrain-auc:0.94082\teval-auc:0.86480\n",
      "[67]\ttrain-auc:0.94114\teval-auc:0.86512\n",
      "[68]\ttrain-auc:0.94122\teval-auc:0.86592\n",
      "[69]\ttrain-auc:0.94147\teval-auc:0.86656\n",
      "[70]\ttrain-auc:0.94160\teval-auc:0.86672\n",
      "[71]\ttrain-auc:0.94170\teval-auc:0.86672\n",
      "[72]\ttrain-auc:0.94195\teval-auc:0.86704\n",
      "[73]\ttrain-auc:0.94233\teval-auc:0.86720\n",
      "[74]\ttrain-auc:0.94257\teval-auc:0.86736\n",
      "[75]\ttrain-auc:0.94272\teval-auc:0.86688\n",
      "[76]\ttrain-auc:0.94284\teval-auc:0.86720\n",
      "[77]\ttrain-auc:0.94327\teval-auc:0.86768\n",
      "[78]\ttrain-auc:0.94374\teval-auc:0.87024\n",
      "[79]\ttrain-auc:0.94390\teval-auc:0.87136\n",
      "[80]\ttrain-auc:0.94421\teval-auc:0.87232\n",
      "[81]\ttrain-auc:0.94434\teval-auc:0.87248\n",
      "[82]\ttrain-auc:0.94446\teval-auc:0.87376\n",
      "[83]\ttrain-auc:0.94463\teval-auc:0.87360\n",
      "[84]\ttrain-auc:0.94472\teval-auc:0.87408\n",
      "[85]\ttrain-auc:0.94498\teval-auc:0.87408\n",
      "[86]\ttrain-auc:0.94516\teval-auc:0.87360\n",
      "[87]\ttrain-auc:0.94530\teval-auc:0.87216\n",
      "[88]\ttrain-auc:0.94560\teval-auc:0.87216\n",
      "[89]\ttrain-auc:0.94581\teval-auc:0.87152\n",
      "[90]\ttrain-auc:0.94592\teval-auc:0.87184\n",
      "[91]\ttrain-auc:0.94605\teval-auc:0.87216\n",
      "[92]\ttrain-auc:0.94616\teval-auc:0.87280\n",
      "[93]\ttrain-auc:0.94643\teval-auc:0.87376\n",
      "[94]\ttrain-auc:0.94655\teval-auc:0.87328\n",
      "0.9360061728395062\n",
      "[0]\ttrain-auc:0.87564\teval-auc:0.89456\n",
      "[1]\ttrain-auc:0.87891\teval-auc:0.89888\n",
      "[2]\ttrain-auc:0.89869\teval-auc:0.92552\n",
      "[3]\ttrain-auc:0.90044\teval-auc:0.92504\n",
      "[4]\ttrain-auc:0.90053\teval-auc:0.92688\n",
      "[5]\ttrain-auc:0.90418\teval-auc:0.94216\n",
      "[6]\ttrain-auc:0.90418\teval-auc:0.94240\n",
      "[7]\ttrain-auc:0.90403\teval-auc:0.94352\n",
      "[8]\ttrain-auc:0.90608\teval-auc:0.94352\n",
      "[9]\ttrain-auc:0.90991\teval-auc:0.94352\n",
      "[10]\ttrain-auc:0.91058\teval-auc:0.94352\n",
      "[11]\ttrain-auc:0.91101\teval-auc:0.94512\n",
      "[12]\ttrain-auc:0.91234\teval-auc:0.94344\n",
      "[13]\ttrain-auc:0.91460\teval-auc:0.94640\n",
      "[14]\ttrain-auc:0.91526\teval-auc:0.94800\n",
      "[15]\ttrain-auc:0.91606\teval-auc:0.94768\n",
      "[16]\ttrain-auc:0.91764\teval-auc:0.94704\n",
      "[17]\ttrain-auc:0.91817\teval-auc:0.94784\n",
      "[18]\ttrain-auc:0.91869\teval-auc:0.94864\n",
      "[19]\ttrain-auc:0.91907\teval-auc:0.94848\n",
      "[20]\ttrain-auc:0.91962\teval-auc:0.94832\n",
      "[21]\ttrain-auc:0.92013\teval-auc:0.94952\n",
      "[22]\ttrain-auc:0.92020\teval-auc:0.94944\n",
      "[23]\ttrain-auc:0.92151\teval-auc:0.95088\n",
      "[24]\ttrain-auc:0.92220\teval-auc:0.95232\n",
      "[25]\ttrain-auc:0.92272\teval-auc:0.95248\n",
      "[26]\ttrain-auc:0.92314\teval-auc:0.95184\n",
      "[27]\ttrain-auc:0.92384\teval-auc:0.95136\n",
      "[28]\ttrain-auc:0.92455\teval-auc:0.95152\n",
      "[29]\ttrain-auc:0.92467\teval-auc:0.95120\n",
      "[30]\ttrain-auc:0.92481\teval-auc:0.95200\n",
      "[31]\ttrain-auc:0.92537\teval-auc:0.95264\n",
      "[32]\ttrain-auc:0.92573\teval-auc:0.95152\n",
      "[33]\ttrain-auc:0.92640\teval-auc:0.95376\n",
      "[34]\ttrain-auc:0.92687\teval-auc:0.95376\n",
      "[35]\ttrain-auc:0.92740\teval-auc:0.95376\n",
      "[36]\ttrain-auc:0.92774\teval-auc:0.95424\n",
      "[37]\ttrain-auc:0.92804\teval-auc:0.95584\n",
      "[38]\ttrain-auc:0.92841\teval-auc:0.95584\n",
      "[39]\ttrain-auc:0.92910\teval-auc:0.95568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[40]\ttrain-auc:0.92915\teval-auc:0.95536\n",
      "[41]\ttrain-auc:0.92950\teval-auc:0.95792\n",
      "[42]\ttrain-auc:0.92984\teval-auc:0.95808\n",
      "[43]\ttrain-auc:0.93016\teval-auc:0.95888\n",
      "[44]\ttrain-auc:0.93057\teval-auc:0.95840\n",
      "[45]\ttrain-auc:0.93073\teval-auc:0.95936\n",
      "[46]\ttrain-auc:0.93132\teval-auc:0.95888\n",
      "[47]\ttrain-auc:0.93166\teval-auc:0.96160\n",
      "[48]\ttrain-auc:0.93192\teval-auc:0.96240\n",
      "[49]\ttrain-auc:0.93215\teval-auc:0.96240\n",
      "[50]\ttrain-auc:0.93251\teval-auc:0.96256\n",
      "[51]\ttrain-auc:0.93297\teval-auc:0.96288\n",
      "[52]\ttrain-auc:0.93318\teval-auc:0.96272\n",
      "[53]\ttrain-auc:0.93325\teval-auc:0.96272\n",
      "[54]\ttrain-auc:0.93338\teval-auc:0.96288\n",
      "[55]\ttrain-auc:0.93392\teval-auc:0.96256\n",
      "[56]\ttrain-auc:0.93412\teval-auc:0.96288\n",
      "[57]\ttrain-auc:0.93470\teval-auc:0.96256\n",
      "[58]\ttrain-auc:0.93486\teval-auc:0.96240\n",
      "[59]\ttrain-auc:0.93532\teval-auc:0.96416\n",
      "[60]\ttrain-auc:0.93595\teval-auc:0.96592\n",
      "[61]\ttrain-auc:0.93622\teval-auc:0.96608\n",
      "[62]\ttrain-auc:0.93648\teval-auc:0.96528\n",
      "[63]\ttrain-auc:0.93687\teval-auc:0.96576\n",
      "[64]\ttrain-auc:0.93726\teval-auc:0.96624\n",
      "[65]\ttrain-auc:0.93736\teval-auc:0.96720\n",
      "[66]\ttrain-auc:0.93774\teval-auc:0.96800\n",
      "[67]\ttrain-auc:0.93808\teval-auc:0.96800\n",
      "[68]\ttrain-auc:0.93850\teval-auc:0.96800\n",
      "[69]\ttrain-auc:0.93889\teval-auc:0.96752\n",
      "[70]\ttrain-auc:0.93914\teval-auc:0.96784\n",
      "[71]\ttrain-auc:0.93987\teval-auc:0.96912\n",
      "[72]\ttrain-auc:0.94005\teval-auc:0.96896\n",
      "[73]\ttrain-auc:0.94004\teval-auc:0.96944\n",
      "[74]\ttrain-auc:0.94018\teval-auc:0.96976\n",
      "[75]\ttrain-auc:0.94047\teval-auc:0.96944\n",
      "[76]\ttrain-auc:0.94068\teval-auc:0.96864\n",
      "[77]\ttrain-auc:0.94080\teval-auc:0.96864\n",
      "[78]\ttrain-auc:0.94092\teval-auc:0.96848\n",
      "[79]\ttrain-auc:0.94112\teval-auc:0.96848\n",
      "[80]\ttrain-auc:0.94132\teval-auc:0.96864\n",
      "[81]\ttrain-auc:0.94152\teval-auc:0.96880\n",
      "[82]\ttrain-auc:0.94159\teval-auc:0.96832\n",
      "[83]\ttrain-auc:0.94165\teval-auc:0.96832\n",
      "[84]\ttrain-auc:0.94184\teval-auc:0.96928\n",
      "0.9362607173046735\n"
     ]
    }
   ],
   "source": [
    "#Supervised model trained with PSSM scores + CoPheeMap info + KMap info\n",
    "Cancer=['BRCA','CCRCC','COAD','GBM','HCC','HNSCC','LSCC','LUAD','OV','PDAC','UCEC'] \n",
    "col_idx=['cophee_emb0', 'cophee_emb1', 'cophee_emb2', 'cophee_emb3', 'cophee_emb4', \n",
    "         'cophee_emb5', 'cophee_emb6', 'cophee_emb7', 'cophee_emb8', 'cophee_emb9', \n",
    "         'cophee_emb10','cophee_emb11', 'cophee_emb12', 'cophee_emb13', 'cophee_emb14', \n",
    "         'cophee_emb15', 'k_emb0', 'k_emb1', 'k_emb2', 'k_emb3','k_emb4', 'k_emb5', \n",
    "         'k_emb6', 'k_emb7', 'k_emb8', 'k_emb9', 'k_emb10',\n",
    "         'k_emb11', 'k_emb12', 'k_emb13', 'k_emb14', 'k_emb15','PSSM']\n",
    "for i in range(10):\n",
    "    data_neg=pd.read_csv('Supplementary_table/Table_S3_negative_KSA_w_features.tsv.zip', compression='zip', sep='\\t')\n",
    "    data_neg=data_neg.drop_duplicates()\n",
    "    data_neg=data_neg.replace(0,np.nan)\n",
    "    data_neg=data_neg[~data_neg['PSSM'].isna()]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    data_train=pd.read_csv('CoPheeKSA/train/w_feature/train_'+str(i)+'.csv',index_col=0)\n",
    "    data_train=data_train.drop_duplicates()\n",
    "    data_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    data_train=data_train[~data_train['PSSM'].isna()]\n",
    "    \n",
    "    kinases=list(set(data_train.kinase))\n",
    "    data_train_neg=pd.DataFrame()\n",
    "    for k in kinases:\n",
    "        tmp=data_train[data_train.kinase==k]\n",
    "        tmp2=data_neg[data_neg.kinase==k]\n",
    "        if len(tmp2) > 10*len(tmp):\n",
    "            idx=random.sample(range(len(tmp2)),10*len(tmp))\n",
    "            tmp2=tmp2.iloc[idx]\n",
    "            \n",
    "        data_train_neg=pd.concat([data_train_neg,tmp2])\n",
    "    idx=random.sample(range(len(data_neg)),2*len(data_train))\n",
    "    data_train_neg=pd.concat([data_train_neg,data_neg.iloc[idx]])\n",
    "    data_train_neg=data_train_neg.drop_duplicates()\n",
    "    \n",
    "    idx_removed=data_train_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    data_test=pd.read_csv('CoPheeKSA/test/w_feature/test_'+str(i)+'.csv',index_col=0)\n",
    "    data_test=data_test.drop_duplicates()\n",
    "    data_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    data_test=data_test[~data_test['PSSM'].isna()]\n",
    "    \n",
    "    idx=random.sample(range(len(data_test)),25)\n",
    "    data_eval=data_test.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_test)) if i not in idx]\n",
    "    data_test=data_test.iloc[idx2,:]\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_eval))\n",
    "    data_neg_eval=data_neg.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_neg)) if i not in idx]\n",
    "    data_neg=data_neg.iloc[idx2,:]\n",
    "    \n",
    "    \n",
    "    data_train=data_train[col_idx]\n",
    "    data_eval=data_eval[col_idx]\n",
    "    data_test=data_test[col_idx]\n",
    "    data_neg=data_neg[col_idx]\n",
    "    data_neg_eval=data_neg_eval[col_idx]\n",
    "    data_train_neg=data_train_neg[col_idx]\n",
    "    \n",
    "    X_train=pd.concat([data_train,data_train_neg])\n",
    "    y_train=np.concatenate((np.repeat(1,len(data_train)),np.repeat(0,len(data_train_neg))))\n",
    "    X_train=X_train[col_idx]\n",
    "    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    #X_train=X_train.replace(np.nan,0)\n",
    "    dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "    \n",
    "    X_eval=pd.concat([data_eval,data_neg_eval])\n",
    "    y_eval=np.concatenate((np.repeat(1,len(data_eval)),np.repeat(0,len(data_neg_eval))))\n",
    "    X_eval=X_eval[col_idx]\n",
    "    X_eval.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    deval = xgb.DMatrix(X_eval,label=y_eval)\n",
    "    param = {'max_depth': 2, 'eta': 0.2, 'objective': 'binary:logistic','verbosity':0}\n",
    "    param['nthread'] = 4\n",
    "    param['eval_metric'] = 'auc'\n",
    "    evallist = [(dtrain, 'train'),(deval, 'eval')]\n",
    "    num_round = 300\n",
    "    bst = xgb.train(param, dtrain, num_round, evallist, early_stopping_rounds=10)\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_test))\n",
    "    data_neg2=data_neg.iloc[idx,:]\n",
    "    X_test=pd.concat([data_test,data_neg2])\n",
    "    X_test=X_test[col_idx]\n",
    "    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    y_test=np.concatenate((np.repeat(1,len(data_test)),np.repeat(0,len(data_neg2))))\n",
    "    dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "    y_pred = bst.predict(dtest)\n",
    "\n",
    "    fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred)\n",
    "    auc_tmp=metrics.auc(fpr, tpr)\n",
    "    print(auc_tmp)\n",
    "    \n",
    "    auc.append(auc_tmp)\n",
    "    label.append('PSSM+CoPheeMap+KMap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7556ba1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.55576\teval-auc:0.50256\n",
      "[1]\ttrain-auc:0.62559\teval-auc:0.64608\n",
      "[2]\ttrain-auc:0.64873\teval-auc:0.69360\n",
      "[3]\ttrain-auc:0.65144\teval-auc:0.71480\n",
      "[4]\ttrain-auc:0.65787\teval-auc:0.71056\n",
      "[5]\ttrain-auc:0.66160\teval-auc:0.70696\n",
      "[6]\ttrain-auc:0.66777\teval-auc:0.71616\n",
      "[7]\ttrain-auc:0.67744\teval-auc:0.75048\n",
      "[8]\ttrain-auc:0.68434\teval-auc:0.74912\n",
      "[9]\ttrain-auc:0.69349\teval-auc:0.75848\n",
      "[10]\ttrain-auc:0.69879\teval-auc:0.76104\n",
      "[11]\ttrain-auc:0.69949\teval-auc:0.75408\n",
      "[12]\ttrain-auc:0.70558\teval-auc:0.75112\n",
      "[13]\ttrain-auc:0.71357\teval-auc:0.77224\n",
      "[14]\ttrain-auc:0.71639\teval-auc:0.77752\n",
      "[15]\ttrain-auc:0.72325\teval-auc:0.77120\n",
      "[16]\ttrain-auc:0.72806\teval-auc:0.77200\n",
      "[17]\ttrain-auc:0.73147\teval-auc:0.78040\n",
      "[18]\ttrain-auc:0.73303\teval-auc:0.78976\n",
      "[19]\ttrain-auc:0.73542\teval-auc:0.79392\n",
      "[20]\ttrain-auc:0.73732\teval-auc:0.78776\n",
      "[21]\ttrain-auc:0.74090\teval-auc:0.78648\n",
      "[22]\ttrain-auc:0.74132\teval-auc:0.79560\n",
      "[23]\ttrain-auc:0.74265\teval-auc:0.80536\n",
      "[24]\ttrain-auc:0.74580\teval-auc:0.81416\n",
      "[25]\ttrain-auc:0.74696\teval-auc:0.81224\n",
      "[26]\ttrain-auc:0.74920\teval-auc:0.81016\n",
      "[27]\ttrain-auc:0.75109\teval-auc:0.81312\n",
      "[28]\ttrain-auc:0.75266\teval-auc:0.81728\n",
      "[29]\ttrain-auc:0.75521\teval-auc:0.82000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[30]\ttrain-auc:0.75753\teval-auc:0.82720\n",
      "[31]\ttrain-auc:0.75787\teval-auc:0.82912\n",
      "[32]\ttrain-auc:0.76040\teval-auc:0.83312\n",
      "[33]\ttrain-auc:0.76092\teval-auc:0.83008\n",
      "[34]\ttrain-auc:0.76206\teval-auc:0.83184\n",
      "[35]\ttrain-auc:0.76281\teval-auc:0.82832\n",
      "[36]\ttrain-auc:0.76363\teval-auc:0.82560\n",
      "[37]\ttrain-auc:0.76380\teval-auc:0.82912\n",
      "[38]\ttrain-auc:0.76465\teval-auc:0.83232\n",
      "[39]\ttrain-auc:0.76621\teval-auc:0.82480\n",
      "[40]\ttrain-auc:0.76740\teval-auc:0.82928\n",
      "[41]\ttrain-auc:0.76908\teval-auc:0.83056\n",
      "0.8348522448979591\n",
      "[0]\ttrain-auc:0.60219\teval-auc:0.70160\n",
      "[1]\ttrain-auc:0.60896\teval-auc:0.69344\n",
      "[2]\ttrain-auc:0.64544\teval-auc:0.80152\n",
      "[3]\ttrain-auc:0.65402\teval-auc:0.80824\n",
      "[4]\ttrain-auc:0.65616\teval-auc:0.79800\n",
      "[5]\ttrain-auc:0.66426\teval-auc:0.78744\n",
      "[6]\ttrain-auc:0.67685\teval-auc:0.82640\n",
      "[7]\ttrain-auc:0.68717\teval-auc:0.82704\n",
      "[8]\ttrain-auc:0.69288\teval-auc:0.85536\n",
      "[9]\ttrain-auc:0.69333\teval-auc:0.83840\n",
      "[10]\ttrain-auc:0.69391\teval-auc:0.83424\n",
      "[11]\ttrain-auc:0.70705\teval-auc:0.85024\n",
      "[12]\ttrain-auc:0.71081\teval-auc:0.84664\n",
      "[13]\ttrain-auc:0.72034\teval-auc:0.86432\n",
      "[14]\ttrain-auc:0.72096\teval-auc:0.86568\n",
      "[15]\ttrain-auc:0.72604\teval-auc:0.86360\n",
      "[16]\ttrain-auc:0.73211\teval-auc:0.85128\n",
      "[17]\ttrain-auc:0.73496\teval-auc:0.85904\n",
      "[18]\ttrain-auc:0.74219\teval-auc:0.85640\n",
      "[19]\ttrain-auc:0.74404\teval-auc:0.86056\n",
      "[20]\ttrain-auc:0.74632\teval-auc:0.85976\n",
      "[21]\ttrain-auc:0.74599\teval-auc:0.85448\n",
      "[22]\ttrain-auc:0.74993\teval-auc:0.86184\n",
      "[23]\ttrain-auc:0.75042\teval-auc:0.86760\n",
      "[24]\ttrain-auc:0.75186\teval-auc:0.86216\n",
      "[25]\ttrain-auc:0.75450\teval-auc:0.86680\n",
      "[26]\ttrain-auc:0.75599\teval-auc:0.86952\n",
      "[27]\ttrain-auc:0.75752\teval-auc:0.86920\n",
      "[28]\ttrain-auc:0.75857\teval-auc:0.87016\n",
      "[29]\ttrain-auc:0.75963\teval-auc:0.86960\n",
      "[30]\ttrain-auc:0.76111\teval-auc:0.87088\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[31]\ttrain-auc:0.76292\teval-auc:0.87792\n",
      "[32]\ttrain-auc:0.76414\teval-auc:0.87696\n",
      "[33]\ttrain-auc:0.76576\teval-auc:0.87384\n",
      "[34]\ttrain-auc:0.76650\teval-auc:0.86912\n",
      "[35]\ttrain-auc:0.76799\teval-auc:0.86640\n",
      "[36]\ttrain-auc:0.76896\teval-auc:0.86624\n",
      "[37]\ttrain-auc:0.76992\teval-auc:0.86776\n",
      "[38]\ttrain-auc:0.77109\teval-auc:0.87224\n",
      "[39]\ttrain-auc:0.77174\teval-auc:0.87272\n",
      "[40]\ttrain-auc:0.77348\teval-auc:0.87016\n",
      "0.8130020015495867\n",
      "[0]\ttrain-auc:0.61031\teval-auc:0.51864\n",
      "[1]\ttrain-auc:0.63433\teval-auc:0.54464\n",
      "[2]\ttrain-auc:0.63768\teval-auc:0.53688\n",
      "[3]\ttrain-auc:0.66209\teval-auc:0.59976\n",
      "[4]\ttrain-auc:0.66925\teval-auc:0.57536\n",
      "[5]\ttrain-auc:0.67365\teval-auc:0.57808\n",
      "[6]\ttrain-auc:0.68260\teval-auc:0.60352\n",
      "[7]\ttrain-auc:0.69115\teval-auc:0.58904\n",
      "[8]\ttrain-auc:0.69116\teval-auc:0.60168\n",
      "[9]\ttrain-auc:0.69508\teval-auc:0.63584\n",
      "[10]\ttrain-auc:0.69819\teval-auc:0.63784\n",
      "[11]\ttrain-auc:0.70472\teval-auc:0.66688\n",
      "[12]\ttrain-auc:0.71140\teval-auc:0.68688\n",
      "[13]\ttrain-auc:0.71722\teval-auc:0.69768\n",
      "[14]\ttrain-auc:0.72072\teval-auc:0.68464\n",
      "[15]\ttrain-auc:0.72494\teval-auc:0.69168\n",
      "[16]\ttrain-auc:0.73101\teval-auc:0.71176\n",
      "[17]\ttrain-auc:0.73602\teval-auc:0.70808\n",
      "[18]\ttrain-auc:0.74029\teval-auc:0.71584\n",
      "[19]\ttrain-auc:0.74129\teval-auc:0.71432\n",
      "[20]\ttrain-auc:0.74489\teval-auc:0.71032\n",
      "[21]\ttrain-auc:0.74646\teval-auc:0.71208\n",
      "[22]\ttrain-auc:0.75121\teval-auc:0.71368\n",
      "[23]\ttrain-auc:0.75459\teval-auc:0.72456\n",
      "[24]\ttrain-auc:0.75674\teval-auc:0.73360\n",
      "[25]\ttrain-auc:0.75778\teval-auc:0.73016\n",
      "[26]\ttrain-auc:0.75880\teval-auc:0.73104\n",
      "[27]\ttrain-auc:0.76046\teval-auc:0.73240\n",
      "[28]\ttrain-auc:0.76152\teval-auc:0.72936\n",
      "[29]\ttrain-auc:0.76384\teval-auc:0.73736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[30]\ttrain-auc:0.76496\teval-auc:0.74408\n",
      "[31]\ttrain-auc:0.76635\teval-auc:0.73688\n",
      "[32]\ttrain-auc:0.76704\teval-auc:0.73848\n",
      "[33]\ttrain-auc:0.76783\teval-auc:0.73960\n",
      "[34]\ttrain-auc:0.77037\teval-auc:0.74392\n",
      "[35]\ttrain-auc:0.77043\teval-auc:0.74152\n",
      "[36]\ttrain-auc:0.77140\teval-auc:0.74568\n",
      "[37]\ttrain-auc:0.77368\teval-auc:0.75464\n",
      "[38]\ttrain-auc:0.77437\teval-auc:0.75576\n",
      "[39]\ttrain-auc:0.77499\teval-auc:0.75384\n",
      "[40]\ttrain-auc:0.77660\teval-auc:0.75832\n",
      "[41]\ttrain-auc:0.77775\teval-auc:0.75976\n",
      "[42]\ttrain-auc:0.77866\teval-auc:0.76360\n",
      "[43]\ttrain-auc:0.77953\teval-auc:0.76024\n",
      "[44]\ttrain-auc:0.78073\teval-auc:0.75624\n",
      "[45]\ttrain-auc:0.78137\teval-auc:0.75592\n",
      "[46]\ttrain-auc:0.78179\teval-auc:0.75528\n",
      "[47]\ttrain-auc:0.78261\teval-auc:0.75464\n",
      "[48]\ttrain-auc:0.78249\teval-auc:0.75352\n",
      "[49]\ttrain-auc:0.78330\teval-auc:0.75400\n",
      "[50]\ttrain-auc:0.78380\teval-auc:0.75464\n",
      "[51]\ttrain-auc:0.78420\teval-auc:0.75560\n",
      "[52]\ttrain-auc:0.78470\teval-auc:0.75352\n",
      "0.7926772687036415\n",
      "[0]\ttrain-auc:0.55909\teval-auc:0.56200\n",
      "[1]\ttrain-auc:0.60089\teval-auc:0.62712\n",
      "[2]\ttrain-auc:0.64288\teval-auc:0.70560\n",
      "[3]\ttrain-auc:0.65773\teval-auc:0.76352\n",
      "[4]\ttrain-auc:0.66363\teval-auc:0.75104\n",
      "[5]\ttrain-auc:0.66530\teval-auc:0.75192\n",
      "[6]\ttrain-auc:0.67299\teval-auc:0.76000\n",
      "[7]\ttrain-auc:0.67969\teval-auc:0.77584\n",
      "[8]\ttrain-auc:0.69365\teval-auc:0.76480\n",
      "[9]\ttrain-auc:0.69887\teval-auc:0.78056\n",
      "[10]\ttrain-auc:0.70254\teval-auc:0.77640\n",
      "[11]\ttrain-auc:0.70655\teval-auc:0.79056\n",
      "[12]\ttrain-auc:0.71120\teval-auc:0.79920\n",
      "[13]\ttrain-auc:0.71766\teval-auc:0.82072\n",
      "[14]\ttrain-auc:0.71904\teval-auc:0.83592\n",
      "[15]\ttrain-auc:0.72423\teval-auc:0.83200\n",
      "[16]\ttrain-auc:0.72892\teval-auc:0.82592\n",
      "[17]\ttrain-auc:0.72896\teval-auc:0.82424\n",
      "[18]\ttrain-auc:0.73445\teval-auc:0.81536\n",
      "[19]\ttrain-auc:0.73794\teval-auc:0.81920\n",
      "[20]\ttrain-auc:0.73969\teval-auc:0.81920\n",
      "[21]\ttrain-auc:0.74212\teval-auc:0.81704\n",
      "[22]\ttrain-auc:0.74424\teval-auc:0.82120\n",
      "[23]\ttrain-auc:0.74608\teval-auc:0.82000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8160261224489795\n",
      "[0]\ttrain-auc:0.56163\teval-auc:0.58096\n",
      "[1]\ttrain-auc:0.61166\teval-auc:0.61048\n",
      "[2]\ttrain-auc:0.63736\teval-auc:0.67472\n",
      "[3]\ttrain-auc:0.65788\teval-auc:0.70896\n",
      "[4]\ttrain-auc:0.66386\teval-auc:0.75736\n",
      "[5]\ttrain-auc:0.67454\teval-auc:0.73680\n",
      "[6]\ttrain-auc:0.67442\teval-auc:0.74064\n",
      "[7]\ttrain-auc:0.67394\teval-auc:0.73392\n",
      "[8]\ttrain-auc:0.68169\teval-auc:0.73096\n",
      "[9]\ttrain-auc:0.69342\teval-auc:0.75136\n",
      "[10]\ttrain-auc:0.69959\teval-auc:0.75176\n",
      "[11]\ttrain-auc:0.70287\teval-auc:0.75200\n",
      "[12]\ttrain-auc:0.71178\teval-auc:0.75640\n",
      "[13]\ttrain-auc:0.71520\teval-auc:0.75232\n",
      "[14]\ttrain-auc:0.72263\teval-auc:0.76040\n",
      "[15]\ttrain-auc:0.72600\teval-auc:0.76568\n",
      "[16]\ttrain-auc:0.73072\teval-auc:0.76312\n",
      "[17]\ttrain-auc:0.73449\teval-auc:0.78096\n",
      "[18]\ttrain-auc:0.73677\teval-auc:0.78440\n",
      "[19]\ttrain-auc:0.73857\teval-auc:0.78264\n",
      "[20]\ttrain-auc:0.74212\teval-auc:0.78616\n",
      "[21]\ttrain-auc:0.74536\teval-auc:0.79064\n",
      "[22]\ttrain-auc:0.74677\teval-auc:0.78768\n",
      "[23]\ttrain-auc:0.74822\teval-auc:0.78512\n",
      "[24]\ttrain-auc:0.75171\teval-auc:0.79504\n",
      "[25]\ttrain-auc:0.75292\teval-auc:0.79808\n",
      "[26]\ttrain-auc:0.75483\teval-auc:0.80160\n",
      "[27]\ttrain-auc:0.75543\teval-auc:0.80448\n",
      "[28]\ttrain-auc:0.75700\teval-auc:0.80800\n",
      "[29]\ttrain-auc:0.75814\teval-auc:0.80816\n",
      "[30]\ttrain-auc:0.75909\teval-auc:0.81056\n",
      "[31]\ttrain-auc:0.76071\teval-auc:0.81288\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[32]\ttrain-auc:0.76235\teval-auc:0.81192\n",
      "[33]\ttrain-auc:0.76315\teval-auc:0.81064\n",
      "[34]\ttrain-auc:0.76687\teval-auc:0.80472\n",
      "[35]\ttrain-auc:0.76798\teval-auc:0.80520\n",
      "[36]\ttrain-auc:0.76976\teval-auc:0.80472\n",
      "[37]\ttrain-auc:0.77059\teval-auc:0.80488\n",
      "[38]\ttrain-auc:0.77155\teval-auc:0.80648\n",
      "[39]\ttrain-auc:0.77273\teval-auc:0.81624\n",
      "[40]\ttrain-auc:0.77379\teval-auc:0.81544\n",
      "[41]\ttrain-auc:0.77480\teval-auc:0.81872\n",
      "[42]\ttrain-auc:0.77561\teval-auc:0.82080\n",
      "[43]\ttrain-auc:0.77631\teval-auc:0.82048\n",
      "[44]\ttrain-auc:0.77687\teval-auc:0.82192\n",
      "[45]\ttrain-auc:0.77792\teval-auc:0.81888\n",
      "[46]\ttrain-auc:0.77855\teval-auc:0.82048\n",
      "[47]\ttrain-auc:0.77883\teval-auc:0.81936\n",
      "[48]\ttrain-auc:0.77930\teval-auc:0.82128\n",
      "[49]\ttrain-auc:0.78062\teval-auc:0.82112\n",
      "[50]\ttrain-auc:0.78141\teval-auc:0.82144\n",
      "[51]\ttrain-auc:0.78234\teval-auc:0.82304\n",
      "[52]\ttrain-auc:0.78313\teval-auc:0.82048\n",
      "[53]\ttrain-auc:0.78402\teval-auc:0.82016\n",
      "[54]\ttrain-auc:0.78442\teval-auc:0.82064\n",
      "[55]\ttrain-auc:0.78483\teval-auc:0.82288\n",
      "[56]\ttrain-auc:0.78505\teval-auc:0.82320\n",
      "[57]\ttrain-auc:0.78624\teval-auc:0.82272\n",
      "[58]\ttrain-auc:0.78645\teval-auc:0.82304\n",
      "[59]\ttrain-auc:0.78655\teval-auc:0.82288\n",
      "[60]\ttrain-auc:0.78692\teval-auc:0.82320\n",
      "[61]\ttrain-auc:0.78752\teval-auc:0.82400\n",
      "[62]\ttrain-auc:0.78777\teval-auc:0.82304\n",
      "[63]\ttrain-auc:0.78955\teval-auc:0.82448\n",
      "[64]\ttrain-auc:0.79010\teval-auc:0.82368\n",
      "[65]\ttrain-auc:0.79034\teval-auc:0.82416\n",
      "[66]\ttrain-auc:0.79065\teval-auc:0.82512\n",
      "[67]\ttrain-auc:0.79081\teval-auc:0.82464\n",
      "[68]\ttrain-auc:0.79104\teval-auc:0.82496\n",
      "[69]\ttrain-auc:0.79181\teval-auc:0.82624\n",
      "[70]\ttrain-auc:0.79235\teval-auc:0.82576\n",
      "[71]\ttrain-auc:0.79213\teval-auc:0.82608\n",
      "[72]\ttrain-auc:0.79452\teval-auc:0.82208\n",
      "[73]\ttrain-auc:0.79523\teval-auc:0.82144\n",
      "[74]\ttrain-auc:0.79568\teval-auc:0.82384\n",
      "[75]\ttrain-auc:0.79594\teval-auc:0.82464\n",
      "[76]\ttrain-auc:0.79617\teval-auc:0.82544\n",
      "[77]\ttrain-auc:0.79638\teval-auc:0.82480\n",
      "[78]\ttrain-auc:0.79645\teval-auc:0.82528\n",
      "[79]\ttrain-auc:0.79674\teval-auc:0.82624\n",
      "0.8300231481481481\n",
      "[0]\ttrain-auc:0.59515\teval-auc:0.59608\n",
      "[1]\ttrain-auc:0.63573\teval-auc:0.63192\n",
      "[2]\ttrain-auc:0.63742\teval-auc:0.62256\n",
      "[3]\ttrain-auc:0.65394\teval-auc:0.64280\n",
      "[4]\ttrain-auc:0.66184\teval-auc:0.62016\n",
      "[5]\ttrain-auc:0.66810\teval-auc:0.62816\n",
      "[6]\ttrain-auc:0.67072\teval-auc:0.64704\n",
      "[7]\ttrain-auc:0.67980\teval-auc:0.65408\n",
      "[8]\ttrain-auc:0.68926\teval-auc:0.64136\n",
      "[9]\ttrain-auc:0.69419\teval-auc:0.64040\n",
      "[10]\ttrain-auc:0.70014\teval-auc:0.65256\n",
      "[11]\ttrain-auc:0.70769\teval-auc:0.66168\n",
      "[12]\ttrain-auc:0.71041\teval-auc:0.67192\n",
      "[13]\ttrain-auc:0.71572\teval-auc:0.67832\n",
      "[14]\ttrain-auc:0.71991\teval-auc:0.68104\n",
      "[15]\ttrain-auc:0.72429\teval-auc:0.68352\n",
      "[16]\ttrain-auc:0.72752\teval-auc:0.68776\n",
      "[17]\ttrain-auc:0.72975\teval-auc:0.69392\n",
      "[18]\ttrain-auc:0.73286\teval-auc:0.69576\n",
      "[19]\ttrain-auc:0.73779\teval-auc:0.69736\n",
      "[20]\ttrain-auc:0.74095\teval-auc:0.69616\n",
      "[21]\ttrain-auc:0.74335\teval-auc:0.69312\n",
      "[22]\ttrain-auc:0.74577\teval-auc:0.70144\n",
      "[23]\ttrain-auc:0.74684\teval-auc:0.69904\n",
      "[24]\ttrain-auc:0.74954\teval-auc:0.71072\n",
      "[25]\ttrain-auc:0.75063\teval-auc:0.71328\n",
      "[26]\ttrain-auc:0.75206\teval-auc:0.71616\n",
      "[27]\ttrain-auc:0.75372\teval-auc:0.72192\n",
      "[28]\ttrain-auc:0.75671\teval-auc:0.71216\n",
      "[29]\ttrain-auc:0.75878\teval-auc:0.72192\n",
      "[30]\ttrain-auc:0.75878\teval-auc:0.72960\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[31]\ttrain-auc:0.75918\teval-auc:0.73520\n",
      "[32]\ttrain-auc:0.76008\teval-auc:0.73648\n",
      "[33]\ttrain-auc:0.76215\teval-auc:0.74000\n",
      "[34]\ttrain-auc:0.76518\teval-auc:0.73248\n",
      "[35]\ttrain-auc:0.76580\teval-auc:0.73232\n",
      "[36]\ttrain-auc:0.76662\teval-auc:0.73440\n",
      "[37]\ttrain-auc:0.76906\teval-auc:0.72688\n",
      "[38]\ttrain-auc:0.77030\teval-auc:0.73344\n",
      "[39]\ttrain-auc:0.77114\teval-auc:0.73232\n",
      "[40]\ttrain-auc:0.77198\teval-auc:0.73424\n",
      "[41]\ttrain-auc:0.77383\teval-auc:0.73576\n",
      "[42]\ttrain-auc:0.77523\teval-auc:0.73976\n",
      "[43]\ttrain-auc:0.77608\teval-auc:0.73736\n",
      "0.8373482696280992\n",
      "[0]\ttrain-auc:0.56024\teval-auc:0.58000\n",
      "[1]\ttrain-auc:0.61056\teval-auc:0.58832\n",
      "[2]\ttrain-auc:0.63830\teval-auc:0.68384\n",
      "[3]\ttrain-auc:0.65482\teval-auc:0.69336\n",
      "[4]\ttrain-auc:0.66085\teval-auc:0.70040\n",
      "[5]\ttrain-auc:0.67445\teval-auc:0.74472\n",
      "[6]\ttrain-auc:0.67653\teval-auc:0.70888\n",
      "[7]\ttrain-auc:0.68449\teval-auc:0.72792\n",
      "[8]\ttrain-auc:0.68990\teval-auc:0.74200\n",
      "[9]\ttrain-auc:0.69498\teval-auc:0.75504\n",
      "[10]\ttrain-auc:0.70164\teval-auc:0.76792\n",
      "[11]\ttrain-auc:0.70617\teval-auc:0.78520\n",
      "[12]\ttrain-auc:0.71336\teval-auc:0.79568\n",
      "[13]\ttrain-auc:0.72071\teval-auc:0.81336\n",
      "[14]\ttrain-auc:0.72420\teval-auc:0.82088\n",
      "[15]\ttrain-auc:0.72407\teval-auc:0.81176\n",
      "[16]\ttrain-auc:0.72858\teval-auc:0.82168\n",
      "[17]\ttrain-auc:0.73098\teval-auc:0.82048\n",
      "[18]\ttrain-auc:0.73623\teval-auc:0.83760\n",
      "[19]\ttrain-auc:0.73839\teval-auc:0.84240\n",
      "[20]\ttrain-auc:0.73953\teval-auc:0.85096\n",
      "[21]\ttrain-auc:0.74391\teval-auc:0.85544\n",
      "[22]\ttrain-auc:0.74754\teval-auc:0.85512\n",
      "[23]\ttrain-auc:0.74894\teval-auc:0.85640\n",
      "[24]\ttrain-auc:0.74946\teval-auc:0.85720\n",
      "[25]\ttrain-auc:0.75080\teval-auc:0.86400\n",
      "[26]\ttrain-auc:0.75274\teval-auc:0.86328\n",
      "[27]\ttrain-auc:0.75450\teval-auc:0.86472\n",
      "[28]\ttrain-auc:0.75607\teval-auc:0.86376\n",
      "[29]\ttrain-auc:0.75753\teval-auc:0.86760\n",
      "[30]\ttrain-auc:0.75859\teval-auc:0.86520\n",
      "[31]\ttrain-auc:0.75945\teval-auc:0.86632\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[32]\ttrain-auc:0.76144\teval-auc:0.86920\n",
      "[33]\ttrain-auc:0.76386\teval-auc:0.86640\n",
      "[34]\ttrain-auc:0.76653\teval-auc:0.86672\n",
      "[35]\ttrain-auc:0.76714\teval-auc:0.86832\n",
      "[36]\ttrain-auc:0.76790\teval-auc:0.87008\n",
      "[37]\ttrain-auc:0.76894\teval-auc:0.86464\n",
      "[38]\ttrain-auc:0.76964\teval-auc:0.87040\n",
      "[39]\ttrain-auc:0.77055\teval-auc:0.86992\n",
      "[40]\ttrain-auc:0.77146\teval-auc:0.87136\n",
      "[41]\ttrain-auc:0.77199\teval-auc:0.87040\n",
      "[42]\ttrain-auc:0.77251\teval-auc:0.86768\n",
      "[43]\ttrain-auc:0.77381\teval-auc:0.87264\n",
      "[44]\ttrain-auc:0.77441\teval-auc:0.87328\n",
      "[45]\ttrain-auc:0.77459\teval-auc:0.87152\n",
      "[46]\ttrain-auc:0.77500\teval-auc:0.87072\n",
      "[47]\ttrain-auc:0.77600\teval-auc:0.87344\n",
      "[48]\ttrain-auc:0.77658\teval-auc:0.87424\n",
      "[49]\ttrain-auc:0.77723\teval-auc:0.87504\n",
      "[50]\ttrain-auc:0.77858\teval-auc:0.86784\n",
      "[51]\ttrain-auc:0.77929\teval-auc:0.86736\n",
      "[52]\ttrain-auc:0.78027\teval-auc:0.86480\n",
      "[53]\ttrain-auc:0.78071\teval-auc:0.86496\n",
      "[54]\ttrain-auc:0.78146\teval-auc:0.86496\n",
      "[55]\ttrain-auc:0.78213\teval-auc:0.86576\n",
      "[56]\ttrain-auc:0.78317\teval-auc:0.86448\n",
      "[57]\ttrain-auc:0.78339\teval-auc:0.86560\n",
      "[58]\ttrain-auc:0.78376\teval-auc:0.86320\n",
      "0.8182372175980975\n",
      "[0]\ttrain-auc:0.56038\teval-auc:0.55344\n",
      "[1]\ttrain-auc:0.61516\teval-auc:0.66208\n",
      "[2]\ttrain-auc:0.61682\teval-auc:0.66184\n",
      "[3]\ttrain-auc:0.64770\teval-auc:0.69984\n",
      "[4]\ttrain-auc:0.66026\teval-auc:0.74400\n",
      "[5]\ttrain-auc:0.66764\teval-auc:0.75472\n",
      "[6]\ttrain-auc:0.67074\teval-auc:0.73616\n",
      "[7]\ttrain-auc:0.68065\teval-auc:0.78912\n",
      "[8]\ttrain-auc:0.68300\teval-auc:0.78736\n",
      "[9]\ttrain-auc:0.69082\teval-auc:0.79088\n",
      "[10]\ttrain-auc:0.69933\teval-auc:0.80480\n",
      "[11]\ttrain-auc:0.70175\teval-auc:0.78888\n",
      "[12]\ttrain-auc:0.71104\teval-auc:0.80800\n",
      "[13]\ttrain-auc:0.71433\teval-auc:0.81552\n",
      "[14]\ttrain-auc:0.71722\teval-auc:0.82888\n",
      "[15]\ttrain-auc:0.72543\teval-auc:0.84072\n",
      "[16]\ttrain-auc:0.72915\teval-auc:0.84376\n",
      "[17]\ttrain-auc:0.73241\teval-auc:0.84968\n",
      "[18]\ttrain-auc:0.73418\teval-auc:0.85456\n",
      "[19]\ttrain-auc:0.73800\teval-auc:0.85296\n",
      "[20]\ttrain-auc:0.74107\teval-auc:0.85520\n",
      "[21]\ttrain-auc:0.74282\teval-auc:0.85312\n",
      "[22]\ttrain-auc:0.74471\teval-auc:0.85280\n",
      "[23]\ttrain-auc:0.74886\teval-auc:0.85992\n",
      "[24]\ttrain-auc:0.75183\teval-auc:0.86728\n",
      "[25]\ttrain-auc:0.75401\teval-auc:0.86776\n",
      "[26]\ttrain-auc:0.75590\teval-auc:0.86904\n",
      "[27]\ttrain-auc:0.75840\teval-auc:0.86952\n",
      "[28]\ttrain-auc:0.75926\teval-auc:0.87048\n",
      "[29]\ttrain-auc:0.76194\teval-auc:0.87288\n",
      "[30]\ttrain-auc:0.76310\teval-auc:0.87352\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[31]\ttrain-auc:0.76406\teval-auc:0.87056\n",
      "[32]\ttrain-auc:0.76513\teval-auc:0.87024\n",
      "[33]\ttrain-auc:0.76597\teval-auc:0.87280\n",
      "[34]\ttrain-auc:0.76794\teval-auc:0.87392\n",
      "[35]\ttrain-auc:0.76922\teval-auc:0.87520\n",
      "[36]\ttrain-auc:0.76956\teval-auc:0.87472\n",
      "[37]\ttrain-auc:0.76982\teval-auc:0.87384\n",
      "[38]\ttrain-auc:0.77148\teval-auc:0.87528\n",
      "[39]\ttrain-auc:0.77268\teval-auc:0.87896\n",
      "[40]\ttrain-auc:0.77326\teval-auc:0.87848\n",
      "[41]\ttrain-auc:0.77342\teval-auc:0.88008\n",
      "[42]\ttrain-auc:0.77404\teval-auc:0.87800\n",
      "[43]\ttrain-auc:0.77711\teval-auc:0.87560\n",
      "[44]\ttrain-auc:0.77740\teval-auc:0.87448\n",
      "[45]\ttrain-auc:0.77807\teval-auc:0.87384\n",
      "[46]\ttrain-auc:0.77868\teval-auc:0.87512\n",
      "[47]\ttrain-auc:0.77903\teval-auc:0.87544\n",
      "[48]\ttrain-auc:0.78056\teval-auc:0.87168\n",
      "[49]\ttrain-auc:0.78154\teval-auc:0.87088\n",
      "[50]\ttrain-auc:0.78223\teval-auc:0.86848\n",
      "0.818574581258488\n",
      "[0]\ttrain-auc:0.60396\teval-auc:0.61056\n",
      "[1]\ttrain-auc:0.63854\teval-auc:0.64112\n",
      "[2]\ttrain-auc:0.66274\teval-auc:0.66912\n",
      "[3]\ttrain-auc:0.67315\teval-auc:0.67424\n",
      "[4]\ttrain-auc:0.67606\teval-auc:0.67800\n",
      "[5]\ttrain-auc:0.67972\teval-auc:0.69064\n",
      "[6]\ttrain-auc:0.68318\teval-auc:0.70200\n",
      "[7]\ttrain-auc:0.68954\teval-auc:0.69592\n",
      "[8]\ttrain-auc:0.69466\teval-auc:0.67944\n",
      "[9]\ttrain-auc:0.69702\teval-auc:0.69048\n",
      "[10]\ttrain-auc:0.70423\teval-auc:0.71152\n",
      "[11]\ttrain-auc:0.71220\teval-auc:0.72360\n",
      "[12]\ttrain-auc:0.71502\teval-auc:0.74008\n",
      "[13]\ttrain-auc:0.72254\teval-auc:0.73752\n",
      "[14]\ttrain-auc:0.72700\teval-auc:0.73616\n",
      "[15]\ttrain-auc:0.72895\teval-auc:0.73872\n",
      "[16]\ttrain-auc:0.73331\teval-auc:0.73784\n",
      "[17]\ttrain-auc:0.73651\teval-auc:0.73304\n",
      "[18]\ttrain-auc:0.73913\teval-auc:0.73288\n",
      "[19]\ttrain-auc:0.74259\teval-auc:0.74192\n",
      "[20]\ttrain-auc:0.74568\teval-auc:0.74192\n",
      "[21]\ttrain-auc:0.74816\teval-auc:0.74160\n",
      "[22]\ttrain-auc:0.75220\teval-auc:0.74496\n",
      "[23]\ttrain-auc:0.75210\teval-auc:0.74304\n",
      "[24]\ttrain-auc:0.75354\teval-auc:0.74592\n",
      "[25]\ttrain-auc:0.75519\teval-auc:0.74336\n",
      "[26]\ttrain-auc:0.75727\teval-auc:0.75296\n",
      "[27]\ttrain-auc:0.75793\teval-auc:0.74968\n",
      "[28]\ttrain-auc:0.75910\teval-auc:0.75304\n",
      "[29]\ttrain-auc:0.76144\teval-auc:0.74712\n",
      "[30]\ttrain-auc:0.76258\teval-auc:0.74888\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[31]\ttrain-auc:0.76426\teval-auc:0.75432\n",
      "[32]\ttrain-auc:0.76523\teval-auc:0.75160\n",
      "[33]\ttrain-auc:0.76619\teval-auc:0.74976\n",
      "[34]\ttrain-auc:0.76703\teval-auc:0.75248\n",
      "[35]\ttrain-auc:0.76960\teval-auc:0.74496\n",
      "[36]\ttrain-auc:0.77039\teval-auc:0.74704\n",
      "[37]\ttrain-auc:0.77142\teval-auc:0.74752\n",
      "[38]\ttrain-auc:0.77209\teval-auc:0.74848\n",
      "[39]\ttrain-auc:0.77337\teval-auc:0.74928\n",
      "[40]\ttrain-auc:0.77437\teval-auc:0.75040\n",
      "0.8013287037037036\n",
      "[0]\ttrain-auc:0.60631\teval-auc:0.53960\n",
      "[1]\ttrain-auc:0.63034\teval-auc:0.63152\n",
      "[2]\ttrain-auc:0.65771\teval-auc:0.67360\n",
      "[3]\ttrain-auc:0.65532\teval-auc:0.62224\n",
      "[4]\ttrain-auc:0.66823\teval-auc:0.70800\n",
      "[5]\ttrain-auc:0.66804\teval-auc:0.68408\n",
      "[6]\ttrain-auc:0.68069\teval-auc:0.69832\n",
      "[7]\ttrain-auc:0.67800\teval-auc:0.67440\n",
      "[8]\ttrain-auc:0.68293\teval-auc:0.66616\n",
      "[9]\ttrain-auc:0.69368\teval-auc:0.70960\n",
      "[10]\ttrain-auc:0.69974\teval-auc:0.68488\n",
      "[11]\ttrain-auc:0.70514\teval-auc:0.71120\n",
      "[12]\ttrain-auc:0.71011\teval-auc:0.74408\n",
      "[13]\ttrain-auc:0.71340\teval-auc:0.73592\n",
      "[14]\ttrain-auc:0.72346\teval-auc:0.75832\n",
      "[15]\ttrain-auc:0.72970\teval-auc:0.77472\n",
      "[16]\ttrain-auc:0.73593\teval-auc:0.77744\n",
      "[17]\ttrain-auc:0.73829\teval-auc:0.78296\n",
      "[18]\ttrain-auc:0.74148\teval-auc:0.78392\n",
      "[19]\ttrain-auc:0.74273\teval-auc:0.78104\n",
      "[20]\ttrain-auc:0.74674\teval-auc:0.78680\n",
      "[21]\ttrain-auc:0.74843\teval-auc:0.78824\n",
      "[22]\ttrain-auc:0.75070\teval-auc:0.79736\n",
      "[23]\ttrain-auc:0.75283\teval-auc:0.79376\n",
      "[24]\ttrain-auc:0.75409\teval-auc:0.79840\n",
      "[25]\ttrain-auc:0.75634\teval-auc:0.80048\n",
      "[26]\ttrain-auc:0.75822\teval-auc:0.79720\n",
      "[27]\ttrain-auc:0.76053\teval-auc:0.79832\n",
      "[28]\ttrain-auc:0.76215\teval-auc:0.80440\n",
      "[29]\ttrain-auc:0.76329\teval-auc:0.80064\n",
      "[30]\ttrain-auc:0.76462\teval-auc:0.79568\n",
      "[31]\ttrain-auc:0.76528\teval-auc:0.80272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[32]\ttrain-auc:0.76559\teval-auc:0.80256\n",
      "[33]\ttrain-auc:0.76736\teval-auc:0.80352\n",
      "[34]\ttrain-auc:0.76879\teval-auc:0.80416\n",
      "[35]\ttrain-auc:0.76950\teval-auc:0.81392\n",
      "[36]\ttrain-auc:0.77106\teval-auc:0.81672\n",
      "[37]\ttrain-auc:0.77198\teval-auc:0.81848\n",
      "[38]\ttrain-auc:0.77239\teval-auc:0.81688\n",
      "[39]\ttrain-auc:0.77415\teval-auc:0.82144\n",
      "[40]\ttrain-auc:0.77482\teval-auc:0.82400\n",
      "[41]\ttrain-auc:0.77498\teval-auc:0.82032\n",
      "[42]\ttrain-auc:0.77546\teval-auc:0.82112\n",
      "[43]\ttrain-auc:0.77614\teval-auc:0.82144\n",
      "[44]\ttrain-auc:0.77719\teval-auc:0.82416\n",
      "[45]\ttrain-auc:0.77882\teval-auc:0.82448\n",
      "[46]\ttrain-auc:0.77984\teval-auc:0.82432\n",
      "[47]\ttrain-auc:0.78029\teval-auc:0.82560\n",
      "[48]\ttrain-auc:0.78074\teval-auc:0.82656\n",
      "[49]\ttrain-auc:0.78179\teval-auc:0.82656\n",
      "[50]\ttrain-auc:0.78207\teval-auc:0.82592\n",
      "[51]\ttrain-auc:0.78210\teval-auc:0.82624\n",
      "[52]\ttrain-auc:0.78262\teval-auc:0.82768\n",
      "[53]\ttrain-auc:0.78294\teval-auc:0.82800\n",
      "[54]\ttrain-auc:0.78392\teval-auc:0.82656\n",
      "[55]\ttrain-auc:0.78438\teval-auc:0.82736\n",
      "[56]\ttrain-auc:0.78479\teval-auc:0.82816\n",
      "[57]\ttrain-auc:0.78524\teval-auc:0.82816\n",
      "[58]\ttrain-auc:0.78581\teval-auc:0.83056\n",
      "[59]\ttrain-auc:0.78618\teval-auc:0.83120\n",
      "[60]\ttrain-auc:0.78665\teval-auc:0.82688\n",
      "[61]\ttrain-auc:0.78693\teval-auc:0.82752\n",
      "[62]\ttrain-auc:0.78784\teval-auc:0.82976\n",
      "[63]\ttrain-auc:0.78863\teval-auc:0.83104\n",
      "[64]\ttrain-auc:0.79009\teval-auc:0.83168\n",
      "[65]\ttrain-auc:0.79025\teval-auc:0.83120\n",
      "[66]\ttrain-auc:0.79044\teval-auc:0.83152\n",
      "[67]\ttrain-auc:0.79105\teval-auc:0.83040\n",
      "[68]\ttrain-auc:0.79155\teval-auc:0.83152\n",
      "[69]\ttrain-auc:0.79151\teval-auc:0.83120\n",
      "[70]\ttrain-auc:0.79185\teval-auc:0.83168\n",
      "[71]\ttrain-auc:0.79189\teval-auc:0.83168\n",
      "[72]\ttrain-auc:0.79235\teval-auc:0.83168\n",
      "[73]\ttrain-auc:0.79256\teval-auc:0.83232\n",
      "[74]\ttrain-auc:0.79266\teval-auc:0.83296\n",
      "[75]\ttrain-auc:0.79358\teval-auc:0.83072\n",
      "[76]\ttrain-auc:0.79412\teval-auc:0.83440\n",
      "[77]\ttrain-auc:0.79467\teval-auc:0.83200\n",
      "[78]\ttrain-auc:0.79504\teval-auc:0.83136\n",
      "[79]\ttrain-auc:0.79522\teval-auc:0.83136\n",
      "[80]\ttrain-auc:0.79570\teval-auc:0.83152\n",
      "[81]\ttrain-auc:0.79644\teval-auc:0.83200\n",
      "[82]\ttrain-auc:0.79642\teval-auc:0.83184\n",
      "[83]\ttrain-auc:0.79734\teval-auc:0.83648\n",
      "[84]\ttrain-auc:0.79769\teval-auc:0.83440\n",
      "[85]\ttrain-auc:0.79819\teval-auc:0.83312\n",
      "[86]\ttrain-auc:0.79832\teval-auc:0.83328\n",
      "[87]\ttrain-auc:0.79910\teval-auc:0.83616\n",
      "[88]\ttrain-auc:0.79921\teval-auc:0.83616\n",
      "[89]\ttrain-auc:0.79919\teval-auc:0.83600\n",
      "[90]\ttrain-auc:0.79958\teval-auc:0.83744\n",
      "[91]\ttrain-auc:0.80070\teval-auc:0.83696\n",
      "[92]\ttrain-auc:0.80075\teval-auc:0.83696\n",
      "[93]\ttrain-auc:0.80085\teval-auc:0.83696\n",
      "[94]\ttrain-auc:0.80160\teval-auc:0.83712\n",
      "[95]\ttrain-auc:0.80271\teval-auc:0.83520\n",
      "[96]\ttrain-auc:0.80312\teval-auc:0.83520\n",
      "[97]\ttrain-auc:0.80336\teval-auc:0.83312\n",
      "[98]\ttrain-auc:0.80356\teval-auc:0.83312\n",
      "[99]\ttrain-auc:0.80356\teval-auc:0.83312\n",
      "[100]\ttrain-auc:0.80357\teval-auc:0.83168\n",
      "0.8290499335829005\n"
     ]
    }
   ],
   "source": [
    "#Supervised model trained with dynamic features\n",
    "Cancer=['BRCA','CCRCC','COAD','GBM','HCC','HNSCC','LSCC','LUAD','OV','PDAC','UCEC'] \n",
    "col_idx=['BRCA_spc_k_s', \n",
    "         'CCRCC_spc_k_s', 'COAD_spc_k_s', 'GBM_spc_k_s',\n",
    "         'HCC_spc_k_s', 'HNSCC_spc_k_s', 'LSCC_spc_k_s', 'LUAD_spc_k_s',\n",
    "         'OV_spc_k_s', 'PDAC_spc_k_s', 'UCEC_spc_k_s', 'BRCA_spc_kactivity_s',\n",
    "         'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "         'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "         'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "         'UCEC_spc_kactivity_s']\n",
    "for i in range(10):\n",
    "    data_neg=pd.read_csv('Supplementary_table/Table_S3_negative_KSA_w_features.tsv.zip', compression='zip', sep='\\t')\n",
    "    data_neg=data_neg.drop_duplicates()\n",
    "    data_neg=data_neg.replace(0,np.nan)\n",
    "    data_neg=data_neg[~data_neg['PSSM'].isna()]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    data_train=pd.read_csv('CoPheeKSA/train/w_feature/train_'+str(i)+'.csv',index_col=0)\n",
    "    data_train=data_train.drop_duplicates()\n",
    "    data_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    \n",
    "    #Avoid overfitting caused by missing values\n",
    "    kinases=list(set(data_train.kinase))\n",
    "    data_train_neg=pd.DataFrame()\n",
    "    for k in kinases:\n",
    "        tmp=data_train[data_train.kinase==k]\n",
    "        tmp2=data_neg[data_neg.kinase==k]\n",
    "        if len(tmp2) > 10*len(tmp):\n",
    "            idx=random.sample(range(len(tmp2)),10*len(tmp))\n",
    "            tmp2=tmp2.iloc[idx]\n",
    "        \n",
    "        c=tmp[['BRCA_spc_kactivity_s',\n",
    "             'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "             'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "             'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "             'UCEC_spc_kactivity_s']].isna().sum()\n",
    "        for idx_col in ['BRCA_spc_kactivity_s',\n",
    "             'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "             'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "             'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "             'UCEC_spc_kactivity_s']:\n",
    "            num=int(c.loc[idx_col]/len(tmp)*len(tmp2))\n",
    "            idx=random.sample(tmp2.index.tolist(),num)\n",
    "            \n",
    "            for z in idx:\n",
    "                tmp2.loc[z, idx_col]=np.nan\n",
    "            \n",
    "        data_train_neg=pd.concat([data_train_neg,tmp2])\n",
    "    idx=random.sample(range(len(data_neg)),2*len(data_train))\n",
    "    data_train_neg=pd.concat([data_train_neg,data_neg.iloc[idx]])\n",
    "    data_train_neg=data_train_neg.drop_duplicates()\n",
    "    \n",
    "    idx_removed=data_train_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    data_test=pd.read_csv('CoPheeKSA/test/w_feature/test_'+str(i)+'.csv',index_col=0)\n",
    "    data_test=data_test.drop_duplicates()\n",
    "    data_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    #data_test=data_test[~data_test['PSSM'].isna()]\n",
    "    \n",
    "    idx=random.sample(range(len(data_test)),25)\n",
    "    data_eval=data_test.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_test)) if i not in idx]\n",
    "    data_test=data_test.iloc[idx2,:]\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_eval))\n",
    "    data_neg_eval=data_neg.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_neg)) if i not in idx]\n",
    "    data_neg=data_neg.iloc[idx2,:]\n",
    "    \n",
    "    data_train=data_train[col_idx]\n",
    "    data_eval=data_eval[col_idx]\n",
    "    data_test=data_test[col_idx]\n",
    "    data_neg=data_neg[col_idx]\n",
    "    data_train_neg=data_train_neg[col_idx]\n",
    "    data_neg_eval=data_neg_eval[col_idx]\n",
    "    \n",
    "    X_train=pd.concat([data_train,data_train_neg])\n",
    "    y_train=np.concatenate((np.repeat(1,len(data_train)),np.repeat(0,len(data_train_neg))))\n",
    "    X_train=X_train[col_idx]\n",
    "    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "    \n",
    "    X_eval=pd.concat([data_eval,data_neg_eval])\n",
    "    y_eval=np.concatenate((np.repeat(1,len(data_eval)),np.repeat(0,len(data_neg_eval))))\n",
    "    X_eval=X_eval[col_idx]\n",
    "    X_eval.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    deval = xgb.DMatrix(X_eval,label=y_eval)\n",
    "    param = {'max_depth': 2, 'eta': 0.2, 'objective': 'binary:logistic','verbosity':0}\n",
    "    param['nthread'] = 4\n",
    "    param['eval_metric'] = 'auc'\n",
    "    evallist = [(dtrain, 'train'),(deval, 'eval')]\n",
    "    num_round = 300\n",
    "    bst = xgb.train(param, dtrain, num_round, evallist, early_stopping_rounds=10)\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_test))\n",
    "    data_neg2=data_neg.iloc[idx,:]\n",
    "    X_test=pd.concat([data_test,data_neg2])\n",
    "    X_test=X_test[col_idx]\n",
    "    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    y_test=np.concatenate((np.repeat(1,len(data_test)),np.repeat(0,len(data_neg2))))\n",
    "    dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "    y_pred = bst.predict(dtest)\n",
    "\n",
    "    fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred)\n",
    "    auc_tmp=metrics.auc(fpr, tpr)\n",
    "    print(auc_tmp)\n",
    "    \n",
    "    auc.append(auc_tmp)\n",
    "    label.append('Dynamic_features')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "52eafdb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.60569\teval-auc:0.62240\n",
      "[1]\ttrain-auc:0.65381\teval-auc:0.71456\n",
      "[2]\ttrain-auc:0.65902\teval-auc:0.68080\n",
      "[3]\ttrain-auc:0.66070\teval-auc:0.71984\n",
      "[4]\ttrain-auc:0.66266\teval-auc:0.71320\n",
      "[5]\ttrain-auc:0.66709\teval-auc:0.73552\n",
      "[6]\ttrain-auc:0.67660\teval-auc:0.76520\n",
      "[7]\ttrain-auc:0.68039\teval-auc:0.74928\n",
      "[8]\ttrain-auc:0.68600\teval-auc:0.74024\n",
      "[9]\ttrain-auc:0.69715\teval-auc:0.76584\n",
      "[10]\ttrain-auc:0.70300\teval-auc:0.76848\n",
      "[11]\ttrain-auc:0.71063\teval-auc:0.77728\n",
      "[12]\ttrain-auc:0.71605\teval-auc:0.78800\n",
      "[13]\ttrain-auc:0.71944\teval-auc:0.80048\n",
      "[14]\ttrain-auc:0.72465\teval-auc:0.82440\n",
      "[15]\ttrain-auc:0.72729\teval-auc:0.82440\n",
      "[16]\ttrain-auc:0.73169\teval-auc:0.82920\n",
      "[17]\ttrain-auc:0.73877\teval-auc:0.83696\n",
      "[18]\ttrain-auc:0.74130\teval-auc:0.84336\n",
      "[19]\ttrain-auc:0.74523\teval-auc:0.84488\n",
      "[20]\ttrain-auc:0.75014\teval-auc:0.85928\n",
      "[21]\ttrain-auc:0.75438\teval-auc:0.86704\n",
      "[22]\ttrain-auc:0.75637\teval-auc:0.87120\n",
      "[23]\ttrain-auc:0.75834\teval-auc:0.87280\n",
      "[24]\ttrain-auc:0.75852\teval-auc:0.87704\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\ttrain-auc:0.76005\teval-auc:0.87992\n",
      "[26]\ttrain-auc:0.76098\teval-auc:0.88880\n",
      "[27]\ttrain-auc:0.76292\teval-auc:0.89184\n",
      "[28]\ttrain-auc:0.76509\teval-auc:0.89696\n",
      "[29]\ttrain-auc:0.76727\teval-auc:0.90144\n",
      "[30]\ttrain-auc:0.77019\teval-auc:0.89424\n",
      "[31]\ttrain-auc:0.77171\teval-auc:0.89120\n",
      "[32]\ttrain-auc:0.77306\teval-auc:0.89616\n",
      "[33]\ttrain-auc:0.77363\teval-auc:0.89776\n",
      "[34]\ttrain-auc:0.77533\teval-auc:0.89504\n",
      "[35]\ttrain-auc:0.77556\teval-auc:0.89344\n",
      "[36]\ttrain-auc:0.77815\teval-auc:0.89040\n",
      "[37]\ttrain-auc:0.77931\teval-auc:0.88848\n",
      "[38]\ttrain-auc:0.78099\teval-auc:0.89360\n",
      "0.8295314285714285\n",
      "[0]\ttrain-auc:0.55983\teval-auc:0.58200\n",
      "[1]\ttrain-auc:0.62951\teval-auc:0.67176\n",
      "[2]\ttrain-auc:0.65550\teval-auc:0.71832\n",
      "[3]\ttrain-auc:0.66560\teval-auc:0.72808\n",
      "[4]\ttrain-auc:0.66544\teval-auc:0.71704\n",
      "[5]\ttrain-auc:0.67276\teval-auc:0.72496\n",
      "[6]\ttrain-auc:0.68416\teval-auc:0.75616\n",
      "[7]\ttrain-auc:0.69176\teval-auc:0.75912\n",
      "[8]\ttrain-auc:0.69628\teval-auc:0.73672\n",
      "[9]\ttrain-auc:0.70493\teval-auc:0.72400\n",
      "[10]\ttrain-auc:0.70729\teval-auc:0.74360\n",
      "[11]\ttrain-auc:0.71484\teval-auc:0.78728\n",
      "[12]\ttrain-auc:0.71940\teval-auc:0.79488\n",
      "[13]\ttrain-auc:0.72499\teval-auc:0.81712\n",
      "[14]\ttrain-auc:0.72724\teval-auc:0.82656\n",
      "[15]\ttrain-auc:0.73571\teval-auc:0.82280\n",
      "[16]\ttrain-auc:0.74062\teval-auc:0.84648\n",
      "[17]\ttrain-auc:0.74436\teval-auc:0.84376\n",
      "[18]\ttrain-auc:0.74649\teval-auc:0.83824\n",
      "[19]\ttrain-auc:0.74822\teval-auc:0.83544\n",
      "[20]\ttrain-auc:0.74994\teval-auc:0.83864\n",
      "[21]\ttrain-auc:0.75370\teval-auc:0.84312\n",
      "[22]\ttrain-auc:0.75842\teval-auc:0.84504\n",
      "[23]\ttrain-auc:0.76096\teval-auc:0.84680\n",
      "[24]\ttrain-auc:0.76259\teval-auc:0.84952\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\ttrain-auc:0.76426\teval-auc:0.83776\n",
      "[26]\ttrain-auc:0.76612\teval-auc:0.84120\n",
      "[27]\ttrain-auc:0.76854\teval-auc:0.84536\n",
      "[28]\ttrain-auc:0.76949\teval-auc:0.84688\n",
      "[29]\ttrain-auc:0.77085\teval-auc:0.85472\n",
      "[30]\ttrain-auc:0.77281\teval-auc:0.85744\n",
      "[31]\ttrain-auc:0.77339\teval-auc:0.85280\n",
      "[32]\ttrain-auc:0.77460\teval-auc:0.85296\n",
      "[33]\ttrain-auc:0.77600\teval-auc:0.85136\n",
      "[34]\ttrain-auc:0.77740\teval-auc:0.85008\n",
      "[35]\ttrain-auc:0.77970\teval-auc:0.86160\n",
      "[36]\ttrain-auc:0.78096\teval-auc:0.85720\n",
      "[37]\ttrain-auc:0.78246\teval-auc:0.85848\n",
      "[38]\ttrain-auc:0.78327\teval-auc:0.85768\n",
      "[39]\ttrain-auc:0.78390\teval-auc:0.85912\n",
      "[40]\ttrain-auc:0.78742\teval-auc:0.85408\n",
      "[41]\ttrain-auc:0.78895\teval-auc:0.85776\n",
      "[42]\ttrain-auc:0.79013\teval-auc:0.85888\n",
      "[43]\ttrain-auc:0.79282\teval-auc:0.85888\n",
      "[44]\ttrain-auc:0.79449\teval-auc:0.86112\n",
      "[45]\ttrain-auc:0.79621\teval-auc:0.86464\n",
      "[46]\ttrain-auc:0.79669\teval-auc:0.86720\n",
      "[47]\ttrain-auc:0.79835\teval-auc:0.86608\n",
      "[48]\ttrain-auc:0.79998\teval-auc:0.86400\n",
      "[49]\ttrain-auc:0.80093\teval-auc:0.86512\n",
      "[50]\ttrain-auc:0.80171\teval-auc:0.86512\n",
      "[51]\ttrain-auc:0.80201\teval-auc:0.86512\n",
      "[52]\ttrain-auc:0.80298\teval-auc:0.86640\n",
      "[53]\ttrain-auc:0.80358\teval-auc:0.86720\n",
      "[54]\ttrain-auc:0.80453\teval-auc:0.86416\n",
      "[55]\ttrain-auc:0.80573\teval-auc:0.86576\n",
      "[56]\ttrain-auc:0.80641\teval-auc:0.87008\n",
      "[57]\ttrain-auc:0.80800\teval-auc:0.86336\n",
      "[58]\ttrain-auc:0.80955\teval-auc:0.85888\n",
      "[59]\ttrain-auc:0.80996\teval-auc:0.86128\n",
      "[60]\ttrain-auc:0.81012\teval-auc:0.85968\n",
      "[61]\ttrain-auc:0.81045\teval-auc:0.86144\n",
      "[62]\ttrain-auc:0.81069\teval-auc:0.86192\n",
      "[63]\ttrain-auc:0.81113\teval-auc:0.86288\n",
      "[64]\ttrain-auc:0.81207\teval-auc:0.86688\n",
      "[65]\ttrain-auc:0.81273\teval-auc:0.86528\n",
      "0.8617058367768595\n",
      "[0]\ttrain-auc:0.60471\teval-auc:0.52464\n",
      "[1]\ttrain-auc:0.64496\teval-auc:0.58120\n",
      "[2]\ttrain-auc:0.64853\teval-auc:0.58144\n",
      "[3]\ttrain-auc:0.65931\teval-auc:0.52584\n",
      "[4]\ttrain-auc:0.66488\teval-auc:0.52208\n",
      "[5]\ttrain-auc:0.67272\teval-auc:0.56032\n",
      "[6]\ttrain-auc:0.67549\teval-auc:0.53152\n",
      "[7]\ttrain-auc:0.69298\teval-auc:0.60216\n",
      "[8]\ttrain-auc:0.69681\teval-auc:0.58608\n",
      "[9]\ttrain-auc:0.70199\teval-auc:0.56672\n",
      "[10]\ttrain-auc:0.70755\teval-auc:0.59376\n",
      "[11]\ttrain-auc:0.71586\teval-auc:0.61176\n",
      "[12]\ttrain-auc:0.71806\teval-auc:0.62816\n",
      "[13]\ttrain-auc:0.72416\teval-auc:0.65184\n",
      "[14]\ttrain-auc:0.73498\teval-auc:0.67208\n",
      "[15]\ttrain-auc:0.73644\teval-auc:0.65216\n",
      "[16]\ttrain-auc:0.74182\teval-auc:0.67024\n",
      "[17]\ttrain-auc:0.74343\teval-auc:0.67632\n",
      "[18]\ttrain-auc:0.74653\teval-auc:0.67792\n",
      "[19]\ttrain-auc:0.74993\teval-auc:0.69512\n",
      "[20]\ttrain-auc:0.75258\teval-auc:0.68608\n",
      "[21]\ttrain-auc:0.75548\teval-auc:0.69128\n",
      "[22]\ttrain-auc:0.76020\teval-auc:0.69176\n",
      "[23]\ttrain-auc:0.76290\teval-auc:0.69008\n",
      "[24]\ttrain-auc:0.76449\teval-auc:0.69880\n",
      "[25]\ttrain-auc:0.76738\teval-auc:0.71592\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[26]\ttrain-auc:0.76935\teval-auc:0.71480\n",
      "[27]\ttrain-auc:0.77088\teval-auc:0.71336\n",
      "[28]\ttrain-auc:0.77272\teval-auc:0.70856\n",
      "[29]\ttrain-auc:0.77445\teval-auc:0.71104\n",
      "[30]\ttrain-auc:0.77599\teval-auc:0.71400\n",
      "[31]\ttrain-auc:0.77828\teval-auc:0.72056\n",
      "[32]\ttrain-auc:0.78043\teval-auc:0.72392\n",
      "[33]\ttrain-auc:0.78065\teval-auc:0.72448\n",
      "[34]\ttrain-auc:0.78119\teval-auc:0.72592\n",
      "[35]\ttrain-auc:0.78255\teval-auc:0.72224\n",
      "[36]\ttrain-auc:0.78532\teval-auc:0.73296\n",
      "[37]\ttrain-auc:0.78619\teval-auc:0.73360\n",
      "[38]\ttrain-auc:0.78880\teval-auc:0.73872\n",
      "[39]\ttrain-auc:0.78984\teval-auc:0.74048\n",
      "[40]\ttrain-auc:0.79073\teval-auc:0.74272\n",
      "[41]\ttrain-auc:0.79198\teval-auc:0.74816\n",
      "[42]\ttrain-auc:0.79452\teval-auc:0.75376\n",
      "[43]\ttrain-auc:0.79627\teval-auc:0.75008\n",
      "[44]\ttrain-auc:0.79709\teval-auc:0.74400\n",
      "[45]\ttrain-auc:0.79814\teval-auc:0.75024\n",
      "[46]\ttrain-auc:0.79888\teval-auc:0.74944\n",
      "[47]\ttrain-auc:0.79970\teval-auc:0.75232\n",
      "[48]\ttrain-auc:0.80082\teval-auc:0.75536\n",
      "[49]\ttrain-auc:0.80317\teval-auc:0.74848\n",
      "[50]\ttrain-auc:0.80394\teval-auc:0.75152\n",
      "[51]\ttrain-auc:0.80470\teval-auc:0.74640\n",
      "[52]\ttrain-auc:0.80501\teval-auc:0.74640\n",
      "[53]\ttrain-auc:0.80644\teval-auc:0.74736\n",
      "[54]\ttrain-auc:0.80707\teval-auc:0.74960\n",
      "[55]\ttrain-auc:0.80810\teval-auc:0.75024\n",
      "[56]\ttrain-auc:0.80894\teval-auc:0.75168\n",
      "[57]\ttrain-auc:0.80901\teval-auc:0.75152\n",
      "[58]\ttrain-auc:0.81083\teval-auc:0.75584\n",
      "[59]\ttrain-auc:0.81147\teval-auc:0.75040\n",
      "[60]\ttrain-auc:0.81199\teval-auc:0.75008\n",
      "[61]\ttrain-auc:0.81308\teval-auc:0.74784\n",
      "[62]\ttrain-auc:0.81353\teval-auc:0.75104\n",
      "[63]\ttrain-auc:0.81393\teval-auc:0.75328\n",
      "[64]\ttrain-auc:0.81489\teval-auc:0.75184\n",
      "[65]\ttrain-auc:0.81720\teval-auc:0.75872\n",
      "[66]\ttrain-auc:0.81755\teval-auc:0.75952\n",
      "[67]\ttrain-auc:0.81785\teval-auc:0.75456\n",
      "[68]\ttrain-auc:0.81835\teval-auc:0.75680\n",
      "[69]\ttrain-auc:0.81874\teval-auc:0.75840\n",
      "[70]\ttrain-auc:0.81897\teval-auc:0.75840\n",
      "[71]\ttrain-auc:0.81907\teval-auc:0.75840\n",
      "[72]\ttrain-auc:0.81943\teval-auc:0.76000\n",
      "[73]\ttrain-auc:0.81968\teval-auc:0.76016\n",
      "[74]\ttrain-auc:0.82023\teval-auc:0.76320\n",
      "[75]\ttrain-auc:0.82097\teval-auc:0.76336\n",
      "[76]\ttrain-auc:0.82123\teval-auc:0.76320\n",
      "[77]\ttrain-auc:0.82207\teval-auc:0.76224\n",
      "[78]\ttrain-auc:0.82296\teval-auc:0.76640\n",
      "[79]\ttrain-auc:0.82343\teval-auc:0.76928\n",
      "[80]\ttrain-auc:0.82425\teval-auc:0.76896\n",
      "[81]\ttrain-auc:0.82553\teval-auc:0.76912\n",
      "[82]\ttrain-auc:0.82607\teval-auc:0.77008\n",
      "[83]\ttrain-auc:0.82660\teval-auc:0.77296\n",
      "[84]\ttrain-auc:0.82702\teval-auc:0.76704\n",
      "[85]\ttrain-auc:0.82902\teval-auc:0.76736\n",
      "[86]\ttrain-auc:0.82939\teval-auc:0.76608\n",
      "[87]\ttrain-auc:0.82981\teval-auc:0.76720\n",
      "[88]\ttrain-auc:0.83042\teval-auc:0.76368\n",
      "[89]\ttrain-auc:0.83056\teval-auc:0.76384\n",
      "[90]\ttrain-auc:0.83133\teval-auc:0.75984\n",
      "[91]\ttrain-auc:0.83207\teval-auc:0.76016\n",
      "[92]\ttrain-auc:0.83239\teval-auc:0.75920\n",
      "0.8491651658984768\n",
      "[0]\ttrain-auc:0.58799\teval-auc:0.64264\n",
      "[1]\ttrain-auc:0.63546\teval-auc:0.74912\n",
      "[2]\ttrain-auc:0.64827\teval-auc:0.73832\n",
      "[3]\ttrain-auc:0.65124\teval-auc:0.75040\n",
      "[4]\ttrain-auc:0.65745\teval-auc:0.77264\n",
      "[5]\ttrain-auc:0.66642\teval-auc:0.75088\n",
      "[6]\ttrain-auc:0.67762\teval-auc:0.77456\n",
      "[7]\ttrain-auc:0.68182\teval-auc:0.75808\n",
      "[8]\ttrain-auc:0.68471\teval-auc:0.76296\n",
      "[9]\ttrain-auc:0.69483\teval-auc:0.76216\n",
      "[10]\ttrain-auc:0.69733\teval-auc:0.77232\n",
      "[11]\ttrain-auc:0.70471\teval-auc:0.80160\n",
      "[12]\ttrain-auc:0.71388\teval-auc:0.81144\n",
      "[13]\ttrain-auc:0.72561\teval-auc:0.84008\n",
      "[14]\ttrain-auc:0.72597\teval-auc:0.83544\n",
      "[15]\ttrain-auc:0.73210\teval-auc:0.83800\n",
      "[16]\ttrain-auc:0.73810\teval-auc:0.84200\n",
      "[17]\ttrain-auc:0.74299\teval-auc:0.85568\n",
      "[18]\ttrain-auc:0.74558\teval-auc:0.85312\n",
      "[19]\ttrain-auc:0.75084\teval-auc:0.86024\n",
      "[20]\ttrain-auc:0.75516\teval-auc:0.86600\n",
      "[21]\ttrain-auc:0.75637\teval-auc:0.86568\n",
      "[22]\ttrain-auc:0.76187\teval-auc:0.84936\n",
      "[23]\ttrain-auc:0.76426\teval-auc:0.85400\n",
      "[24]\ttrain-auc:0.76679\teval-auc:0.86072\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\ttrain-auc:0.76730\teval-auc:0.85720\n",
      "[26]\ttrain-auc:0.76889\teval-auc:0.85640\n",
      "[27]\ttrain-auc:0.77369\teval-auc:0.86576\n",
      "[28]\ttrain-auc:0.77467\teval-auc:0.86944\n",
      "[29]\ttrain-auc:0.77594\teval-auc:0.86704\n",
      "[30]\ttrain-auc:0.77746\teval-auc:0.86464\n",
      "[31]\ttrain-auc:0.77872\teval-auc:0.86432\n",
      "[32]\ttrain-auc:0.78022\teval-auc:0.86624\n",
      "[33]\ttrain-auc:0.78142\teval-auc:0.86896\n",
      "[34]\ttrain-auc:0.78256\teval-auc:0.86992\n",
      "[35]\ttrain-auc:0.78372\teval-auc:0.86800\n",
      "[36]\ttrain-auc:0.78538\teval-auc:0.86192\n",
      "[37]\ttrain-auc:0.78745\teval-auc:0.86024\n",
      "[38]\ttrain-auc:0.79006\teval-auc:0.86528\n",
      "[39]\ttrain-auc:0.79123\teval-auc:0.86480\n",
      "[40]\ttrain-auc:0.79245\teval-auc:0.86672\n",
      "[41]\ttrain-auc:0.79410\teval-auc:0.87184\n",
      "[42]\ttrain-auc:0.79424\teval-auc:0.86928\n",
      "[43]\ttrain-auc:0.79742\teval-auc:0.87248\n",
      "[44]\ttrain-auc:0.79749\teval-auc:0.87072\n",
      "[45]\ttrain-auc:0.80014\teval-auc:0.86832\n",
      "[46]\ttrain-auc:0.80057\teval-auc:0.86752\n",
      "[47]\ttrain-auc:0.80180\teval-auc:0.86880\n",
      "[48]\ttrain-auc:0.80361\teval-auc:0.86912\n",
      "[49]\ttrain-auc:0.80501\teval-auc:0.86944\n",
      "[50]\ttrain-auc:0.80712\teval-auc:0.87376\n",
      "[51]\ttrain-auc:0.80861\teval-auc:0.87568\n",
      "[52]\ttrain-auc:0.80902\teval-auc:0.87568\n",
      "[53]\ttrain-auc:0.80950\teval-auc:0.87376\n",
      "[54]\ttrain-auc:0.81002\teval-auc:0.87456\n",
      "[55]\ttrain-auc:0.81199\teval-auc:0.87840\n",
      "[56]\ttrain-auc:0.81278\teval-auc:0.87440\n",
      "[57]\ttrain-auc:0.81365\teval-auc:0.87696\n",
      "[58]\ttrain-auc:0.81390\teval-auc:0.87600\n",
      "[59]\ttrain-auc:0.81434\teval-auc:0.87760\n",
      "[60]\ttrain-auc:0.81601\teval-auc:0.87664\n",
      "[61]\ttrain-auc:0.81648\teval-auc:0.87872\n",
      "[62]\ttrain-auc:0.81715\teval-auc:0.87888\n",
      "[63]\ttrain-auc:0.81754\teval-auc:0.87904\n",
      "[64]\ttrain-auc:0.81884\teval-auc:0.87888\n",
      "[65]\ttrain-auc:0.81907\teval-auc:0.87984\n",
      "[66]\ttrain-auc:0.81973\teval-auc:0.88096\n",
      "[67]\ttrain-auc:0.81979\teval-auc:0.87984\n",
      "[68]\ttrain-auc:0.82123\teval-auc:0.88176\n",
      "[69]\ttrain-auc:0.82123\teval-auc:0.88176\n",
      "[70]\ttrain-auc:0.82211\teval-auc:0.88208\n",
      "[71]\ttrain-auc:0.82218\teval-auc:0.88208\n",
      "[72]\ttrain-auc:0.82229\teval-auc:0.88288\n",
      "[73]\ttrain-auc:0.82429\teval-auc:0.88720\n",
      "[74]\ttrain-auc:0.82491\teval-auc:0.88832\n",
      "[75]\ttrain-auc:0.82558\teval-auc:0.88848\n",
      "[76]\ttrain-auc:0.82593\teval-auc:0.88976\n",
      "[77]\ttrain-auc:0.82618\teval-auc:0.88928\n",
      "[78]\ttrain-auc:0.82646\teval-auc:0.89056\n",
      "[79]\ttrain-auc:0.82744\teval-auc:0.88944\n",
      "[80]\ttrain-auc:0.82846\teval-auc:0.89104\n",
      "[81]\ttrain-auc:0.82950\teval-auc:0.89264\n",
      "[82]\ttrain-auc:0.82962\teval-auc:0.89136\n",
      "[83]\ttrain-auc:0.83005\teval-auc:0.89136\n",
      "[84]\ttrain-auc:0.83037\teval-auc:0.89040\n",
      "[85]\ttrain-auc:0.83044\teval-auc:0.88992\n",
      "[86]\ttrain-auc:0.83076\teval-auc:0.88928\n",
      "[87]\ttrain-auc:0.83194\teval-auc:0.89328\n",
      "[88]\ttrain-auc:0.83232\teval-auc:0.89376\n",
      "[89]\ttrain-auc:0.83310\teval-auc:0.89520\n",
      "[90]\ttrain-auc:0.83331\teval-auc:0.89296\n",
      "[91]\ttrain-auc:0.83431\teval-auc:0.89168\n",
      "[92]\ttrain-auc:0.83439\teval-auc:0.89072\n",
      "[93]\ttrain-auc:0.83520\teval-auc:0.89008\n",
      "[94]\ttrain-auc:0.83565\teval-auc:0.89072\n",
      "[95]\ttrain-auc:0.83585\teval-auc:0.89056\n",
      "[96]\ttrain-auc:0.83620\teval-auc:0.89056\n",
      "[97]\ttrain-auc:0.83627\teval-auc:0.89184\n",
      "[98]\ttrain-auc:0.83816\teval-auc:0.88992\n",
      "0.8548669387755101\n",
      "[0]\ttrain-auc:0.55328\teval-auc:0.53008\n",
      "[1]\ttrain-auc:0.61702\teval-auc:0.62344\n",
      "[2]\ttrain-auc:0.65594\teval-auc:0.68640\n",
      "[3]\ttrain-auc:0.66568\teval-auc:0.68128\n",
      "[4]\ttrain-auc:0.66917\teval-auc:0.68232\n",
      "[5]\ttrain-auc:0.67883\teval-auc:0.71072\n",
      "[6]\ttrain-auc:0.68770\teval-auc:0.71824\n",
      "[7]\ttrain-auc:0.68971\teval-auc:0.69704\n",
      "[8]\ttrain-auc:0.69140\teval-auc:0.69784\n",
      "[9]\ttrain-auc:0.70260\teval-auc:0.74344\n",
      "[10]\ttrain-auc:0.70966\teval-auc:0.76216\n",
      "[11]\ttrain-auc:0.71213\teval-auc:0.77144\n",
      "[12]\ttrain-auc:0.71823\teval-auc:0.77840\n",
      "[13]\ttrain-auc:0.72197\teval-auc:0.79760\n",
      "[14]\ttrain-auc:0.72899\teval-auc:0.80504\n",
      "[15]\ttrain-auc:0.73577\teval-auc:0.80872\n",
      "[16]\ttrain-auc:0.74025\teval-auc:0.80392\n",
      "[17]\ttrain-auc:0.74291\teval-auc:0.80728\n",
      "[18]\ttrain-auc:0.74677\teval-auc:0.80144\n",
      "[19]\ttrain-auc:0.74762\teval-auc:0.80712\n",
      "[20]\ttrain-auc:0.75097\teval-auc:0.80376\n",
      "[21]\ttrain-auc:0.75482\teval-auc:0.81248\n",
      "[22]\ttrain-auc:0.75741\teval-auc:0.82016\n",
      "[23]\ttrain-auc:0.76048\teval-auc:0.82424\n",
      "[24]\ttrain-auc:0.76249\teval-auc:0.83096\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[25]\ttrain-auc:0.76383\teval-auc:0.82944\n",
      "[26]\ttrain-auc:0.76729\teval-auc:0.82608\n",
      "[27]\ttrain-auc:0.77273\teval-auc:0.84000\n",
      "[28]\ttrain-auc:0.77409\teval-auc:0.84272\n",
      "[29]\ttrain-auc:0.77615\teval-auc:0.84936\n",
      "[30]\ttrain-auc:0.77659\teval-auc:0.85336\n",
      "[31]\ttrain-auc:0.77811\teval-auc:0.85880\n",
      "[32]\ttrain-auc:0.77893\teval-auc:0.85688\n",
      "[33]\ttrain-auc:0.78071\teval-auc:0.85912\n",
      "[34]\ttrain-auc:0.78253\teval-auc:0.85792\n",
      "[35]\ttrain-auc:0.78354\teval-auc:0.85984\n",
      "[36]\ttrain-auc:0.78591\teval-auc:0.86560\n",
      "[37]\ttrain-auc:0.78789\teval-auc:0.87104\n",
      "[38]\ttrain-auc:0.78903\teval-auc:0.87504\n",
      "[39]\ttrain-auc:0.79012\teval-auc:0.87392\n",
      "[40]\ttrain-auc:0.79080\teval-auc:0.87584\n",
      "[41]\ttrain-auc:0.79123\teval-auc:0.87824\n",
      "[42]\ttrain-auc:0.79288\teval-auc:0.87392\n",
      "[43]\ttrain-auc:0.79408\teval-auc:0.87584\n",
      "[44]\ttrain-auc:0.79508\teval-auc:0.87280\n",
      "[45]\ttrain-auc:0.79586\teval-auc:0.87360\n",
      "[46]\ttrain-auc:0.79641\teval-auc:0.86944\n",
      "[47]\ttrain-auc:0.79730\teval-auc:0.87456\n",
      "[48]\ttrain-auc:0.79788\teval-auc:0.87488\n",
      "[49]\ttrain-auc:0.79905\teval-auc:0.87392\n",
      "[50]\ttrain-auc:0.79970\teval-auc:0.86736\n",
      "0.8445462962962963\n",
      "[0]\ttrain-auc:0.60797\teval-auc:0.62840\n",
      "[1]\ttrain-auc:0.62921\teval-auc:0.62056\n",
      "[2]\ttrain-auc:0.65157\teval-auc:0.68408\n",
      "[3]\ttrain-auc:0.65360\teval-auc:0.69464\n",
      "[4]\ttrain-auc:0.65734\teval-auc:0.68880\n",
      "[5]\ttrain-auc:0.66584\teval-auc:0.72416\n",
      "[6]\ttrain-auc:0.67307\teval-auc:0.74704\n",
      "[7]\ttrain-auc:0.68438\teval-auc:0.73072\n",
      "[8]\ttrain-auc:0.69438\teval-auc:0.71952\n",
      "[9]\ttrain-auc:0.70077\teval-auc:0.71744\n",
      "[10]\ttrain-auc:0.70438\teval-auc:0.72744\n",
      "[11]\ttrain-auc:0.70879\teval-auc:0.72248\n",
      "[12]\ttrain-auc:0.71173\teval-auc:0.73536\n",
      "[13]\ttrain-auc:0.71966\teval-auc:0.73976\n",
      "[14]\ttrain-auc:0.72969\teval-auc:0.74488\n",
      "[15]\ttrain-auc:0.73441\teval-auc:0.74072\n",
      "[16]\ttrain-auc:0.73817\teval-auc:0.73144\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.786147339876033\n",
      "[0]\ttrain-auc:0.60258\teval-auc:0.59952\n",
      "[1]\ttrain-auc:0.63607\teval-auc:0.57376\n",
      "[2]\ttrain-auc:0.64984\teval-auc:0.62424\n",
      "[3]\ttrain-auc:0.66199\teval-auc:0.59208\n",
      "[4]\ttrain-auc:0.66481\teval-auc:0.58144\n",
      "[5]\ttrain-auc:0.67322\teval-auc:0.64328\n",
      "[6]\ttrain-auc:0.68573\teval-auc:0.68320\n",
      "[7]\ttrain-auc:0.68450\teval-auc:0.64328\n",
      "[8]\ttrain-auc:0.69160\teval-auc:0.65488\n",
      "[9]\ttrain-auc:0.69696\teval-auc:0.67096\n",
      "[10]\ttrain-auc:0.70278\teval-auc:0.68200\n",
      "[11]\ttrain-auc:0.70942\teval-auc:0.68768\n",
      "[12]\ttrain-auc:0.71904\teval-auc:0.70704\n",
      "[13]\ttrain-auc:0.72093\teval-auc:0.71680\n",
      "[14]\ttrain-auc:0.72717\teval-auc:0.72312\n",
      "[15]\ttrain-auc:0.72728\teval-auc:0.71232\n",
      "[16]\ttrain-auc:0.73470\teval-auc:0.73080\n",
      "[17]\ttrain-auc:0.73748\teval-auc:0.74232\n",
      "[18]\ttrain-auc:0.73950\teval-auc:0.75000\n",
      "[19]\ttrain-auc:0.74313\teval-auc:0.75896\n",
      "[20]\ttrain-auc:0.74476\teval-auc:0.76216\n",
      "[21]\ttrain-auc:0.74851\teval-auc:0.76120\n",
      "[22]\ttrain-auc:0.75084\teval-auc:0.77432\n",
      "[23]\ttrain-auc:0.75421\teval-auc:0.77936\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[24]\ttrain-auc:0.75488\teval-auc:0.78968\n",
      "[25]\ttrain-auc:0.75754\teval-auc:0.79096\n",
      "[26]\ttrain-auc:0.75940\teval-auc:0.78328\n",
      "[27]\ttrain-auc:0.76107\teval-auc:0.78488\n",
      "[28]\ttrain-auc:0.76259\teval-auc:0.78680\n",
      "[29]\ttrain-auc:0.76406\teval-auc:0.79456\n",
      "[30]\ttrain-auc:0.76709\teval-auc:0.79184\n",
      "[31]\ttrain-auc:0.76835\teval-auc:0.80624\n",
      "[32]\ttrain-auc:0.77169\teval-auc:0.81432\n",
      "[33]\ttrain-auc:0.77346\teval-auc:0.81336\n",
      "[34]\ttrain-auc:0.77458\teval-auc:0.81608\n",
      "[35]\ttrain-auc:0.77537\teval-auc:0.81624\n",
      "[36]\ttrain-auc:0.77625\teval-auc:0.81256\n",
      "[37]\ttrain-auc:0.77880\teval-auc:0.81544\n",
      "[38]\ttrain-auc:0.77983\teval-auc:0.81688\n",
      "[39]\ttrain-auc:0.78248\teval-auc:0.82136\n",
      "[40]\ttrain-auc:0.78366\teval-auc:0.82232\n",
      "[41]\ttrain-auc:0.78502\teval-auc:0.82280\n",
      "[42]\ttrain-auc:0.78727\teval-auc:0.82888\n",
      "[43]\ttrain-auc:0.78882\teval-auc:0.83416\n",
      "[44]\ttrain-auc:0.79049\teval-auc:0.83496\n",
      "[45]\ttrain-auc:0.79145\teval-auc:0.82608\n",
      "[46]\ttrain-auc:0.79192\teval-auc:0.82688\n",
      "[47]\ttrain-auc:0.79240\teval-auc:0.82720\n",
      "[48]\ttrain-auc:0.79354\teval-auc:0.83488\n",
      "[49]\ttrain-auc:0.79424\teval-auc:0.83520\n",
      "[50]\ttrain-auc:0.79487\teval-auc:0.83312\n",
      "[51]\ttrain-auc:0.79671\teval-auc:0.83856\n",
      "[52]\ttrain-auc:0.79760\teval-auc:0.84496\n",
      "[53]\ttrain-auc:0.79845\teval-auc:0.84560\n",
      "[54]\ttrain-auc:0.80018\teval-auc:0.84832\n",
      "[55]\ttrain-auc:0.80143\teval-auc:0.84688\n",
      "[56]\ttrain-auc:0.80164\teval-auc:0.84576\n",
      "[57]\ttrain-auc:0.80207\teval-auc:0.84448\n",
      "[58]\ttrain-auc:0.80311\teval-auc:0.84416\n",
      "[59]\ttrain-auc:0.80404\teval-auc:0.84496\n",
      "[60]\ttrain-auc:0.80493\teval-auc:0.84624\n",
      "[61]\ttrain-auc:0.80541\teval-auc:0.84544\n",
      "[62]\ttrain-auc:0.80543\teval-auc:0.84688\n",
      "[63]\ttrain-auc:0.80665\teval-auc:0.84736\n",
      "0.8495970405601796\n",
      "[0]\ttrain-auc:0.56060\teval-auc:0.57960\n",
      "[1]\ttrain-auc:0.61665\teval-auc:0.69272\n",
      "[2]\ttrain-auc:0.65244\teval-auc:0.79680\n",
      "[3]\ttrain-auc:0.65896\teval-auc:0.78408\n",
      "[4]\ttrain-auc:0.65858\teval-auc:0.77520\n",
      "[5]\ttrain-auc:0.66876\teval-auc:0.80600\n",
      "[6]\ttrain-auc:0.67311\teval-auc:0.81592\n",
      "[7]\ttrain-auc:0.68147\teval-auc:0.80424\n",
      "[8]\ttrain-auc:0.68378\teval-auc:0.81208\n",
      "[9]\ttrain-auc:0.68913\teval-auc:0.81216\n",
      "[10]\ttrain-auc:0.69642\teval-auc:0.82520\n",
      "[11]\ttrain-auc:0.70407\teval-auc:0.83632\n",
      "[12]\ttrain-auc:0.71337\teval-auc:0.84192\n",
      "[13]\ttrain-auc:0.71868\teval-auc:0.84272\n",
      "[14]\ttrain-auc:0.72801\teval-auc:0.86040\n",
      "[15]\ttrain-auc:0.73442\teval-auc:0.86952\n",
      "[16]\ttrain-auc:0.74041\teval-auc:0.86872\n",
      "[17]\ttrain-auc:0.74235\teval-auc:0.87136\n",
      "[18]\ttrain-auc:0.74767\teval-auc:0.87216\n",
      "[19]\ttrain-auc:0.75052\teval-auc:0.87056\n",
      "[20]\ttrain-auc:0.75305\teval-auc:0.86336\n",
      "[21]\ttrain-auc:0.75681\teval-auc:0.86768\n",
      "[22]\ttrain-auc:0.76018\teval-auc:0.86280\n",
      "[23]\ttrain-auc:0.75999\teval-auc:0.86424\n",
      "[24]\ttrain-auc:0.76220\teval-auc:0.86168\n",
      "[25]\ttrain-auc:0.76254\teval-auc:0.86384\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[26]\ttrain-auc:0.76489\teval-auc:0.85824\n",
      "[27]\ttrain-auc:0.76724\teval-auc:0.86528\n",
      "[28]\ttrain-auc:0.76857\teval-auc:0.86544\n",
      "0.8307817451335446\n",
      "[0]\ttrain-auc:0.60677\teval-auc:0.63032\n",
      "[1]\ttrain-auc:0.63765\teval-auc:0.68656\n",
      "[2]\ttrain-auc:0.65073\teval-auc:0.73672\n",
      "[3]\ttrain-auc:0.65818\teval-auc:0.73728\n",
      "[4]\ttrain-auc:0.67215\teval-auc:0.73736\n",
      "[5]\ttrain-auc:0.68369\teval-auc:0.77560\n",
      "[6]\ttrain-auc:0.68685\teval-auc:0.78360\n",
      "[7]\ttrain-auc:0.69216\teval-auc:0.79128\n",
      "[8]\ttrain-auc:0.69748\teval-auc:0.80256\n",
      "[9]\ttrain-auc:0.70310\teval-auc:0.81616\n",
      "[10]\ttrain-auc:0.70361\teval-auc:0.80224\n",
      "[11]\ttrain-auc:0.71200\teval-auc:0.82136\n",
      "[12]\ttrain-auc:0.71868\teval-auc:0.82312\n",
      "[13]\ttrain-auc:0.72833\teval-auc:0.83728\n",
      "[14]\ttrain-auc:0.73347\teval-auc:0.83472\n",
      "[15]\ttrain-auc:0.73956\teval-auc:0.83672\n",
      "[16]\ttrain-auc:0.74468\teval-auc:0.84856\n",
      "[17]\ttrain-auc:0.74439\teval-auc:0.84776\n",
      "[18]\ttrain-auc:0.74372\teval-auc:0.84928\n",
      "[19]\ttrain-auc:0.74802\teval-auc:0.85304\n",
      "[20]\ttrain-auc:0.75466\teval-auc:0.85672\n",
      "[21]\ttrain-auc:0.75770\teval-auc:0.85800\n",
      "[22]\ttrain-auc:0.75914\teval-auc:0.85944\n",
      "[23]\ttrain-auc:0.76043\teval-auc:0.86536\n",
      "[24]\ttrain-auc:0.76352\teval-auc:0.86824\n",
      "[25]\ttrain-auc:0.76530\teval-auc:0.86696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[26]\ttrain-auc:0.76741\teval-auc:0.86488\n",
      "[27]\ttrain-auc:0.76954\teval-auc:0.85752\n",
      "[28]\ttrain-auc:0.77076\teval-auc:0.86536\n",
      "[29]\ttrain-auc:0.77479\teval-auc:0.87504\n",
      "[30]\ttrain-auc:0.77577\teval-auc:0.86976\n",
      "[31]\ttrain-auc:0.77744\teval-auc:0.87184\n",
      "[32]\ttrain-auc:0.77843\teval-auc:0.87424\n",
      "[33]\ttrain-auc:0.77843\teval-auc:0.87504\n",
      "[34]\ttrain-auc:0.78016\teval-auc:0.87464\n",
      "[35]\ttrain-auc:0.78289\teval-auc:0.86848\n",
      "[36]\ttrain-auc:0.78504\teval-auc:0.86896\n",
      "[37]\ttrain-auc:0.78631\teval-auc:0.87168\n",
      "[38]\ttrain-auc:0.78738\teval-auc:0.86768\n",
      "0.8199197530864197\n",
      "[0]\ttrain-auc:0.60541\teval-auc:0.57160\n",
      "[1]\ttrain-auc:0.63057\teval-auc:0.62976\n",
      "[2]\ttrain-auc:0.65869\teval-auc:0.73448\n",
      "[3]\ttrain-auc:0.65389\teval-auc:0.69840\n",
      "[4]\ttrain-auc:0.65699\teval-auc:0.69384\n",
      "[5]\ttrain-auc:0.66052\teval-auc:0.71832\n",
      "[6]\ttrain-auc:0.67525\teval-auc:0.72840\n",
      "[7]\ttrain-auc:0.68593\teval-auc:0.75136\n",
      "[8]\ttrain-auc:0.69539\teval-auc:0.73136\n",
      "[9]\ttrain-auc:0.70249\teval-auc:0.74976\n",
      "[10]\ttrain-auc:0.71198\teval-auc:0.75280\n",
      "[11]\ttrain-auc:0.71988\teval-auc:0.77416\n",
      "[12]\ttrain-auc:0.72721\teval-auc:0.77888\n",
      "[13]\ttrain-auc:0.72815\teval-auc:0.78328\n",
      "[14]\ttrain-auc:0.73233\teval-auc:0.78336\n",
      "[15]\ttrain-auc:0.73796\teval-auc:0.80024\n",
      "[16]\ttrain-auc:0.73889\teval-auc:0.80584\n",
      "[17]\ttrain-auc:0.74357\teval-auc:0.80464\n",
      "[18]\ttrain-auc:0.74732\teval-auc:0.80576\n",
      "[19]\ttrain-auc:0.74989\teval-auc:0.80704\n",
      "[20]\ttrain-auc:0.75503\teval-auc:0.80344\n",
      "[21]\ttrain-auc:0.75751\teval-auc:0.80680\n",
      "[22]\ttrain-auc:0.76120\teval-auc:0.81504\n",
      "[23]\ttrain-auc:0.76217\teval-auc:0.81136\n",
      "[24]\ttrain-auc:0.76345\teval-auc:0.81616\n",
      "[25]\ttrain-auc:0.76448\teval-auc:0.81648\n",
      "[26]\ttrain-auc:0.76582\teval-auc:0.82160\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/klml322/anaconda3/lib/python3.10/site-packages/xgboost/core.py:617: FutureWarning: Pass `evals` as keyword args.\n",
      "  warnings.warn(msg, FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[27]\ttrain-auc:0.76959\teval-auc:0.80864\n",
      "[28]\ttrain-auc:0.77222\teval-auc:0.81552\n",
      "[29]\ttrain-auc:0.77326\teval-auc:0.81792\n",
      "[30]\ttrain-auc:0.77485\teval-auc:0.81536\n",
      "[31]\ttrain-auc:0.77538\teval-auc:0.80720\n",
      "[32]\ttrain-auc:0.77857\teval-auc:0.80400\n",
      "[33]\ttrain-auc:0.78008\teval-auc:0.80672\n",
      "[34]\ttrain-auc:0.78166\teval-auc:0.80400\n",
      "[35]\ttrain-auc:0.78247\teval-auc:0.80816\n",
      "[36]\ttrain-auc:0.78526\teval-auc:0.80704\n",
      "0.8352327617437506\n"
     ]
    }
   ],
   "source": [
    "#Supervised model trained with dynamic features + CoPheeMap info + KMap info\n",
    "Cancer=['BRCA','CCRCC','COAD','GBM','HCC','HNSCC','LSCC','LUAD','OV','PDAC','UCEC'] \n",
    "col_idx=['cophee_emb0', 'cophee_emb1', 'cophee_emb2', 'cophee_emb3', 'cophee_emb4', \n",
    "         'cophee_emb5', 'cophee_emb6', 'cophee_emb7', 'cophee_emb8', 'cophee_emb9', \n",
    "         'cophee_emb10','cophee_emb11', 'cophee_emb12', 'cophee_emb13', 'cophee_emb14', \n",
    "         'cophee_emb15', 'k_emb0', 'k_emb1', 'k_emb2', 'k_emb3','k_emb4', 'k_emb5', \n",
    "         'k_emb6', 'k_emb7', 'k_emb8', 'k_emb9', 'k_emb10',\n",
    "         'k_emb11', 'k_emb12', 'k_emb13', 'k_emb14', 'k_emb15','BRCA_spc_k_s', \n",
    "         'CCRCC_spc_k_s', 'COAD_spc_k_s', 'GBM_spc_k_s',\n",
    "         'HCC_spc_k_s', 'HNSCC_spc_k_s', 'LSCC_spc_k_s', 'LUAD_spc_k_s',\n",
    "         'OV_spc_k_s', 'PDAC_spc_k_s', 'UCEC_spc_k_s', 'BRCA_spc_kactivity_s',\n",
    "         'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "         'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "         'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "         'UCEC_spc_kactivity_s']\n",
    "for i in range(10):\n",
    "    data_train=pd.read_csv('CoPheeKSA/train/w_feature/train_'+str(i)+'.csv',index_col=0)\n",
    "    data_train=data_train.drop_duplicates()\n",
    "    data_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    \n",
    "    #Avoid overfitting caused by missing values\n",
    "    data_neg=pd.read_csv('Supplementary_table/Table_S3_negative_KSA_w_features.tsv.zip', compression='zip', sep='\\t')\n",
    "    data_neg=data_neg.drop_duplicates()\n",
    "    \n",
    "    kinases=list(set(data_train.kinase))\n",
    "    data_train_neg=pd.DataFrame()\n",
    "    for k in kinases:\n",
    "        tmp=data_train[data_train.kinase==k]\n",
    "        tmp2=data_neg[data_neg.kinase==k]\n",
    "        if len(tmp2) > 10*len(tmp):\n",
    "            idx=random.sample(range(len(tmp2)),10*len(tmp))\n",
    "            tmp2=tmp2.iloc[idx]\n",
    "        \n",
    "        c=tmp[['BRCA_spc_kactivity_s',\n",
    "             'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "             'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "             'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "             'UCEC_spc_kactivity_s']].isna().sum()\n",
    "        for idx_col in ['BRCA_spc_kactivity_s',\n",
    "             'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "             'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "             'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "             'UCEC_spc_kactivity_s']:\n",
    "            num=int(c.loc[idx_col]/len(tmp)*len(tmp2))\n",
    "            idx=random.sample(tmp2.index.tolist(),num)\n",
    "            \n",
    "            for z in idx:\n",
    "                tmp2.loc[z, idx_col]=np.nan\n",
    "            \n",
    "        data_train_neg=pd.concat([data_train_neg,tmp2])\n",
    "    idx=random.sample(range(len(data_neg)),2*len(data_train))\n",
    "    data_train_neg=pd.concat([data_train_neg,data_neg.iloc[idx]])\n",
    "    data_train_neg=data_train_neg.drop_duplicates()\n",
    "    \n",
    "    idx_removed=data_train_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    data_test=pd.read_csv('CoPheeKSA/test/w_feature/test_'+str(i)+'.csv',index_col=0)\n",
    "    data_test=data_test.drop_duplicates()\n",
    "    data_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    #data_test=data_test[~data_test['PSSM'].isna()]\n",
    "    \n",
    "    idx=random.sample(range(len(data_test)),25)\n",
    "    data_eval=data_test.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_test)) if i not in idx]\n",
    "    data_test=data_test.iloc[idx2,:]\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_eval))\n",
    "    data_neg_eval=data_neg.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_neg)) if i not in idx]\n",
    "    data_neg=data_neg.iloc[idx2,:]\n",
    "    \n",
    "    \n",
    "    data_train=data_train[col_idx]\n",
    "    data_eval=data_eval[col_idx]\n",
    "    data_test=data_test[col_idx]\n",
    "    data_neg=data_neg[col_idx]\n",
    "    data_train_neg=data_train_neg[col_idx]\n",
    "    data_neg_eval=data_neg_eval[col_idx]\n",
    "    \n",
    "    X_train=pd.concat([data_train,data_train_neg])\n",
    "    y_train=np.concatenate((np.repeat(1,len(data_train)),np.repeat(0,len(data_train_neg))))\n",
    "    X_train=X_train[col_idx]\n",
    "    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    #X_train=X_train.replace(np.nan,0)\n",
    "    dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "    \n",
    "    X_eval=pd.concat([data_eval,data_neg_eval])\n",
    "    y_eval=np.concatenate((np.repeat(1,len(data_eval)),np.repeat(0,len(data_neg_eval))))\n",
    "    X_eval=X_eval[col_idx]\n",
    "    X_eval.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    deval = xgb.DMatrix(X_eval,label=y_eval)\n",
    "    param = {'max_depth': 2, 'eta': 0.2, 'objective': 'binary:logistic','verbosity':0}\n",
    "    param['nthread'] = 4\n",
    "    param['eval_metric'] = 'auc'\n",
    "    evallist = [(dtrain, 'train'),(deval, 'eval')]\n",
    "    num_round = 300\n",
    "    bst = xgb.train(param, dtrain, num_round, evallist, early_stopping_rounds=10)\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_test))\n",
    "    data_neg2=data_neg.iloc[idx,:]\n",
    "    X_test=pd.concat([data_test,data_neg2])\n",
    "    X_test=X_test[col_idx]\n",
    "    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    y_test=np.concatenate((np.repeat(1,len(data_test)),np.repeat(0,len(data_neg2))))\n",
    "    dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "    y_pred = bst.predict(dtest)\n",
    "\n",
    "    fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred)\n",
    "    auc_tmp=metrics.auc(fpr, tpr)\n",
    "    print(auc_tmp)\n",
    "    \n",
    "    auc.append(auc_tmp)\n",
    "    label.append('Dynamic_features+CoPheeMap+KMap')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "bdf10b11",
   "metadata": {},
   "outputs": [],
   "source": [
    "auc_results={}\n",
    "auc_results['label']=np.array(label)\n",
    "auc_results['AUROC']=np.array(auc)\n",
    "auc_results=pd.DataFrame(auc_results)\n",
    "\n",
    "#auc_results.to_csv('CoPheeKSA/auc_results.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "8dea851c",
   "metadata": {},
   "outputs": [],
   "source": [
    "auc_results=pd.read_csv('CoPheeKSA/auc_results.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "9f16f68c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp=pd.concat([auc_results[auc_results['label']=='PSSM'],\n",
    "               auc_results[auc_results['label']=='CoPheeMap+KMap'],\n",
    "               auc_results[auc_results['label']=='PSSM+CoPheeMap+KMap'],\n",
    "               auc_results[auc_results['label']=='Dynamic_features'],\n",
    "               auc_results[auc_results['label']=='Dynamic_features+CoPheeMap+KMap'],\n",
    "               auc_results[auc_results['label']=='CoPheeKSA']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f9289faa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'CoPheeKSA',\n",
       " 'CoPheeMap+KMap',\n",
       " 'Dynamic_features',\n",
       " 'Dynamic_features+CoPheeMap+KMap',\n",
       " 'PSSM',\n",
       " 'PSSM+CoPheeMap+KMap'}"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(auc_results.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "4cca2d34",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "p-value annotation legend:\n",
      "ns: 5.00e-02 < p <= 1.00e+00\n",
      "*: 1.00e-02 < p <= 5.00e-02\n",
      "**: 1.00e-03 < p <= 1.00e-02\n",
      "***: 1.00e-04 < p <= 1.00e-03\n",
      "****: p <= 1.00e-04\n",
      "\n",
      "Dynamic_features v.s. Dynamic_features+CoPheeMap+KMap: t-test independent samples with Bonferroni correction, P_val=6.786e-02 stat=-2.631e+00\n",
      "Dynamic_features+CoPheeMap+KMap v.s. CoPheeKSA: t-test independent samples with Bonferroni correction, P_val=3.026e-10 stat=-1.348e+01\n",
      "PSSM v.s. PSSM+CoPheeMap+KMap: t-test independent samples with Bonferroni correction, P_val=6.725e-03 stat=-3.688e+00\n",
      "PSSM+CoPheeMap+KMap v.s. CoPheeKSA: t-test independent samples with Bonferroni correction, P_val=5.760e-04 stat=-4.798e+00\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs0AAAJ7CAYAAAAY3KzpAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy88F64QAAAACXBIWXMAAA9hAAAPYQGoP6dpAADWWklEQVR4nOzdeVxU1fsH8OewzYCyCaigIO6aKe6m4pZr7loqalpumfvWV8vKJTPL1DSXslwqzVRySdRcE819S61ccl8wF1wAlQGBz+8PfnObkWVAwRng8369eIn33jlz5l7m3uee+5xzFAAIERERERGlyc7aFSAiIiIisnUMmomIiIiILGDQTERERERkAYNmIiIiIiILGDQTEREREVnAoJmIiIiIyAIGzUREREREFjBoJiIiIiKygEEzEREREZEFDJqJiIiIiCxg0ExERJSDNGzYMFPLLa0jooxxsHYFiIiIKH179uyR2NhYadKkibZs27Zt4uzsnOZyEUlzXd26dZ9f5YlyCQUA1q4EERERpe3q1asycuRI8fHxkQMHDkitWrXk7t278tlnn8k777yTYvn06dMlKSkp1ddMnz5dihQpYu2PRJTjMD2DiIjIxvn7+0toaKi4u7vL0aNHxcPDQ5YvXy7FihVLdXmRIkXSfA0DZqKnw6CZiIjIxkVEREhISIjcv39fqlatKvfu3ZOQkJB0l6e3jogyj+kZRERENs40p7lhw4YSHh6eIqf5yeUikuY65jQTZR47AhIREdm41IJc0w5+GVluaR0RpY8tzUREREREFjCnmYiIiIjIAgbNREREREQWMGgmIiIiIrKAQTMRERERkQUcPYOIKBsBEIPBYO1qEBFlKb1eL0opa1fjuWLQTESUTQBInz595MSJE9auChFRlgoKCpIFCxbkqcCZ6RlERNnEYDAwYCaiXOn48eN57ikaW5qJiJ6DLVu2aLO0ERHlVLGxsdKsWTNrV8MqGDQTET0Hzs7ODJqJiHIwpmcQEREREVnAoJmIiIiIyAIGzUREREREFjBoJiIiIiKyQAGAtStBRJQbmU5skhcnAiCi3Ccvn9cYNBMRERERWcD0DCIiIiIiCxg0ExERERFZwKCZiIiIiMgCBs1ERERERBYwaCYiIiIisoBBMxERERGRBQyaiYiIiIgsYNBMRERERGQBg2YiIiKip9SwYcNMLX+WdWRdDtauABEREVFOsmfPHomNjZUmTZpoy7Zt2ybOzs5pLheRp1pXt27d7P44lEGcRpuIiIgoE65evSojR44UHx8fOXDggNSqVUvu3r0rn332mbzzzjsplk+fPl2SkpJSfY2ldUWKFLH2x6X/x/QMIiIiokzw9/eX0NBQcXd3l6NHj4qHh4csX75cihUrluryIkWKpPkaS+vIdjBoJiIiIsqEiIgICQkJkfv370vVqlXl3r17EhISku7yp11HtoPpGURERESZYJrT3LBhQwkPD0+R0/zkchF5qnXMabYd7AhIRERElAmpBbKmnfgysvxZ1pF1sKWZiIiIiMgC5jQTEREREVnAoJmIiIiIyAIGzUREREREFjBoJiIiIiKygKNnEBERkc0BIAaDwdrVoDTo9XpRSlm7Gs8Vg2YiIiKyKQCkT58+cuLECWtXhdIQFBQkCxYsyFOBM9MziIiIyKYYDAYGzDbu+PHjee5JAFuaiYiIyGZt2bJFmzWPrC82NlaaNWtm7WpYBYNmIiIislnOzs4MmskmMD2DiIiIiMgCBs1ERERERBYwaCYiIiIisoBBMxERERGRBQoArF0JIiIiIiPTiU3y4iQatiwvHxsGzUREREREFjA9g4iIiIjIAgbNREREREQWMGgmIiIiIrKAQTMRERERkQUMmomIiIiILGDQTERERERkAYNmIiIiIiILGDQTEREREVnAoJkoh2rYsGGmlhMREdHTc7B2BYgo4/bs2SOxsbHSpEkTbdm2bdvE2dk5zeV169a1RlWJiIhyFbY0E+UgAQEBMn/+fBk4cKDExMTIwIEDZcGCBVK0aNFUlwcGBlq7ykSUxRo2bChDhw6V0aNHS4ECBaRw4cIyYcIEbf2ECRMkICBAdDqd+Pn5ydChQ61XWaJchEEzUQ7i7+8voaGh4u7uLkePHhUPDw9Zvny5FCtWLNXlRYoUsXaViSgbfP/995IvXz45cOCATJ06VT766CPZunWr/Pzzz/LFF1/I/Pnz5ezZs7J27VqpWLGitatLlCsoALB2JYgoYyIiImTUqFHi6ekpBw8elJo1a8q9e/dk+vTpaS5n4EyUuzRs2FASExPl999/15bVrFlTXn75ZSlYsKDMnz9f/vrrL3F0dLRiLYlyH7Y0E+Ugly5dkr59+8pXX30lrq6u8tVXX0nfvn3TXU5EuU+lSpXM/u/r6yu3bt2STp06SWxsrJQoUUL69esna9askYSEBCvVkih3YUdAohwktU59pp3/MrKciHK+J1uRlVKSlJQk/v7+cubMGdm6dats27ZNBg4cKJ9//rns3LmTLc9Ez4gtzUQ5VHh4eKaWE1He4OzsLG3btpUvv/xSwsPDZd++ffLnn39au1pEOR5bmomIiHKJ7777ThITE6VWrVri4uIiS5YsEWdnZylWrJi1q0aU47GlmYiIKJfw8PCQb7/9VurWrSuVKlWS7du3S1hYmHh5eVm7akQ5HkfPICIiIiKygOkZlCEAxGAwWLsaRDmSXq8XpZS1q0FP4HnN9vG7Q7aEQTNZBED69OkjJ06csHZViHKkoKAgWbBgAS/+NoTntZyB3x2yJcxpJosMBgMvLETP4Pjx42zRtDE8r+UM/O6QLWFLM2XKli1bxNnZ2drVIMoRYmNjpVmzZtauBlnA85rt4XeHbBGDZsoUZ2dnXlyIKFfheY2IMoLpGUREREREFjBoJiIiIiKygEEzEREREZEFDJqJiIiIiCxg0ExEREREZAGn0SaLTGfN4uxMRBnH747t4rGxbTw+ZIsYNBMRERERWcD0DCIiIiIiCxg0ExERERFZwKCZiIiIiMgCBs1ERERERBYwaCYiIiIissDB2hXIzZKSkuT69evi6urK4XKIiIiIbBAAiYmJET8/P7GzS7s9mUFzNrp+/br4+/tbuxpEREREZMHVq1elaNGiaa5n0JyNXF1dRST5ILi5uVm5NkRERET0pOjoaPH399fitrQwaM5GxpQMNzc3Bs1ERERENsxSKi07AhIRERERWcCgmYiIiIjIAgbNREREREQWMGgmIiIiIrKAQTMRERERkQUMmomIiIiILGDQTERERERkAYNmIiIiIiILGDQTEREREVnAGQGJiIiIciEAYjAYsrzMuLg4ERHR6XQWZ9HLLL1en+VlZhUGzURERES5kMFgkDZt2li7GpkSFhYmzs7O1q5GqpieQURERERkAVuaiYiIiHIhvV4vYWFhWVqmwWCQTp06iYhIaGio6PX6LC0/q8vLSgyaiYiIiHIhpVS2pjro9XqbTaXIDkzPICIiIiKygEEzEREREZEFTM8gIiIisgHZMURcVjOtn63XVSRrh7Bj0ExERERkA3LaEHHGDoG2LCuHsLPZ9IxDhw5Jy5YtxdPTU/Llyyc1a9aUZcuWZaqMa9euSf/+/SUgIECcnJzEz89PevXqJVevXk11+8DAQFFKpfrz9ttvZ8XHIiIiIqIcyCZbmsPDw6V58+bi5OQkISEh4u7uLqtXr5bu3bvLpUuXZOzYsRbLOH/+vNSpU0du3bolTZs2lS5dusjZs2fl+++/l40bN8revXulZMmSKV7n7u4uw4cPT7G8evXqWfHRiIiIiFIFQPs9pGuMODggna2tAxBJTEj+3d5BxBYn70tIULL8J1cRMd+nz0ohK0vLAgkJCVKuXDm5du2a7Nu3T6pUqSIiIjExMVK7dm05c+aMnDx5UkqXLp1uOa1bt5YNGzbIrFmzZOjQodry0NBQ6dy5szRv3lw2bdpk9prAwEAREbl06VKWfJbo6Ghxd3eXqKgocXNzy5IyiYiIKHe6d+9ejkh5yElCQ0PF09Mz3W0yGq/ZXHrGb7/9JufPn5du3bppAbOIiKurq3z44YeSkJAgixcvTrcMg8EgmzdvlkKFCsmQIUPM1nXq1EkqV64smzdvlgsXLmTLZyAiIiKi3MXm0jPCw8NFRKRZs2Yp1hmX7dy5M90y7ty5IwkJCVKsWLFUe0wWL15cjh07Jjt27JASJUqYrYuLi5Pvv/9eIiIixNPTU+rUqSNBQUFP+WmIiIiIMkan02m/Mz3j6ZmmZ5ju02dlc0Hz2bNnRURSTb/w9PQUb29vbZu0eHp6ir29vVy+fFkApAicL168KCIi//zzT4rX3rhxQ958802zZS1atJAlS5aIt7d3uu8bFxcncXFx2v+jo6PT3Z6IiIjIyDReMQZ99Gyyarg5ERtMz4iKihKR5A55qXFzc9O2SYuLi4s0aNBAbt68KfPmzTNbt3r1ajl27JiIiNy/f99sXe/evSU8PFxu374t0dHRsn//fnnllVdk06ZN0rZtW4vJ5FOmTBF3d3ftx9/fP93tiYiIiChnsLmOgM2aNZOtW7fK2bNnpVSpUinWlyxZUq5du2bWopua48ePS3BwsDx48ECaN28ulSpVknPnzskvv/wiL774opw4cUIGDBiQIqh+UlJSkjRo0EB2794t69evl1atWqW5bWotzf7+/uwISERERBZl9eQmBoMhx3UsDA0NFb1en2XlZWRyk4x2BLS59AxjC3NarcnGD2ZJUFCQHDp0SMaPHy87duyQHTt2SKlSpWT+/Ply//59+d///ic+Pj4Wy7Gzs5NevXrJ7t27Zc+ePekGzTqdLktzZ4jo+cmOmbgAaDfSOp0uSx8TimTtTFdEZH1KqSybiEMk+RwRFhaWZeWJ5O3zms0FzcZc5rNnz0q1atXM1t27d08iIyOlTp06GSqrXLlysmLFihTLjTnLGR172ZjL/OjRowxtT0Q5T06biUska2e6IqLcJ6uDcCMXF5csLzMnsLmc5gYNGoiIyJYtW1KsMy4zbvM0YmJiJCwsTAoUKCBNmzbN0GsOHDggIv+N40xEREREeYvN5TQnJCRI2bJlJSIiQvbv3y+VK1cWEfPJTf7++28pU6aMiIhERkZKZGSkeHt7m41uERsbK46OjuLg8F9jelxcnPTo0UNCQ0NTTHpy8uRJ8fPzEw8PD7P67N69W5o2bSoA5J9//pGAgIAMfxZObkKUc2RHeoZpPmFW5+mJ2PZjTCKinCLH5jQ7ODjIggULpHnz5lKvXj3p2rWruLm5yerVq+XixYvy8ccfawGziMicOXNk4sSJMn78eJkwYYK2/MiRI9KxY0dp2rSp+Pv7S3R0tGzYsEGuXLki/fr1SzHpycqVK2Xq1KnSuHFjCQwMFJ1OJ3/99Zds2bJF7Ozs5Ouvv85UwExEOUt2PcY00uv1TKUgolxh3759MmfOHBk8eLDUrl3b2tV5bmwuaBYRadSokezevVvGjx8vK1eulPj4eKlQoYJMmjRJunfvnqEyAgICpGHDhvL777/LzZs3xcXFRapWrSozZsyQV199NdX3PHXqlBw9elR27twpBoNBChUqJF26dJERI0ZIzZo1s/pjEhEREeUoBoNBZs2aJZGRkTJr1iypUqVKlj9Fs1U2l56RmzA9gyhvi42N1ToXstMeEeUGixcvlmXLlmmTx3Xv3j3FpHA5TUbjNZvrCEhEREREticiIkKWL1+uTfYGQJYvXy4RERFWrtnzwaCZiIiIiNIFQGbPnp3m8ryQuMCgmYiIiIjSdeXKFTl8+LAkJiaaLU9MTJTDhw/LlStXrFSz54dBMxERERGlKyAgINVJ4ZRSUqNGjTwxwhiDZiIiIiJKl1JKOnfunGI5AOncuXOeGDPeJoecIyKyJDsmI8lqpvWz9bqKcLIUIkobAFm5cqUopczyl5VSsmLFCqlcuXKuP38waCaiHMlgMGjDueUExpkBbRmHxSOitBhzmp8EQMtpLlasmBVq9vwwPYOIiIiI0mXMabazMw8d7e3t80xOM1uaiSjHazmgpTg42t7pDIAkJiT3NLd3sLfJR5cJjxNk41cbrV0NIrJxSikZMmSI9O7dO9Xltnh+y2q2d5UhIsokB0cHmwyaRUQcnRytXQUioixRpEgRCQkJMZsRMCQkRPz8/KxdteeC6RlERERElCFdu3YVLy8vERHx8vKSkJAQK9fo+WHQTEREREQZotfrZdiwYVKoUCEZNmyY6PV6a1fpubHN55lEREREZJNq164ttWvXtnY1nju2NBMRERERWcCWZiLKkUwH1094nGDFmuRspvvOdJ8SEZE5Bs1ElCPFxcVpv3PItKwRFxcnLi4u1q4GEZFNYnoGEREREZEFbGkmohxJp9Npv9vq5CY5genkJqb7lIiIzPEqQ0Q5kunsU7Y8uUlOkhdm9CIielq8yhBRjmerHQFzyjTaRERkGYNmIsrx2BGQiIiyGzsCEhERERFZwJZmIsqR9Hq9hIWFWbsa6TIYDNKpUycREQkNDbX56WZtvX5ERNbEoJmIciSllDg7O1u7Ghmm1+tzVH2JiMgcg+ZcCIAYDIYsLc84kYROp8vyzkx6vd4mO0gRERERGTFozoUMBoO0adPG2tXIsLCwMLbAERERkU1jR0AiIiIiIgvY0pwLZXUHqezuzMTOR0RERGTrGDRbWVbnH+dE2fH5mSdNREREWYlBs5XltPxjY4uzrWOeNGVWdtzAmpbHm0MiopyNQTMRkWT/DWx23HDy5pCI6Plh0GxDPCqME2XnZO1qpABABI+T/6McbbZlC0nxcv/vj6xdDSIiIsqFGDTbEGXnJMre9oLm5BBZZ+VaEGWv7Jhh8HmMcU5ERM8Hg2YiIsm+GQZdXFyyvEwiInr+OE4zEREREZEFDJqJiIiIiCxg0ExEREREZAFzmq0MwH+/J8VbsSY5n+n+M92vRERERM+KQbOVGXvWiwiHS8tCcXFx7IBFREREWYbpGUREREREFrCl2cp0uv/GP3Yv/y4nN3kGSIqXqFOfioj5fiUiIiJ6Vgyarcw0ADUGfPTsbDWwJyIiopzJZtMzDh06JC1bthRPT0/Jly+f1KxZU5YtW5apMq5duyb9+/eXgIAAcXJyEj8/P+nVq5dcvXo1W9+XiIiIiHIXBRscZiA8PFyaN28uTk5OEhISIu7u7rJ69Wq5ePGiTJ48WcaOHWuxjPPnz0udOnXk1q1b0rRpUwkKCpKzZ8/KunXrxMfHR/bu3SslS5bM8vc1FR0dLe7u7hIVFSVubm6pbgNADAZDpsp93gwGg3Tq1ElEREJDQ3PE1L16vZ6tzURERGRRRuI1ERsMmhMSEqRcuXJy7do12bdvn1SpUkVERGJiYqR27dpy5swZOXnypJQuXTrdclq3bi0bNmyQWbNmydChQ7XloaGh0rlzZ2nevLls2rQpy9/XVEYPgq2LjY2VNm3aiIhIWFhYtkw1TERERGQNGY3XbC4947fffpPz589Lt27dtMBVRMTV1VU+/PBDSUhIkMWLF6dbhsFgkM2bN0uhQoVkyJAhZus6deoklStXls2bN8uFCxey9H2JiJ60b98+6d69u+zbt8/aVSEiomdgc0FzeHi4iIg0a9YsxTrjsp07d6Zbxp07dyQhIUGKFSuW6iP64sWLi4jIjh07svR9iYhMGQwGmTVrlty8eVNmzZpl86lYRESUNpsLms+ePSsikmoahKenp3h7e2vbpMXT01Ps7e3l8uXLqc4Md/HiRRER+eeff7L0fePi4iQ6Otrsh4jyrp9++knu3LkjIsk388uXL7dyjYiI6GnZ3JBzUVFRIiLi7u6e6no3Nze5du1aumW4uLhIgwYN5LfffpN58+bJoEGDtHWrV6+WY8eOiYjI/fv3s/R9p0yZIhMnTkx3m+chqzsXmpaVHS1l7LRHuVFERIQsX75cu3EHIMuXL5emTZtKkSJFrFw7IiLKLJsLmrPKjBkzJDg4WAYPHixhYWFSqVIlOXfunPzyyy9SqVIlOXHihNjb22fpe7733nsycuRI7f/R0dHi7++fpe+REQaDQeu4l9WMo2hkJXYupNwGgMyePTvN5VOmTOGNIhFRDmNz6RnGll5jy++TjD0cLQkKCpJDhw5J586d5ejRozJr1iw5c+aMzJ8/X3r06CEiIj4+Pln6vjqdTtzc3Mx+iCjvuXLlihw+fFgSExPNlicmJsrhw4flypUrVqoZERE9LZtraTbmFJ89e1aqVatmtu7evXsSGRkpderUyVBZ5cqVkxUrVqRY/uabb4qISPXq1bPlfa1Nr9dLWFhYlpUHQOLi4kQk+cYgq1vIcsK4z0SZERAQINWrV5ejR49KUlKSttze3l6qVq0qAQEBVqwdERE9DZtraW7QoIGIiGzZsiXFOuMy4zZPIyYmRsLCwqRAgQLStGnT5/a+z5NSSpydnbPsx8XFRTw9PcXT01NcXFyytGxnZ2c+pqZcRyklQ4YMSfG3ndZyIiKyfTYXNDdu3FhKlCghy5Yt0zrsiSQHu5MmTRIHBwetpVhEJDIyUk6fPi2RkZFm5cTGxkpCQoLZsri4OOnTp4/cvXtXxo8fb9bCmdn3JSJKT5EiRSQkJEQLkJVSEhISIn5+flauGRERPQ2bC5odHBxkwYIFkpSUJPXq1ZO33npL3nnnHQkKCpK///5bJkyYIGXKlNG2nzNnjpQvX17mzJljVs6RI0fEz89PunfvLu+++64MHDhQypQpI6GhodKvX78Uk55k9n2JiCzp2rWreHl5iYiIl5eXhISEWLlGRET0tGwup1lEpFGjRrJ7924ZP368rFy5UuLj46VChQoyadIk6d69e4bKCAgIkIYNG8rvv/8uN2/eFBcXF6latarMmDFDXn311Wx7XyIiI71eL8OGDZM5c+bI4MGDmb9PRJSDKaQ2+wdliYzOZU5ERERE1pHReM3m0jOIiIiIiGwNg2YiIiIiIgsYNBMRERERWcCgmYiIiIjIAgbNREREREQWMGgmIiIiIrKAQTMRERERkQUMmomIiIiILGDQTERERERkAYNmIiIiIiILGDQTEREREVnAoJmIiIiIyAIGzUREREREFjBoJiIiIiKygEEzEREREZEFDJqJiIiIiCxg0ExEREREZAGDZiIiIiIiCxg0ExERERFZwKCZiIiIiMgCBs1ERERERBY4WLsCRERElHMBEIPBkOVlxsXFiYiITqcTpVSWla3X67O0PMo7GDQTERHRUzMYDNKmTRtrVyPDwsLCxNnZ2drVoByI6RlERERERBawpZmIiIieml6vl7CwsCwt02AwSKdOnUREJDQ0VPR6fZaVnZVlUd7CoJmIiIiemlIqW9Md9Ho90ynIJjA9g4iIiIjIAgbNREREREQWZCpojo2Nlejo6HS3ASDR0dESGxv7TBUjIiIiIrIVGQ6aIyMjxc/PT1q1aiVJSUlpbpeUlCStWrWSokWLyr1797KkkkRERERE1pThoHnhwoUSHR0tc+bMETu7tF9mb28vc+fOlfv378v8+fOzpJJERERERNaU4aB5/fr1Uq1aNQkKCrK4baVKlaRGjRqyfv36Z6ocEREREZEtyHDQfPLkSXnppZcyXHDNmjXl1KlTT1UpIiIiIiJbkuGgOSYmRtzc3DJcsJubmzx48OCpKkVEREREZEsyHDR7eHjIzZs3M1zwzZs3xcPD42nqRERERERkUzIcNL/44ouyfft2AWBxWwCyfft2qVChwjNVjoiIiIjIFmQ4aG7Xrp1cvnxZvvzyS4vbzpkzRy5fviwdOnR4psoREREREdmCDAfNb731lvj7+8s777wjEyZMkJiYmBTbPHjwQCZOnCijRo2SYsWKSb9+/bK0skRERERE1uCQ0Q2dnZ3ll19+kWbNmsmkSZNk2rRpUq1aNSlatKiIiERERMjhw4clNjZWvL29Ze3ataLX67Ot4kREREREz0uGg2YRkaCgIDl69KiMHTtWVqxYIb///rvZeicnJ+nRo4dMnjxZihQpkqUVJSIiIiKylkwFzSIiRYoUke+//16+/vprOXTokNy4cUNERAoXLiw1atQQZ2fnLK8kEREREZE1ZTpoNnJ2dpb69etnZV2IiIiIiGxShjsCPslgMMiePXvk559/llWrVsmePXvEYDBkWcUOHTokLVu2FE9PT8mXL5/UrFlTli1blqky7t+/L+PGjZNKlSqJq6ureHt7S40aNWTOnDmp1jUwMFCUUqn+vP3221n10YiIiIgoh8l0S3NERIS8//77smLFComPjzdb5+TkJCEhIfLxxx8/U05zeHi4NG/eXCvP3d1dVq9eLd27d5dLly7J2LFjLZZx//59qVatmly4cEGCg4Olf//+EhcXJ7/++qsMGTJE1qxZI1u3bhU7O/P7Bnd3dxk+fHiK8qpXr/7Un4eIiIiIcjaFjMxW8v8OHz4sLVu2lMjISNHr9VK1alXx9/cXAHLt2jU5evSoGAwG8fb2ll9//VWqVauW6QolJCRIuXLl5Nq1a7Jv3z6pUqWKiCRP4127dm05c+aMnDx5UkqXLp1uOVOnTpUxY8bIiBEjZMaMGdry+Ph4CQ4OlkOHDsnOnTvNUkwCAwNFROTSpUuZrndqoqOjxd3dXaKiojI1BTkREVFeFhsbK23atBERkbCwMPaXomyV0Xgtw+kZ0dHR0rZtW7l3756MHTtW/v33X9m9e7f89NNPsnz5ctm9e7f8+++/8v7778u9e/ekbdu2Eh0dnemK//bbb3L+/Hnp1q2bFjCLiLi6usqHH34oCQkJsnjxYovlXLhwQUREWrZsabbcyclJmjZtKiIit27dynT9iIiIiCjvyXB6xqxZs+TGjRvyzTffSN++fVPdxt3dXSZNmiSBgYHSr18/+fLLL+WDDz7IVIXCw8NFRKRZs2Yp1hmX7dy502I5xim8N23aJE2aNNGWP378WLZt2ybOzs5Su3btFK+Li4uT77//XiIiIsTT01Pq1KkjQUFBmfoMREREtghAlvY/yi6mdcwJ9dXr9aKUsnY1KJtlOD2jRo0akpCQIH/88UeGCq5SpYrY29vL4cOHM1WhTp06yc8//yyHDx9ONb3Dx8dHlFIWW4ljY2OlQYMGcujQIWnQoIHUqFFD4uLiZNOmTXLv3j359ttvpX379mavCQwMlMuXL6coq0WLFrJkyRLx9vZO9z3j4uIkLi5O+390dLT4+/szPYOIiGyCadoDZR2mkORsWZ6ecfbsWWnYsGGGK9CwYUM5d+5chrc3ioqKEpHkVuvUuLm5adukx9nZWcLDw+X111+XnTt3yrRp02T27Nla6kdwcHCK1/Tu3VvCw8Pl9u3bEh0dLfv375dXXnlFNm3aJG3bthVL9xdTpkwRd3d37cff3z8Dn5iIiIiIbF2G0zPi4+MzdRel1+tTjK7xPEVGRkq7du3k1q1bsmHDBqlbt64YDAZZt26djBo1StavXy+HDx8WT09P7TXjxo0zK6NWrVqyfv16adCggezevVs2btworVq1SvM933vvPRk5cqT2f2NLMxERka2Z0eGS6BySrF2NVAEi8YnJ6Q5O9hBbzHyIS7CTkWsCrV0Neo4yHDT7+vrK33//neGCT506JYULF850hYwtzGm1Jhub0C0ZOXKk7N27V44fPy6VKlXSyu7Xr58kJibKgAEDZObMmTJx4sR0y7Gzs5NevXrJ7t27Zc+ePekGzTqdTnQ6ncW6ERERWZvOIUl0DhkeQOu50zvabt2S2eYNB2WfDKdn1K9fXzZv3ixnz561uO3Zs2dl06ZN0qBBg0xXyDiUXGrvc+/ePYmMjLQ43JyIyIYNG6RAgQJawGzq5ZdfFhGRI0eOZKhOxlzmR48eZWh7IiIiIspdMhw0Dxs2TB4/fiytWrWSM2fOpLndP//8I61atZLHjx/L0KFDM10hY6C9ZcuWFOuMyzISjMfHx0t0dHSqKSK3b98WEclwq/CBAwdE5L9xnImIiIgob8lw0Fy5cmUZP368nDt3TipXriyvv/66LFq0SLZu3Spbt26VRYsWSffu3SUoKEjOnTsnH374odk4yxnVuHFjKVGihCxbtkyOHTumLY+JiZFJkyaJg4ODvPnmm9ryyMhIOX36tERGRpqVU7duXUlISJBJkyaZLY+Li9OWNWrUSFt+8uRJuX//for67N69W2bMmCE6nU46duyY6c9DRERERDlfpqbRHjdunHh5ecnYsWNl2bJl8tNPP5mtByD58+eXqVOnypAhQ56uQg4OsmDBAmnevLnUq1dPunbtKm5ubrJ69Wq5ePGifPzxx1KmTBlt+zlz5sjEiRNl/PjxMmHCBG35p59+Knv37pWPP/5YtmzZonUE3Lx5s1y4cEGqVatmNt70ypUrZerUqdK4cWMJDAwUnU4nf/31l2zZskXs7Ozk66+/loCAgKf6TERERESUs2UqaBYRGTRokLz++usSGhoqe/fulRs3boiISOHChaVOnTry2muviYeHxzNVqlGjRrJ7924ZP368rFy5UuLj46VChQoyadIk6d69e4bKqFy5shw5ckSmTJki27dvlzlz5oiDg4OUKlVKJk6cKO+8847o9Xqz9zx16pQcPXpUdu7cKQaDQQoVKiRdunSRESNGSM2aNZ/pMxERERFRzpXhyU0o8zI6WDYREaUvq2eyA6BNRqXT6bJ8NjdbnSHu0aNH0rZtWxERmdHhok2PnmHr4hKUjFxTXERE1q1bJy4uLlauET2tjMZrmW5pzozw8PBMTYhCRESUGoPBkKNmsrPVGeJMZ601Bnz07OLi4hg05wEZ7giYGXv27JHGjRtL48aNs6N4IiIiIqLnKlMtzY8fP5Zly5bJkSNHxMHBQYKDg81GlDh27Ji8++67snXrVgEg1atXz/IKExFR3qPX6yUsLCzLyjMYDNKpUycREQkNDTXr45IVsrq8rGI61CrTM56NaXoGJzbLGzIcNMfExEj9+vXlxIkTYkyDnjVrlrRv315WrVol48aNk08++USSkpKkatWqMmHCBGndunW2VZyIiPIOpVS2pTvo9XqbTKXIDqZ51joHMGjOIraYv05ZL8NB82effSbHjx+XoKAg6d69uwCQJUuWyNq1a6VTp06yatUqKVWqlEybNk3rZEBERERElBtkOGj+5ZdfpFixYnLgwAFxcnISEZHBgwdL2bJlZfXq1fLKK6/I6tWr+YiCiIiIiHKdDHcEvHDhgrzyyitawCwi4uzsrKVgTJs2jQEzEREREeVKGQ6aY2NjpXDhwimWFypUSEREypYtm3W1IiIiIiKyIVk25JydXbaMXkdEREREZHWZGnLur7/+kpUrV6ZYJpI8ZE9qkwt27tz5GapHRERERGR9mQqaV61aJatWrTJbZgyUQ0JCUixXSjFoJiIiIqIcL8NB8/jx47OzHkRERERENotBMxERERGRBZlKzyAiIqLcIS7BTkSSrF2NVAEi8YnJs+w52UNsccK95P1HeQmDZiIiojxo5JpAa1eBKEfJcNBcokSJdNfb2dmJu7u7BAUFSc+ePaVhw4bPWjciIiIiIpuQ4aD50qVLGdrujz/+kO+//16GDx8u06dPf9p6ERERURbT6/USFhZm7WpYZDAYpFOnTiKSPKStXq+3co3SZ+v1o6yR4aA5KSn9vCcAEhkZKXv37pXx48fLzJkzpX79+tKuXbtnriQRERE9O6WUODs7W7samaLX63NcnSl3yrKcZqWU+Pj4SLt27aRBgwZSpkwZ+eabbxg0E5kAIAaDIcvLjIuLExERnU4nKgt7zOj1+iwtj4iIKKfKlo6AHh4e0q5dO1m3bl12FE+UYxkMBmnTpo21q5FhYWFhbOEhIiISkWwbL6Vw4cJy//797CqeiIiIiOi5ybYh5+7cuSP58+fPruKJcqTs6ISTnR1m2LmFiIgoWbYEzfHx8bJx40apVKlSdhRPlGNldyccdpghW5AduftZzbR+tl5XI/YxILKuLA+a//rrLxk9erRcvXqVU28TEeVBOS133/ikxtaxjwGRdWXZ5CZJSUly9+5defjwoQCQDh06SK9evZ65gkRERERE1pZlk5sopcTd3V2Cg4PljTfeYMBMREQyw/2S6FT64/xbAyASL8mpDk4CsdWshzjYycioQGtXg4gkCyc3SU1CQoI4OGRbX0MiIrJxOpUkOgVrVyMlJaIXG6xXCrZ3w0GUV2XLkHMnT56UUaNGSdGiRbOjeCIiIiKi5yrLmoEfPHggy5cvl4ULF8rBgwcFgDg5OWVV8UREREREVvPMQfPu3btl0aJFEhoaKo8ePRIAUqVKFenVq5d069YtK+pIRERERGRVTxU037x5U77//ntZtGiRnD17VgBI4cKF5eHDh9KzZ0/57rvvsriaRERERETWk6mOgBs2bJCFCxfKxo0bJSEhQfR6vXTu3Fl69uwpzZo1E0dHR6ZkEBER5SHZMZlNdk4+w0li6GllOGguWrSo3Lx5U0RE6tatKz179pTOnTuLm5tbtlWOiIiIbFt2T2aT1ZPPcJIYeloZDppv3LghdnZ2MmrUKHnvvffEw8MjG6tFRERERGQ7Mhw0v/7667J69WqZNm2afPnll9K6dWvp0aOHtGzZkmMxExER5VF6vV7CwsKytEwAEhcXJyIiOp0uS9Mp9Hp9lpVFeUuGo90ffvhB5s6dK8uWLZOFCxfKqlWrZPXq1eLp6SkhISHy+uuvZ2c9iYiIyAYppbIl3cHFxSXLyyR6Fpma3MTV1VX69+8vBw8elBMnTsiQIUNEKSXz5s2TunXrilJKzpw5I1euXMmu+hIRERERPXdPPSPgiy++KDNnzpTr16/L8uXLpWnTpqKUkt9//11KlCghTZs2lZ9++ikr60pEREREZBXPPI22o6OjdO7cWTZt2iSXLl2SCRMmSEBAgGzfvp0pG0RERESUK2RpD76iRYvKuHHjZNy4cbJt2zZZtGhRVhZPREQ5AADt9zhwPNxnYbr/TPcrET1/2TbsRZMmTaRJkybZVTwREdko46gHIiIjo4pbsSa5S1xcHDvHEVnRM6dnEBERERHldjY7wPKhQ4dk/Pjxsm/fPomPj5cKFSrI8OHDpVu3bhku4/79+zJjxgxZu3atXLx4UXQ6nRQvXlzeeOMN6du3b6pjNWbF+xIR5WU6nU77fYb7RdEpphU8rTgorbXedL8S0fNnk0FzeHi4NG/eXJycnCQkJETc3d1l9erV0r17d7l06ZKMHTvWYhn379+XatWqyYULFyQ4OFj69+8vcXFx8uuvv8qQIUNkzZo1snXrVrGz+6+xPSvel4gorzOdiEKnwKA5i2TlBB9ElHk2FzQnJCRI3759RSklu3btkipVqoiIyPjx46V27doyfvx46dSpk5QuXTrdcr755hu5cOGCjBgxQmbMmKEtj4+Pl+DgYPntt99k9+7dUr9+/Sx9XyIiIiLKfWwup/m3336T8+fPS7du3bTAVSR5YpUPP/xQEhISZPHixRbLuXDhgoiItGzZ0my5k5OTNG3aVEREbt26leXvS0RERES5j80FzeHh4SIi0qxZsxTrjMt27txpsZwKFSqIiMimTZvMlj9+/Fi2bdsmzs7OUrt27Sx/XyIiIiLKfWwuPePs2bMiIqmmQXh6eoq3t7e2TXr69u0rS5YskenTp8vhw4elRo0aEhcXJ5s2bZJ79+7JsmXLpEiRIln6vnFxcWZDLUVHR1usJxFRbhYHOxFJsnY1UgBE4iU5R9hJILaaLpy8/4jIFthc0BwVFSUiIu7u7qmud3Nzk2vXrlksx9nZWcLDw6V///6ydOlSrZXYzs5OBg8eLMHBwVn+vlOmTJGJEydarBsRUV4xMirQ2lUgIsoSNhc0Z5XIyEhp166d3Lp1SzZs2CB169YVg8Eg69atk1GjRsn69evl8OHD4unpmWXv+d5778nIkSO1/0dHR4u/v3+WlU/PFwAxGAzWroZFpnXMCfXV6/UcBYCIiHIcmwuajS29xpbfJ0VHR6fZGmxq5MiRsnfvXjl+/LhUqlRJK7tfv36SmJgoAwYMkJkzZ2otw1nxvjqdjuNo5iIGg0HatGlj7WpkSqdOnaxdBYvCwsLE2dnZ2tWgbKTX6yUsLMza1UiXwWDQvi+hoaGpjttva3JCHYlyM5sLmo05xWfPnpVq1aqZrbt3755ERkZKnTp1LJazYcMGKVCggBYwm3r55ZdFROTIkSNZ/r5ERHmdUipH3Rjp9focVV8isg6bC5obNGggU6ZMkS1btkhISIjZui1btmjbWBIfHy8Gg0Hi4+PFycnJbN3t27dFxHx2pax6X8qdZoy6JDpH2+vMJPL/HZoe/3+HJkfb7NAU99hORk4PtHY1iIiInprNdctt3LixlChRQpYtWybHjh3TlsfExMikSZPEwcFB3nzzTW15ZGSknD59WiIjI83KqVu3riQkJMikSZPMlsfFxWnLGjVq9NTvS3mLzjFJdE6wyR+9DuKWP0nc8ieJXmf9+qT6Y6M3HERERBllc0Gzg4ODLFiwQJKSkqRevXry1ltvyTvvvCNBQUHy999/y4QJE6RMmTLa9nPmzJHy5cvLnDlzzMr59NNPxdXVVT7++GOpVauWjBw5UgYOHCgvvPCCbN68WapVqyZ9+/Z96vclIiIiorzD5oJmkeQW4N27d0twcLCsXLlS5s2bJ15eXrJ06VJ5//33M1RG5cqV5ciRI9KrVy+5ceOGzJkzR7777jvJly+fTJw4UXbt2pWiU0VWvC8RERER5T4KAKxdidzKOOJGVFSUuLm5Wbs6lEmxsbHa6Blz370gOid+VZ5WXLySQZ+WEBGOnkG2wfT7zb9Jorwto/GaTbY0ExERERHZEgbNREREREQWMGgmIiIiIrKAQTMRERERkQUMmomIiIiILGDQTERERERkAYNmIiIisjn79u2T7t27y759+6xdFSIRYdBMRERENsZgMMisWbPk5s2bMmvWLDEYDNauEhGDZiIiIrItP/30k9y5c0dERO7cuSPLly+3co2IGDQTERGRDYmIiJDly5eLccJiALJ8+XKJiIiwcs0or3OwdgWIiIgsAZClj+hNy8qOR/96vV6UUllebm4HQGbPnp3m8ilTpnC/ktUwaCYiIptnMBikTZs22VJ2p06dsrzMsLAwcXZ2zvJyc7srV67I4cOHUyxPTEyUw4cPy5UrV6RYsWJWqBkR0zOIiIjIRgQEBEj16tXFzs48PLG3t5caNWpIQECAlWpGxJZmIiLKAfR6vYSFhWVZeQAkLi5ORER0Ol2WP/LX6/VZWl5eoZSSIUOGSO/evVNdztQMsiYGzUREZPOUUlme7uDi4pKl5VHWKFKkiISEhMiyZcsEgCilJCQkRPz8/KxdNcrjmJ5BRERENqVr167i5eUlIiJeXl4SEhJi5RoRMWgmIiIiG6PX62XYsGFSqFAhGTZsGNNdyCYwPYOIiIhsTu3ataV27drWrgaRhi3NREREREQWMGgmIiIiIrKAQTMRERERkQUMmomIiIiILGDQTERERERkAYNmIiIiIiILGDQTEREREVnAoJmIiIiIyAIGzUREREREFjBoJiIiIiKygEEzEREREZEFDJqJiIiIiCxwsHYFiGwVAO33uHhlxZrkfKb7z3S/EhER5RQMmonSEBcXp/0+ckZxK9Ykd4mLixMXFxdrV4OIiChTmJ5BRERERGQBW5qJ0qDT6bTfZ4y8KDonphU8rbh4pbXWm+5XIiKinIJBM1EalPovD1fnBAbNWcR0vxIREeUUTM8gIiIiIrKAQTMRERERkQUMmomIiIiILGDQTERERERkAYNmIiIiIiILGDQTEREREVnAoJmIiIiIyAKbDZoPHTokLVu2FE9PT8mXL5/UrFlTli1bluHXN2zYUJRS6f4sWbLE7DWBgYFpbvv2229n9UckIiIiohzCJic3CQ8Pl+bNm4uTk5OEhISIu7u7rF69Wrp37y6XLl2SsWPHWizjzTfflIYNG6ZY/vjxY5kyZYrY2dlJ48aNU6x3d3eX4cOHp1hevXr1p/koRERERJQL2FzQnJCQIH379hWllOzatUuqVKkiIiLjx4+X2rVry/jx46VTp05SunTpdMt58803U12+atUqASAtW7YUPz+/FOs9PDxkwoQJz/oxiIiIiCgXsbn0jN9++03Onz8v3bp10wJmERFXV1f58MMPJSEhQRYvXvzU5S9YsEBERPr06fPMdSUiIiKivMHmWprDw8NFRKRZs2Yp1hmX7dy586nKvnbtmmzZskUKFy4srVq1SnWbuLg4+f777yUiIkI8PT2lTp06EhQU9FTvR0RERES5g80FzWfPnhURSTX9wtPTU7y9vbVtMmvx4sWSlJQkb775pjg4pP7Rb9y4kSK1o0WLFrJkyRLx9vZOt/y4uDiJi4vT/h8dHf1U9SQiIiIi22Jz6RlRUVEiktwhLzVubm7aNpkBQEvrSCs1o3fv3hIeHi63b9+W6Oho2b9/v7zyyiuyadMmadu2rQBI9z2mTJki7u7u2o+/v3+m60lEREREtsfmgubs8ttvv8nFixelQYMGUqpUqVS3GTdunDRo0EC8vb3F1dVVatWqJevXr5fg4GDZt2+fbNy4Md33eO+99yQqKkr7uXr1anZ8FCIiIiJ6zmwuPcPYwpxWa3J0dHSardDpMXYA7Nu3b6ZeZ2dnJ7169ZLdu3fLnj170syFFhHR6XSi0+kyXTeyfXGP7UQkydrVSBUgEv9YiYiIkyNEKStXKBXJ+4+IiCjnsrmg2ZjLfPbsWalWrZrZunv37klkZKTUqVMnU2Xeu3dP1qxZIx4eHvLqq69muk7GXOZHjx5l+rWUO4ycHmjtKhAREZEV2VzzT4MGDUREZMuWLSnWGZcZt8mopUuXSlxcnHTv3l2cnZ0zXacDBw6ISPKMgURERESU9yhY6t32nCUkJEjZsmUlIiJC9u/fL5UrVxYRkZiYGKldu7acOXNG/v77bylTpoyIiERGRkpkZKR4e3unObpF5cqV5fjx43L06FGzsZ9NnTx5Uvz8/MTDw8Ns+e7du6Vp06YCQP755x8JCAjI8GcxppJERUWJm5tbhl9HtgGAGAwGa1fDIoPBIJ06dRIRkdDQUNHr9VauUfr0er0oW8whISKiPCmj8ZrNpWc4ODjIggULpHnz5lKvXj3p2rWruLm5yerVq+XixYvy8ccfawGziMicOXNk4sSJMn78+FRn8jty5IgcP35cqlatmmbALCKycuVKmTp1qjRu3FgCAwNFp9PJX3/9JVu2bBE7Ozv5+uuvMxUwU86nlHqqJxPWpNfrc1ydiYiIcgKbC5pFRBo1aiS7d++W8ePHy8qVKyU+Pl4qVKggkyZNku7du2eqrIULF4qI5Q6AjRo1klOnTsnRo0dl586dYjAYpFChQtKlSxcZMWKE1KxZ86k/DxERERHlbDaXnpGbMD2DnofY2Fhp06aNiIiEhYWxpZmIiCgTMhqv2VxHQCIiIiIiW8OgmYiIiIjIAgbNREREREQWMGgmIiIiIrKAQTMRERERkQUMmomIiIiILGDQTERERERkAYNmIiIiIiILGDQTEREREVnAoJmIiIiIyAIGzUREREREFjBoJiIiIiKygEEzEREREZEFDJqJiIiIiCxg0ExEREREZAGDZiIiIiIiCxg0ExERERFZwKCZiIiIiMgCBs1ERERERBYwaCYiIiIisoBBMxERERGRBQyaiYiIiIgsYNBMRERERGQBg2YiIiIiIgsYNBMRERERWeBg7QoQ5SUAxGAwZGmZpuVlddl6vV6UUllaJpGt2Ldvn8yZM0cGDx4stWvXtnZ1iMjGKQCwdiVyq+joaHF3d5eoqChxc3OzdnXIBsTGxkqbNm2sXY0MCwsLE2dnZ2tXgyjLGQwGefPNNyUyMlK8vb3lu+++E71eb+1qEZEVZDReY3oGERHlOT/99JPcuXNHRETu3Lkjy5cvt3KNiMjWsaU5G7GlmZ6UHekZACQuLk5ERHQ6XZamUzA9g3KjiIgI6d27tyQmJmrLHBwcZOHChVKkSBEr1oyIrCGj8RpzmomeI6VUtqQ7uLi4ZHmZRLkRAJk9e3aay6dMmcIbRSJKFdMziIgoz7hy5YocPnzYrJVZRCQxMVEOHz4sV65csVLNiMjWMWgmIqI8IyAgQKpXry52duaXP3t7e6lRo4YEBARYqWZEZOsYNBMRUZ6hlJIhQ4akSMFIazkRkRGDZiIiylOKFCkiISEhWoCslJKQkBDx8/Ozcs2IyJYxaCYiojyna9eu4uXlJSIiXl5eEhISYuUaEZGtY9BMRER5jl6vl2HDhkmhQoVk2LBhnNiEiCzikHNERJQn1a5dm9NnE1GGsaWZiIiIiMgCBs1ERERERBYwaCYiIiIisoBBMxERERGRBQyaiYiIiIgsYNBMRERERGQBh5zLRgBERCQ6OtrKNSEiIiKi1BjjNGPclhYGzdkoJiZGRET8/f2tXBMiIiIiSk9MTIy4u7unuV7BUlhNTy0pKUmuX78urq6uopSydnWeSXR0tPj7+8vVq1fFzc3N2tWhJ/D42C4eG9vFY2PbeHxsV247NgAkJiZG/Pz8xM4u7cxltjRnIzs7OylatKi1q5Gl3NzccsUXJLfi8bFdPDa2i8fGtvH42K7cdGzSa2E2YkdAIiIiIiILGDQTEREREVnAoJkyRKfTyfjx40Wn01m7KpQKHh/bxWNju3hsbBuPj+3Kq8eGHQGJiIiIiCxgSzMRERERkQUMmomIiIiILGDQTERERERkAYNmIiIiIiILGDQTEREREVnAoJmIiIiIyAIGzXnY/v375a+//rJ2NYiIshRHUiVKX2JiorWrkCMxaM6D7t69K927d5c6derIxIkTJTo62tpVIsoxGJDZrqSkJElKShKllLWrQmTT7O3tJTExUc6dOyciyd8dsoxBcx60b98++emnn8Te3l7WrVsn27dvt3aV6P8B0E5eDM5sS1JSkiQmJjIgs1GJiYliZ2cndnZ2cuHCBQkNDZVjx45Zu1r0/0zPbWR9q1evFl9fX6lfv7788ccfWsszj1H6GDTnQTVq1BBfX1+pWLGiFC1aVD777DO5efOmtauV5yUkJIhSSuzs7CQuLo7BmQ0BIHZ2dmJvby+XLl2Sr7/+WsLCwuTcuXPaRYYXG+uyt7eXqKgo6d+/vwQHB0uvXr3km2++kbt371q7anme6bktISHB2tUhEZk9e7ZERkbKjRs3pHPnzvLll1+KiIidHcPC9HDv5EGxsbHi4+MjXl5e0rBhQzl48KAsX76cF30rc3BwEBGRzz//XFq3bi2dOnWSDz74QCIiIqxcs7zLeIFXSklCQoKMGTNGypQpIwMHDpR27dpJjRo15K233tJaOcl61q5dKy+++KL88ssv0r59e5kxY4Z89tlnUqBAAWtXLc8zntu++OIL6dy5s/Tr109+/PFHuX37tojwhvN5MrYot2rVSpRSMmXKFHn06JH873//kylTpsi1a9dEhMckTaBc6fbt29i0aRPu3bsHAEhMTDRbX7NmTbRv3x7r1q1DQEAAAgMDcerUKSvUNG8zPS7//PMPgoKC4OjoCD8/P3h6ekIphYoVK2Ljxo0ptqfnJzY2FqtXr0ahQoXQv39/LFy4EPPnz0elSpWglMLgwYNx+/Zta1czz7p8+TKqVKmCEiVKYOXKlYiKirJ2lfI803PV8ePH8cILL8DOzg4FCxaEvb09lFKoVasWrl69asVa5l1fffUVlFLYuHEj/v77bzRs2BBKKbRu3Ro3b97UtktKSrJiLW0Pg+ZcaNasWVBKQSmFfv364eHDh9q6+Ph4AMCoUaNQuHBhPH78GBMmTIBSCiNGjIDBYLBWtfOsU6dO4fjx4xg/fjyqVKmCpUuX4tatWzh37hzGjx8PpRQKFSqEGzduWLuqec7Vq1fx4osv4s0338SLL76Ifv364f79+9r6v/76C8HBwXB0dMRXX32FuLg4K9Y290vrAv7tt99CKYWFCxemWGd6vNIrg7Le2bNncerUKXz44YeoUaMGvv/+e1y6dAmHDx9Gt27doJRC06ZNecOZRZKSksz+vhMSElLdBgB27doFpRRGjhyJpKQkREZG4rXXXoNSCvXr18fWrVufW71zEgbNuczDhw9RpUoVKKVQsGBBeHl5oUWLFjh+/LjZdpMmTYKTkxP++OMPnD17FjVq1ED+/Pmxa9cuK9U8bzp69CiUUnj55ZdRunRpTJ8+PcU2AwYMgFIKffr0AcDW5ufp3r17KFiwIIoUKYISJUrgyJEjAJIvPMbjEBYWhmLFiqFmzZq4dOmSNaubaz1+/Djd9VOnToVSChMmTAAAHDhwAGvWrEHfvn0RFBSE1157DbNmzQLAoPl5+f3336GUwmuvvQZ3d3d8/fXXZusfPHiA9u3bQymFzz//nOe1Z2S6/86dO4eFCxdi4cKFuHLlSqrbX7t2DYULF0b37t21Zbdv38bw4cPh6uoKV1dXrFixQvu+8HuTjEFzLvDkH7XxMXLBggUxa9YsuLq6onz58tiyZYv2ms2bN0MphfDwcADJrdOOjo7o0KFDipYZenZpnXD+/fdf9OvXD0opODk54eLFiwCSgwRjK8Hly5dRvnx5KKXwzz//AGDgnJVSa40B/jtmP//8s/bk5vz58wCS979x/f379/Hmm29CKYVNmzaZvZaezZP7MTQ0FBMmTMDs2bOxY8cObfmdO3fg4+MDpRT8/Pzg5eUFpRR8fX1Rrlw56PV6KKXw+++/p1ouPb209uX169fRuHFj2NnZoUyZMoiOjgaQ/N0xnr/Cw8MRGBiIkiVLIiYm5rnVOTcx3f/37t3DwIED4e/vDwcHB/j7+2Pt2rWpvu7KlSsoVaoUAgMDUzwh++KLL6CUgqenJyZMmMCYwASD5hwsNjYWa9euxdKlS7WLuZHxIj579mxs3rwZZcuWhbu7O77//nvExsbi8uXLcHFxwUcffQQAiIiIQLt27aCUwk8//WSNj5MrmV4gTJme6Hbs2IGgoCAopRAaGprqduPGjYNSChMnTszeCuchTx6bCxcu4MaNG4iNjTXbLiEhAR06dIBSClOnTjVbZzw+CxYsgFIKw4cPz/6K50H79u1DpUqVYGdnh/z580MpBWdnZ4wZM0Zr3Q8LC0P//v3Rtm1b9OrVC5s3b8b58+cRFxeHZcuWQSmFzp07W/mT5B6Wzm1JSUn49ddf4erqCqUU/vjjD7P1Rl27doVSCitXrkx1PWXMwoUL4e7uDm9vb3Tu3Bnffvstbt68meJ8ZqpJkyZwdnbGgQMHtGWffvop9Hq99qRaKYVu3brhzp07z+Nj2DwGzTnU7NmzUalSJbi7u0MphUWLFiE2NlY74Rw+fBh+fn5wc3PD5cuXcfHiRQQHB0On0+Gdd97BpUuX4Ovri169emlfqlWrVsHLywvVqlXDtWvXrPnxcgXTFsyLFy/im2++wcqVK3H8+HGzO/uoqCh8/PHHUEphzJgxWotLUlKSVsaePXtgZ2eHDz74AABbmjMqrf1kemE+ePAgmjdvDn9/f/j5+aF169ZmrZhA8uN+Ozs71KtXDxcuXACQ3D/AmDYQHh4OpRSmTZuWPR8kD9u7dy8CAwNRrlw5zJw5E8eOHcO+ffvQvXt3KKXQv39/s+2NLZqm/vzzT63DZkJCAgOzZ2R6brt8+TJCQ0OxZcuWFOlJ9+/fx6BBg6CUwqeffmq2zvjdWbp0qZaiQU9nxYoVcHd3R+3atbFu3TqLLcPG8+KYMWPg4uKCK1euYMOGDQgMDIRSCo0aNcKPP/6IEydOoHLlylBK4YUXXsCGDRuex8exaQyacxhji4ujoyMaNWqECRMm4MCBA3j06FGKAMHYiaxHjx4Akh+XvfXWW9qFpkCBAujYsaO2vcFgwMCBA7XcwLQeW1OyjASucXFxGDp0KFxcXLQe4y4uLiku9MePH0eNGjVQsmRJbN++PUX5xpayd999N2s/RC5luu8MBkOqObEPHjzA6NGj4eTkhPLly6Nt27bo1KkTXF1d4enpiZ07d5qVY7z4DxkyJMV7Gb83v/zyS/Z9qFwurfNN7969odfrsW7dOrPj8ddff6FQoUJQSmHNmjVplrtv3z40bdoUhQsXxv79+7O62rlSRs9tw4YN01qSlVIoWrRoiqcxR44cga+vL2rVqoVDhw5przWaM2cOlFJYvHhxln6G3Ci170hMTAzq168PLy8v/P3332bbGjv+p3WTOHHiRCil4O/vD6UUypUrh88++0xLEwSSv2cDBw5EwYIFsXfv3qz9QDkQg+Yc5MiRI6hQoQJ8fX0xb968NDsdGU94ERERqFatGpRS+O233wAkt8JMmzYNLi4uUEqhVKlSZqNr7Nq1C+7u7qhTpw5zzDIorc6TBw4cQHBwMLy8vNCvXz8sWLAABw4cQOfOnVO0vMTHx+Pbb7+Fk5MTmjRpol1cAODMmTNo3LgxvLy8cOzYsWz/PLnJxIkT0bBhwxTB0oMHDzBu3DgULVoUb7/9Nnbv3q2tCwsLg06nQ4cOHcyGYbxy5Qp8fX21JwIHDx7Enj17MHHiRDg7O6N+/frM/csC33zzjZamdP78eRQsWBCvvvqqtv7ChQuYPXs2atWqBaUUunbtavboOC4uDlFRUdi7dy8+/PBDVK5cGe7u7pgyZYoWRFDaTAOsv/76K9Ubzt27d6NWrVooUKAA+vTpg7lz52L16tWoU6eOdvNovA4ZDAZMmjQJSil07NjR7Lpy5swZ1KlTB4UKFdKe4FD6Hj16hMmTJ+PgwYMAkjv9OTo6omnTpto2psF1QkIC4uLi8PjxY+3YGr8Hxs6ahQoVwtChQ83SNEz/Dvhk8z8MmnMA4x/sgAED4ODggB9++MFsvfGuPbU/cmOuZXBwsNn6xYsXY/To0di4cWOKkyLvJjNm165dWi6ysUOl0f379/HKK6+gXLlymDNnjtm4l4sWLYJSCvnz5zdrGbh48aIWUPv7+2P8+PEYOXIkmjVrBqUUBg0ahIcPH/LRcgY8fPhQe3xvfHJiGtD++++/CAgIwMCBAxEZGaktDw0NRZEiRaCUgr29PaZNm2Z2Uzlv3jwopeDj44P8+fOjbNmyyJ8/P1q3bs3xZp+C6cU9KioKISEhUEqhZcuWePDgAc6dOwcnJyd89NFHSEpKwvLly9G6dWsopVCpUiWtYx/w3+P+S5cuoVChQtDr9fDw8MBLL72Effv2PffPlpPt3bsXZcqUQb58+VLcqF+/fh2NGjVCUFAQvvrqK9y6dUtbN23aNG385bNnz2rLz58/j5deekkbYu6TTz7BpEmT0KRJEyilMGrUKMTHx/PcZsGtW7dQuHBhlClTBo8ePQKQnHpUrFgxBAcHayNlPHr0CNHR0fjhhx/QoUMHtGjRAvXq1cOECRPMbhz3798PPz8/BAcHm70Pg+S0MWjOQcqUKYPSpUubTVhiepJJrRXFYDCgTZs2Wt6zUWJiIk9Sz8gYzCqlUK9evRTrX3vtNbNxYy9cuIAWLVpAKYUyZcpAKYUuXbqYveaXX35BYGAgPDw8EBwcjObNm6NWrVpYsWJFtn+e3MY4DmzJkiXh6+urPW0xMm1dPn36NF5++WUopdC4cWNMmTIFZcuWxQsvvGAWcMXExKBq1aoICAjAtGnTcPr0abNWbF5sMu/27dsIDw9HeHg4ihcvjsmTJ2udxv7++28UKFAAZcuWxeDBg5EvXz54enpi/vz5Kco5fPgwHjx4AACYPn06PvjgA7MRg0yHCaS0xcfHo3Tp0tq5bfDgwWadyRISElCmTBmt4x6QfMNvvJkJCAiAUgrjx4/XXvf48WMtxaxEiRIoUaIEGjZsiIoVK7LjeSbs3r0bSil88skn2rK7d+8iJCQE9vb2aN68OYYPH45evXppKRfGm3zj72+//baWfhEREQFXV1cUKFBAGyubMUH6GDTboEuXLmHjxo1aC1hiYiIiIiJQs2ZNFCxYUEvLePz4MRITE7Fnzx4MGDAAISEhaNCgAT766COziTCMPZgDAgJw9+5dq3ym3MR44f3ss8/g5OSEevXqmU2sYGw9M73QzJw5E/nz50eRIkXw6aefIjw8HMHBwVBKYfPmzdp2N2/exNChQ6HT6fDdd9+leNzPPPP/pBUAGZd/++23cHBwwKhRo+Dh4YHu3bvj33//TbH9/v37ERQUhEKFCmHSpEk4f/48Hj9+jCFDhmh55KataWvXrtV6lBu/TwkJCTw2GfDkMbt//z58fHwQEBCAcuXKISQkJMVrGjVqBKUUHB0dMXbs2FQnYJo4cSKqVKmitYo+eeHnsckY434aNGgQ7Ozs0KBBAzg5OWHbtm1m6007W06fPh0uLi4oUqQIpkyZgtDQUFSqVAm+vr5aCgGQfG5r06YNChUqhGXLlmktpU++d16X3ogkY8eOhVJKS6MwLj9w4ABeffVVLTB2c3ND3bp1MWzYMOzYsQOXL1/Gjz/+iMDAQLi5uZkNQ9euXTt4e3vjxIkTz+HT5XwMmm3I9evXMWDAAFSoUCHV4ZGGDh0KpRSaNGmC999/H++++y5q166tfVHy5cun/d64cWOzx8XGCTI+/PBDALybtCQjrVIzZsyAn58fhgwZAm9vbwQEBGhPAUxt2bIFBQoUQMOGDbFx40btYjF58mRt9iXTFIA9e/agePHiqF27Nv766y8Alid3yMuMrYtP/k0vWLAAHh4e+Oijj9ClSxc4OztjxYoV2nE1/jto0CDodDp8++23ZsfcOFNmyZIlU/Qab9++PXQ6Xaoz0JFlpvvZmO/q7u6udaQ0HZlk69at2vjLT7p48SI++OADFC5cGL17904xcgZbllOydG5LTEzExIkT4e3tjcGDB2sTZKXW4LJ8+XJ4eHigcePGZql+r7/+OpRSGDhwoPa6pKQkbNu2DS4uLujQoYPW2slZNFNKTEzEmjVrzG4s4uLiULZsWVSqVCnV/kYPHz5EeHg4QkND8c8//6Ta58k4dOmwYcO01zRo0ABKKZw5cybbPk9uwqDZRkyaNAl6vR5+fn5o2bIlPv/8c2zfvh3x8fHaCe6ff/5Br169tMBYKYXy5csjJCQECxcuxD///IO1a9dqj/6//PJLrfy//voLJUqUgFIKJ0+etNbHzBFMg6/jx4+btTKaTlN68OBBKKXw66+/YvDgwVBK4f333zcrIz4+Hs2bN0e+fPm0R85Gn376KTw9PaGUMpsJ8OHDh5gyZQrs7OzwySefpBkU5nWbNm1CzZo1MXLkSLOOYMYWq9OnT8PR0RFLly7F6tWr4enpiZdfftks1zIxMREBAQGoVq1aiot33759Ub9+fe0m1PTv4NixY3BxcUGjRo20CWd4fCw7f/48evfubdZvIioqCtWrV4dSCvPmzQOQcl8aR/2pU6cOFi1ahD///BOzZs1C27Zt4ezsjLZt25odV0qdabB8/vz5FNNXG/f7t99+C3d3dyxbtkybWvm7774z2zYmJgZBQUHw8/PDn3/+aVb+6NGjtX4bq1ev1pY/ePAAgwYN0qadp5RiYmK0OROMkyUByQMBGKe9flJ6557Hjx9rqZurVq3SyjDe4GzatMmszw2lj0GzlcXGxmLgwIHQ6XR47bXXsGHDhlTHGTWKiYnBb7/9hq+//hoHDx7EyZMnUzzmWrlyJZRSqFq1qtny8ePHo2PHjqm2hlLKmZWME8TUr19f681vus3FixdRoEABTJkyBf/++y+KFSsGnU6njbiQlJSE+/fvo1q1anjhhRfM3ishIQEvvfQS6tati0KFCqFChQpmx/HkyZNo0KABihYtmiIXNy968tHtd999Z/aEpU2bNmZjiycmJuLGjRsoWrQoBg8ejKSkJPTr1w92dnb44osvtNSZ27dv4+WXX4afn5/W8vXw4UMsWrQInp6e+PLLLzF58mSEhYWlqNPw4cOhlMLcuXOz74PnIgkJCZg9ezaUUvjf//5ndh5as2YNlFJo3ry5ttx0nPI7d+5g4sSJKFCggHbcdTodChUqhNmzZ1vh0+RcpjNYdu7c2exRvzG4PXnyJBwcHBAWFoa9e/fCy8sLFStWxOXLl7Vy7t69i2LFiqFNmzba64HkwLhs2bJo3bo1nJyc0LZtW7Nz24kTJ1CiRAlUq1bNLH2D/jNv3jy4ubmhRYsWWpqmcWg+Y6pMeozH0vQm6fLly6hfvz50Op1ZB1rKHAbNVrZjxw7Y2dmhXbt2Zhf9x48fa3eCpq2baUlISNBy/U6dOgV7e3u0aNECDx8+1C48fMSfutT27fLly7WLc/HixeHl5YWffvrJ7OQfGRmJYsWKacNhffLJJ6l27nvllVeglMKyZcsAAEePHsXAgQNRvHhxnDlzBnv37k0RFCYkJGD+/Pmwt7dPcxrUvObx48faUHwHDx5EgwYNoNPp0LFjRzg5OeGll14yS6O4d+8eKlasiGbNmiEhIQHh4eEoXbo0ypcvbzak35QpU+Dk5IQGDRpgzJgx6NOnD3x9fdGiRYt0b2Bv3LjB1rJUpJebeuHCBbRr1w4FCxY0y+VPTExE69atodfrsWTJErPXPDkE2tKlS7FkyRIsW7YsRQc1Mpfauc04Nq+x016RIkVw8OBBs47kJ06cgI+PDwYNGgTgv9TAiRMnateR69eva8P+HT9+HElJSTh+/Dj69euHF198EUePHsWWLVtSHJfHjx9rqU9bt27Nxk+f8xiD3Lt376JLly5aX5mEhAQ0a9YMPj4+2pPHjHr06BG2bduGDh06QK/XY/To0enOEkjpY9BsZWPGjNFOOkDyl8Y0uE0t9+zJnEzTk9Ljx4+1VAG2gGVOaGioWa/k9u3bQymFV155BX379oW9vT3+97//me3vV155BTVr1gSQPIyZMcfc9LHatm3btN7LpUuXRmBgIBwdHTF8+HCzC9WTNzV37twxGw4tLzJe9Ddt2gQXFxctJx8AvvjiC+j1enTr1g3ff/89ihcvDjc3N6xatQpRUVEAgM6dO2ut/A8fPtQu1qNGjdL27dWrV7UbHjs7O+TLlw9vvvmmWUsoUy/+k9FRKC5cuGB242+0Zs0auLm5ISQkxKyh4NChQ3BwcMDLL7+s5WNm5H0YLFv2888/a/n3169f1/JY3377bVSrVg2FChVKMSNfiRIltAaAvXv34oUXXoCPjw+OHj2qbTN37ly4ubnBzc0NNWrUQPny5aHT6fDee++ZHbsnj1FkZGSqnXLpv+/Khg0bULhwYVSsWBHr1q2DTqdDnz59LMYHAHDt2jVcvHgR33//Pfr27YtixYrB0dERH3/88XP5DLkZg+bnaNmyZZgyZQqmTZum9VTt2bMnlFL48ccfAZhfXFauXIkPPvgA//vf/1LM0pOaEydOYOTIkXBxcUHr1q2Zp5QJxtkTTcfAPnToEFxcXODi4oKrV6/ijTfe0EZNOH/+PABgxIgR2lTlAPD9999DKYXq1aubBcTbt29Hz549UbduXTRr1owtLJnUsWNH2NnZISIiQlt2/fp1bZir9evXY9euXWjcuDFcXFwwYsQIAMmtanq9XuvkcvToUQQHB8PHx8fsxgZI7jOwa9cusw4x7EiWNtMOXk/e/I0ePRo6nc7s8bvx3Hb79m30798fOp0OS5YsMQuojC2aT065nBbezKTN+Lc7cuTIFKP0LF68GI6OjqhduzZOnz6NOnXqwM7ODh9//LF2LuvUqRNKly6tvcbYcblXr17aE5jY2FisW7cOderUQfny5VG/fn2zYf4syes3PE8OG2tcBiTvG2MKWJUqVbSxyQcOHIgNGzbg3r17ZucnYzkPHz7ExIkT4e7uDnt7ewQGBqJHjx5mqTX09Bg0Pwc///wzSpYsqU2YoJSCp6cn5s2bh3379mljV/78889YvHgxxo4dazbGovGnatWqZjP23L59G3fv3sVvv/2Gd999Vxs8vkuXLryLzyDjSTs4OBjFixfHzZs3zdJhjBecUaNGAUgeMcPR0RHVq1fHqVOnMHv2bOTPn1+b3CQhIUEb+sfYqcnUk/nkDMr+k9YF9MqVKyhYsCAaN24MIOWNpXECi1u3buHBgwdax6UxY8Zg0KBB8PX1NctJ/vrrr5EvXz5069YtzQlJOKZv+j744AN4e3ub5UbGx8dro73Mnj0ber0eb7/9dqo9/Xfv3o1ixYqhXr16Zh2Tr127hsKFCyMoKEgLuHkcno5xv7344ouoUKECYmNjtZsb0/PUihUrcOPGDW2IxebNm+PBgwcYMGAAfH19tVSm8+fP4+WXX4aDgwM2btxo9j00GAxmHWVN359SZ3q+u3XrFu7cuZMibeLAgQNaCkyJEiW0MbCNnSyDg4Px/vvvY9++fWbfs3379mHevHlYsmQJO/5nMQbN2cjYmcvYmWzKlClYvXo13n33Xe0P/8qVK1rrivFHr9ejbNmy6NKlC37++WfMnDkTLVu2hFJK63QBJD8aMw5CX7BgQQQHB+PXX3+14ifOma5fvw6lFHr06KEtM57wb968qZ2oDh8+DACYP38+ihQpgpIlS2q9xE1bjsPDw1GwYEH4+vqmefOS11tY0mMcY9x4gV+xYgXs7Oy0x8cJCQnaBfvRo0fayArG1Jo7d+7g008/haurqza735o1a7TyL1++rAXWP/zwA1srM8G4r6ZMmQKlFIYMGQIA+Oqrr6CUQu/evREbG4uIiAh07twZer3e7Jxk/F5FR0djzJgxsLOzw9SpU83yNI2dBfv378/A6xkZR/gx5iYD/x2DHTt2wMPDA76+vtpTyffffx/58uVD+/bt0a1btxTj93733XfaEHOmcwGY4rkt46KiovD222+jVKlS2k3kzJkzzbaZPHky8uXLh9deew0nT57Ehg0bMGbMGLz88stmk5YYGxaGDx/OVuVsxKA5m3zzzTdQSsHDwwNz587F6dOnzdaPGjVKO5klJCRg1apVGDBgAD799FNs3rxZG8LHKCIiAv7+/vDx8TGbc37y5MmYN28etm/f/tw+W04WHh6eYkrdr7/+GkopLF++3Gy58eJi7LVsOuvfgQMHULVqVe2E9eTjZGMgl9rMZZS648ePw9fXFwMGDDC78Pbu3RvOzs4petobj8/vv/+OUqVKoUiRIlpLJ5A8oUzVqlXh7++fIh1mxYoV6N+/f57PGc8s0xuMatWqwdHREf7+/rCzs8Mrr7yC9evXazc7v/zyCwoWLIimTZummiq2bt06KKVQs2ZNs9kZHzx4gD59+nCyhUzav3+/Nqzlk5NhGG9cnrxBHDZsmNnNz4MHD7Bp0yZ4eXlp57bVq1dr29+/f1/r6/Hk2OWUMcZjsH79ehQtWhReXl5o1aoV3nzzTQQGBkIphXfeeUd7Cnb+/Hk0adIE9vb22jjmQHLr/rlz57By5UoMHz4cdevWhYuLC3r27MlO/9mIQXM2+f3336GUQtmyZbWAOTExUXv88ttvv0Gv16Njx47pDu5uHBEjKioKQUFB8PT0ZOrFU9qwYQOUUmjfvr22D5OSktC6dWt4enri3LlzZtubXmCMHfyWLl2qLbt48SJ69+6N//3vf9oYscZg7+LFi9ixY0c2f6Lc5c8//4Svry9KlSqlBbmPHj2Cr68vatWqlW6LsDEnfdCgQVqrpcFgwPnz5/Hnn39q37HUOs+ypTlzjPuwYsWKcHR0hJubG2bNmpVizN+oqCiMGDECSil8++232j43Hot//vkHjo6OsLOzw4ABA1INrHlsMsY4zGj//v21FLDExETUrFkTpUqVStEqbDyGZ86cQdmyZWFnZ2c2jvymTZvQo0cPDBo0SOu/YTwWu3btYp+MZxQTE4M6deqgZMmSWLZsmXbzfuXKFbz++utwd3fHlClTtO/KkiVL4Orqinr16mnXriefwkRHR6f4DlLWY9CcjYxpF++++y4A8wvArFmzUjw2A8xHxjD9Uuzfvx/58+fHSy+9hJiYGD62fApnz55Fhw4d4OrqimXLluHx48eIiYlB/vz50a5du1Qv0MYL/caNG7Uhmkz3/YMHD8xm80sNj1XGPH78WHvM37NnT9y/f1+7+fzggw8ApNyXxv8bOzPp9fpMPXXhsbHM+L0w/X4cPnwYderU0SZMMo6V/OTwmHv27EGFChVQrly5FC3Ho0aNQrVq1VC/fn14eHhok8QY8dhk3B9//IE6deogICBAa428du0alFLo06cPgLRvQKZPnw6lFNq2bWu2PCYmRhuFJi28qXk6xqeXpjOKxsTEYNOmTVoqZteuXbUgODo6Whs0YO7cual+J3ksng8Gzdno6tWrKFSoEAoVKqQ9foyKisKqVasQGBiI4sWLmz1ONjK9WDx8+BBr167VhvsxfTxDmRcaGgofHx/Ur18fly9fxrZt26CUwqJFiyy+tlu3blBKYcKECQCYu5cdrl+/jkaNGiF//vxYsWIFPvroIyilsGvXLm2btMYt//rrr6HT6dCqVStO4JMFnrxxN/Xo0SPcvXsXZ86cgVIKFSpU0J62mL7m8ePH+OKLL6DT6dC8eXPs3r0b586dw8yZM1GuXDl89913OHnyJEf6eUaPHz/GokWLoNPp0KVLF0RGRmoj+axatSrV1xiP071799CwYUMopbQx4RmAPZu0rg2mI5q4ublp56l9+/Zh9OjR8Pb2hrOzM6ZMmaK9xngstm7dipIlS6J48eLs3GdFDJqzmbFFuVu3bjh48CDGjRuHggULwtvbO8Ug/kbGx8s///wzhg4dioIFC6JAgQL4/vvvn2fVcxXjiefOnTsYMGAAlFL47LPPEBISAp1OZzbZxZOM+WF///038ufPD51Op+WbMXDOWklJSfjll1/g6OiIJk2awN/fHwEBAZg/fz4uXbqUaiqTMYc2MjISnTt31tIB6OmZ/l3/+eef+OKLL7BgwYIUE5IA/02ZPHr0aLMyjOsjIiK0bZydneHm5galFJo2bWo24gK/S8/G2MHVxcUFc+fORdOmTVGwYMEULfimAbHxGIWGhsLBwQFBQUGc+OIZPDmE3NWrVxEdHZ3i5rN79+5wcnLCTz/9hPnz56NChQpQSqF79+7aUI5A8pB+pt+LQYMGQSmFn3/+Ofs/DKWKQXM2e/jwIapVq6YNGWOcujSt3KNz586hRo0a8PT0RIECBeDj44M33niDuUpZaMeOHShXrhz8/Pzg4+MDnU6Hli1bYv78+Thy5Eiqs8AZT1wffvghlFJo0aLF8652jpaZIdxMp/k1DtGolIKLiwsaN26McePGYdOmTbh69WqKMletWgWlFNq1a6f1B6Cnc/fuXbzxxhtwdHQ0G93nySmwExIStIaAPXv2AIDZuM1GU6ZMQc+ePdGsWTM2AGST9evXw9vbG5UqVUK+fPlQpEgR9O7dG2vXrsWFCxfMZjQFzAPoHj16mD1Jo6e3Y8cONGjQACVKlIC/vz969OihfTcA4Mcff4RSCqVKlYJSCrVq1Up1SvGpU6ea9Y05f/58ipsger4YND8HYWFh2riKpj2OTYfOMjV06FD06NEDn332Gf7+++/nWdUcKyMBmXFfGwwGTJo0Cfnz50f+/PlRtmxZ5MuXTwsKfH198cYbb+Cnn37CX3/9ZRZ83b17F2XKlMHYsWPNyqSMyWhr4v79++Hn54eSJUtqI8Q0btwYOp1OO06FChVC27ZtMW3aNGzduhWnT59GUlISdu7cmc2fIvfbtWsXqlatCj8/P4wYMQKhoaHYv38/+vfvDzs7O0yaNMks39WYCtC1a1ez78T169dTjFBiup6ty5ZlZB8Z92lkZKQ2MlOBAgXMRsEwTjX//vvvY+vWrbh69arZzc2uXbvg4+ODzz77LNs+S2738OFDjB49GnZ2dqhUqRJee+01dOjQAQ4ODvDw8MDatWuRkJCA06dPo3r16qk+oQGSc5inTp0KpRQ++ugjK3wSSguD5ufEOC7sV199BSDllMnAfye+hIQEtpI9pT///FNLb0mtk4QxuP7jjz/QtGlTeHh4YMWKFTh27BiWLFmCvn37ao/KjD/lypXD66+/jm+++QYA+PjyKS1YsACNGjXS8vhTG8nCKDY2FuPGjdMuKsaL+7///osNGzbgww8/1CbzMf48OW08A7Knc/v2bbz22msoUaIEFi9ejPv372vrjMMzvvjiiyn6V9SsWRNKKSxevBj37t3D0qVL0bx5c3z11Vfa8TM9x1Hm/PPPP6nepD+5bM+ePahUqRJKliyJ5cuX4/Dhw9pY/x4eHtr3xcPDAy1atMCIESOwfv16AEj1KRtl3C+//AIPDw906dLFbCKyjRs3IjAwEDVq1NBajmfNmgU7Ozu0atXKrHFsz549GDlypNYoYBy9hGwDg+bn5Pjx49DpdChbtmyKoIGeXWJiIsaNGwd7e3ts27bNbHlaY1Z+88030Ov16Ny5s1lHpLt37+LIkSOYP38+unXrhoCAABQtWhT79+83K5fHL3M+/fRTKKUwceJEbZlpS9epU6fMtj937hwqVqwIX19frFu3LtUyL1++jFWrVuHYsWPZU+lcLK2/36ioKHTp0sVsWLFjx46hTp062lS+9vb2CAkJwbVr17RtDh8+DGdnZ9jZ2aFcuXLQ6XTw8fFJMS46ZU5SUhLeeOMN+Pn5aecgY2fY1M5tDx8+xOeffw47OzsMGTJEu+lJSEjAtWvXsGPHDnzyySeoV68e7O3tUahQIbPh5lKb2pksi4+PR/Xq1VG8eHFERERoy69cuYI5c+bA29tbm1AJSL5BMc7C6Ofnhw4dOqBRo0YoVqyYNurJnTt3rPVxKA0Mmp+jMWPGQCmFoUOHWrsqudKuXbtQoEAB9OrVK8W4pLdv38bnn39uNm3y1atX0aVLFzg6OmLx4sVptu4/OT0sZY7pBbhixYoICAjAb7/9pi27cOECWrRoAScnJ7MWl4SEBCxcuBBKKYSEhGh5/en1TOfFPqUn94lpoJXWvjQGWnFxcZg8eTJ0Oh1KlSqFGTNmYP/+/ejQoQPc3NxSdLj88ccf0bdvXzRs2FB7qkbPbvny5VBK4b333ksxLfmdO3cwZ84cs2WnT59Gw4YN4e3tjdDQ0FTLTEhISHMaeUKK+RCSkpLS/L4kJSUhKioKRYoUweuvvw4geWbTVatWoU2bNlBKoXLlymbTzht9/fXXaNCgAYoXL45q1aqhQ4cOZvnPZFsYND9HN2/eRGBgIAoXLpzmDE309AwGA8aMGQO9Xo+VK1dqyydOnKgNi2U68xgArF27Fr6+vnjppZdStHQ+2RLHWZaennHfbdq0CUopDBs2DNHR0Vr+ZWBgID766CNcuXLF7HU3b95Ey5Yt4e7unu6IGPwepbRy5Uqz1IonzZ8/H61bt0aHDh0wduxYs8fJRqtWrUKBAgXQuHFjbN++XdvPxqEA27Rpg+PHj6d4nekTBH5vskabNm1QqFAhbNy4UVs2adIkODg44IUXXjCbRTYhIQE//vgjXFxc0KFDB1y8eBFA2k8XeIzM9enTBwULFtTGFjdtUImOjsa1a9dSjM9/5coV5M+fHy+//DK2bduGwYMHI1++fPD09DSbGTYpKSlFh8z4+HgkJSWZtVCTbWLQ/JwZcwJ79uzJx/tZLDExEadOnULZsmXRvHlzzJkzB/7+/nB0dESPHj2wefNm7eRnvPjfv38fI0eOhIODAz744IMUrTiUNRISErQLc4cOHaDT6ZA/f37ky5cPb7/9Nnbt2pXqiAsAsHnzZiilUL16dc6GmQE7d+5EmTJl4Obmluq47keOHEHVqlVhb2+PsmXLokyZMlpHsVWrVmnfjZiYGNSuXRve3t7abJnGdcZUG1dXV3z44YepDgXI81vWOnz4MJycnNC3b18sWLAAAQEBcHR0RPfu3bFly5YUT8oiIiLQo0cP5M+fH1988QWPRwYY/77ff//9VJ8KT548GUWKFIGvry/q1KmDTZs2ma1v1qwZHB0dUaRIEe31T96Q3LlzB6NGjTL7zvDY5BwMmp+zuLg4zJ49mx39slBSUpJZwGUcFk4phQYNGmD58uUp0jVM7dq1C4ULF0azZs3SbZmjlNKaaMR0velF49ixYyhXrhzs7OxQsGBBLFiwwOzikVpZxp7kqQ3JROb27t2LwoULo2bNmli4cGGKv/v79++jQYMG8Pf3x6xZs3DmzBkAwJYtWxAUFISSJUtiy5Yt2vYVK1ZEUFCQWc7/rVu3EBQUhN69e8PX1xeNGjVi7mU2efL706NHD20YxkaNGuHHH39M99z266+/ws7ODl27dmUnv1SYnr+e3NclSpRA/vz5tcA4JCQErq6uqFu3LmrXrg1nZ2d4e3tj2bJlAJJbixcsWABHR0ezCc1M/fnnn2jZsiXs7e1TndiMbB+DZrJplnrZm96hx8TEoE+fPlBKIV++fChYsCDWrFljtn1aU2UfPXo0S+qbl2Tmke6///6LV199FUop1KxZE02bNoVSyiwXMzPDBlLq/ve//0EplebEScZpymfNmmW2/MiRI+jQoYPWASk6OhpRUVHo06cPdDqdNorG/v378fbbb6NkyZI4fPgwe/Y/A0vnNtP1Dx48QN++faGUgqOjI0qWLKml+BmlNlrQ/fv3sXfv3iysde5hun+NjSWm+3D16tVQSmk5xuXLl8cnn3yi9XExDiVbunRp7aby5MmTaN++PZycnPDxxx9raRh37txBaGgomjVrhkKFCmHq1KkchSmHYtBMNunJAOrevXvpBlWfffYZXF1d4enpiREjRmDGjBlwdHTEe++9p7WwZCTgYm5f5n3zzTeYPHkyfvjhhxR54QAwb948KKVQtGhRfPDBBzh16hQSEhLg4+ODoKAgbeQLS0EzA+a0GffNuHHj4OjoqI2y8OWXX2Lu3LnaxfuNN96Al5eXtq/Pnj2LL7/8EkFBQVBKoUuXLmatxps3b0bFihW1vPPChQvD0dER48aNMws6+L3JuCf/zp/Mb31y/fTp0+Hq6goPDw/873//026MZs6cqW2bke8Gh/lL9mRDy7hx49CkSRM0b94cXbt21Z6+AECDBg3g5OSEypUro0OHDinKGD58OJRSGDFihLb81KlTKFmypNZA0LFjR9SoUQOurq7w9vbGd99995w+KWUHBs1kU548+W/ZsgUNGjRAUFAQXnrpJSxatEh7HJmYmIhHjx5h5MiRUEqhU6dOWL9+PR4+fIjExET06tULBQoUMHvcTE/vyWH2tm7diuLFi5uNlezi4oIlS5ZoNyoGgwGLFy9Gz549sW3bNrPWlQULFmgjAhgxMH56CQkJOHz4MAoUKICXX34ZL7zwApRSeOutt7TjUb9+fRQrVgz79+/HL7/8gldeeQVKKVStWtVsaDhj+tjjx4/x999/o1OnTnj55ZfRuXPndKecp4zbunUrmjdvjoYNG6Jr165YvXq12fro6GjtyVnnzp2xfv167TjWrFkTZcuW5SgLz2Dq1KnIly8f9Ho9PDw8kD9/fiil0LBhQ224xePHj2vntmnTpgFI/m4Yb0Bu376N4sWLI3/+/Dhy5IhW9okTJ/DBBx8gMDAQFSpUwEsvvYTRo0en6DxIOQ+DZrIZT7awjB8/Hnq9HqVLl0a1atXg4+MDpRQaN26Mu3fvattt374dq1evNut5nJSUhEOHDqFgwYLo1q0beyVnoevXrwMAGjZsiAYNGmDhwoU4fPgwvvjiC/j4+MDDwwPTp0/XjufDhw/N8ilNj3OtWrXg5uZmNlMmZZ7xZuOPP/7QpuYNDAzEV199hatXr5q1RBu/Q3q9Ht7e3li8eHGK8n766SezPGbAvEWU45Rnjmkrb2JiIt555x04OTmhWLFiWqukUgrDhw/XHv/Hx8fj119/xdKlS83Gwwb+G4Vm1KhRqaYWUNo2bdqE0qVLa7neP//8M+7fv49jx46ZpSgZp4ofOnQolFJ48803zcoxHlPjk7T27duneK9Hjx4hJibG7HpFORuDZrIpCQkJ+Pbbb7Fv3z4EBwdjwIABOHHiBBITE3Hp0iU0adIESin0799fu5A8+djReDGPj4/H+PHjoZTSOmvQszEO3zdgwAC88MILKTq7rF69GgEBAShevDg2b94MIPVcS+Pj/B07dkCv12uzZNHTS0xMhJeXF5ydneHl5YXKlSvj5MmTZtusW7cOAQEBWmCQmiVLlkAphSlTpqS6no/5n05CQgKWL1+OixcvIigoCEOHDsWxY8fw+PFjbdpyOzs7jB8/XguynhyVxPS71LlzZyilEBYW9lw/R062bds2rUV52rRp2tjvRuHh4ShSpAiqVKmipWk8ePAArq6uKFCgAA4fPgwg+ViaHou6detCKYW1a9cCMB9ykTczuQuDZrIpxqF+mjRpgmLFiqXolX/8+HG0aNECzs7OWLZsmVlrV2onp/Pnz+P9999HVFRUttc9L1iwYAEcHBzg7e2Nxo0ba8uNQXBsbCxmz54NpRQGDRqU7uNI47FjPuyzMwaye/bswZo1azBx4kTY2dnhs88+MxtG8dq1a+jXrx/s7e0xbdo0s1F8zp07h6lTp6J48eJo3LgxTp8+/dw/R27WqlUrKKVQp04dVK1aNcXNx/bt21G9enX4+fmlmAEztXPb33//bZZ6Q5ZdvXoVvXv3hk6nMxs72Zg2dv/+ffj4+MDJycns6eSXX34JpRT69u1rVp7xGBpb/gMCAnhTmcsxaKbnLr1hyq5du6a1hIWEhABIziEz3X758uVwd3dHx44dLQ559uT7UvpSe+z+5DLjxb9Lly4AUga9p06dQoUKFVC1atV0h8MyxcA5a/3zzz946aWXUKJEiRSjJxw6dAj16tWDUgrNmjXD3LlzMWLECNSvXx8ODg5o3Lgxh8N6SqmdY4zfnYMHD2p5/2+//TaA5BZJ0ydj3377LRwdHTF06NAUHQQpa/z6668oVqwYqlatapaHDCR3fNXr9WjVqhUeP35sdl4qX748dDqd1rL/5HmyQ4cO6NGjR4rrFeUuDJrpuTI9CT15YjGehL755hsopVCmTBntrt00OI6IiEDFihXh4OCQItcvLTyJWWbaQnL9+nXs2bMHV65c0R4RG4/dgQMH4OjoCL1erz1GNr2AREVFoWXLllBK4cKFC8/xE5Dx7zwuLg7fffcdnJycMGjQIERGRpptFxERgY4dO2qPqn18fFCuXDl8/fXX1qh2rpDejZ/xu/X2229rw5ilxpiTXr58+QznjDO3PGOM340HDx7g3XffhVIK48ePx+PHjxEVFYVly5ahXLlyKFCggNmMssbjapxkqXnz5toTtMTERO3Ycu6FvIFBMz0XT57YFy5ciI8//hhfffWVNmmF8aT28OFDLUdsxYoVZuuMJ7A33ngDSiltaC2yLCMXV4PBgGHDhqFw4cJwd3eHu7s7OnfunCL4NXaOeeedd7Syn8y3dHR0TJFTS8/P1atX8dprr8HDw8Pscb9pcHflyhX8888/2Lt3r9lyPmLOuCe/Vz/++CNmzZqFlStX4p9//gHw3z6/d+8e9Ho9HBwctLHhTfd1UlKSdu4zzsJITye1v2HjOerQoUOoXr06fH198cUXX6Bfv35wdnZGkSJF0s0Rb9u2LZRSmD17tll5lHcwaKYskZSUlKGg7Oeff0ZAQIA2q5XxcWVoaKjZCWjjxo1QSqFly5baaA2mnStat24NpVSGW5rzOtN9e+XKlRTLgOTHlmXLloWnpyfat2+P0aNHo1u3bnBwcMArr7yijacMJLdUGqeK3bVrl1k5+/fvh6enJ+rVq8chlp5SVo1OsXHjRnh5eaFt27a4dOkSgNQ7ZhoxWE4ptXNbasFSaGgoAgMDzYZgLFGiBA4dOmT2+mnTpkEphZ49e2rLTNfXrl0bfn5+nJ30KT35N3zv3r1UO1TOnDkTrq6ucHBwgIODg9nQl8Ztnizz/PnzUEqhYsWKPD55FINmylInTpzA1q1bzQJcIPmR2Pjx47UOZPPmzcOpU6ewY8cO1KlTB35+fti2bZvZa0JCQqCUQr9+/bTy7t27h2+//RY6nQ6vv/76c/tcucFvv/2G6tWro3379trjeuOF4dixY6hWrRpq166NpUuX4t9//9Ve179/f7i4uGDYsGFmx3XmzJlQSqF48eIYPXo09uzZg7lz56Jhw4ZQSqU6lBlZZnrRv3r1KjZv3pzpVkfjcb179y6GDx8OJycnLFy4MMX3ktJnGjidPn06xQ0ikJyONGbMGHh4eKBx48aYMWMGjhw5gkWLFqFYsWKoWbNmiteVLl0aSil89dVX2vG+e/eu1ol2+PDh2fvB8oAff/wRwcHBqFSpEipVqoRJkyaZnddOnz6NTp06acfBKK3viPE4/fjjjxzCNA9j0ExZwmAwaLMj1alTJ8XMcBs2bICvry+6detmNmV1ZGQkXn75ZSil0K1bN7M0gL///huurq5QSmkTK3Ts2BEeHh4IDAxM9QJGqfvjjz/g7u4OpRRKliyZYlaqRYsWoUaNGti+fbu27MKFC2jevDmUUtDpdChatKjZY/6EhATUqFEDSil4e3ujWrVq8PX1RenSpfHzzz8/t8+WU6X3aDc2NhZDhgxBkSJF4OLigkGDBqUYHiujDh48iNKlS6NYsWLs4PcUDAaDWS7yxYsXAfx3/H7++Wf4+fmhZ8+eZk9jbty4oaVaPHn8jE/S7O3t0aJFCwwbNgwhISHw8PDQpiinlNIbLckY1F67dg3t27eHUgrVq1dHw4YNtUmY6tSpY3aMfvzxR3h6euKVV16xOFoMUzEIYNBMWeTQoUPaI0knJyd8/vnnZo/md+/ejd69e5vlTX700UdwcnJCwYIFUaVKFTg5OeHbb781u9P/6KOPoJRCcHAwunbtio4dO2L06NEpHrdR+k6dOoXChQujXLlycHBwQLNmzcxaL2/dumWW6vLRRx/BwcEBRYsWxZQpUzBjxgwopfDqq6+aXfxXr14NvV6PKlWq4P79+zh+/LjZ+7KTUkoLFizQWqqezGcFgJ07d6JMmTLw9PREv379MHPmTNy6deupUycePXqEKVOmYODAgfzePIV169Zp57bChQvjm2++MVv/zTffYOjQoWZ/65MmTdLObcWLF4ePjw+WL19u9jpjZ9lKlSqhZ8+eqFu3Lt555x12KEuF6b69d+8e5s2bhw8++AADBw7U+r0YTZw4EXq9Hu+++y7Onj0LIPn81r17d+0cZmzUuXnzJt566y3Y2dlh1qxZZjOWEqWGQTNlmXLlyqFcuXKoVq0aihcvjgMHDpitNwbDR44cQXBwMBwcHNC1a1ds3boVv/zyCwoVKoTg4GD88ccf2mtu3bqFgIAAlC1bNsX0vRym7D8ZGXqvatWqaNKkCQYOHAilFD799NMU28THx+ODDz6Ak5MTXn31VWzfvh1xcXG4e/cuqlatioIFC2Lu3Llmr+nYsSOUUpg3b562jMcmdcuWLdOm6k3NgwcP0Lp1a7i7u+O7777LsrxJpmU8vZs3b8LZ2RnBwcEoWLAg6tatixMnTmjrExMTtUD34MGDqFWrFuzt7dG1a1ds3LgRy5Ytg7OzM9q3b68FcQBw8uRJKKVQrVq1FDPG8fuTuk8++QReXl7w8vJCvnz5ULZsWfTv319Lu7h69SpcXV1RrVo1bR8az4tnz55Fjx494OjoiC+++EJbv3XrVpQqVQoVKlQwewpKlBoGzZQhllq5oqOj0a5dO5QtWxaTJ0+GTqfDkCFDUkxOAgDdu3eHu7s7Jk+erHVKi4mJQZkyZaCUwieffGI2GcnChQuhlMJbb72ltQTwovIf01YYY6dJwPxxYlxcHHr27ImAgADs2rULRYsWxYsvvqjd2BjLOHPmDPz9/VGvXj1tOl8gOY3G2PGvSZMm+Pvvv7V1x44dg6urK4oVK6ZNvcwOZeaMx+LevXvo0qULPvnkE7MRR4z/rlixAkopfPzxx6m+/lmx5T+l9P5WExMT8eDBA9SuXRvBwcGYMmUKlFKYNGlSinGUY2Nj0bZtW7i7u+PTTz/F1atXASRPGlOsWDEULFgQs2bNMjuWgwYNglIKH374ofZ+PEYpnThxAg0aNIBer0fXrl3xww8/4Pbt24iNjTW7sTx9+jRcXV3Rr18/ACnH+N+2bRsKFy6MZs2aaWPIx8bGYuLEiXB0dMTAgQM55TWli0EzZdijR4+wdu1aLe/4yYtN165d8eKLL2Lt2rVo2LAhChQogI0bN5pts2PHDiil8Oabb6Yo/6WXXkK+fPlQrFgxbNmyRVseHx+P+vXrw8PDA6tXrwbA/LInRUZGomvXrihSpIg2HNKTBg4cCF9fXxw8eBCTJk3SOhyZPrJfunQplFKYPHmy2WsvXbqEKlWqoH79+lBKYerUqQD+Ow7vvPMOlFIYMmSI2XL6jzEYMqYtme4j4zpjOpJxOt5nDaB4HDLGYDBg8+bNuHfvHgDz/f7o0SM0a9YMVatWxbp161CxYkWUKVMmxdTvy5cvh1IKI0aMMFseHR0NX19f2NnZoVatWtizZ49Z2W5ubvDz89Mm2uANp7n4+Hh07doV+fPnx+TJk8064RmPk/HvfN++fVBKoVatWqmWdePGDTRq1Ag6nU5rsAGSn36WKVMGwcHBqTb0EBkxaKYMMV4QlFJo2rSp2Uxvxke/33zzDRwdHfHvv/9iyZIlcHZ2RseOHc1OTuHh4XBwcMBnn31mVv4PP/wAvV6PMWPGQCmFJUuWmK03Btv16tUza02l5NZLY76e8WfixInafjIGxatWrYKDgwP+/vtvRERE4MUXX4Svr6/ZuKSHDx+Gs7MzevbsqbU0HzlyBC1atECtWrVw7do1zJgxQ7tYGf+NjIxE/vz5odfr2bM8A/bs2QMHBwdMnz4dQPJ3KCkpSQuajTeNTwZQGQ2inwyWTVvPGEibM94oKqXQu3dvs31u/H3IkCEICAjAtWvX8Pnnn8POzg4DBw40y+//4YcfoJTCggULzMqfO3cuAgICMGDAALMbIuPTsq+++krLtaX/GP/W586dC6UUPvroI7P1af0dBwUFwd3dHb/++qvZdsZj2atXLyilsHv3brPXsZMsZQSDZrLIeKevlIKXlxc8PDxQo0YNbNq0yWy7lStXQimFNWvW4P79++jSpQscHR2xaNEiLbD+/fffUbx4cVSoUAEHDx7EuXPnsHDhQgQFBWHAgAGIjIxMM+jq0KEDGjduzJaAVISGhsLf3x+VKlVCvXr14OrqimbNmpntq/Xr18PR0VHLPTamvXTp0kVLq4iIiNAmjmnQoAG6du2KypUrw8PDI8WIG8aLmvFitHnzZm0sYErf7t27kS9fPvj4+Jh1/DLOhtmxY8cUrzENEoyTZqTGdLu7d+8iNDQUAwYMSHfShrzq+vXraNeundbJzziKj3HSJGNgu2jRItjZ2eHPP//Ev//+iwYNGsDHxwehoaFaWZs3b4aXlxfatWuHP//8Ezdu3MDixYvx4osvYsiQIbh3716aM2QWLlwYTZo0QXR0dPZ/6BymZcuW0Ov12r5L68bR+HdvDLK7d++u3Sya3ggZy+PNPT0NBs2ULuMJ6vfff0f16tXh7u6OmTNnwsfHB97e3li6dKm2zYkTJ8xmS1q7di38/Pzw0ksv4c8//9TK+/jjj7Xpe319feHk5IQXXnjBrKPfkzPMAcmdpMiccR/dvHkTAwYMgLOzM6ZPn4733nsPHh4eqFOnjhYsnT17Fs7Ozpg0aRKA5EeVrVq1gouLC+bPn6+Vef36dbRp0wYFCxaEp6cnatasia1bt6b6vk/+Tv+xlHf/4YcfphiTNy4uTgveNm/enKKchw8fYsiQIejVq1eKqbGfzGHftWsXBgwYAJ1Oh4IFC6ZIJ6Bka9asQbly5fDCCy9g+PDhyJ8/P4oXL45jx45pN/vLly+Hvb299gTsxx9/hIuLi1nnvhs3bmhpSgUKFECJEiWg0+lQsWLFFB0Hn5zh1LT/AP3n/Pnz8PX1xYsvvpjh11y/fh2NGzfWUmWMx/DGjRv44osv4OTkhFGjRmVXlSmXY9BMGTZx4kQopTB27Fhs3LgRzZo1g06nw/jx47WOe8WLF0ffvn0BAHfu3MHIkSO1jjPGbR4+fIh169ahR48eaNWqFWbMmGG1z5Sb/P777yhWrBhq1KiBXbt2ITw8HIULF4abmxt++eUXPHjwALVq1UKHDh0AJLe+bNmyBS4uLqhfv77Z2NqxsbG4detWivFi2Ukp80JDQ7Fz584UgdGVK1dQuXJlKKXMRlX47rvvoJRCYGCg2SPkP/74A2PHjoWnpyfeeecdrSPakzctf/31FyZNmoSAgAAopTBq1Cje2KTCuE/u37+vnafmzZuHb775BpUqVULhwoXx+eefA0gOuPLly4cvvvgCAHD58mX06NEDOp0Oc+bM0VoyHz58iM8++wytW7dGvXr1tNdntC78fpmLiYmBt7c3ypUrp42PnRGnT5+Gj48PlFKoWbMmunXrhqZNm0Kv1+Oll14y68hMlBkMmimFJy+wxhP52bNnUa9ePeh0Omzbtg13795F586dYW9vjy5duuDMmTMIDg7GSy+9pAXIe/bsQVBQUIoAwMh0KCyOiGFZap2EjMsePHiAjz/+GEopDBs2DEByGkCDBg3g5uaGgQMHonXr1ggKCtJaKaOjozFw4EA4ODhoLdBAyos3Oydl3q+//mo2rXLZsmXNJocBgPnz50MphVdeecVs+eDBg6HX67U8/saNG6NChQpa3m1qKUrXrl3DggULULNmTW0KeqbLmHvy3Gb8/969e1GhQgWUKVMGv//+Oy5fvoyaNWvC3t4eH3/8MY4cOYJq1aqhVatW2mu3bNkCPz8/1KpVS+vEZ8p0zF+e257Ow4cPUa9ePfj6+qa6j1Pz4MEDxMfH4/jx4xg0aBDc3Ny0YUuf7OBMlFkMmkmTlJRkMThasGAB9Ho9mjRpgvj4eMTExGjDMHXo0AH+/v546aWXtF7oCQkJmDZtGpydnfHGG2+YdSA0Si0Vg8wlJSWZBbJnz57F+fPntf1sdOLECdSpUweFChXChg0bACT30G/evDkKFiyoTaZg2mpjHDKuYMGC7AzzFFL73kRERKBq1apo2LAhPvroI4wYMQKFChVC6dKlzfoCPHr0SJvkwnSkmaioKKxfvx4tWrRAmTJlUL58eTRt2tRsxkbTR/zbtm1D69atYW9vjzJlypiNPkNpn9tMO4lNnToV9vb22pOy8+fPa0PCdezYEYGBgWjdurXW+e/+/fsYO3YsnJycMHr06FTzkXlue3bGlJfx48enuY3pOPWfffYZ3nvvPW1dVFQULly4gJiYmOyuKuUBDJoJgHnL4u3btzFnzhwsWrQImzdv1jqJAckpF126dIFSymxmrO+++w4lSpSAUgpubm5mrzl58iSqVq2KkiVLamOX0tM5d+4c2rVrB19fXxQsWBAvvPACPv/8c63DS1xcHBYtWgSdTocuXbrg/PnzAJIfJ8+aNQvu7u4YNmyY2WyNcXFxmD9/fopWULLMNBCLi4vT8u6nT5+OcuXKmeXpr1mzRpsS3nQ0i40bN8LZ2RllypTRlpkOTxcTE2P2vXkyAIyKisKQIUPg4uKCmTNnZv2HzOFM91VkZCR++OEHrFq1CgcOHDD7Hpw8eRJNmzaFq6sr1q5dqwVho0aNQqlSpaCUQo0aNczGZz58+DD8/PxQo0YNbYINylq3bt2Cvb093N3dzSYfeXKMcyB5ZlrjtPOPHz/mEzLKcgyaycwnn3yCfPnywcnJSXusXK5cOZw4cUJ7xBgWFgZfX1+UL19eC8oAYNOmTRg7dix+/PHHFAP///HHH2xxeUrG/bZ06VJ4enoiMDAQr7/+OoYNG4Zq1apBKYVevXppQ8xduHABnTp1Qr58+fDtt9+alXXu3DmzYbLo6Tz5tzxjxgxUrFgRtWvXxuzZs9GjRw9MmDBBW2/87hhHoZk2bZrZ6/v16wellBb0pjWDX1pBwJ9//snply2YPHky3Nzc4ODgAKUU7O3t0a5dO7OnLt999x3c3d3RqlUrbZr5R48e4ZdffkGPHj3www8/mKVaGAwG7Nu3j7nI2Wz69OlaypHxRtT0OxgbG4u1a9eiYsWKeOGFF3DmzBlrVZVyOQbNhKSkJMTFxWHkyJFwdnZGSEgIlixZggsXLmDq1Knw9fVFzZo1sXPnTgDJAcDw4cO1maxML9aWZrRibt/TiYiIQOXKlVG+fHmsW7dOy5c0GAwICQmBo6Oj2SgM69evh4+PD+rXr2/Wc5+enenf98OHD9GlSxc4OzujfPnyWgqMUgrLli0DkBwAG19z48YN2Nvbo3Tp0jh9+rRWzvHjx7UnNVk1dTYln9uio6MxYMAAODk5ISQkBAsWLMC+ffswdOhQODk5oU2bNvjjjz8AJE/D/MYbb8De3h5ff/21WV5ybGxsui2XPLdln7i4OLRs2RJ2dnZa3wDjkHGbN2/Ge++9h6JFi6Jo0aJYuXKllWtLuRmDZgKQ3BLs7u6ODh06mD0KvnHjBkaNGgWlFDp16qS1Zh4+fBiVKlWCl5eX2SNoyh6ffPIJlFLajIhA8rFZvXo1Xn75ZSilEBISouXt3blzB8OHD4ednR0++eQT7eLP1v6scebMGXz55Ze4evUqfH19MXXqVFy9ehW3b9/G4MGDtRx/U8aAyziBibGzptGYMWPg6+uLP//8k8cpC+3YsQNubm7o1q2b2URLN27cQM+ePbUnNcag99dff0WxYsVQpUoVHD9+3FrVpifcuHEDw4YN025KdTodvL294enpCScnJ7Rv357pf5TtGDTnIeldiCdMmACllDYUT3x8PH7//XeMGDECbm5ucHJywo8//mj2mqlTp8LOzg5vvvkmx1B+Rmm1YBlbKAcPHgxPT088fvwYMTEx2LZtG/r27auNwbt06dIUr92/fz8qVaqEEiVKaGP+0rOLjY1F3bp1tZEt+vfvb7Y+Pj4eL774IpRSWse9hIQEs++fv78/vL29zTr2PZnSRBmX3rmtR48ecHV11UYSiY+Px549ezB8+HC4ubnB2dkZK1as0LZ/8OAB3n33Xa3zmXEkILINmzZtwrhx49ChQwe8/fbbGDNmDA4cOGDtalEewaA5D0jvkaKxd/cbb7wBDw8PXLx4EVeuXMGnn36qdX55++23Ux0+6fLly2jWrJk2CyBl3pPH5sSJE7hx44ZZT/zHjx+jT58+2n7+9NNP4efnBzs7O7z77rtmr3/06JH2eP/BgweYMGEC7O3tzUZmoIwx3rCYBmTGZT/99JN2DIy5y4mJidp346effoJSCg0bNtRem5SUpK0PDQ3VcjSfTGfiY/6Ms5Qu8fjxY7Rv3x7FixcHkDyT4tSpU7VzW//+/c1uVozH4tChQ6hRowacnJzMbmyIKG9j0JzLmV6Qd+3ahS+//BI//PADjh8/bhYMjB49WruI1K9fX5tG+eTJk2blXblyxaxV+dtvv0XHjh35WOwZLV26FBUqVEDhwoVRqFAhNGrUCLt379ZuVmbOnAmlFIoWLQqlFNq1a5fqPv/0008RGhqqHffr169z2vFMSm/oRdOAtlu3blBKYeLEiQBSBnCtWrWCUkrrjPnk+iFDhuDYsWNZWfU8xfTctnfvXvzwww8ICwszyxUH/jsO7777Lho1agSlFOrXr59igoubN29qo5okJSVhypQpaN26NUfFsEGp3dASPQ8MmvOAixcv4pVXXoG9vT0cHR21nLC33npLG+f3999/h6urqxaYGcf4NXXjxg2UKlXKbBIMDunzbO7cuYM+ffrAzs4O9evXx4ABA9CzZ094eXnB29sbc+bMAZDcauzn5welVKqzjP3777947733UgwFaMTjlDGm++natWuYOHEi3nnnHXz00UdaxyOjffv2oVChQihQoIB2c5OYmKiVceDAAW0oOWPrv+l6I174n57x3Obg4AB7e3sopeDh4YG5c+dqE/gsXboUSik4OjqiaNGi2rTyT5ZTsWJFLF++XFvGdBkiehKD5lzur7/+Qo0aNeDr64vx48dj69atOH78OF5//XXY2dnhnXfewe3bt3Hnzh28/vrrUEqlmNY6ISEB27dv/7/27jug5v3/A/jzfU7jNJG0FFlXhLJHyCZ7ZOZyy4hLNtfXvLjGtUUke19ky4hrz3CtMjOSEYUiqU7nvH5/9Pt8dMq8l4xej3/u9/uZfc7HOZ/XeZ/X+/Wi+vXrk42NDW3bti3LeTgo+3dWrlxJxsbG5OPjQ5cvX5aXX7p0ifLly0eOjo5yo4qgoCC5VqzUdlmtVlNoaCh17dqVLC0tycvLK0twxz7d2LFjydDQkIyMjOQvk6VKlaLg4GCd7aQvKoMHDyairO+Dfv36kRCC+vTp89bzcKmyTyd9yQgLC6NSpUqRnZ0djRo1inbt2kWbN2+munXrkqGhIU2cOJFSU1MpLi6OqlWrRkKILLn/Go2G9u7dS3Xq1CFra2s6dOhQlvPxZxtjTMJB8w9KerBIHasWLFhAKSkp8vrjx4+Tq6sr6enpUWBgIBGlp284ODiQgYEB/e9//6MLFy7QwYMHadSoUeTk5ESWlpY0e/Zszrn8DLRaLcXHx1O5cuWoaNGiOg/mCxcukL+/v9xGOeODXqrna2pqShUqVKBSpUpRvnz5SKlU0pgxY/gB/x9FRUVR+/btycrKirp27Urr16+nO3fu0IkTJyhXrlxUqlQpOnnypLz93bt3qXTp0iSEkOv6pqWlycFwdHQ0mZubU48ePfjefCbSZ1vfvn3J2NiYli1bptOk5MSJE2Rvb08WFhZy+bEdO3aQEILy5s1LixYtoqioKDp+/DhNmDCBSpQoQTY2NhQQEMCfbYyx9+Kg+TuUcXTqfT/tarVacnBwoAYNGsjLYmJi6K+//qL69euTEIIqV64sP+yJiA4dOkTFixeXi/8bGRmRvr4+1ahRQyf/kn9SzkpqkiA9eN9Wszrj6/b06VOytbWlHj16EFF6296lS5fKo2L16tXTaR5DlN79LSQkhFq0aEHVqlWjunXrkq+vr852PHr572g0GgoMDCSVSkXjx4+n+/fvy+uuXLlC9vb2ZGhoSJ06ddLZb/78+SSEoKZNm+osl4Jkzon9eB/72RYTE0O5cuUiT09PnWV//fUXeXh4yJMwMzYumTdvHhUoUICEEGRkZERGRkakp6dHbm5unFvOGPsoHDR/J6KiomjmzJnyQyXjTP13iYyMpLx588oP+f3795OPjw8ZGhqStbW1Tgk5rVYrP6Ti4uIoODiYZs+eTTNnzqQDBw7I232oeUlOlJycLHd6k+okZxzVv3//Pp0/f54SExN1AoGrV6+SpaUlVatWjQ4dOkTt27cnhUJBRYoUyZJT/uzZM537LQVkUk66tIzvzYe9b8R31apVtHjxYvn/P378mDw9PUkIQeXLlyd7e3syMDCgjRs3ytu8evVKDtR279791nO8b3JhThcdHU0rV64kojeB8oc+2y5dukRmZmY0atQoIkovQ9atWzf5sy3jrzMZj3X//n2aNm0ajRs3jsaNG6dTilGqJMQYY+/CQfN3onLlyjpdxqQPd61WS/Pnzyd/f3/atWuXTvtdtVpNtra2VKRIEerbty/Z2tqSUqmkkSNHZjn+jRs3iOj9o5T80H+3IUOGkBCC/Pz85GWvXr2i3r17k5mZGZmbm5Obm5tOQEZEVLZsWXnykrGxMU2dOjXLse/evUtDhw7VaQ2b+eHOwfKHZX6Nnjx5kmVZxslfwcHBZG9vT5aWljRs2DCKjIykLVu2yKOYUqUFovQOjEIIKlas2Je9iB9MUlISOTo6khCC/vnnHyLS/ZxZuXIlrVq1is6fP6/TefTmzZskhKBy5crR//73v3eWYCRKr/jzoSCcP9sYYx+Dg+ZvnPRhHhYWRkIIql69Oj158oSIiLZt20YODg5yNQwhBHXr1k2n69WgQYPkdS1atKDHjx9nOceaNWvI3NxcJ02DSDcwZ+kyvxbSw/jBgwdUvnx5EkJQeHg4ERE1a9aMrKysqHHjxtSmTRtSKBQkhNAZ3Vq1apVcseRt9+b48eNUrVo1yp8/f5b7wz5O5nu2adMmqlixIpUuXZratWtH+/bty7LPvXv3qEKFCmRvb09r166Vc2ajoqLI3t6ezMzMaPr06Tr7BAQEZEmnYe8mfWHx9/cnIQS1bt1aXrd161bKnz+//NllZmZGQ4YM0dlfGt1XKBTvLHs5f/58qlChQpYUGS5Zxhj7Nzho/sa8bcRQ+mCXWr7OmDGDiIiqVKlCtWvXpgULFtCqVauoefPmcik5SWhoKBUuXJiKFCmSpeZyXFwczZkzhxwdHalOnTpcdeE9tFqtzr15/PhxllSVRYsWkRCCmjRpQuHh4WRtbU0BAQFyCsXatWspV65c9NNPP8nL0tLSqFatWnL3MWn08tatW7RkyRKqWLEi2dvb08KFC3mS0r+QeQTxt99+I0NDQypatKicu29ubi4355HSaubNm0dCCFq+fLnO/teuXSMzMzMyNDQkR0fHLLV+33ZOlu59TVykDoshISGUnJxMJUuWpHr16tGMGTMoKCiInJ2dSQhBs2fPJqL013jbtm2kp6dHZcqUyVIeLi4ujqZPn04ODg7UtGlTrlXOGPssOGj+RmR80MbHx9OmTZvo8OHDdO7cOXl5bGwsGRgYUOHChalv377k7Oyss16tVlPx4sVJoVDIObFxcXH0559/khCCnJ2dae3atXTlyhVasWIFde3alXLlykVVq1bVqQjAdGW8N7dv36YePXpQuXLlqESJEtS5c2e5Y9jr16+pWbNmJIQgFxcX6tevX5ZjSe15pS5yROmpMWXKlCEhBDk4OFCVKlWoWLFipK+vT0WLFuUW2P/Ry5cvKTAwkJ49e0ZlypShPn360JUrV0itVtOaNWvkkf6M91lqK58xaE5JSaGBAwdS48aNycvLi4QQdPz4cZ1z8chlVhmD5YSEBNq7dy9dvnxZZ/RXSm+pUKECjRs3jsqVKyenaxARnT17lqysrMjQ0JBiY2OJKD2fv2fPniSEoAYNGtCBAwfo0aNHtG7dOurWrRvlzp2b3N3d6ezZs9l3sYyxHxoHzV9Z5ofs+PHj5dxjpVJJxsbGNGvWLPkBM378eDm4GjRokLyflO+3adMmEkJQtWrVdI47bNgwsrCwICGEXH/WxMSEhgwZwiNjH0GtVss1d6XR4nz58pFCoaBChQrR6dOniYho+/bt8s/K48ePJyLdyZMRERFUsWJFMjMz08lRvnbtGs2cOZPc3NyoZs2a1KhRI3lUTcJ5yx/2ttdI+iJTtWpVql+/fpb1gwcPJiEEDRs2TF4WFhZGhoaGVKtWLbp48SJdvHiRJk6cSDY2NuTv709EpNNanmWV+V5MnDiRChYsSKampiSEICcnJ1q4cKG8vl27dmRoaEiWlpY0efJkebk0Ij1lyhQSQtDPP/+sc47WrVvLxzQ2NiaVSkUqlYoGDx7Mn22Msc+Kg+ZvxMaNG8nOzo709fWpWbNmNHToUOrSpQupVCrKmzcvzZo1S962cOHCJISQJ41l/tm+VatWJISQu8kRpT9c7t27R7NnzyZ/f38KCAjQKanFD5d3W7hwIVlZWZGZmRn9+uuv8shveHg4+fj4kBBCp/SVNPol5WBmfm0DAgJIoVDoPPwzSkpK0pn0xGkZn+748eN06dIlIiI6cOAAWVhYkEqlor59+xIRUWpqqvy63rlzh0qVKkVCCLp9+zYRpU/ilBqXqFQqOShr06aNTu4535sPCw4Opvz585O+vj7VqVOHfH19qVatWqRSqcjW1laupXzx4kXKmzcvCSFo06ZNRJT++krBd1xcnJzGERYWJh8/Pj6eTp8+TSNHjqRx48bR9OnTdeZ18GcbY+xz4aD5K4uJiaGGDRuSEILc3d1pw4YNcm6xVqulefPmkYmJCTVs2FCeCCYV6q9UqZIcXGUsaXX+/HkyNTWlQoUKyZMGM5ZAy4jLlL3fqVOnSAhBenp6WZofpKWl0c2bN8na2poKFSokTwAMDw+nwoULk4GBgfz6Z3ydY2Ji5PxzKbUjY8609OsDl8D6d6TJlTt37pSX9erVi4QQ5O3tTURZJ4IFBASQEIKaNWumcyx/f3/y8/OjX375RS4nxz7Ow4cPqXr16iSEoNq1a9Nff/0lT9aLjY2lCRMmyBMAExISiIhozJgxJISgDh066BxLul9r164lIQRVqVIly7rM+LONMfa5cdD8lUnNRHLlykXr16+Xl0sB8KVLl8je3p4cHR3p4cOH8vp69eqREIKWLl1KRFkrXUg/OQ8cOFBn+du2Ze8WHx9Pfn5+pFAoaNKkSfJy6UvIgwcPyMHBgSwtLXVqJk+ePJmEEPTLL78QUdYgbcuWLWRiYkLFixfXKRPI/rulS5eSEIJWrFghL4uOjiYbGxvS09Oja9euEZFuUJWYmEgNGjTQ+SLzLjxy+XF27dpFuXLlovz589PWrVt11qnVajp16hRZWVlR5cqV5S+XCQkJVLx4cVIqlfI8i7S0NJ3PrHbt2pEQglatWiUfS5Kxjj1jjH1uHDR/ZWq1mmbMmEH6+vrUt29fiouLI6I3D2a1Wk2WlpZkbm5Od+7ckZdHRETIE2ekYDpj7mxsbCyZmpqSUqnUScNgn+78+fNkb29PTk5OdPDgQZ11q1atIkNDQ/L09CSNRiPfn9jYWLmznzQRSa1Wyw//169fk6+vL40YMSJbr+V7lPnLXcZGPG/b7tChQzo1s6X3xLRp00gIQZ07d9bZT1q/Y8cOMjIyIhsbm7celwOxT/P8+XMaPXo0KRQKGj9+vFy2Twpyb9++Tfr6+mRnZ0evXr2SX1/pS0+jRo10jie9t44ePSrXNmeMsezEQfNXJD2Mr127Rg0aNCBzc3PavHmzvP7evXvUu3dvuRyZRHq4SOsyVmIgevNw2bFjR5Yyc+zTpaamypOQ+vbtS2lpafTkyRMKCAggW1tbcnBwoNDQUHl76f5IlRmqVq2qczxpPY8wv9vHBKg3btyQU5kyjv7GxcWRnZ0d+fj4ENGbXwVSU1OpbNmypFAo5PuVedS4TZs2VKZMGYqNjeVfYj6Dc+fOkYuLCxUqVIj27NkjL7937x517dqV9PX15QmvGe953bp1SQghdy3NfJ8WLlxIly9fzoYrYIyxNzho/kasWLGCzM3NqXnz5nTlyhU6cOAAeXl5kVKppMaNG8sd+4jePFxevnxJpqam5ODgQOfPnyci3Z8yJfxz8oe9rc1xxtfx7t27VLFiRcqbNy8NHz5cbq3s6uqqMykp434ajYbat29PQgi5TfDbzsEjmG9kfH0SEhLo6NGjdPjwYVq5ciVdvnxZrrc7Z84ceTRS+mlfcu/ePbk2uTSqKf03ODhYbhIkyXjvM6bYsH9Peg8kJyfT3LlzSalUUq9evej27dt05MgRuWRfixYt5MmXRG/u/6FDh0hfX59Kly4t12DOXBddWsYYY9mFg+Zs8q5Z9tJD4tGjR9SlSxdSKBTUsGFDsrKyIn19fZowYcJ7jyc1YWjTps2X+cNzgIyBWlJSEkVHR9OLFy+y5EpKk5CkhhgZy2VlThmQHuZSqoCTk1M2XMn3K3PwM3nyZKpUqRJZWlrKr3muXLmodu3adP36dbp9+7ZcuaRatWq0f/9+nf3r169PBQoUoJs3b2Y5V4sWLUgIQQsWLCCit3+p5KoYH+9Dr1VkZCR5eHiQiYkJtWzZUv5s++OPP967X48ePbKUAsyIfwlgjGU3Dpq/sMzB1O7du2ndunW0a9cuuUi/ZOfOnXLpqyZNmlB8fLy87n2joAUKFCBfX19+0P9HU6dOJWdnZ3J0dKQSJUrQr7/+qlNeLCYmhjp16qTTmYzow2kW27dv1ykhx3RlDJiXL19Otra2ZGZmRnXr1qXBgwfTli1baPz48VSlShV5dH/v3r2k0WjI39+fjI2NycLCglavXi0fZ8iQIWRgYCC3tc44mnzu3DkSQlDevHnp5cuX2XuxP5DMn20HDhygvXv30qlTp3Q68Gm1Wtq4cSNZWVnJFUoSExPl9Zk/26T/HxkZScbGxtSvXz8OkBlj3wQOmj+Dj/mJ8MSJE+Tq6kqGhobyyJm1tTUFBgbKD4mEhAT63//+RwqFgqZOnSo/KN6VXiEFyS9evPhMV/Jj+VDqg/T6Xr16lapXr04qlYrq1atH7dq1o2LFipEQgmrUqCF3V9RoNHTo0CEyNzcnNzc3uWPZu86R+UHPX2re7fTp03J5skaNGtHq1auztHV//Pgx9e7dmwwMDMjV1VWuw7xp0yYqUqQIqVQqGjFiBCUnJ9OMGTOyVNAgenNPpk+fTkeOHMmei/vBHT9+nMqVK6fz2VayZEk6duyY/GUxOjqafv31V1IoFHLFH6J3f7ZlLM/IGGPfCg6a/4OMH/gvX76k27dvU1RUlE5tXqL08mJWVlZUqlQpmjp1Km3ZsoWmTp1KxsbGZGRkRAsXLpRHXk6dOkXly5en/PnzZ/nJ+X3elpObk2V8LWJiYujUqVN09uxZ+ef6jOsHDhxISqWS/vzzT7nSSHR0NI0YMYKEEFSzZk2KiooiovQvKAMHDiQhBI0aNYpHwD6DJ0+ekIGBAQkhaOjQoTqjlES69apv375NXbp0ISEE9ejRQ97m/Pnz1KRJExJCULdu3eRUmvnz5xPRuytg8P17u/d9lmQcYV6zZg3lzp2bXF1dacKECbR27Vo5raJw4cI6o/8HDhygggULkqura5b24+/Def+MsW8FB83/QuYH7cSJE6lq1aqUP39+yp07N/n5+en8HN+0aVOytLTMEgSvXLmSChUqREWLFpVnlms0Gpo9ezYZGBhQ165deaTlE2W8N0lJSTRo0CAqUaIE5cmTh5RKJdWoUUNn+4iICLKwsCA3N7csx0pISKDOnTtnqVASHh5ORYoUoSJFisgl6Dj4+m+kutZS8xGidwduJ0+epPz585Otra1OTeUXL15Q3759SaFQUJ48eXTKzr2vRB17I/NrsnnzZpo7dy5NmDBBbhaT8RewunXrkq2tLR05ckQObF+/fk1z584lMzMzqlChAp05c4aI0idZjh07Vs5Tlhqa8H1gjH0vOGj+DzZv3kwODg5kYmJCdevWpT59+pC/vz+tWrVKzpW8d+8eKRQKatWqlbyflAObmJhI/v7+pKenR3379qVnz54REdHNmzepWbNmZG5uThs3buRRln9h9erVlC9fPsqTJw+1adOGxo0bR/v27aOVK1fS69ev5df0woULpK+vLzchyZxCER4eTqamplS1alW6c+cOEaXfP39/fxJCUNu2beWHP/v3EhISyNnZmUxMTOQWyu/6d5+QkED9+/cnpVJJ06ZN0xmJTElJoZUrV8oTCOvUqcN5y//CmjVrqGjRovKXTXt7e2revLlOdZHDhw/rNPAhevP+iY+Pp+HDh5MQgkaPHi1/Abpw4QJVqlSJHB0dadeuXdl6TYwx9l9x0PyJpJ8m58+fTxYWFlStWjVas2aNTre+jA4fPkxKpZI6depERFlHzy5evEilS5cmJycnnYmB69evJwMDAypbtqzcepa9nzRitWDBAsqbNy+5u7vT5s2bs/zcn9HRo0fJxMSEPDw8dI4hiY+Pp5YtW1K+fPno3r178vJ79+5RqVKlqHnz5hw0fyYbNmwgIQQ1bNhQDs7eFTgvXryYhBDk6+v71u1CQkLIxcWFKlasSK9fv+bRzI8UHR1N7dq1Iz09Papbty7NnDlTLvV39+5dnW137dpFSqWSxowZQ0Rv6mFLzp49S/b29lSnTh15Qm1qaqp875o2bZqlXCBjjH3LOGj+F27cuEEFCxYkZ2dnudWrRAqKM05kUSqVVLBgQfmhk/EBr9FoqFmzZiSEkH/GlPYbOHCgzqQZ9mF3796lAgUKUOnSpeWJekTvzot88eIFFS5cWOf1zzza3Lp1axJC0Llz53SOl3miGvtvNBoNNW7cmIQQNHfu3HduQ/SmAka9evV0qpdIwfGLFy+oXbt2ZGBgwPfpE/j6+pKJiQkNHjyYrl+//t5tDx48mKV5T8YvJzExMVSjRg0yMDDQGaGOjIykzp07Z5mkyRhj3zoF2CebNm0a7t27h4ULF6JKlSoAACICACiVSgCAQqGAVquFtbU1vLy8EB0djW3btsnrNBoNNBoNFAoFHBwcAABmZmbyOaytrTFz5kx4e3tn56V995YtW4bo6GhMmzYNZcuWBZB+b4QQUCh0/7lrNBqYmZmhT58+AIBRo0YBAPT09HS2e/DgAQoUKABra2t5mRACdnZ28nHYf6dQKDBmzBgIIbBo0SJERkYCALRabZZt79+/DwCoUqUK9PX15fefEAJEBDMzM9jY2CAtLQ2PHz/Ovov4Dkmv3caNGxEUFARvb29MnToVP/30k876zGrVqoUyZcrg1KlT2LJli7ytdL+sra2hUqmgr6+P58+fy/sVKVIEK1euRJcuXb7kZTHG2GfHQfMnICIkJibi4sWLcHR0RIUKFZCWlgYg/WGdmRSkde/eHWZmZggICMCxY8cApAfXSqUS4eHhCAkJgYuLC2xtbd96TvZx0tLScOjQIeTJkwelSpWSl7/t3gBv7s+gQYPg6uqK0NBQjBw5ElFRUQCAW7duYcyYMQgLC0P37t2RP3/+tx5H+qLE/rvKlSujT58+uHz5MhYtWgQg/T5R+q9i8j3buHEjAKB8+fIAdO+xEAJxcXGIjIyElZUVrKyssvkqvi/SaxccHAwA6Nu3r/zFPuP6jKTAeMSIEQCAiRMn4tmzZ1AoFPI9unDhAsLCwlCtWrUsn21CiLd+GWKMsW/aVxrh/m7dvXuXjIyMqFixYnJ71w9JSUmhiRMnkhCCihQpQhs2bKBr167Rxo0bqVmzZmRsbEzLly//wn/5jy82NpYcHBzI3t6eXr16RUQfnpkv/bR/4sQJKleuHAkhKH/+/FS/fn0qX748KZVKatq0qVyKjn150dHRZG1tTTY2NnT48GGddfHx8TRs2DBSKBQ0dOjQt+5/584dGjx4sFx+juuYf1hMTAwVK1aMSpcu/cn7tm3bVs5FDw0NpdjYWNqyZQt5eHiQqakpbdu27Qv8xYwxlv04aP5Er1+/plKlSpGzszNdu3btk/bt3bs35c6dW27DrFKpyNzcnAIDA7/QX5vzVK5cmQwNDWnfvn2fvG94eDgNHTqUSpUqRaVLl6YqVarwl5mvZM6cOSSEkCfQEhGFhYWRn58fWVpaUtu2beWa25m/GF27do3q1asnT1BjHxYXF0cqlYocHR0/Kgc8Y+3sJ0+eUPPmzcnIyEjutGhkZETm5uYUFBT0pf90xhjLNoKIf///FGlpaWjatClCQ0OxY8cONGnS5J3bajQaKJVKHDt2DK9fv0b9+vVx6dIlbNy4EampqciXLx969eoFU1NTAOk/eWbOu2Xp6P/zkt8nNTUVI0eOxMyZMzFp0iQMGDAAhoaGb91Wujd79uzBvXv30LNnT51z3b9/X841z7g9yx5JSUmoWbMmwsPDMX36dJiYmGDWrFm4evUqfH19MXnyZJ05AJklJydDpVIB4PfVhxARUlNT0ahRI9y8eRNr165FzZo1P2rf58+fI0+ePHj69CmOHTuGffv2ITU1FY6OjvDz85Pv0ce8fxlj7FvHQfO/sHPnTjRv3hyNGjXC5s2b5YdzRhkfEm3atIG5uTn8/f3f+qBPS0uDUqnkh8o7ZH7gvi8IWrduHfr06YNixYohICAAFSpUeO+xO3XqhOvXr+Po0aMwNjbOEhxzsPz1SO8zAwMDpKamwsXFBYGBgahcuTKADwfDHCynvwZCiA9+tiQnJ2PAgAEICgrCpEmTMHDgwA9+4QwICEBUVBSmTp2qsz4tLU2eTMufbYyxH0nOfqL8S40bN0bZsmWxZ88eLFmyBK9evQKQ/jCRJrcIIRAfH48hQ4Zgy5YtqFGjxlsDZiKCnp4eP1TeQwiB8PBwzJgxI8s6+v8JYpJ69eqhYcOGOHPmDJYuXYro6GgA6cGDNGkTAGJjYzFgwABs2LABPXv2hLGxMYCsk/o4YP56mjZtirp160Kr1WLVqlU4f/48KleuDK1W+1EBcU4PmIH010AIAbVaDeDtlV6ICCqVCo0aNYKpqSmWLFmCixcv6qzP+F+lUok7d+5g+vTpePDgAVJSUnTeg1LArNVq+bONMfZjyfaEkB/EpUuXyNTUlGxsbOjPP//UWafVaiksLIz69+9PefPmJW9vb506pezTvHr1Sq5lfenSJSJKr6WcsZ5yWlqaXMM3NDSUKlasSEII8vLyohcvXsj5l2q1mg4fPky9evWiPHny0C+//CI3XmDfnsxt5DPX0GYfNn78eLl5T0bSaym9N7RaLXXt2lXudCnVOc9c3/zQoUNUtWpVcnBw0KldzhhjPzpOz/gPFi1ahN9//x2PHj1C3bp1Ub16ddja2uLEiRM4ffo0rl+/Dl9fX0ydOvW9+Zc5FX1EnqO0TUhICLp164bixYvj8OHDOvuOHTsWt2/fxowZM2BlZQWtVoujR4+iR48eiIyMRNGiRVGjRg3Y2tri3LlzuHbtGqKiojB48GBMmjQJ+vr62XG57D/I+JM/+zSNGzfGnj17sGHDBnh6eiIlJQUGBgby++f+/fuwt7cHAJw/fx4jRozA3r17UbFiRcyfPx9FixZFrly5cP78eRw8eBDLly/Hw4cP8ccff6BHjx78awxjLOf4qiH7D+DEiRPUsGFDsrGxISEEKRQKsrOzo4YNG9KpU6fk7d7VDjinythOPGNHt8wyjhD7+fmREIL27NlDRESbN2+Wu/l16NCBHj16pLPv1atXacSIEVSsWDG5YomTkxN5eXnR1atX5e343rAfkfQeO3nyJBUqVIhcXFzkUoxERMeOHaMCBQpQpUqV6PXr1/Lyq1evUqtWrUgIQUIIsrOzI2dnZ7K3tycDAwOqWLEihYWFZfv1MMbY18YjzZ9BSkoK7t69i4cPHyJ37txQKpUoU6YMgDdNADi/Mqtbt26hd+/eaNGiBXr16vXOEStplDEiIgIdOnTA06dPUaJECRw8eBBubm7w9fVF3bp139ocBgCePXuGuLg4mJqaIiUlBYUKFQLA94blHEOHDsWcOXMwe/Zs+Pr6om3btti6dSvKlSuHzp07o3v37nIVHyC9esnevXsRGhqK8+fPw9LSEpaWlmjRogVatWoFAFkazjDG2I+Og+YviCsvvNvevXvh4eEBAChdujQ2b96MIkWKZNku48/yWq0Wbm5uCAsLQ548eTBp0iS4ubnB2dn5k8/P94b9COgDKU4pKSkwNDREcnIy3NzcEBkZiVevXsHa2hre3t5o166d/AX/Xcd/W2DM7x/GWE7EQwRfED9U3q1AgQIAgEKFCiEiIgJBQUHyDH8J/X9lEQCYO3cu8uTJg/DwcNjZ2SEhIQFNmzaFs7NzlgoaH4PvDfueSRVE3hUwS5V8pLJxoaGhiIyMxMuXL1GkSBFs27YNI0eOlAPm971/pDbm0nkBfv8wxnImDprZF/O+B7GZmRlKlCgBS0tLuLu7IzAwEP/884/OvkII/P333yhZsiT69++PunXrYvPmzRg1ahSEEBg2bBgAvDd4YOxHo9FooFAooFAocPv2bWzcuBEXLlzQ2UapVEKhUMhl+lq2bIkKFSqgTJkyePDgATQaDYyMjOQvqu96/0jLpf9yKgZjLCfjT0D22WWsVZ1x9DhjEG1qagoigoODA2rWrImkpCTMnj0biYmJ8r4AMG7cOKhUKixatAizZ89G/fr15Z+V165di9OnT0OpVOrUYGbsR6ZUKpGQkABfX19Ur14d3t7eCAoKwrNnzwCkv880Gg3atGmD8uXLIykpCTNmzEBwcDAmTZoEIyMjjBw5EgC4cgxjjH0CzmlmX8yQIUNw69Yt9OvXD7Vr15aXS/mQbdu2RUxMDNauXYtu3brh77//xqZNm9CyZUu5ecW9e/eQmJiIkiVLAngzAr1//354e3vDxsYGZ86c+VqXyFi227p1K/z8/KBWq9G6dWu4urqiY8eOWcpa+vr6QqFQoHv37nB1dZVTKnx8fLBmzRosXrwYP//8M+cnM8bYR+KgmX12sbGxaNGiBU6dOgUgPW951qxZaN68OYA3Py+PHTsWf/75Jx48eIATJ06gc+fOKF26NDZt2gQbG5ssx5X+qQohQETw8/PD/PnzceTIEVSvXj37LpCxr+TevXto2bIlEhISMGXKFDRs2BDm5uY620iTZ9VqNVJTU2FiYgIAUKvV0NfXx/nz59GxY0ckJibi2rVrOlUzGGOMvRunZ7D/RErFyChfvnwoW7YsTE1NUahQIRQqVAjt27fHpEmT5FEtIQSsra2h1Wpx48YN1KtXD23atMHJkyexfv36tx5XCAEhhJzDPGDAAFy4cIEDZvbDeddYRmhoKC5cuICRI0eibdu2OgHzixcvALxpY61UKmFiYiIfS0rFKFu2LBo3bozcuXMjJibmS14GY4z9UDhoZv+KRqMBkD4xKCUlBU+ePNHJXx4yZAgKFy6MO3fuwM/PD02bNsWYMWPg6+uLuLg4AEDt2rWh0Whw//59GBsbw8vLC46Ojpg9ezZu3LjxznNLk5GKFi2KMmXK/KvqGYx9i6Tc/HdNzHv+/DkAIDo6GgAQFhaGrVu3okePHnB3d0fbtm3h7++vc4yMx5K+jP7xxx8IDw9H0aJFv8yFMMbYD4jTM9gnyVwXdtmyZdi9ezfi4uIwaNAgNG3aVF43b9489OvXD56enliwYAFmzpyJyZMno1GjRpg4cSKsrKzg6uoqrweAkSNHYvLkyRgwYAAmT54sl8xi7EeW+X0VHByMK1euwMLCAqVKlUKtWrUApDfqcXJyQlxcHGxtbZGSkoJnz57BxsYGuXLlwp07d5CamiqnLElzA96F25MzxtjH45Fm9kmkB/vmzZtRrFgx9O/fH2fPnoVarYapqSmSkpLkUV9vb2/Ur18fwcHBOHDgACZOnIh58+YhLCwMbdu2xY0bN2BoaAi1Wo2kpCQAQKdOnVCxYkUsXbqUJ/ixHEN6X506dQouLi5o3749pk2bhn79+qFx48YYPnw4oqKiYGFhgaVLl6Jnz56oUKECmjdvjj179uDYsWO4ePEili1bBiC9rjnw4RJxHDAzxtgn+MJtutkPJj4+ngYOHEgKhYLc3d3J39+frl+/nmU7jUZDRETbtm0jlUpFRYsWpaSkJCIiCgkJIScnJ3JwcCAhBJUuXVpn3/nz55MQgmrWrEkvX7788hfF2DfgxIkT5OjoSE5OTjR79my6cOECnTx5kry8vEgIQb169dLZ/sWLF1mOcfnyZRJCUN++fSktLY20Wm12/fmMMfbD46CZfZItW7aQmZkZeXh40IULF+TlWq1WDpSl/y/x9vYmIQSNGzdOXnb37l1yd3enPHnyUJcuXSghIUHe58GDB9S+fXsKCAjIhitiLHulpaW9dbmPjw+pVCravn27znspPDycrK2tSQhBW7ZseedxT548SQ0aNCAbGxs6derU5/6zGWMsx+OcZpYFZcqvlCQnJ6Ny5cqIjo7GpUuXYG9vD+BNKavMpEoZ586dQ8uWLREfH4+LFy+icOHCAIDbt28jLi4O9vb2sLOz0zn3h3IxGfveLVq0CHny5IGnpydu376NqlWrokaNGggODgYA3LlzByEhIVi9ejXCwsLQoUMHzJs3DxYWFgCA1NRUJCcnIyIiArt378aOHTtw584dDB8+HIMHD+bGJYwx9plxQhuT0f9XoZCC1VevXsk1XrVaLYgIenp6sLOzg7Gxsbyf9HC+evUq4uPjQURwdHSUA+Hy5cvDx8cHEyZMwLhx47BixQoAQOHCheUAWpKxXa9UWo5bZLPvXcYGIi9evICvry/Wr18PDw8PeHh4gIgQHx8PV1dXEBE2bNiA1atXIyQkBKVLl9apRS5N3nv06BEqV66MhIQEqFQqODk5Yc+ePahSpcrXvFTGGPthcdDMAEAe2RVCIDo6Ghs2bEBYWBg8PT3Rtm1bKBQKvH79Gnnz5sX+/fuxf/9+VK1aFS9fvsTt27exePFiHDx4EAqFAgkJCbCzs8O4cePQrVs3AEDPnj0REhKCVatWwcfHB+7u7u8c0ZbwSDP7USiVSsTFxSEiIgJEhNOnT+OPP/5A48aNYWJigpSUFJiammL16tV4/Pgxli1bBgMDAwQGBqJnz546x7p48SKcnJxQsGBBDBs2DM+ePYO7uzvq168PIOuXX8YYY58Hp2cwmVqtxv/+9z+sWLECGo0GiYmJ6NixozwyDAArVqzAb7/9hidPnsDOzg5KpVKuGWtvbw83Nzc8evQIR44cQe7cubF161bUqFEDQggsWbIEvr6+qFq1Ko4cOcIjyOyHlTm9KCEhAcWKFYORkRGMjY3h6uqKdevW6exTp04dHDp0CHp6ehg6dCjGjBmTpeTi+PHjsXXrVixbtgwuLi5ZvnhyS2zGGPtyeCiCAQD2798PZ2dnBAYGonHjxpg9ezZevnyJxYsXA3jTFMHLywtBQUFo0qQJAMDJyQljxozB5s2bce3aNaxbtw6HDh1Cr169EB8fj1WrVskP9c6dO8PNzQ3Hjx/HiRMnvs6FMpYNpIBZet/kypUL/fr1Q3R0NB49eoSOHTsCSP+iKjU0GTFiBID0jpoTJ07UCZjv3r2L0aNHY8GCBShbtqyc1iS9t6TzcMDMGGNfDo80Mzx9+hRNmjTBvXv3MH78eLRq1Qp58+YF8O6Rq+TkZCQmJsLS0lKnQYKUB33u3DlUrFgRzs7OCAsLg0qlghACZ86cgaGhIcqUKZOt18hYdrp9+zYmTpyI7t27o2rVqgDSc5nr1q2Lc+fOISAgAL17984yUuzr64tFixahatWq6N69OypWrIgDBw7g77//xr59+1C/fn3MmDGDO/kxxthXwDnNOci7KlKsXbsWYWFhCAgIQPfu3QGk50Wq1WoYGBi8dX+VSgWVSiVPDiQiaDQaeeLgs2fPAAAVKlTQOUbFihXlY/EkP/Yj0mg02LVrF5YtW4a8efOiRIkSyJ07N8zNzTFy5Ei0bt0a27ZtQ8eOHZE7d24QEbRaLZRKJSZPnoz8+fNjzpw58nwAAwMD5M6dG1OnTkXfvn2/8tUxxljOxSPNOUDmYPnhw4cwMzODmZkZiAjdu3fHsmXLcObMGZQvXz7L6PK1a9dgZWUFIpJHoKV/NkKILNsfPnwY/fv3x6NHjxASEoIKFSpk05Uyln3elz98584dDBw4ECdPnsSqVavQoEEDAOnvxRYtWmD//v1YtGgROnfuLO+TcdQ5IiICFy5cABFBqVSiVatWUKlUHzwvY4yxL4dzmn9g0giWFDCvX78e7u7uqFevHooVK4YhQ4bg6dOn+OmnnwCkjzgDwPPnzxEVFYXly5ejZs2aaN68OZycnODu7o6ZM2ciLS1NZ5RYeoBHRERg3rx56NOnD27duoVhw4bBxcXlK1w5Y/+d9P55F+nf/Z07d+S8ZOnLZKFChfDLL78gOTkZy5Ytw4MHDwCk5zqPHTsWaWlpWLZsGaKiogC8+eVF4uzsDC8vL3Tu3BkdO3aESqWCRqPROS9jjLFs9qW7p7CvI2NHsVOnTlH58uVJCEGFCxemSpUqkZWVFQkhqG7duvT48WPKkycPCSGoRo0a5ObmRgULFiQhBAkhyMnJiRwdHcnAwICEEDR79mwiIkpNTaV9+/aRv78/NW7cmEqVKkUmJibk6upKx48f/1qXzthn9ezZMyJK73KZmpoqL1er1TRs2DAyNDSksLAwebnU2TI2NpZ8fX3J0NCQVq1apdMJsF+/fiSEoClTpnzU38DtsBlj7OvjkeYflEKhQHx8PDp37oyqVatCq9Vi1qxZCAkJwenTpxEaGopixYrhwIEDCAsLw7Zt21C+fHmcO3cO4eHhKFWqFEaOHIm///4bV69exdGjR7Fo0SIAwPz58xETEwN9fX08efIEs2bNwp07d1CsWDEsWbIE58+fR7Vq1T44UsfYt2706NH46aefcOzYMQghoK+vD7VajYiICOjp6cHBwQFCCCxduhSJiYkA3lS0sLS0xM8//wwbGxsEBQXhxo0b8nGHDRsGa2trrFu3DmfOnAGA975XOPefMca+AV87amdfzvTp00kIQS4uLnTu3Lks6//66y8SQpCHhwcRpY+cnTlzhp4+fUovX77UGd1KS0uj58+fU4UKFcjW1pYuXrxIROkj2jdv3qTbt2/rjG6r1eovfHWMfTnSv/3JkyeTEIL8/PyIiGjBggUkhCAfHx96/fo1PXjwgNq1a0cqlYp2794t7y+9F168eEG//fYbKRQKmjp1KiUmJsrbzJ07l4QQ5Ovrq/PeYYwx9m3ikeYfWI8ePeDi4oLIyEjExMQAAFJTU+URLanShTSDX09PDxUqVICFhQVMTU3lSX7SxCOVSoXHjx/j+fPnsLKyApA+ol20aFEUKlRIbn0NQC5Bx9j3bPjw4ShXrhwCAwNRoEAB9OnTB40aNULr1q2hVCphZ2cHLy8vmJubY+bMmXjy5AmAN3WazczM4ObmBiJCcHAwLly4IB/b29sbPj4+6NOnD3fvY4yx7wB/Uv/AzM3NMWLECCQlJWHOnDmIi4uDgYEBFAoFkpKSsGHDBgBAgwYNdH7+1Wq1cgk5pVIJpVKJZ8+eYdSoUXjw4AGGDh0KGxubt56TH/7sRyCEkL8ApqamAkjv6jdr1iysXLkSTZo0gb6+PgCgVq1a8PLywv79+7F9+3Z5wp60n5OTE/T09HD27FmsWbNGDqxNTEywePFilC5dWp5AyBhj7NvFEc4Prk2bNvDw8MC+ffuwadMmAMDp06cxZMgQrF27Fo0bN4aHh4fOPgqFAkIIKJVKpKWl4ejRo/j9998xf/581K1bFx06dPgal8LYFyUFrtJ/FQoFzp07BzMzMzg4OODly5dQKBSwtLQEEcnbmZubw9PTEyVLlsSMGTNw5coVAJDrky9cuBBlypRB9erVsW7dOiQkJOicN3PlDMYYY98mrtOcA5w+fRrVqlVDsWLF0KZNG2zduhVXr17Fr7/+imnTpsHIyEhn+0ePHuH48eN4+vQp9u7di4iICERGRsLX1xczZszIsj1j3zNpRPltv5K8fv0aycnJiI2NhZOTE0qWLImtW7eiaNGiOuUc09LSMG/ePAwfPhy1atXC6NGjYWNjg507dyIwMBDDhw9HpUqVkDdvXjm1iTHG2PeFg+Ycon///pg7dy4UCgWqVq2KgIAAuZV15mYJCQkJcHJywuPHj1GiRAmUK1cOw4cPh7Oz81u3Z+x7lfHfcnh4OPbv3y+PLGdsSKJQKPDbb79h2rRpGDp0KP7880/5GNL6hw8fYs6cOZg2bRpUKhX09fXx8uVL1KtXD2vWrEG+fPmynJMxxtj3g4PmHOL+/fuoVKkSXr58iePHj6NMmTJITU2Fnp6ezggb/X9XsrNnz+LVq1fInz8/ihYtCuD9I3KMfa+eP3+OgQMHYu3atXKTEgAYMmQIRowYgdy5cwNID3bt7Oyg1Wqxbds2VKtWDWq1Ws5tlkyZMgVXr15FTEwMvLy80KVLl+y8HMYYY18IB805iL+/PwYMGICOHTtiyZIlUKlUOq1734dHx9iP6OjRoxgwYABiYmLQvn17VKtWDQ4ODli2bBkWLVqEcePGoV+/fjA3NwcArFy5Er/88gs6dOiANWvWyO+dR48ewcDAQG4zD+i2xeb3D2OMff94yDAH6d69O8qVK4fg4GBs3br1k/blBz770cTFxcHf3x/x8fGYOHEixo4dC09PT1SuXBlly5YFEWH9+vU4dOiQvE+XLl1QsWJF/PXXX1ixYgXi4+OxZs0aeHt7Y+PGjVCr1QDeBMzc+poxxn4cHDTnIMbGxhg7dizUajUWLVqE2NhYndJajP2I3vXv28DAAEqlEgsXLsQvv/yCXLly4eLFi3Bzc0Pv3r1RunRpXL16FevWrcODBw/k/ebPnw+VSoVu3bqhatWq6NatG/755x+4urrKqRrSCDMHy4wx9uPgoDmHadasGdq1a4eDBw8iICAAAOcosx9D5kwzKT9ZoVDII74ZmZubY+HChahXrx5SU1MxadIkVK5cGU+ePMGMGTMQFBSE5s2bY9euXdi9e7e8X/ny5bF48WL4+PjAxsYGs2fPxpMnT1ClSpUve4GMMca+Km7blgMNHToUW7Zs0cm/ZOx7tXHjRjRo0AC5cuXSWS51pQwKCsKOHTugr6+PEiVKoEWLFqhUqRIAyPvs3LkTM2bMQPXq1TFixAjUrl0bQgiULVsWW7duxfbt21GpUiW54kynTp3QqVMnnYmAaWlp3AmTMcZ+YDzEmAOVL18esbGx8PPz+9p/CmP/2pEjR1C8eHF0794dhw8fzrL+n3/+Qfny5fHrr7/i5s2biIiIwOTJk1GjRg1s3rxZHplOTEzE9OnToVAosHDhQtSpU0c+htSg5NChQwgODpa7/En09fW5dTxjjOUQHDTnUObm5tBoNNy+l32XTp48ifbt2yN37tyYNWsWKleurLM+ISEBgwYNQmxsLGbOnInt27fj+vXr2Lt3L0qUKIFhw4Zh//79AABTU1MkJiYif/78MDMzA5CekxwbG4t169bB29sbpqamOHbsGBITE7P8LZzexBhjOQN/2udgSqWS2/ey79KWLVvw+PFj+Pn5wcfHB9bW1jrr161bhyNHjmDIkCHo168ffvrpJwBA3rx5UbhwYdy+fRvr16/Hy5cv8eLFC1SqVAnXrl3Drl27kJCQgNOnT2PMmDFITEzEr7/+imPHjuHAgQOwsLD4GpfLGGPsG8C/JzLGvhtSKTcjIyPo6emhWLFiAIC5c+dCqVTC29sbRkZGOHXqFCwsLNC3b18AQGRkJHbv3o0lS5bg0qVLaNeuHaZOnSqPLLdr1w5hYWHw8fHBuHHjkJycjKdPn+J///sfXF1d5SoYnLfMGGM5Fzc3YYx9VzQaDS5cuIAGDRrA1dUVMTExuHr1Knr06IHp06fDzMwM7u7uiIqKwvr16/H48WMEBgZiz549KFu2LAICAuRKFykpKTA0NERaWhpu3LiB33//HU+fPoWlpSWGDh2KChUqfOWrZYwx9q3gIRPG2HeDiKBUKqFUKmFhYYGDBw+iYMGCmD9/Ppo2bQpTU1MAQK1atTBhwgSMHDkSx48fh6mpKZYuXYpffvlF53hbtmxBnTp1YGVlhZIlS2LDhg14/fo1jIyMAHDreMYYY2/wk4Ax9t2QmvHUq1cPDx48gIWFBXLnzg13d3fY29vLOfoVKlSAg4MDDhw4AC8vL8TGxmYJmFevXo1OnTph6dKlOsulgFmj0UChUHDAzBhjDAAHzYyx74gUyG7fvh1r165Fv379cOnSJezYsUOnskW5cuXQsGFDKBQKlChRAikpKfK6W7duYdq0aRgzZgzq1KmDVq1avfVc3M2PMcZYRpzTzBj7bt28eRNdunTBkydPsHr1alStWlVed/bsWQwaNAjHjh1D/fr10aJFC0RGRuLcuXM4ceIE3N3dMWfOHDg7O3/FK2CMMfa94KCZMfbdkapopKamYt26dejZsyd69OiBcePG6XS6fPjwIfz8/BAaGopXr17B0tISefPmxYABA+Dr6/sVr4Axxtj3hoNmxth37f79+xg4cCD279+PlStXolmzZgB0y8NFR0cjOTkZcXFxqFixorxco9FwGgZjjLGPwkEzY+yr+VzVKXbv3o2ff/4Zbm5u8Pf3R8GCBeXRaAA6/xvgYJkxxtin44mAjLGvImN1ivv37yM0NBS3bt36pGNI3/mrVKmCn3/+GXv27MHff/8NtVqtEyRn7nzJATNjjLFPxUEzY+yLetePWUqlEsnJyejXrx+qVKmCVq1aYdasWYiLi/voY0vBcJ48edCpUycULFgQ48ePx40bNz7L384YY4xJuLkJY+yLWLJkCTw8PGBnZ6eTDiGlShw5cgQ9evRAbGwsPD094ezsjE6dOiFPnjz/6nylSpWCj48PoqOj5fbajDHG2OfCOc2Msc9u3bp18PLygru7Ow4ePJhl/atXr9ChQwccPXoUc+bMQcuWLZErV67/fF61Wg19ff3/fBzGGGMsMx5pZox9NtIosoeHB9q1awcXFxdotVoIISCEkNeHhIQgJCQEEyZMQNeuXbPs/29JAbNWq+VOfowxxj4rDpoZY5+N1OY6d+7cWLp0KYyNjd9axeL69esA0lMqgDdB7r8NmDMH2xwwM8YY+9z4ycIY+6ykgNXY2BgnTpyAgYEBZs6cCSC9YkbGjDBjY2MAWScLSqXoPkTaTwqYnz9/nmUdY4wx9jlw0MwY+2KICIaGhpgyZQpSUlKgr68PIQRsbGwAAIGBgQB0S8ARkRx437x5873HzhgsBwcHY+TIkdi5cyeArGXmGGOMsf+Cg2bG2H+Slpb2znVubm4YNGgQ4uLiMHz4cHl5165dYW1tjS1btiA0NFTnOEIIJCUloV+/fpg8eTKePn2qc8yMo8upqak4evQoRo4cic6dO2PTpk0wNTX93JfIGGOMcdDMGPtvpJbUwcHBOHLkCGJjY3XW9+jRAy4uLpgzZw4iIyMBAAYGBpgyZQoAwNfXF8ePH5ePc+HCBUycOBGrV69G3rx5s6RwSCPIERERmDp1Kjp37ozAwED07dsXMTExqFWr1he/ZsYYYzkPl5xjjP0ne/bsQe/evREVFQUA+OmnnzBt2jQ0a9ZM3iYoKAi9evVCo0aNsGvXLnm5n58fFi9ejJSUFFSvXh0GBgaIiYnBlStX4O3tjWnTpsHCwkLnfA8ePMCePXsQFBSEM2fOwMPDA/Pnz0fBggWz54IZY4zlSBw0M8Y+ChFBq9Xq5B8/fPgQzZo1g7m5OerUqYPnz59j7dq1MDc3x9y5c9GwYUMAwOvXr+Hp6Yndu3cjJCQEHh4eAIAXL17g6NGjmDdvHm7fvg2lUgl7e3sMHz4cderUkc8rhEBaWhoOHz6M2bNnY/fu3ShSpAjmzZuH+vXrZ/+LwRhjLMfhoJkx9kEZO/qlpqZCrVbDxMQEM2fOxKJFi7Bq1SpUqFABALB161a0bt0atWvXRnBwsNzhb/fu3WjTpg0cHBzkknNSqbmkpCRotVrEx8fD3t4eQNYg/cWLFxg1ahSWLFmCSZMmoX///tn9MjDGGMvBOGhmjL1T5vrHs2bNwrJly2BqaopOnTohLCwMRYoUwdixYwGkT+bT09NDp06d8Ndff2HatGkYPHiwvH/Pnj2xePFizJo1C/37939nB7+MQXpG4eHhKFasGAwNDb/A1TLGGGPvxkEzY+ytMnbVS0pKgo+PD7Zv3w5HR0c8ffpUnvC3Zs0adOzYEWq1GkqlEgqFAo8fP0b+/PlRuHBh7NixA8WLFwcAXLp0Ca1atcKdO3fw/Pnzz9I6mzHGGMsOXD2DMfZWCoUCN27cwNy5c/Hs2TMcOXIE48aNQ2hoKCIiItCnTx8AwMaNGwGkt7BWKBTQaDSwtrbG2LFjERkZiQULFsjHLFOmDNq2bQsbGxtER0dzAxLGGGPfDR5pZoy9VXJyMurVq4cTJ06gevXqKFmypNyMBADUajXKlSuHiIgI7N+/H3Xq1IFGo9Fph12gQAG8fv0a69evlyf2vX79GkZGRl/lmhhjjLF/i0eaGcvhpJbVGb8/a7VaqFQq9O3bF7a2tjh+/DhsbW3ldWlpadDX18fIkSMBABMmTADwprOf1Khk5syZePr0KWbMmCGfRwqY39cUhTHGGPvWcNDMWA5FRPLIMKDbdloKcDt06IBatWrpTAgkIrkRSYcOHdC4cWMcPnwYixcvlveV1nt6eqJv376YNGmSfB6JtA1jjDH2PeD0DMZyoIzVKR48eIAlS5bg5cuXMDc3R7du3WBnZydve+rUKbRs2RJqtRoPHjyASqWCVqsFEUGpVCIsLAy1atWCg4MDwsLCkCtXLp31ksyVOBhjjLHvCY80M5YDScHs77//jiJFimDKlClYuHAhxo4di4YNG2LTpk3ytlWqVIGPjw+eP3+OUaNGAYBOQFypUiX06NEDN2/elNM1FAqFTsCs1Wo5YGaMMfZd46CZsRzo3r176NChAxYsWIAOHTpg+fLluHTpEo4fP47o6Gj8/vvvOHXqlLy9r68vSpUqhZkzZ+LWrVtQKpXQaDRyGsfQoUNhZmaG1NRUaDSaLOfLnJrBGGOMfW/4ScZYDqPVarF7925s27YNffv2xcSJE9GuXTs4Ojoid+7cMDMzw82bNzF37lx5n4IFC6J3794AgAEDBgCAXJNZo9HA3t4e169fR1BQ0FubkjDGGGPfOw6aGftBvW3EF0gf9TUxMcG8efMwevRo5M+fH0+ePEHbtm3h7OwMa2tr5MuXD8HBwQgODpb369q1Kxo1aoSQkBDs2bNHPocUJNvY2MiTCxljjLEfDU8EZOwHk7GTHwDExsYib968Ossy1kretGkTBgwYgOTkZPj4+KBnz564fPkyWrduDXd3d2zevBl58uQBAISEhKBZs2YoWrQobty4kb0XxhhjjH1FPNLM2A9C+v4rBcebN29GpUqVULduXXTs2BH79++Xt5UC5ujoaEyZMgUA4O/vj7Fjx6JIkSIoV64c8ufPj3PnzmHp0qXyfk2aNMG8efPkkWbGGGMsp+CRZsZ+ABnTJABg+PDhmD17NhwcHKBUKnHjxg2YmZlhxYoVaNmyJVJTU2FgYICAgAD4+flh2bJl6Nq1q7z/9evXUbFiRaSmpsLW1hYhISEoWbLke8/JGGOM/ch4pJmxH4BSqURiYiIWLlyI58+fY/fu3ejevTu2b9+O8PBwrF69Gi9fvoSfnx80Gg0MDAwAAHFxcVmOlZqaioULF6JGjRrw9PREVFQU4uPjdbbJXIOZMcYY+9HxSDNj36HMecsA0Lx5c+zcuRNVqlSBqakpQkNDddYPGTIEM2fOxNChQ/Hnn38CAM6cOYMaNWqgatWqmDNnDgBg586dmDt3LkaMGAE/Pz8kJydDpVJlz4Uxxhhj3ygOmhn7jp04cQJmZmYoXbo0Dh48CE9PTyQlJaF79+6YO3cu1Go1hBDQ09PD3bt30axZM0RERODWrVsoVKgQkpKS8Mcff2DKlCkwNDSEnp4eXr16hdatW2P+/PmwsrICAKSlpXHba8YYYzkap2cw9p1avXo1qlevjnv37gEAateujXbt2iElJQWvXr0CkJ62oaenByKCo6OjXGu5f//+AABjY2NMmjQJc+bMQY8ePeDp6Yldu3YhODhYDpgBcMDMGGMsx+ORZsa+U8uWLUO3bt2wfPlydOnSBQBw//59VKxYEXFxcQgPD0fx4sWh0WgghIBCoZBHkfft24f9+/ejTp067zw+T/RjjDHG3uCRZsa+EZm/vxJRlmUZtytcuDAA4OzZswDS85zt7e0xePBgaDQa/PHHHwDedO7TarUwMTGBn58fVCoVvLy83npcqTU2B8yMMcbYGxw0M/YVSQEqAAghdNYJISCEwM2bN/Hw4UMAkEeNAaBUqVKwtbWVUzHS0tIApKdeuLq6Yu3atdi3b5+8nzRxsGnTpmjcuDGsrKwQFxcnB8vScTNPMGSMMcYYp2cw9lVkTH148eIFLl26BK1Wi6ioKJQtWxZ2dnawsLCAv78/BgwYgIYNG2LlypXIly+ffIzo6GjUqlULQghcu3YNenp68oS9TZs2oW3btnBzc8PRo0cBpI8ka7VaKJVKxMfHI3fu3F/j0hljjLHvEg8pMZaNMqc+TJkyBfXr10erVq1Qq1YtdO3aFdWrV4enpydu3LiBZs2awdvbG3v37kXLli3x999/y8dycHBAkSJFoFarcffuXQBvJuy1adMGzZs3x/HjxxEYGCifWzqvFDBLo9OMMcYYez8OmhnLJhlrK69YsQJ2dnaYNGkSzMzM0LVrV2zevBnjxo1DiRIlcOjQIbRv3x43b97EokWLMGfOHFy4cAHt2rXDmjVr5GO6uLggJiZGPi4RQaPRAADGjBkDABg1ahQSExPfmqPMVTEYY4yxj8NBM2PZRKFQICwsDDVq1IC3tzdcXFywYMECrFy5EtOnT0fLli0xevRobNu2Db169cKVK1fw22+/ISIiAn5+fli1ahXy5MmD7t27Y+TIkUhJSYGtrS3UajWOHTsGID0vWalUgohQrlw5TJs2DVu2bIGpqelXvnrGGGPs+8bDTIxlk9jYWNSoUQNqtRpDhgzB8OHDYWFhIa/XarUQQsDKygpDhw7Fq1evsGrVKsydOxdBQUFo3bo1ChcujFGjRmHy5Ml4/Pgx6tatCwDyZEAighBC/u/gwYN1ljPGGGPs3+GRZsaySb58+TBu3DgAQFxcnBwwS+kUCoVCDmwLFSqE3r17w87ODjt37sSBAwcAAK6urli3bh369OmDZcuWoU+fPgCAq1ev6pwrYwUMDpgZY4yx/46DZsay0a+//oqSJUtiw4YN2Lx5M4CspeYkJUuWhKenJ548eYJ//vlHrn5hZmaGGTNmYPny5XKeckREBBITE996LA6YGWOMsf+Og2bGspG5uTnGjh2LpKQkBAUFIT4+Xm488rZtS5cuDa1Wi8jISJ3g18DAAD///DNWrFiBMmXK4OXLl3K7bMYYY4x9fhw0M5bN2rRpAw8PD4SGhmL16tUAsjYUkYLosmXLAgBu3boFtVqtUyUDAGrUqIHixYvj4sWLePbsGY8qM8YYY18IB82MZTOFQoExY8ZACIFFixYhMjISAN462nz//n0AQJUqVaCvr6/TvY+IYGZmBhsbG6SlpeHx48fZdxGMMcZYDsNBM2NfQeXKldGnTx9cvnwZixYtApAeTBMRiEgeUd64cSMAoHz58gB085OFEIiLi0NkZCSsrKxgZWWVzVfBGGOM5RwcNDP2lQwbNgxWVlZYuXIljhw5AiA9EBZCICEhAb/99hvWrl2LIUOGoGXLlln2v3v3LqZMmYLdu3ejSZMmMDc3z+YrYIwxxnIODpoZ+0rs7e0xYsQIPH78GAsXLpSXnzlzBqNHj8bSpUvRpk0b9OzZEwCyTPJLSUnBxYsXMXr0aCxevBhmZmbZ+vczxhhjOYkgnm7P2FeTlJSEmjVrIjw8HNOnT4eJiQlmzZqFq1evwtfXF5MnT35vMJycnAyVSgVAt003Y4wxxj4vDpoZ+8p27tyJ5s2bw8DAAKmpqXBxcUFgYCAqV64M4MPBMAfLjDHG2JfHQTNj34D69evj8OHDWLZsGby8vAC8qabBATFjjDH29XHQzNg34PHjx7C2tpb/f1paGvT09L7iX8QYY4yxjDhoZuwbwsEyY4wx9m3ioJkxxhhjjLEP4GRJxhhjjDHGPoCDZsYYY4wxxj6Ag2bGGGOMMcY+gINmxhhjjDHGPoCDZsYYY4wxxj6Ag2bGGGOMMcY+gINmxhhjjDHGPoCDZsYYY4wxxj6Ag2bGGGOMMcY+gINmxhhjjDHGPoCDZsYYY4wxxj7g/wBqrR/YulnJ7gAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 800x400 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(figsize=(8, 4))\n",
    "x = \"label\"\n",
    "y = \"AUROC\"\n",
    "#hue = \"label\"\n",
    "ax = sns.boxplot(data=tmp, x=x, y=y,palette=['royalblue','gold','darkseagreen','orangered','orange']+[sns.color_palette()[8]])\n",
    "add_stat_annotation(ax, data=tmp, x=x, y=y,\n",
    "                    box_pairs=[('PSSM', 'PSSM+CoPheeMap+KMap'),\n",
    "                               ('Dynamic_features','Dynamic_features+CoPheeMap+KMap'),\n",
    "                               ('PSSM+CoPheeMap+KMap','CoPheeKSA'),\n",
    "                               ('Dynamic_features+CoPheeMap+KMap','CoPheeKSA')],\n",
    "                    test='t-test_ind', text_format='star', loc='outside', verbose=2)\n",
    "plt.xticks(rotation=30,size=14,ha='right')\n",
    "plt.yticks(size=14)\n",
    "#plt.title('PSSM zscores',size=14)\n",
    "plt.xlabel('')\n",
    "plt.ylabel('AUROC',size=14)\n",
    "#plt.legend(['PSSM','CoPheeMap+KMap','PSSM+CoPheeMap+KMap','Dynamic features','Dynamic features+CoPheeMap+KMap',\n",
    "#            'CoPheeKSA'],loc='upper left', bbox_to_anchor=(1.03, 1),fontsize=12)\n",
    "plt.savefig('CoPheeKSA/comparision_auroc_KSA.jpeg',bbox_inches='tight',dpi=300,pad_inches=0.1,orientation='portrait')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "fb9404d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\ttrain-auc:0.87802\teval-auc:0.88560\n",
      "[1]\ttrain-auc:0.88141\teval-auc:0.88400\n",
      "[2]\ttrain-auc:0.89189\teval-auc:0.90080\n",
      "[3]\ttrain-auc:0.89877\teval-auc:0.90272\n",
      "[4]\ttrain-auc:0.89865\teval-auc:0.90168\n",
      "[5]\ttrain-auc:0.90154\teval-auc:0.91376\n",
      "[6]\ttrain-auc:0.90182\teval-auc:0.91480\n",
      "[7]\ttrain-auc:0.90244\teval-auc:0.91744\n",
      "[8]\ttrain-auc:0.90297\teval-auc:0.91696\n",
      "[9]\ttrain-auc:0.90458\teval-auc:0.91744\n",
      "[10]\ttrain-auc:0.90615\teval-auc:0.91704\n",
      "[11]\ttrain-auc:0.90972\teval-auc:0.91648\n",
      "[12]\ttrain-auc:0.91130\teval-auc:0.91336\n",
      "[13]\ttrain-auc:0.91121\teval-auc:0.91288\n",
      "[14]\ttrain-auc:0.91177\teval-auc:0.91320\n",
      "[15]\ttrain-auc:0.91367\teval-auc:0.91912\n",
      "[16]\ttrain-auc:0.91639\teval-auc:0.93312\n",
      "[17]\ttrain-auc:0.91722\teval-auc:0.93272\n",
      "[18]\ttrain-auc:0.91852\teval-auc:0.92976\n",
      "[19]\ttrain-auc:0.92137\teval-auc:0.93216\n",
      "[20]\ttrain-auc:0.92185\teval-auc:0.93392\n",
      "[21]\ttrain-auc:0.92300\teval-auc:0.93368\n",
      "[22]\ttrain-auc:0.92416\teval-auc:0.93504\n",
      "[23]\ttrain-auc:0.92500\teval-auc:0.93376\n",
      "[24]\ttrain-auc:0.92603\teval-auc:0.93552\n",
      "[25]\ttrain-auc:0.92657\teval-auc:0.93616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Pass `evals` as keyword args.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[26]\ttrain-auc:0.92743\teval-auc:0.93600\n",
      "[27]\ttrain-auc:0.92800\teval-auc:0.93728\n",
      "[28]\ttrain-auc:0.92872\teval-auc:0.93664\n",
      "[29]\ttrain-auc:0.92972\teval-auc:0.93664\n",
      "[30]\ttrain-auc:0.93036\teval-auc:0.93776\n",
      "[31]\ttrain-auc:0.93169\teval-auc:0.93904\n",
      "[32]\ttrain-auc:0.93193\teval-auc:0.93936\n",
      "[33]\ttrain-auc:0.93251\teval-auc:0.94080\n",
      "[34]\ttrain-auc:0.93330\teval-auc:0.94064\n",
      "[35]\ttrain-auc:0.93383\teval-auc:0.93712\n",
      "[36]\ttrain-auc:0.93406\teval-auc:0.94144\n",
      "[37]\ttrain-auc:0.93449\teval-auc:0.94432\n",
      "[38]\ttrain-auc:0.93485\teval-auc:0.94432\n",
      "[39]\ttrain-auc:0.93548\teval-auc:0.94288\n",
      "[40]\ttrain-auc:0.93608\teval-auc:0.94304\n",
      "[41]\ttrain-auc:0.93644\teval-auc:0.94160\n",
      "[42]\ttrain-auc:0.93707\teval-auc:0.94352\n",
      "[43]\ttrain-auc:0.93745\teval-auc:0.94512\n",
      "[44]\ttrain-auc:0.93768\teval-auc:0.94560\n",
      "[45]\ttrain-auc:0.93800\teval-auc:0.94544\n",
      "[46]\ttrain-auc:0.93828\teval-auc:0.94576\n",
      "[47]\ttrain-auc:0.93871\teval-auc:0.94640\n",
      "[48]\ttrain-auc:0.93926\teval-auc:0.94736\n",
      "[49]\ttrain-auc:0.93986\teval-auc:0.94880\n",
      "[50]\ttrain-auc:0.94067\teval-auc:0.94928\n",
      "[51]\ttrain-auc:0.94131\teval-auc:0.95024\n",
      "[52]\ttrain-auc:0.94184\teval-auc:0.95040\n",
      "[53]\ttrain-auc:0.94201\teval-auc:0.95072\n",
      "[54]\ttrain-auc:0.94223\teval-auc:0.95136\n",
      "[55]\ttrain-auc:0.94255\teval-auc:0.95184\n",
      "[56]\ttrain-auc:0.94305\teval-auc:0.95168\n",
      "[57]\ttrain-auc:0.94328\teval-auc:0.95376\n",
      "[58]\ttrain-auc:0.94363\teval-auc:0.95296\n",
      "[59]\ttrain-auc:0.94392\teval-auc:0.95344\n",
      "[60]\ttrain-auc:0.94425\teval-auc:0.95376\n",
      "[61]\ttrain-auc:0.94434\teval-auc:0.95344\n",
      "[62]\ttrain-auc:0.94463\teval-auc:0.95408\n",
      "[63]\ttrain-auc:0.94504\teval-auc:0.95328\n",
      "[64]\ttrain-auc:0.94541\teval-auc:0.95472\n",
      "[65]\ttrain-auc:0.94551\teval-auc:0.95488\n",
      "[66]\ttrain-auc:0.94574\teval-auc:0.95472\n",
      "[67]\ttrain-auc:0.94600\teval-auc:0.95488\n",
      "[68]\ttrain-auc:0.94607\teval-auc:0.95600\n",
      "[69]\ttrain-auc:0.94623\teval-auc:0.95632\n",
      "[70]\ttrain-auc:0.94670\teval-auc:0.95520\n",
      "[71]\ttrain-auc:0.94689\teval-auc:0.95472\n",
      "[72]\ttrain-auc:0.94716\teval-auc:0.95568\n",
      "[73]\ttrain-auc:0.94719\teval-auc:0.95648\n",
      "[74]\ttrain-auc:0.94775\teval-auc:0.95616\n",
      "[75]\ttrain-auc:0.94804\teval-auc:0.95696\n",
      "[76]\ttrain-auc:0.94828\teval-auc:0.95728\n",
      "[77]\ttrain-auc:0.94843\teval-auc:0.95808\n",
      "[78]\ttrain-auc:0.94872\teval-auc:0.95872\n",
      "[79]\ttrain-auc:0.94903\teval-auc:0.96032\n",
      "[80]\ttrain-auc:0.94931\teval-auc:0.95952\n",
      "[81]\ttrain-auc:0.94947\teval-auc:0.96000\n",
      "[82]\ttrain-auc:0.94969\teval-auc:0.95952\n",
      "[83]\ttrain-auc:0.94995\teval-auc:0.95952\n",
      "[84]\ttrain-auc:0.95034\teval-auc:0.96032\n",
      "[85]\ttrain-auc:0.95057\teval-auc:0.96000\n",
      "[86]\ttrain-auc:0.95083\teval-auc:0.96000\n",
      "[87]\ttrain-auc:0.95109\teval-auc:0.95968\n",
      "[88]\ttrain-auc:0.95116\teval-auc:0.95984\n",
      "[89]\ttrain-auc:0.95147\teval-auc:0.96080\n",
      "[90]\ttrain-auc:0.95167\teval-auc:0.96096\n",
      "[91]\ttrain-auc:0.95174\teval-auc:0.96080\n",
      "[92]\ttrain-auc:0.95201\teval-auc:0.96208\n",
      "[93]\ttrain-auc:0.95216\teval-auc:0.96208\n",
      "[94]\ttrain-auc:0.95234\teval-auc:0.96288\n",
      "[95]\ttrain-auc:0.95243\teval-auc:0.96272\n",
      "[96]\ttrain-auc:0.95256\teval-auc:0.96352\n",
      "[97]\ttrain-auc:0.95277\teval-auc:0.96368\n",
      "[98]\ttrain-auc:0.95341\teval-auc:0.96288\n",
      "[99]\ttrain-auc:0.95367\teval-auc:0.96352\n",
      "[100]\ttrain-auc:0.95385\teval-auc:0.96368\n",
      "[101]\ttrain-auc:0.95404\teval-auc:0.96368\n",
      "[102]\ttrain-auc:0.95418\teval-auc:0.96352\n",
      "[103]\ttrain-auc:0.95425\teval-auc:0.96224\n",
      "[104]\ttrain-auc:0.95451\teval-auc:0.96416\n",
      "[105]\ttrain-auc:0.95476\teval-auc:0.96464\n",
      "[106]\ttrain-auc:0.95493\teval-auc:0.96464\n",
      "[107]\ttrain-auc:0.95514\teval-auc:0.96512\n",
      "[108]\ttrain-auc:0.95528\teval-auc:0.96496\n",
      "[109]\ttrain-auc:0.95547\teval-auc:0.96624\n",
      "[110]\ttrain-auc:0.95555\teval-auc:0.96624\n",
      "[111]\ttrain-auc:0.95562\teval-auc:0.96672\n",
      "[112]\ttrain-auc:0.95578\teval-auc:0.96768\n",
      "[113]\ttrain-auc:0.95607\teval-auc:0.96720\n",
      "[114]\ttrain-auc:0.95618\teval-auc:0.96704\n",
      "[115]\ttrain-auc:0.95628\teval-auc:0.96688\n",
      "[116]\ttrain-auc:0.95639\teval-auc:0.96752\n",
      "[117]\ttrain-auc:0.95647\teval-auc:0.96768\n",
      "[118]\ttrain-auc:0.95643\teval-auc:0.96768\n",
      "[119]\ttrain-auc:0.95660\teval-auc:0.96688\n",
      "[120]\ttrain-auc:0.95665\teval-auc:0.96672\n",
      "[121]\ttrain-auc:0.95686\teval-auc:0.96688\n",
      "[122]\ttrain-auc:0.95702\teval-auc:0.96768\n",
      "0.9614927569035763\n"
     ]
    }
   ],
   "source": [
    "#Train a final model CoPheeKSA using Dynamic features + PSSM + CoPheeMap + KMap\n",
    "Cancer=['BRCA','CCRCC','COAD','GBM','HCC','HNSCC','LSCC','LUAD','OV','PDAC','UCEC'] \n",
    "col_idx=['cophee_emb0', 'cophee_emb1', 'cophee_emb2', 'cophee_emb3', 'cophee_emb4', \n",
    "         'cophee_emb5', 'cophee_emb6', 'cophee_emb7', 'cophee_emb8', 'cophee_emb9', \n",
    "         'cophee_emb10','cophee_emb11', 'cophee_emb12', 'cophee_emb13', 'cophee_emb14', \n",
    "         'cophee_emb15', 'k_emb0', 'k_emb1', 'k_emb2', 'k_emb3','k_emb4', 'k_emb5', \n",
    "         'k_emb6', 'k_emb7', 'k_emb8', 'k_emb9', 'k_emb10',\n",
    "         'k_emb11', 'k_emb12', 'k_emb13', 'k_emb14', 'k_emb15','BRCA_spc_k_s', \n",
    "         'CCRCC_spc_k_s', 'COAD_spc_k_s', 'GBM_spc_k_s',\n",
    "         'HCC_spc_k_s', 'HNSCC_spc_k_s', 'LSCC_spc_k_s', 'LUAD_spc_k_s',\n",
    "         'OV_spc_k_s', 'PDAC_spc_k_s', 'UCEC_spc_k_s', 'BRCA_spc_kactivity_s',\n",
    "         'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "         'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "         'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "         'UCEC_spc_kactivity_s','PSSM']\n",
    "for i in random.sample(range(10),1):\n",
    "    data_train=pd.read_csv('CoPheeKSA/train/w_feature/train_'+str(i)+'.csv',index_col=0)\n",
    "    data_train=data_train.drop_duplicates()\n",
    "    data_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    data_train=data_train[~data_train['PSSM'].isna()]\n",
    "    \n",
    "    data_neg=pd.read_csv('Supplementary_table/Table_S3_negative_KSA_w_features.tsv.zip', compression='zip', sep='\\t')\n",
    "    data_neg=data_neg.drop_duplicates()\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    kinases=list(set(data_train.kinase))\n",
    "    data_train_neg=pd.DataFrame()\n",
    "    for k in kinases:\n",
    "        tmp=data_train[data_train.kinase==k]\n",
    "        tmp2=data_neg[data_neg.kinase==k]\n",
    "        if len(tmp2) > 10*len(tmp):\n",
    "            idx=random.sample(range(len(tmp2)),10*len(tmp))\n",
    "            tmp2=tmp2.iloc[idx]\n",
    "        \n",
    "        c=tmp[['BRCA_spc_kactivity_s',\n",
    "             'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "             'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "             'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "             'UCEC_spc_kactivity_s']].isna().sum()\n",
    "        for idx_col in ['BRCA_spc_kactivity_s',\n",
    "             'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "             'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "             'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "             'UCEC_spc_kactivity_s']:\n",
    "            num=int(c.loc[idx_col]/len(tmp)*len(tmp2))\n",
    "            idx=random.sample(tmp2.index.tolist(),num)\n",
    "            \n",
    "            for z in idx:\n",
    "                tmp2.loc[z, idx_col]=np.nan\n",
    "            \n",
    "        data_train_neg=pd.concat([data_train_neg,tmp2])\n",
    "    idx=random.sample(range(len(data_neg)),2*len(data_train))\n",
    "    data_train_neg=pd.concat([data_train_neg,data_neg.iloc[idx]])\n",
    "    data_train_neg=data_train_neg.drop_duplicates()\n",
    "    \n",
    "    idx_removed=data_train_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    data_test=pd.read_csv('CoPheeKSA/test/w_feature/test_'+str(i)+'.csv',index_col=0)\n",
    "    data_test=data_test.drop_duplicates()\n",
    "    data_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    data_test=data_test[~data_test['PSSM'].isna()]\n",
    "    \n",
    "    idx=random.sample(range(len(data_test)),25)\n",
    "    data_eval=data_test.iloc[idx,:]\n",
    "    idx2=[i for i in range(len(data_test)) if i not in idx]\n",
    "    data_test=data_test.iloc[idx2,:]\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_eval))\n",
    "    data_eval_neg=data_neg.iloc[idx]\n",
    "    idx_removed=data_eval_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    data_neg.index=range(len(data_neg))\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_test))\n",
    "    data_test_neg=data_neg.iloc[idx]\n",
    "    idx_removed=data_test_neg.index\n",
    "    data_neg=data_neg[~data_neg.index.isin(idx_removed)]\n",
    "    \n",
    "    data_train=data_train[col_idx]\n",
    "    data_eval=data_eval[col_idx]\n",
    "    data_test=data_test[col_idx]\n",
    "    data_neg=data_neg[col_idx]\n",
    "    data_train_neg=data_train_neg[col_idx]\n",
    "    data_eval_neg=data_eval_neg[col_idx]\n",
    "    \n",
    "    X_train=pd.concat([data_train,data_train_neg])\n",
    "    y_train=np.concatenate((np.repeat(1,len(data_train)),np.repeat(0,len(data_train_neg))))\n",
    "    X_train=X_train[col_idx]\n",
    "    X_train.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    dtrain = xgb.DMatrix(X_train, label=y_train)\n",
    "    \n",
    "    X_eval=pd.concat([data_eval,data_eval_neg])\n",
    "    y_eval=np.concatenate((np.repeat(1,len(data_eval)),np.repeat(0,len(data_eval_neg))))\n",
    "    X_eval=X_eval[col_idx]\n",
    "    X_eval.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    deval = xgb.DMatrix(X_eval,label=y_eval)\n",
    "    param = {'max_depth': 2, 'eta': 0.2, 'objective': 'binary:logistic','verbosity':0}\n",
    "    param['nthread'] = 4\n",
    "    param['eval_metric'] = 'auc'\n",
    "    evallist = [(dtrain, 'train'),(deval, 'eval')]\n",
    "    num_round = 300\n",
    "    bst = xgb.train(param, dtrain, num_round, evallist, early_stopping_rounds=10)\n",
    "    #bst.save_model('CoPheeKSA/model/best_xgboost.model')\n",
    "    \n",
    "    idx=random.sample(range(len(data_neg)),10*len(data_test))\n",
    "    data_neg2=data_neg.iloc[idx,:]\n",
    "    X_test=pd.concat([data_test,data_neg2])\n",
    "    X_test=X_test[col_idx]\n",
    "    X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "    y_test=np.concatenate((np.repeat(1,len(data_test)),np.repeat(0,len(data_neg2))))\n",
    "    dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "    y_pred = bst.predict(dtest)\n",
    "\n",
    "    fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred)\n",
    "    auc_tmp=metrics.auc(fpr, tpr)\n",
    "    print(auc_tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "id": "188ab859",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Store the test data\n",
    "idx=random.sample(range(len(data_neg)),100*len(data_test))\n",
    "data_neg2=data_neg.iloc[idx,:]\n",
    "X_test=pd.concat([data_test,data_neg2])\n",
    "X_test=X_test[col_idx]\n",
    "X_test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "y_test=np.concatenate((np.repeat(1,len(data_test)),np.repeat(0,len(data_neg2))))\n",
    "dtest = xgb.DMatrix(X_test, label=y_test)\n",
    "y_pred = bst.predict(dtest)\n",
    "y_test=pd.DataFrame(y_test)\n",
    "\n",
    "#X_test.to_csv('CoPheeKSA/model/X_test.csv')\n",
    "#y_test.to_csv('CoPheeKSA/model/y_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "46059b4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_test=pd.read_csv('CoPheeKSA/model/X_test.csv',index_col=0)\n",
    "#y_test=pd.read_csv('CoPheeKSA/model/y_test.csv',index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "aa3f8d63",
   "metadata": {},
   "outputs": [],
   "source": [
    "bst = xgb.Booster()\n",
    "bst.load_model('CoPheeKSA/model/best_xgboost.model')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 436,
   "id": "bc212a26",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RAF1\n",
      "IKBKB\n",
      "CASK\n",
      "PAK3\n",
      "MKNK1\n",
      "PAK2\n",
      "CAMK2A\n",
      "LATS2\n",
      "CDC7\n",
      "PINK1\n",
      "MAPK7\n",
      "PAK1\n",
      "CAMK2B\n",
      "MAPK6\n",
      "LATS1\n",
      "DAPK1\n",
      "DYRK2\n",
      "PAK4\n",
      "CAMK2G\n",
      "MAPK3\n",
      "PRKACA\n",
      "PRKD2\n",
      "IKBKE\n",
      "DAPK2\n",
      "MAPK1\n",
      "MARK3\n",
      "CAMK2D\n",
      "MARK2\n",
      "PRKD1\n",
      "TBK1\n",
      "DAPK3\n",
      "PRKCD\n",
      "RPS6KA1\n",
      "ULK1\n",
      "GRK5\n",
      "MELK\n",
      "STK3\n",
      "GRK4\n",
      "BUB1\n",
      "PRKCE\n",
      "PRKCG\n",
      "NEK7\n",
      "GRK6\n",
      "SMG1\n",
      "NEK6\n",
      "TAF1\n",
      "RPS6KA3\n",
      "PRKCB\n",
      "PKN1\n",
      "AKT2\n",
      "SIK2\n",
      "NEK2\n",
      "PDK2\n",
      "TRPM7\n",
      "STK4\n",
      "CDK12\n",
      "CLK2\n",
      "AKT3\n",
      "MAP3K8\n",
      "WNK1\n",
      "ARAF\n",
      "WNK3\n",
      "PRKCA\n",
      "CDK9\n",
      "AKT1\n",
      "ILK\n",
      "TTK\n",
      "PDK1\n",
      "NEK1\n",
      "EIF2AK2\n",
      "ATR\n",
      "RPS6KA5\n",
      "CDK8\n",
      "CDK5\n",
      "MTOR\n",
      "PLK4\n",
      "ROCK1\n",
      "SRPK1\n",
      "SGK3\n",
      "DYRK1A\n",
      "TGFBR2\n",
      "CDK4\n",
      "MAP3K7\n",
      "PIM1\n",
      "MAP3K5\n",
      "CDK6\n",
      "OXSR1\n",
      "SGK1\n",
      "ROCK2\n",
      "DYRK1B\n",
      "CDK7\n",
      "CDK3\n",
      "PLK2\n",
      "CAMK4\n",
      "PLK3\n",
      "CHEK1\n",
      "CDK2\n",
      "MAP3K1\n",
      "MAP3K3\n",
      "NUAK1\n",
      "PLK1\n",
      "HIPK2\n",
      "ATM\n",
      "NEK9\n",
      "CDK1\n",
      "CHEK2\n",
      "MAP3K2\n",
      "LRRK2\n",
      "CHUK\n",
      "BRAF\n",
      "MAP2K4\n",
      "MAP2K6\n",
      "MAPKAPK3\n",
      "MAPKAPK2\n",
      "MAPK15\n",
      "MAP2K7\n",
      "IRAK4\n",
      "MAP4K1\n",
      "PDPK1\n",
      "STK38\n",
      "GSK3B\n",
      "MAP2K3\n",
      "MAPK10\n",
      "CSNK2A1\n",
      "STK11\n",
      "AURKA\n",
      "MAPK8\n",
      "GSK3A\n",
      "CSNK2A2\n",
      "MAP2K1\n",
      "MAPK9\n"
     ]
    }
   ],
   "source": [
    "#Predict KSAs using CoPheeKSA\n",
    "#Warning: Long running time and original data will be replaced by the new data\n",
    "bst = xgb.Booster()\n",
    "bst.load_model('CoPheeKSA/model/best_xgboost.model')\n",
    "mypath='CoPheeKSA/prediction_data/'\n",
    "onlyfiles = [f for f in listdir(mypath) if f!='.DS_Store']\n",
    "onlyfiles = [f for f in onlyfiles if isfile(''.join([mypath, f]))]\n",
    "\n",
    "path2='CoPheeKSA/PSSM/ST/'\n",
    "onlyfiles2 = [f for f in listdir(path2) if f!='.DS_Store']\n",
    "onlyfiles2 = [f for f in onlyfiles2 if 'family' not in f]\n",
    "ST_k=[f.split('_')[0] for f in onlyfiles2]\n",
    "col_idx=['cophee_emb0', 'cophee_emb1', 'cophee_emb2', 'cophee_emb3', 'cophee_emb4', \n",
    "         'cophee_emb5', 'cophee_emb6', 'cophee_emb7', 'cophee_emb8', 'cophee_emb9', \n",
    "         'cophee_emb10','cophee_emb11', 'cophee_emb12', 'cophee_emb13', 'cophee_emb14', \n",
    "         'cophee_emb15', 'k_emb0', 'k_emb1', 'k_emb2', 'k_emb3','k_emb4', 'k_emb5', \n",
    "         'k_emb6', 'k_emb7', 'k_emb8', 'k_emb9', 'k_emb10',\n",
    "         'k_emb11', 'k_emb12', 'k_emb13', 'k_emb14', 'k_emb15','BRCA_spc_k_s', \n",
    "         'CCRCC_spc_k_s', 'COAD_spc_k_s', 'GBM_spc_k_s',\n",
    "         'HCC_spc_k_s', 'HNSCC_spc_k_s', 'LSCC_spc_k_s', 'LUAD_spc_k_s',\n",
    "         'OV_spc_k_s', 'PDAC_spc_k_s', 'UCEC_spc_k_s', 'BRCA_spc_kactivity_s',\n",
    "         'CCRCC_spc_kactivity_s', 'COAD_spc_kactivity_s', 'GBM_spc_kactivity_s',\n",
    "         'HCC_spc_kactivity_s', 'HNSCC_spc_kactivity_s', 'LSCC_spc_kactivity_s',\n",
    "         'LUAD_spc_kactivity_s', 'OV_spc_kactivity_s', 'PDAC_spc_kactivity_s',\n",
    "         'UCEC_spc_kactivity_s','PSSM']\n",
    "for f in onlyfiles:\n",
    "    kinase=f.split('.')[0]\n",
    "    if kinase in ST_k:\n",
    "        print(kinase)\n",
    "        data=pd.read_csv(''.join([mypath,f]),index_col=0)\n",
    "        data.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "        data.index=data.sites\n",
    "        score=[]\n",
    "        data=data[col_idx]\n",
    "        y_test=np.repeat(0,len(data))\n",
    "        dtest = xgb.DMatrix(data, label=y_test)\n",
    "        prediction = bst.predict(dtest)\n",
    "        data['prediction']=np.array(prediction)\n",
    "        data.to_csv(''.join(['CoPheeKSA/prediction/',f]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "id": "886898f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "131\n",
      "RAF1.csv\n",
      "19\n",
      "IKBKB.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CASK.csv\n",
      "69\n",
      "PAK3.csv\n",
      "20\n",
      "MKNK1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "441\n",
      "PAK2.csv\n",
      "0\n",
      "CAMK2A.csv\n",
      "0\n",
      "LATS2.csv\n",
      "43\n",
      "CDC7.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "PINK1.csv\n",
      "1\n",
      "MAPK7.csv\n",
      "423\n",
      "PAK1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "CAMK2B.csv\n",
      "21\n",
      "MAPK6.csv\n",
      "29\n",
      "LATS1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "DAPK1.csv\n",
      "0\n",
      "DYRK2.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "191\n",
      "PAK4.csv\n",
      "11\n",
      "CAMK2G.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28\n",
      "MAPK3.csv\n",
      "744\n",
      "PRKACA.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "120\n",
      "PRKD2.csv\n",
      "28\n",
      "IKBKE.csv\n",
      "0\n",
      "DAPK2.csv\n",
      "141\n",
      "MAPK1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "973\n",
      "MARK3.csv\n",
      "17\n",
      "CAMK2D.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "MARK2.csv\n",
      "45\n",
      "PRKD1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44\n",
      "TBK1.csv\n",
      "0\n",
      "DAPK3.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "12\n",
      "PRKCD.csv\n",
      "1\n",
      "RPS6KA1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95\n",
      "ULK1.csv\n",
      "0\n",
      "GRK5.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "MELK.csv\n",
      "0\n",
      "STK3.csv\n",
      "0\n",
      "GRK4.csv\n",
      "0\n",
      "BUB1.csv\n",
      "0\n",
      "PRKCE.csv\n",
      "0\n",
      "PRKCG.csv\n",
      "0\n",
      "NEK7.csv\n",
      "140\n",
      "GRK6.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "SMG1.csv\n",
      "318\n",
      "NEK6.csv\n",
      "2\n",
      "TAF1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "461\n",
      "RPS6KA3.csv\n",
      "12\n",
      "PRKCB.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "PKN1.csv\n",
      "25\n",
      "AKT2.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "85\n",
      "SIK2.csv\n",
      "112\n",
      "NEK2.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "PDK2.csv\n",
      "104\n",
      "TRPM7.csv\n",
      "0\n",
      "STK4.csv\n",
      "0\n",
      "CDK12.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2131\n",
      "CLK2.csv\n",
      "15\n",
      "AKT3.csv\n",
      "287\n",
      "MAP3K8.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "WNK1.csv\n",
      "5\n",
      "ARAF.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "WNK3.csv\n",
      "4\n",
      "PRKCA.csv\n",
      "10\n",
      "CDK9.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "384\n",
      "AKT1.csv\n",
      "202\n",
      "ILK.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\n",
      "TTK.csv\n",
      "0\n",
      "PDK1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26\n",
      "NEK1.csv\n",
      "0\n",
      "EIF2AK2.csv\n",
      "0\n",
      "ATR.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26\n",
      "RPS6KA5.csv\n",
      "49\n",
      "CDK8.csv\n",
      "2009\n",
      "CDK5.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "482\n",
      "MTOR.csv\n",
      "9\n",
      "PLK4.csv\n",
      "0\n",
      "ROCK1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n",
      "SRPK1.csv\n",
      "813\n",
      "SGK3.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "515\n",
      "DYRK1A.csv\n",
      "49\n",
      "TGFBR2.csv\n",
      "0\n",
      "CDK4.csv\n",
      "1024\n",
      "MAP3K7.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "PIM1.csv\n",
      "14\n",
      "MAP3K5.csv\n",
      "0\n",
      "CDK6.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "560\n",
      "OXSR1.csv\n",
      "1\n",
      "SGK1.csv\n",
      "94\n",
      "ROCK2.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8\n",
      "DYRK1B.csv\n",
      "17\n",
      "CDK7.csv\n",
      "36\n",
      "CDK3.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "334\n",
      "PLK2.csv\n",
      "44\n",
      "CAMK4.csv\n",
      "17\n",
      "PLK3.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32\n",
      "CHEK1.csv\n",
      "4\n",
      "CDK2.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2101\n",
      "MAP3K1.csv\n",
      "1379\n",
      "MAP3K3.csv\n",
      "53\n",
      "NUAK1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "54\n",
      "PLK1.csv\n",
      "6\n",
      "HIPK2.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "247\n",
      "ATM.csv\n",
      "19\n",
      "NEK9.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "48\n",
      "CDK1.csv\n",
      "2844\n",
      "CHEK2.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "MAP3K2.csv\n",
      "37\n",
      "LRRK2.csv\n",
      "0\n",
      "CHUK.csv\n",
      "0\n",
      "BRAF.csv\n",
      "1\n",
      "MAP2K4.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "MAP2K6.csv\n",
      "5\n",
      "MAPKAPK3.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "270\n",
      "MAPKAPK2.csv\n",
      "75\n",
      "MAPK15.csv\n",
      "20\n",
      "MAP2K7.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18\n",
      "IRAK4.csv\n",
      "0\n",
      "MAP4K1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "253\n",
      "PDPK1.csv\n",
      "18\n",
      "STK38.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41\n",
      "GSK3B.csv\n",
      "135\n",
      "MAP2K3.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9\n",
      "MAPK10.csv\n",
      "237\n",
      "CSNK2A1.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "574\n",
      "STK11.csv\n",
      "69\n",
      "AURKA.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "MAPK8.csv\n",
      "429\n",
      "GSK3A.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "363\n",
      "CSNK2A2.csv\n",
      "320\n",
      "MAP2K1.csv\n",
      "7\n",
      "MAPK9.csv\n",
      "333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
     ]
    }
   ],
   "source": [
    "#Threshold when LLR>0.55\n",
    "thred_55=0.767626\n",
    "\n",
    "#Collect positive predicted KSAs\n",
    "mypath='CoPheeKSA/prediction/'\n",
    "onlyfiles = [f for f in listdir(mypath) if f!='.DS_Store']\n",
    "ks1=[i.split('_')[0] for i in onlyfiles]\n",
    "print(len(ks1))\n",
    "combined_pos=pd.DataFrame()\n",
    "for f in onlyfiles:\n",
    "    print(f)\n",
    "    kinase=f.split('.')[0]\n",
    "    data=pd.read_csv(''.join([mypath,'/',f]),index_col=0)\n",
    "    data=data.sort_values('prediction',ascending=False)\n",
    "    data_pos=data[data.prediction>thred_55]\n",
    "    #data_pos=data_pos.iloc[:1000]\n",
    "    data_pos['kinase']=np.repeat(f.split('.')[0],len(data_pos))\n",
    "    print(len(data_pos))\n",
    "    combined_pos=pd.concat([combined_pos,data_pos])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "id": "3f5b2c38",
   "metadata": {},
   "outputs": [],
   "source": [
    "#combined_pos.to_csv('CoPheeKSA/results/K_S_CoPhee_llr55_w_features.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}