Analyzer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "from pandas import DataFrame, Series\n",
    "import os, sys, re\n",
    "import numpy as np\n",
    "\n",
    "from decimal import Decimal\n",
    "from tabulate import tabulate\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')  # action='once'\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Utility Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from analyzer import interpret\n",
    "\n",
    "def col_values(df, col='age', n=10):\n",
    "    df_subset = df.sample(n=n, random_state=1) # for each column, sample n values (usually n=1)\n",
    "    return df_subset[col].values\n",
    "\n",
    "def summary(df, n=1): \n",
    "    msg = \"\"\n",
    "    msg += \"> sample sizes: {}\\n\".format(df.shape[0])\n",
    "    msg += \"> n(features):  {}\\n\".format(df.shape[1])\n",
    "    # msg += \"> list of features:\\n{}\\n\".format(df.columns.values)\n",
    "    print(msg)\n",
    "\n",
    "    interpret(df, n=n, verbose=True)\n",
    "def show_dict(adict, topn=-1, by='', header=[], n_samples=-1, ascending=False, print_=False): \n",
    "    # print(adict)\n",
    "    \n",
    "    # convert to two-column dataframe format \n",
    "    if not header: \n",
    "        header = ['key', 'value']\n",
    "    else: \n",
    "        assert len(header) == 2\n",
    "    D = {h:[] for h in header}\n",
    "    for k, v in adict.items(): \n",
    "        D[header[0]].append(k)\n",
    "        D[header[1]].append(v)\n",
    "    \n",
    "    df = DataFrame(D, columns=header)\n",
    "    msg = ''\n",
    "    if topn > 0: \n",
    "        assert by in df.columns\n",
    "        df = df.sort_values([by, ], ascending=ascending)\n",
    "        msg = tabulate(df[:topn], headers='keys', tablefmt='psql')\n",
    "    else: \n",
    "        if n_samples < 0: \n",
    "            msg = tabulate(df, headers='keys', tablefmt='psql')\n",
    "        else: \n",
    "            n = min(df.shape[0], n_samples)\n",
    "            msg = tabulate(df.sample(n=n), headers='keys', tablefmt='psql')\n",
    "    if print_: print(msg)\n",
    "    return msg"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Determine and Retrieve Patient Cohort(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "1. Find rows whose column match a substring \n",
    "   https://davidhamann.de/2017/06/26/pandas-select-elements-by-string/\n",
    "   \n",
    "   https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.contains.html\n",
    "\n",
    "\"\"\"\n",
    "import json \n",
    "from cohort_search import gen_code_set, gen_query_str\n",
    "\n",
    "# use module cohort_search to generate the desired query strings \n",
    "# run CohortRetrieval on Databricks (https://dbc-25924283-13f6.cloud.databricks.com/#notebook/1971793/command/1973735)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Base/Positive Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[load] Loading default input data: andromeda-pond-hepatitis-c.csv\n",
      "(canonicalize) Operations: fill n/a + dehyphenate + replace_values + trim_tail + fill others (non-target classes)\n",
      "> dim(ts0): (71224, 127)\n",
      "> N(loinc_set): 733 | example codes (source):\n",
      "['64717', '21626', '422410', '143388', '265249', '163485', '268748', '178566', '139501', '265116', '264747', '303644', '433045', '264648', '24588', '93187', '109009', '295980', '301804', '298935', '264663', '177915', '192997', '17426', '193433', '169490', '320184', '64634', '339143', '502211']\n",
      "\n",
      "> dim(performance matrix): (733, 5)\n",
      "> Set the baseline class sample size: Nmax: 5593 (Nmax0: 5593, median: 3.0)\n",
      "> top 10 sample sizes:\n",
      "+----+---------+---------+\n",
      "|    | code    |   n_pos |\n",
      "|----+---------+---------|\n",
      "|  0 | unknown |    5593 |\n",
      "|  1 | 67686   |    3120 |\n",
      "|  2 | 110114  |    2521 |\n",
      "|  3 | 19752   |    2423 |\n",
      "|  4 | 17426   |    2405 |\n",
      "|  5 | 17517   |    2368 |\n",
      "|  6 | 7187    |    2037 |\n",
      "|  7 | 7773    |    2037 |\n",
      "|  8 | 21600   |    2029 |\n",
      "|  9 | 19208   |    1971 |\n",
      "+----+---------+---------+\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\nConclusion \\n----------\\n\\nCohort: hepatitis-c\\n\\n1. max sample size with known code: 67686 => 57202\\n2. loinc_set: target loinc codes\\n   size(loinc_set): 733\\n3. ts0: dim=(71224, 127)\\n\\n\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from analyzer import load_src_data, load_data, save_data, load_performance, stratify\n",
    "# from transformer import canonicalize  # ... obsolete\n",
    "from transformer import resolve_duplicate\n",
    "import loinc as lc\n",
    "# from loinc import canonicalize\n",
    "\n",
    "######################################\n",
    "# params\n",
    "cohort = domain = 'hepatitis-c'\n",
    "token_default = 'unknown'\n",
    "col_target = 'test_result_loinc_code'\n",
    "######################################\n",
    "\n",
    "# load source data\n",
    "ts0 = load_src_data(cohort=cohort, warn_bad_lines=False, canonicalized=True, processed=False)\n",
    "\n",
    "print(\"> dim(ts0): {}\".format(ts0.shape))  \n",
    "loinc_set = codes0 = ts0[col_target].unique()\n",
    "codes0_subset = np.random.choice(codes0, 30)\n",
    "print(\"> N(loinc_set): {} | example codes (source):\\n{}\\n\".format(len(loinc_set), list(codes0_subset)))\n",
    "\n",
    "# stratify the data by class labels (e.g. LOINC codes)\n",
    "ds = stratify(ts0, col=col_target) # code|mean|std|n_pos\n",
    "Nmax0 = ds[0][1]  \n",
    "# ... here we have not added extra control data yet (i.e. patients without the target disease such hepatitis C)\n",
    "\n",
    "# compare to the previously generated performance dataframe\n",
    "df_perf = load_performance(input_dir='result', cohort=cohort)\n",
    "df_perf = df_perf.sort_values(by=['n_pos', ], ascending=False)\n",
    "Nmax = np.max(df_perf['n_pos'].values)\n",
    "Nm = np.median(df_perf['n_pos'].values)\n",
    "\n",
    "ss_dict = dict(zip(df_perf['code'].values, df_perf['n_pos'].values))\n",
    "print(\"> Set the baseline class sample size: Nmax: {} (Nmax0: {}, median: {})\".format(Nmax, Nmax0, Nm))\n",
    "\n",
    "topn = 10\n",
    "print(\"> top {} sample sizes:\\n{}\\n\".format(topn, show_dict(ss_dict, topn=topn, \n",
    "        by='n_pos', header=['code', 'n_pos'])))\n",
    "\n",
    "######################################\n",
    "\"\"\"\n",
    "Conclusion \n",
    "----------\n",
    "\n",
    "Cohort: hepatitis-c\n",
    "\n",
    "1. max sample size with known code: 67686 => 57202\n",
    "2. loinc_set: target loinc codes\n",
    "   size(loinc_set): 733\n",
    "3. ts0: dim=(71224, 127)\n",
    "\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Control Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[load] Loading default input data: andromeda_pond-10p.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 256: expected 127 fields, saw 128\\nSkipping line 511: expected 127 fields, saw 129\\nSkipping line 5917: expected 127 fields, saw 133\\nSkipping line 7086: expected 127 fields, saw 128\\nSkipping line 7750: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 10545: expected 127 fields, saw 131\\nSkipping line 14112: expected 127 fields, saw 128\\nSkipping line 14113: expected 127 fields, saw 128\\nSkipping line 15206: expected 127 fields, saw 135\\n'\n",
      "b'Skipping line 20277: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 25255: expected 127 fields, saw 136\\nSkipping line 30914: expected 127 fields, saw 164\\n'\n",
      "b'Skipping line 34649: expected 127 fields, saw 130\\nSkipping line 37064: expected 127 fields, saw 131\\nSkipping line 39673: expected 127 fields, saw 128\\nSkipping line 39674: expected 127 fields, saw 128\\nSkipping line 39676: expected 127 fields, saw 128\\nSkipping line 40260: expected 127 fields, saw 132\\nSkipping line 40291: expected 127 fields, saw 129\\nSkipping line 40614: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 42848: expected 127 fields, saw 130\\nSkipping line 45037: expected 127 fields, saw 128\\nSkipping line 47998: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 54420: expected 127 fields, saw 128\\nSkipping line 56728: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 57592: expected 127 fields, saw 136\\nSkipping line 60874: expected 127 fields, saw 128\\nSkipping line 61935: expected 127 fields, saw 130\\nSkipping line 65484: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 67176: expected 127 fields, saw 130\\nSkipping line 68783: expected 127 fields, saw 130\\nSkipping line 71167: expected 127 fields, saw 128\\nSkipping line 71168: expected 127 fields, saw 128\\nSkipping line 71669: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 79844: expected 127 fields, saw 129\\nSkipping line 80627: expected 127 fields, saw 130\\nSkipping line 81933: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 83305: expected 127 fields, saw 128\\nSkipping line 83908: expected 127 fields, saw 128\\nSkipping line 87276: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 94673: expected 127 fields, saw 139\\nSkipping line 96401: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 98670: expected 127 fields, saw 130\\nSkipping line 99670: expected 127 fields, saw 128\\nSkipping line 99922: expected 127 fields, saw 128\\nSkipping line 99939: expected 127 fields, saw 128\\nSkipping line 102858: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 107404: expected 127 fields, saw 128\\nSkipping line 107406: expected 127 fields, saw 141\\nSkipping line 109634: expected 127 fields, saw 132\\nSkipping line 109870: expected 127 fields, saw 130\\nSkipping line 111305: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 114793: expected 127 fields, saw 128\\nSkipping line 114944: expected 127 fields, saw 130\\nSkipping line 115536: expected 127 fields, saw 128\\nSkipping line 116055: expected 127 fields, saw 138\\nSkipping line 118241: expected 127 fields, saw 128\\nSkipping line 118454: expected 127 fields, saw 128\\nSkipping line 120201: expected 127 fields, saw 131\\nSkipping line 120459: expected 127 fields, saw 128\\nSkipping line 120854: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 125696: expected 127 fields, saw 142\\nSkipping line 127415: expected 127 fields, saw 130\\nSkipping line 130262: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 134281: expected 127 fields, saw 128\\nSkipping line 134616: expected 127 fields, saw 130\\nSkipping line 134871: expected 127 fields, saw 128\\nSkipping line 134874: expected 127 fields, saw 128\\nSkipping line 134875: expected 127 fields, saw 128\\nSkipping line 136206: expected 127 fields, saw 128\\nSkipping line 136207: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 145335: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 147956: expected 127 fields, saw 130\\nSkipping line 149636: expected 127 fields, saw 128\\nSkipping line 149637: expected 127 fields, saw 128\\nSkipping line 150689: expected 127 fields, saw 130\\nSkipping line 152328: expected 127 fields, saw 128\\nSkipping line 154010: expected 127 fields, saw 132\\nSkipping line 154375: expected 127 fields, saw 128\\nSkipping line 155356: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 158896: expected 127 fields, saw 128\\nSkipping line 159673: expected 127 fields, saw 144\\nSkipping line 159785: expected 127 fields, saw 128\\nSkipping line 160951: expected 127 fields, saw 130\\nSkipping line 161398: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 164840: expected 127 fields, saw 128\\nSkipping line 165813: expected 127 fields, saw 128\\nSkipping line 166823: expected 127 fields, saw 133\\nSkipping line 168093: expected 127 fields, saw 130\\nSkipping line 169465: expected 127 fields, saw 140\\nSkipping line 169945: expected 127 fields, saw 128\\nSkipping line 170949: expected 127 fields, saw 130\\nSkipping line 171191: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 175752: expected 127 fields, saw 128\\nSkipping line 177830: expected 127 fields, saw 137\\n'\n",
      "b'Skipping line 180481: expected 127 fields, saw 129\\nSkipping line 184493: expected 127 fields, saw 130\\nSkipping line 186800: expected 127 fields, saw 128\\nSkipping line 187274: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 188602: expected 127 fields, saw 128\\nSkipping line 188603: expected 127 fields, saw 128\\nSkipping line 192530: expected 127 fields, saw 154\\nSkipping line 195633: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 196800: expected 127 fields, saw 128\\nSkipping line 197287: expected 127 fields, saw 128\\nSkipping line 201382: expected 127 fields, saw 130\\nSkipping line 202988: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 207558: expected 127 fields, saw 128\\nSkipping line 209219: expected 127 fields, saw 128\\nSkipping line 210263: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 215757: expected 127 fields, saw 128\\nSkipping line 215758: expected 127 fields, saw 128\\nSkipping line 216040: expected 127 fields, saw 128\\nSkipping line 221135: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 222531: expected 127 fields, saw 128\\nSkipping line 227829: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 232840: expected 127 fields, saw 133\\nSkipping line 233156: expected 127 fields, saw 131\\nSkipping line 234373: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 239683: expected 127 fields, saw 128\\nSkipping line 239790: expected 127 fields, saw 138\\nSkipping line 243126: expected 127 fields, saw 128\\nSkipping line 243291: expected 127 fields, saw 136\\nSkipping line 245639: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 246178: expected 127 fields, saw 128\\nSkipping line 247192: expected 127 fields, saw 130\\nSkipping line 247193: expected 127 fields, saw 130\\nSkipping line 249361: expected 127 fields, saw 128\\nSkipping line 250834: expected 127 fields, saw 129\\nSkipping line 251834: expected 127 fields, saw 128\\nSkipping line 252175: expected 127 fields, saw 138\\nSkipping line 253283: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 259185: expected 127 fields, saw 130\\nSkipping line 261653: expected 127 fields, saw 145\\nSkipping line 261862: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 262581: expected 127 fields, saw 129\\nSkipping line 265801: expected 127 fields, saw 128\\nSkipping line 266868: expected 127 fields, saw 128\\nSkipping line 269181: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 272485: expected 127 fields, saw 128\\nSkipping line 274597: expected 127 fields, saw 128\\nSkipping line 274599: expected 127 fields, saw 128\\nSkipping line 274994: expected 127 fields, saw 130\\nSkipping line 275102: expected 127 fields, saw 132\\nSkipping line 275957: expected 127 fields, saw 131\\nSkipping line 278217: expected 127 fields, saw 130\\nSkipping line 278553: expected 127 fields, saw 142\\nSkipping line 278555: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 282462: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 287243: expected 127 fields, saw 132\\nSkipping line 287458: expected 127 fields, saw 133\\nSkipping line 288701: expected 127 fields, saw 128\\nSkipping line 288808: expected 127 fields, saw 128\\nSkipping line 290212: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 295908: expected 127 fields, saw 130\\nSkipping line 298155: expected 127 fields, saw 128\\nSkipping line 299334: expected 127 fields, saw 130\\nSkipping line 301792: expected 127 fields, saw 128\\nSkipping line 301794: expected 127 fields, saw 128\\nSkipping line 301960: expected 127 fields, saw 133\\nSkipping line 303059: expected 127 fields, saw 128\\nSkipping line 303136: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 304466: expected 127 fields, saw 128\\nSkipping line 306384: expected 127 fields, saw 128\\nSkipping line 306385: expected 127 fields, saw 128\\nSkipping line 306387: expected 127 fields, saw 128\\nSkipping line 307786: expected 127 fields, saw 128\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 312711: expected 127 fields, saw 132\\nSkipping line 313105: expected 127 fields, saw 128\\nSkipping line 314345: expected 127 fields, saw 128\\nSkipping line 315877: expected 127 fields, saw 129\\nSkipping line 317175: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 323634: expected 127 fields, saw 130\\nSkipping line 323650: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 331833: expected 127 fields, saw 148\\nSkipping line 332177: expected 127 fields, saw 142\\nSkipping line 334586: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 336654: expected 127 fields, saw 128\\nSkipping line 341873: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 345036: expected 127 fields, saw 129\\nSkipping line 346459: expected 127 fields, saw 131\\nSkipping line 346788: expected 127 fields, saw 134\\nSkipping line 348949: expected 127 fields, saw 130\\nSkipping line 348994: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 353627: expected 127 fields, saw 130\\nSkipping line 355166: expected 127 fields, saw 128\\nSkipping line 355565: expected 127 fields, saw 138\\nSkipping line 359641: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 362022: expected 127 fields, saw 130\\nSkipping line 364782: expected 127 fields, saw 128\\nSkipping line 364783: expected 127 fields, saw 128\\nSkipping line 365843: expected 127 fields, saw 130\\nSkipping line 367065: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 368947: expected 127 fields, saw 130\\nSkipping line 373001: expected 127 fields, saw 128\\nSkipping line 373002: expected 127 fields, saw 128\\nSkipping line 373003: expected 127 fields, saw 128\\nSkipping line 376067: expected 127 fields, saw 130\\nSkipping line 376113: expected 127 fields, saw 133\\nSkipping line 376159: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 381295: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 388782: expected 127 fields, saw 129\\nSkipping line 389691: expected 127 fields, saw 128\\nSkipping line 391519: expected 127 fields, saw 128\\nSkipping line 393074: expected 127 fields, saw 132\\nSkipping line 393155: expected 127 fields, saw 129\\nSkipping line 393244: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 396327: expected 127 fields, saw 130\\nSkipping line 399819: expected 127 fields, saw 128\\nSkipping line 400963: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 402709: expected 127 fields, saw 130\\nSkipping line 405260: expected 127 fields, saw 136\\nSkipping line 407250: expected 127 fields, saw 130\\nSkipping line 407912: expected 127 fields, saw 130\\nSkipping line 408842: expected 127 fields, saw 128\\nSkipping line 409792: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 412915: expected 127 fields, saw 130\\nSkipping line 414485: expected 127 fields, saw 134\\nSkipping line 415442: expected 127 fields, saw 131\\nSkipping line 416260: expected 127 fields, saw 130\\nSkipping line 416411: expected 127 fields, saw 128\\nSkipping line 416573: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 420483: expected 127 fields, saw 129\\nSkipping line 422377: expected 127 fields, saw 128\\nSkipping line 423881: expected 127 fields, saw 144\\n'\n",
      "b'Skipping line 428093: expected 127 fields, saw 128\\nSkipping line 428094: expected 127 fields, saw 128\\nSkipping line 428095: expected 127 fields, saw 128\\nSkipping line 428096: expected 127 fields, saw 128\\nSkipping line 432522: expected 127 fields, saw 130\\nSkipping line 432767: expected 127 fields, saw 142\\nSkipping line 433321: expected 127 fields, saw 134\\nSkipping line 433759: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 436526: expected 127 fields, saw 128\\nSkipping line 440404: expected 127 fields, saw 128\\nSkipping line 441412: expected 127 fields, saw 131\\nSkipping line 441415: expected 127 fields, saw 128\\nSkipping line 442439: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 443039: expected 127 fields, saw 128\\nSkipping line 445531: expected 127 fields, saw 131\\nSkipping line 446322: expected 127 fields, saw 128\\nSkipping line 446323: expected 127 fields, saw 128\\nSkipping line 446686: expected 127 fields, saw 130\\nSkipping line 449618: expected 127 fields, saw 130\\nSkipping line 450521: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 451372: expected 127 fields, saw 137\\nSkipping line 454099: expected 127 fields, saw 128\\nSkipping line 454782: expected 127 fields, saw 128\\nSkipping line 458104: expected 127 fields, saw 128\\nSkipping line 458160: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 460740: expected 127 fields, saw 159\\nSkipping line 462517: expected 127 fields, saw 161\\nSkipping line 463513: expected 127 fields, saw 129\\nSkipping line 463613: expected 127 fields, saw 128\\nSkipping line 463614: expected 127 fields, saw 128\\nSkipping line 463616: expected 127 fields, saw 128\\nSkipping line 466446: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 469324: expected 127 fields, saw 128\\nSkipping line 470219: expected 127 fields, saw 130\\nSkipping line 472302: expected 127 fields, saw 128\\nSkipping line 473338: expected 127 fields, saw 132\\nSkipping line 474458: expected 127 fields, saw 128\\nSkipping line 474460: expected 127 fields, saw 128\\nSkipping line 474491: expected 127 fields, saw 130\\nSkipping line 474607: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 476889: expected 127 fields, saw 130\\nSkipping line 477136: expected 127 fields, saw 130\\nSkipping line 478454: expected 127 fields, saw 130\\nSkipping line 479658: expected 127 fields, saw 128\\nSkipping line 479679: expected 127 fields, saw 140\\nSkipping line 481418: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 484166: expected 127 fields, saw 133\\nSkipping line 484404: expected 127 fields, saw 130\\nSkipping line 485409: expected 127 fields, saw 128\\nSkipping line 486345: expected 127 fields, saw 129\\nSkipping line 487976: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 493548: expected 127 fields, saw 139\\nSkipping line 495699: expected 127 fields, saw 133\\nSkipping line 495721: expected 127 fields, saw 132\\nSkipping line 496167: expected 127 fields, saw 128\\nSkipping line 496168: expected 127 fields, saw 128\\nSkipping line 499468: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 500659: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 510909: expected 127 fields, saw 128\\nSkipping line 512907: expected 127 fields, saw 129\\nSkipping line 513598: expected 127 fields, saw 130\\nSkipping line 514166: expected 127 fields, saw 130\\nSkipping line 514392: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 517848: expected 127 fields, saw 130\\nSkipping line 518741: expected 127 fields, saw 130\\nSkipping line 523177: expected 127 fields, saw 130\\nSkipping line 523781: expected 127 fields, saw 136\\nSkipping line 524509: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 524631: expected 127 fields, saw 138\\nSkipping line 525484: expected 127 fields, saw 139\\nSkipping line 526480: expected 127 fields, saw 130\\nSkipping line 528519: expected 127 fields, saw 130\\nSkipping line 530284: expected 127 fields, saw 130\\nSkipping line 532113: expected 127 fields, saw 128\\nSkipping line 532593: expected 127 fields, saw 142\\n'\n",
      "b'Skipping line 533117: expected 127 fields, saw 130\\nSkipping line 537468: expected 127 fields, saw 128\\nSkipping line 539629: expected 127 fields, saw 131\\nSkipping line 539963: expected 127 fields, saw 128\\nSkipping line 540755: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 541818: expected 127 fields, saw 130\\nSkipping line 543025: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 553775: expected 127 fields, saw 128\\nSkipping line 554475: expected 127 fields, saw 128\\nSkipping line 556114: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 558520: expected 127 fields, saw 128\\nSkipping line 559628: expected 127 fields, saw 137\\nSkipping line 563251: expected 127 fields, saw 138\\nSkipping line 565138: expected 127 fields, saw 133\\nSkipping line 565217: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 567504: expected 127 fields, saw 128\\nSkipping line 569508: expected 127 fields, saw 130\\nSkipping line 573733: expected 127 fields, saw 128\\nSkipping line 573734: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 580198: expected 127 fields, saw 128\\nSkipping line 581088: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 581983: expected 127 fields, saw 131\\nSkipping line 586929: expected 127 fields, saw 128\\nSkipping line 586968: expected 127 fields, saw 130\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 590629: expected 127 fields, saw 130\\nSkipping line 595449: expected 127 fields, saw 130\\nSkipping line 595463: expected 127 fields, saw 130\\nSkipping line 596527: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 599747: expected 127 fields, saw 131\\nSkipping line 599973: expected 127 fields, saw 141\\nSkipping line 600752: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 606853: expected 127 fields, saw 129\\nSkipping line 608275: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 615975: expected 127 fields, saw 129\\nSkipping line 619136: expected 127 fields, saw 128\\nSkipping line 620929: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 628790: expected 127 fields, saw 128\\nSkipping line 628793: expected 127 fields, saw 128\\nSkipping line 628794: expected 127 fields, saw 128\\nSkipping line 629569: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 631936: expected 127 fields, saw 128\\nSkipping line 632185: expected 127 fields, saw 128\\nSkipping line 632331: expected 127 fields, saw 128\\nSkipping line 635612: expected 127 fields, saw 129\\nSkipping line 635984: expected 127 fields, saw 129\\nSkipping line 636495: expected 127 fields, saw 139\\nSkipping line 636980: expected 127 fields, saw 128\\nSkipping line 638899: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 640245: expected 127 fields, saw 128\\nSkipping line 644758: expected 127 fields, saw 130\\nSkipping line 645686: expected 127 fields, saw 136\\nSkipping line 646663: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 648656: expected 127 fields, saw 128\\nSkipping line 648763: expected 127 fields, saw 137\\nSkipping line 654170: expected 127 fields, saw 129\\nSkipping line 654656: expected 127 fields, saw 128\\nSkipping line 654882: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 659273: expected 127 fields, saw 130\\nSkipping line 660811: expected 127 fields, saw 128\\nSkipping line 663182: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 664691: expected 127 fields, saw 128\\nSkipping line 664692: expected 127 fields, saw 128\\nSkipping line 666253: expected 127 fields, saw 130\\nSkipping line 666956: expected 127 fields, saw 128\\nSkipping line 668542: expected 127 fields, saw 132\\nSkipping line 669572: expected 127 fields, saw 130\\nSkipping line 670583: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 673019: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 680359: expected 127 fields, saw 130\\nSkipping line 682044: expected 127 fields, saw 128\\nSkipping line 682843: expected 127 fields, saw 128\\nSkipping line 682845: expected 127 fields, saw 128\\nSkipping line 682846: expected 127 fields, saw 128\\nSkipping line 683593: expected 127 fields, saw 136\\nSkipping line 686324: expected 127 fields, saw 132\\nSkipping line 687061: expected 127 fields, saw 128\\nSkipping line 688199: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 689445: expected 127 fields, saw 136\\nSkipping line 689462: expected 127 fields, saw 131\\nSkipping line 690340: expected 127 fields, saw 131\\nSkipping line 691100: expected 127 fields, saw 130\\nSkipping line 692160: expected 127 fields, saw 131\\nSkipping line 693623: expected 127 fields, saw 132\\nSkipping line 694435: expected 127 fields, saw 129\\nSkipping line 695574: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 699736: expected 127 fields, saw 130\\nSkipping line 699749: expected 127 fields, saw 128\\nSkipping line 699843: expected 127 fields, saw 130\\nSkipping line 700159: expected 127 fields, saw 129\\nSkipping line 700657: expected 127 fields, saw 129\\nSkipping line 701407: expected 127 fields, saw 128\\nSkipping line 701641: expected 127 fields, saw 128\\nSkipping line 702711: expected 127 fields, saw 131\\nSkipping line 704662: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 707160: expected 127 fields, saw 128\\nSkipping line 709629: expected 127 fields, saw 136\\nSkipping line 709773: expected 127 fields, saw 136\\nSkipping line 710938: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 716125: expected 127 fields, saw 138\\nSkipping line 716999: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 723123: expected 127 fields, saw 130\\nSkipping line 724744: expected 127 fields, saw 131\\nSkipping line 725023: expected 127 fields, saw 130\\nSkipping line 726973: expected 127 fields, saw 130\\nSkipping line 727314: expected 127 fields, saw 130\\nSkipping line 727384: expected 127 fields, saw 130\\nSkipping line 728993: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 732912: expected 127 fields, saw 128\\nSkipping line 734452: expected 127 fields, saw 129\\nSkipping line 734576: expected 127 fields, saw 130\\nSkipping line 735174: expected 127 fields, saw 128\\nSkipping line 735175: expected 127 fields, saw 128\\nSkipping line 735176: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 737858: expected 127 fields, saw 128\\nSkipping line 738873: expected 127 fields, saw 152\\nSkipping line 739068: expected 127 fields, saw 128\\nSkipping line 739793: expected 127 fields, saw 130\\nSkipping line 745324: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 748904: expected 127 fields, saw 128\\nSkipping line 752897: expected 127 fields, saw 129\\nSkipping line 753963: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 754323: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 762331: expected 127 fields, saw 128\\nSkipping line 762853: expected 127 fields, saw 131\\nSkipping line 764377: expected 127 fields, saw 130\\nSkipping line 768304: expected 127 fields, saw 130\\nSkipping line 768313: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 775732: expected 127 fields, saw 133\\nSkipping line 775975: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 781443: expected 127 fields, saw 128\\nSkipping line 782868: expected 127 fields, saw 134\\nSkipping line 785402: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 787618: expected 127 fields, saw 128\\nSkipping line 789317: expected 127 fields, saw 128\\nSkipping line 794024: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 802444: expected 127 fields, saw 128\\nSkipping line 802658: expected 127 fields, saw 136\\nSkipping line 803213: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 803972: expected 127 fields, saw 132\\nSkipping line 804923: expected 127 fields, saw 128\\nSkipping line 808104: expected 127 fields, saw 140\\nSkipping line 809853: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 813758: expected 127 fields, saw 130\\nSkipping line 815380: expected 127 fields, saw 128\\nSkipping line 816576: expected 127 fields, saw 128\\nSkipping line 816577: expected 127 fields, saw 128\\nSkipping line 816579: expected 127 fields, saw 128\\nSkipping line 816824: expected 127 fields, saw 133\\nSkipping line 817264: expected 127 fields, saw 128\\nSkipping line 817392: expected 127 fields, saw 130\\nSkipping line 818218: expected 127 fields, saw 128\\nSkipping line 818347: expected 127 fields, saw 129\\nSkipping line 818643: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 820148: expected 127 fields, saw 128\\nSkipping line 820484: expected 127 fields, saw 128\\nSkipping line 823079: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 833840: expected 127 fields, saw 132\\nSkipping line 834920: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 837174: expected 127 fields, saw 128\\nSkipping line 840426: expected 127 fields, saw 128\\nSkipping line 840711: expected 127 fields, saw 156\\n'\n",
      "b'Skipping line 844829: expected 127 fields, saw 128\\nSkipping line 846606: expected 127 fields, saw 128\\nSkipping line 846607: expected 127 fields, saw 128\\nSkipping line 849484: expected 127 fields, saw 128\\nSkipping line 850688: expected 127 fields, saw 128\\nSkipping line 851714: expected 127 fields, saw 128\\nSkipping line 852023: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 854799: expected 127 fields, saw 128\\nSkipping line 860487: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 861645: expected 127 fields, saw 130\\nSkipping line 866276: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 871110: expected 127 fields, saw 130\\nSkipping line 871262: expected 127 fields, saw 130\\nSkipping line 871709: expected 127 fields, saw 129\\nSkipping line 874016: expected 127 fields, saw 130\\nSkipping line 874872: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 877453: expected 127 fields, saw 136\\nSkipping line 880547: expected 127 fields, saw 130\\nSkipping line 881191: expected 127 fields, saw 128\\nSkipping line 882488: expected 127 fields, saw 128\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 885532: expected 127 fields, saw 128\\nSkipping line 888688: expected 127 fields, saw 130\\nSkipping line 890608: expected 127 fields, saw 130\\nSkipping line 891400: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 893962: expected 127 fields, saw 134\\nSkipping line 894471: expected 127 fields, saw 128\\nSkipping line 897155: expected 127 fields, saw 131\\nSkipping line 897175: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 905330: expected 127 fields, saw 128\\nSkipping line 907408: expected 127 fields, saw 128\\nSkipping line 907851: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 910890: expected 127 fields, saw 132\\nSkipping line 910933: expected 127 fields, saw 129\\nSkipping line 911173: expected 127 fields, saw 135\\nSkipping line 911180: expected 127 fields, saw 144\\nSkipping line 913057: expected 127 fields, saw 129\\nSkipping line 914740: expected 127 fields, saw 128\\nSkipping line 917499: expected 127 fields, saw 128\\nSkipping line 917500: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 920059: expected 127 fields, saw 128\\nSkipping line 921861: expected 127 fields, saw 130\\nSkipping line 921879: expected 127 fields, saw 132\\nSkipping line 923819: expected 127 fields, saw 128\\nSkipping line 925875: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 926722: expected 127 fields, saw 128\\nSkipping line 929795: expected 127 fields, saw 128\\nSkipping line 930199: expected 127 fields, saw 132\\nSkipping line 931058: expected 127 fields, saw 131\\nSkipping line 931750: expected 127 fields, saw 132\\nSkipping line 932181: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 935801: expected 127 fields, saw 129\\nSkipping line 937116: expected 127 fields, saw 130\\nSkipping line 937597: expected 127 fields, saw 137\\n'\n",
      "b'Skipping line 946919: expected 127 fields, saw 128\\nSkipping line 947309: expected 127 fields, saw 131\\nSkipping line 947476: expected 127 fields, saw 134\\nSkipping line 949202: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 951615: expected 127 fields, saw 128\\nSkipping line 951616: expected 127 fields, saw 128\\nSkipping line 953021: expected 127 fields, saw 130\\nSkipping line 953188: expected 127 fields, saw 128\\nSkipping line 953189: expected 127 fields, saw 128\\nSkipping line 955726: expected 127 fields, saw 128\\nSkipping line 958321: expected 127 fields, saw 136\\n'\n",
      "b'Skipping line 959300: expected 127 fields, saw 132\\nSkipping line 961244: expected 127 fields, saw 128\\nSkipping line 962033: expected 127 fields, saw 128\\nSkipping line 964960: expected 127 fields, saw 137\\nSkipping line 966195: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 967625: expected 127 fields, saw 140\\nSkipping line 969537: expected 127 fields, saw 128\\nSkipping line 970568: expected 127 fields, saw 128\\nSkipping line 973128: expected 127 fields, saw 144\\nSkipping line 973225: expected 127 fields, saw 128\\nSkipping line 973226: expected 127 fields, saw 128\\nSkipping line 973227: expected 127 fields, saw 128\\nSkipping line 973228: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 975555: expected 127 fields, saw 130\\nSkipping line 975740: expected 127 fields, saw 128\\nSkipping line 977221: expected 127 fields, saw 130\\nSkipping line 978449: expected 127 fields, saw 130\\nSkipping line 980204: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 985641: expected 127 fields, saw 128\\nSkipping line 991529: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 992376: expected 127 fields, saw 130\\nSkipping line 994068: expected 127 fields, saw 128\\nSkipping line 998631: expected 127 fields, saw 139\\nSkipping line 998699: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1001791: expected 127 fields, saw 130\\nSkipping line 1004087: expected 127 fields, saw 129\\nSkipping line 1007740: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1008240: expected 127 fields, saw 132\\nSkipping line 1009698: expected 127 fields, saw 151\\nSkipping line 1013764: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 1022827: expected 127 fields, saw 128\\nSkipping line 1022953: expected 127 fields, saw 130\\nSkipping line 1022956: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1025345: expected 127 fields, saw 129\\nSkipping line 1025477: expected 127 fields, saw 128\\nSkipping line 1027321: expected 127 fields, saw 129\\nSkipping line 1028377: expected 127 fields, saw 130\\nSkipping line 1028512: expected 127 fields, saw 134\\nSkipping line 1031121: expected 127 fields, saw 130\\nSkipping line 1031281: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1033363: expected 127 fields, saw 130\\nSkipping line 1034643: expected 127 fields, saw 131\\nSkipping line 1038364: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 1042599: expected 127 fields, saw 133\\nSkipping line 1044686: expected 127 fields, saw 131\\nSkipping line 1047254: expected 127 fields, saw 130\\nSkipping line 1047716: expected 127 fields, saw 128\\nSkipping line 1048140: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1049299: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1057863: expected 127 fields, saw 129\\nSkipping line 1058441: expected 127 fields, saw 132\\nSkipping line 1058542: expected 127 fields, saw 133\\nSkipping line 1059607: expected 127 fields, saw 130\\nSkipping line 1062174: expected 127 fields, saw 128\\nSkipping line 1062175: expected 127 fields, saw 128\\nSkipping line 1062177: expected 127 fields, saw 128\\nSkipping line 1062178: expected 127 fields, saw 128\\nSkipping line 1062583: expected 127 fields, saw 137\\nSkipping line 1065261: expected 127 fields, saw 130\\nSkipping line 1065482: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1066477: expected 127 fields, saw 128\\nSkipping line 1068341: expected 127 fields, saw 128\\nSkipping line 1069919: expected 127 fields, saw 133\\nSkipping line 1071497: expected 127 fields, saw 137\\n'\n",
      "b'Skipping line 1074017: expected 127 fields, saw 130\\nSkipping line 1075580: expected 127 fields, saw 128\\nSkipping line 1075581: expected 127 fields, saw 128\\nSkipping line 1075582: expected 127 fields, saw 128\\nSkipping line 1075584: expected 127 fields, saw 128\\nSkipping line 1076485: expected 127 fields, saw 133\\nSkipping line 1076761: expected 127 fields, saw 129\\nSkipping line 1080711: expected 127 fields, saw 130\\nSkipping line 1081527: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1082261: expected 127 fields, saw 128\\nSkipping line 1085155: expected 127 fields, saw 128\\nSkipping line 1085533: expected 127 fields, saw 128\\nSkipping line 1086388: expected 127 fields, saw 134\\nSkipping line 1086390: expected 127 fields, saw 128\\nSkipping line 1086391: expected 127 fields, saw 128\\nSkipping line 1087517: expected 127 fields, saw 130\\nSkipping line 1088214: expected 127 fields, saw 130\\nSkipping line 1089294: expected 127 fields, saw 131\\nSkipping line 1089295: expected 127 fields, saw 139\\nSkipping line 1089529: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1092061: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1100150: expected 127 fields, saw 130\\nSkipping line 1103339: expected 127 fields, saw 132\\nSkipping line 1103947: expected 127 fields, saw 141\\n'\n",
      "b'Skipping line 1110867: expected 127 fields, saw 128\\nSkipping line 1112321: expected 127 fields, saw 128\\nSkipping line 1112919: expected 127 fields, saw 144\\nSkipping line 1112959: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1117680: expected 127 fields, saw 128\\nSkipping line 1118207: expected 127 fields, saw 128\\nSkipping line 1119898: expected 127 fields, saw 129\\nSkipping line 1120043: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1127381: expected 127 fields, saw 130\\nSkipping line 1128936: expected 127 fields, saw 132\\nSkipping line 1129557: expected 127 fields, saw 128\\nSkipping line 1131078: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1133802: expected 127 fields, saw 130\\nSkipping line 1134363: expected 127 fields, saw 136\\nSkipping line 1138358: expected 127 fields, saw 128\\nSkipping line 1138438: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1140620: expected 127 fields, saw 128\\nSkipping line 1140621: expected 127 fields, saw 128\\nSkipping line 1141552: expected 127 fields, saw 133\\nSkipping line 1142399: expected 127 fields, saw 132\\nSkipping line 1144487: expected 127 fields, saw 128\\nSkipping line 1145480: expected 127 fields, saw 128\\nSkipping line 1145734: expected 127 fields, saw 130\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 1149200: expected 127 fields, saw 128\\nSkipping line 1151607: expected 127 fields, saw 128\\nSkipping line 1153131: expected 127 fields, saw 129\\nSkipping line 1153303: expected 127 fields, saw 135\\nSkipping line 1153807: expected 127 fields, saw 134\\nSkipping line 1154441: expected 127 fields, saw 130\\nSkipping line 1155012: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1158360: expected 127 fields, saw 130\\nSkipping line 1159215: expected 127 fields, saw 128\\nSkipping line 1161997: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1165521: expected 127 fields, saw 131\\nSkipping line 1166351: expected 127 fields, saw 128\\nSkipping line 1167166: expected 127 fields, saw 134\\nSkipping line 1167592: expected 127 fields, saw 148\\nSkipping line 1167916: expected 127 fields, saw 138\\nSkipping line 1169383: expected 127 fields, saw 130\\nSkipping line 1171852: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1172217: expected 127 fields, saw 130\\nSkipping line 1178310: expected 127 fields, saw 128\\nSkipping line 1178311: expected 127 fields, saw 128\\nSkipping line 1178312: expected 127 fields, saw 128\\nSkipping line 1178314: expected 127 fields, saw 128\\nSkipping line 1178343: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1181100: expected 127 fields, saw 134\\nSkipping line 1181652: expected 127 fields, saw 128\\nSkipping line 1184602: expected 127 fields, saw 129\\nSkipping line 1186004: expected 127 fields, saw 128\\nSkipping line 1186022: expected 127 fields, saw 130\\nSkipping line 1187222: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1188547: expected 127 fields, saw 128\\nSkipping line 1193152: expected 127 fields, saw 128\\nSkipping line 1193551: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1198888: expected 127 fields, saw 128\\nSkipping line 1198900: expected 127 fields, saw 130\\nSkipping line 1201835: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1206156: expected 127 fields, saw 132\\nSkipping line 1207675: expected 127 fields, saw 148\\nSkipping line 1207795: expected 127 fields, saw 132\\nSkipping line 1208458: expected 127 fields, saw 132\\nSkipping line 1209027: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 1213873: expected 127 fields, saw 135\\nSkipping line 1214849: expected 127 fields, saw 132\\nSkipping line 1214870: expected 127 fields, saw 132\\nSkipping line 1215494: expected 127 fields, saw 132\\nSkipping line 1216262: expected 127 fields, saw 130\\nSkipping line 1216268: expected 127 fields, saw 131\\nSkipping line 1220048: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1224975: expected 127 fields, saw 130\\nSkipping line 1226533: expected 127 fields, saw 128\\nSkipping line 1228522: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1229723: expected 127 fields, saw 130\\nSkipping line 1235478: expected 127 fields, saw 132\\nSkipping line 1236284: expected 127 fields, saw 133\\nSkipping line 1236303: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1238827: expected 127 fields, saw 128\\nSkipping line 1238937: expected 127 fields, saw 130\\nSkipping line 1239378: expected 127 fields, saw 130\\nSkipping line 1239987: expected 127 fields, saw 130\\nSkipping line 1243413: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1246566: expected 127 fields, saw 130\\nSkipping line 1246879: expected 127 fields, saw 128\\nSkipping line 1246880: expected 127 fields, saw 128\\nSkipping line 1247783: expected 127 fields, saw 132\\nSkipping line 1249181: expected 127 fields, saw 128\\nSkipping line 1252433: expected 127 fields, saw 152\\nSkipping line 1253786: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1254639: expected 127 fields, saw 150\\nSkipping line 1256303: expected 127 fields, saw 128\\nSkipping line 1256660: expected 127 fields, saw 136\\nSkipping line 1259392: expected 127 fields, saw 128\\nSkipping line 1260783: expected 127 fields, saw 130\\nSkipping line 1262085: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 1263629: expected 127 fields, saw 130\\nSkipping line 1269809: expected 127 fields, saw 128\\nSkipping line 1270293: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 1270714: expected 127 fields, saw 130\\nSkipping line 1271086: expected 127 fields, saw 128\\nSkipping line 1275690: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 1282973: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1287328: expected 127 fields, saw 128\\nSkipping line 1288039: expected 127 fields, saw 128\\nSkipping line 1293806: expected 127 fields, saw 139\\n'\n",
      "b'Skipping line 1297117: expected 127 fields, saw 133\\nSkipping line 1299947: expected 127 fields, saw 136\\nSkipping line 1301678: expected 127 fields, saw 128\\nSkipping line 1301679: expected 127 fields, saw 128\\nSkipping line 1301681: expected 127 fields, saw 128\\nSkipping line 1302721: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1305446: expected 127 fields, saw 128\\nSkipping line 1307701: expected 127 fields, saw 128\\nSkipping line 1310032: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1312390: expected 127 fields, saw 130\\nSkipping line 1315384: expected 127 fields, saw 129\\nSkipping line 1316368: expected 127 fields, saw 128\\nSkipping line 1316976: expected 127 fields, saw 136\\nSkipping line 1318013: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1326118: expected 127 fields, saw 128\\nSkipping line 1326119: expected 127 fields, saw 128\\nSkipping line 1326120: expected 127 fields, saw 128\\nSkipping line 1326121: expected 127 fields, saw 128\\nSkipping line 1327338: expected 127 fields, saw 128\\nSkipping line 1327668: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1335170: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1339176: expected 127 fields, saw 130\\nSkipping line 1339284: expected 127 fields, saw 133\\nSkipping line 1342571: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1344910: expected 127 fields, saw 134\\nSkipping line 1346446: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1353095: expected 127 fields, saw 130\\nSkipping line 1353142: expected 127 fields, saw 128\\nSkipping line 1353149: expected 127 fields, saw 130\\nSkipping line 1354458: expected 127 fields, saw 128\\nSkipping line 1354459: expected 127 fields, saw 128\\nSkipping line 1355589: expected 127 fields, saw 133\\nSkipping line 1355689: expected 127 fields, saw 128\\nSkipping line 1357750: expected 127 fields, saw 128\\nSkipping line 1359284: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1361890: expected 127 fields, saw 150\\nSkipping line 1363753: expected 127 fields, saw 128\\nSkipping line 1365940: expected 127 fields, saw 132\\nSkipping line 1366174: expected 127 fields, saw 134\\nSkipping line 1368821: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 1368934: expected 127 fields, saw 128\\nSkipping line 1370839: expected 127 fields, saw 135\\nSkipping line 1370935: expected 127 fields, saw 136\\nSkipping line 1370946: expected 127 fields, saw 130\\nSkipping line 1371002: expected 127 fields, saw 141\\nSkipping line 1374300: expected 127 fields, saw 130\\nSkipping line 1376701: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1378320: expected 127 fields, saw 130\\nSkipping line 1378428: expected 127 fields, saw 129\\nSkipping line 1378461: expected 127 fields, saw 128\\nSkipping line 1379886: expected 127 fields, saw 128\\nSkipping line 1382757: expected 127 fields, saw 128\\nSkipping line 1384664: expected 127 fields, saw 129\\nSkipping line 1384698: expected 127 fields, saw 143\\nSkipping line 1384886: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1388472: expected 127 fields, saw 139\\nSkipping line 1389500: expected 127 fields, saw 133\\nSkipping line 1391040: expected 127 fields, saw 136\\n'\n",
      "b'Skipping line 1400886: expected 127 fields, saw 136\\n'\n",
      "b'Skipping line 1401936: expected 127 fields, saw 128\\nSkipping line 1401939: expected 127 fields, saw 128\\nSkipping line 1402444: expected 127 fields, saw 130\\nSkipping line 1403135: expected 127 fields, saw 128\\nSkipping line 1408177: expected 127 fields, saw 130\\nSkipping line 1408714: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1413374: expected 127 fields, saw 148\\nSkipping line 1413458: expected 127 fields, saw 128\\nSkipping line 1413514: expected 127 fields, saw 145\\nSkipping line 1415415: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1423260: expected 127 fields, saw 130\\nSkipping line 1423473: expected 127 fields, saw 128\\nSkipping line 1425063: expected 127 fields, saw 130\\nSkipping line 1425136: expected 127 fields, saw 128\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 1426387: expected 127 fields, saw 128\\nSkipping line 1426397: expected 127 fields, saw 132\\nSkipping line 1429280: expected 127 fields, saw 131\\nSkipping line 1434262: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1436489: expected 127 fields, saw 128\\nSkipping line 1437192: expected 127 fields, saw 133\\nSkipping line 1437854: expected 127 fields, saw 128\\nSkipping line 1438348: expected 127 fields, saw 130\\nSkipping line 1439732: expected 127 fields, saw 130\\nSkipping line 1442219: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1444192: expected 127 fields, saw 128\\nSkipping line 1445893: expected 127 fields, saw 129\\nSkipping line 1450159: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1452365: expected 127 fields, saw 128\\nSkipping line 1452367: expected 127 fields, saw 128\\nSkipping line 1452368: expected 127 fields, saw 128\\nSkipping line 1452369: expected 127 fields, saw 128\\nSkipping line 1452370: expected 127 fields, saw 128\\nSkipping line 1452371: expected 127 fields, saw 128\\nSkipping line 1452723: expected 127 fields, saw 134\\nSkipping line 1452913: expected 127 fields, saw 128\\nSkipping line 1453171: expected 127 fields, saw 133\\nSkipping line 1455382: expected 127 fields, saw 128\\nSkipping line 1455675: expected 127 fields, saw 128\\nSkipping line 1455800: expected 127 fields, saw 133\\nSkipping line 1457906: expected 127 fields, saw 128\\nSkipping line 1458736: expected 127 fields, saw 166\\nSkipping line 1458816: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1460396: expected 127 fields, saw 128\\nSkipping line 1462176: expected 127 fields, saw 133\\nSkipping line 1462684: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1468122: expected 127 fields, saw 132\\nSkipping line 1470648: expected 127 fields, saw 133\\nSkipping line 1470671: expected 127 fields, saw 130\\nSkipping line 1471734: expected 127 fields, saw 130\\nSkipping line 1473097: expected 127 fields, saw 129\\nSkipping line 1474575: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1477489: expected 127 fields, saw 128\\nSkipping line 1480271: expected 127 fields, saw 130\\nSkipping line 1480746: expected 127 fields, saw 128\\nSkipping line 1481890: expected 127 fields, saw 132\\nSkipping line 1482048: expected 127 fields, saw 133\\nSkipping line 1483478: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 1486770: expected 127 fields, saw 144\\nSkipping line 1486887: expected 127 fields, saw 128\\nSkipping line 1489151: expected 127 fields, saw 132\\nSkipping line 1489655: expected 127 fields, saw 130\\nSkipping line 1491672: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1492915: expected 127 fields, saw 128\\nSkipping line 1495491: expected 127 fields, saw 128\\nSkipping line 1496796: expected 127 fields, saw 130\\nSkipping line 1497501: expected 127 fields, saw 128\\nSkipping line 1498420: expected 127 fields, saw 128\\nSkipping line 1498505: expected 127 fields, saw 130\\nSkipping line 1498552: expected 127 fields, saw 132\\nSkipping line 1498912: expected 127 fields, saw 128\\nSkipping line 1499797: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 1504860: expected 127 fields, saw 128\\nSkipping line 1504861: expected 127 fields, saw 128\\nSkipping line 1504959: expected 127 fields, saw 128\\nSkipping line 1504960: expected 127 fields, saw 128\\nSkipping line 1504961: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1508922: expected 127 fields, saw 141\\nSkipping line 1510114: expected 127 fields, saw 130\\nSkipping line 1511692: expected 127 fields, saw 128\\nSkipping line 1512791: expected 127 fields, saw 128\\nSkipping line 1513674: expected 127 fields, saw 128\\nSkipping line 1513699: expected 127 fields, saw 130\\nSkipping line 1514954: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1521206: expected 127 fields, saw 129\\nSkipping line 1522294: expected 127 fields, saw 139\\n'\n",
      "b'Skipping line 1525186: expected 127 fields, saw 128\\nSkipping line 1526748: expected 127 fields, saw 128\\nSkipping line 1531808: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1536710: expected 127 fields, saw 132\\nSkipping line 1537915: expected 127 fields, saw 136\\nSkipping line 1539308: expected 127 fields, saw 138\\n'\n",
      "b'Skipping line 1543365: expected 127 fields, saw 128\\nSkipping line 1547103: expected 127 fields, saw 129\\nSkipping line 1548158: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1552495: expected 127 fields, saw 128\\nSkipping line 1554376: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1563542: expected 127 fields, saw 128\\nSkipping line 1563602: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1566006: expected 127 fields, saw 133\\nSkipping line 1567050: expected 127 fields, saw 128\\nSkipping line 1569432: expected 127 fields, saw 128\\nSkipping line 1569434: expected 127 fields, saw 128\\nSkipping line 1569435: expected 127 fields, saw 128\\nSkipping line 1569436: expected 127 fields, saw 128\\nSkipping line 1569917: expected 127 fields, saw 128\\nSkipping line 1570289: expected 127 fields, saw 146\\nSkipping line 1570986: expected 127 fields, saw 131\\nSkipping line 1572639: expected 127 fields, saw 130\\nSkipping line 1572650: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1574197: expected 127 fields, saw 128\\nSkipping line 1576298: expected 127 fields, saw 130\\nSkipping line 1577152: expected 127 fields, saw 128\\nSkipping line 1577694: expected 127 fields, saw 137\\nSkipping line 1579361: expected 127 fields, saw 128\\nSkipping line 1580599: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1589041: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1596064: expected 127 fields, saw 128\\nSkipping line 1596065: expected 127 fields, saw 128\\nSkipping line 1597223: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1600269: expected 127 fields, saw 132\\nSkipping line 1601166: expected 127 fields, saw 128\\nSkipping line 1606499: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1608862: expected 127 fields, saw 128\\nSkipping line 1609654: expected 127 fields, saw 130\\nSkipping line 1614397: expected 127 fields, saw 128\\nSkipping line 1614455: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1618958: expected 127 fields, saw 130\\nSkipping line 1619089: expected 127 fields, saw 128\\nSkipping line 1619090: expected 127 fields, saw 128\\nSkipping line 1619092: expected 127 fields, saw 128\\nSkipping line 1619427: expected 127 fields, saw 130\\nSkipping line 1620630: expected 127 fields, saw 150\\n'\n",
      "b'Skipping line 1624811: expected 127 fields, saw 130\\nSkipping line 1629930: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1631605: expected 127 fields, saw 130\\nSkipping line 1633095: expected 127 fields, saw 128\\nSkipping line 1633096: expected 127 fields, saw 128\\nSkipping line 1633098: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1640169: expected 127 fields, saw 128\\nSkipping line 1640170: expected 127 fields, saw 128\\nSkipping line 1641808: expected 127 fields, saw 130\\nSkipping line 1642356: expected 127 fields, saw 128\\nSkipping line 1643315: expected 127 fields, saw 148\\nSkipping line 1643336: expected 127 fields, saw 132\\nSkipping line 1644054: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1649014: expected 127 fields, saw 130\\nSkipping line 1653941: expected 127 fields, saw 128\\nSkipping line 1653943: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1656411: expected 127 fields, saw 130\\nSkipping line 1659464: expected 127 fields, saw 129\\nSkipping line 1660694: expected 127 fields, saw 130\\nSkipping line 1660796: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1667650: expected 127 fields, saw 132\\nSkipping line 1670212: expected 127 fields, saw 143\\nSkipping line 1670214: expected 127 fields, saw 128\\nSkipping line 1670277: expected 127 fields, saw 128\\nSkipping line 1671058: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 1674764: expected 127 fields, saw 131\\nSkipping line 1676618: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1681738: expected 127 fields, saw 130\\nSkipping line 1684820: expected 127 fields, saw 129\\nSkipping line 1684977: expected 127 fields, saw 128\\nSkipping line 1688385: expected 127 fields, saw 128\\nSkipping line 1688386: expected 127 fields, saw 128\\nSkipping line 1688388: expected 127 fields, saw 128\\nSkipping line 1688389: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1693737: expected 127 fields, saw 130\\nSkipping line 1694441: expected 127 fields, saw 128\\nSkipping line 1696340: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1698347: expected 127 fields, saw 130\\nSkipping line 1700766: expected 127 fields, saw 143\\nSkipping line 1704391: expected 127 fields, saw 134\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 1706227: expected 127 fields, saw 130\\nSkipping line 1706285: expected 127 fields, saw 130\\nSkipping line 1712181: expected 127 fields, saw 128\\nSkipping line 1712397: expected 127 fields, saw 130\\nSkipping line 1712578: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 1714574: expected 127 fields, saw 128\\nSkipping line 1715874: expected 127 fields, saw 129\\nSkipping line 1716176: expected 127 fields, saw 131\\nSkipping line 1716986: expected 127 fields, saw 128\\nSkipping line 1720583: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1721845: expected 127 fields, saw 138\\nSkipping line 1727001: expected 127 fields, saw 128\\nSkipping line 1729172: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1729847: expected 127 fields, saw 130\\nSkipping line 1730161: expected 127 fields, saw 128\\nSkipping line 1730162: expected 127 fields, saw 128\\nSkipping line 1730163: expected 127 fields, saw 128\\nSkipping line 1730164: expected 127 fields, saw 128\\nSkipping line 1731539: expected 127 fields, saw 130\\nSkipping line 1734799: expected 127 fields, saw 151\\nSkipping line 1735282: expected 127 fields, saw 131\\nSkipping line 1735356: expected 127 fields, saw 128\\nSkipping line 1736705: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1744730: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1745997: expected 127 fields, saw 128\\nSkipping line 1745998: expected 127 fields, saw 128\\nSkipping line 1745999: expected 127 fields, saw 128\\nSkipping line 1746000: expected 127 fields, saw 128\\nSkipping line 1746001: expected 127 fields, saw 128\\nSkipping line 1746450: expected 127 fields, saw 128\\nSkipping line 1749222: expected 127 fields, saw 128\\nSkipping line 1751714: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1755358: expected 127 fields, saw 132\\nSkipping line 1755574: expected 127 fields, saw 128\\nSkipping line 1757806: expected 127 fields, saw 141\\nSkipping line 1760289: expected 127 fields, saw 129\\nSkipping line 1761453: expected 127 fields, saw 136\\n'\n",
      "b'Skipping line 1764391: expected 127 fields, saw 130\\nSkipping line 1766528: expected 127 fields, saw 130\\nSkipping line 1766594: expected 127 fields, saw 129\\nSkipping line 1769153: expected 127 fields, saw 128\\nSkipping line 1769154: expected 127 fields, saw 128\\nSkipping line 1769155: expected 127 fields, saw 128\\nSkipping line 1769156: expected 127 fields, saw 128\\nSkipping line 1769157: expected 127 fields, saw 128\\nSkipping line 1769158: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1770820: expected 127 fields, saw 131\\nSkipping line 1775772: expected 127 fields, saw 130\\nSkipping line 1777051: expected 127 fields, saw 128\\nSkipping line 1778074: expected 127 fields, saw 128\\nSkipping line 1778568: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1779668: expected 127 fields, saw 128\\nSkipping line 1782279: expected 127 fields, saw 132\\nSkipping line 1783222: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1786865: expected 127 fields, saw 128\\nSkipping line 1786866: expected 127 fields, saw 128\\nSkipping line 1787508: expected 127 fields, saw 143\\nSkipping line 1789074: expected 127 fields, saw 129\\nSkipping line 1791254: expected 127 fields, saw 137\\n'\n",
      "b'Skipping line 1796215: expected 127 fields, saw 128\\nSkipping line 1799439: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1805536: expected 127 fields, saw 128\\nSkipping line 1809855: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1818679: expected 127 fields, saw 137\\n'\n",
      "b'Skipping line 1821686: expected 127 fields, saw 130\\nSkipping line 1826621: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1831152: expected 127 fields, saw 128\\nSkipping line 1831153: expected 127 fields, saw 128\\nSkipping line 1831154: expected 127 fields, saw 128\\nSkipping line 1831155: expected 127 fields, saw 128\\nSkipping line 1831156: expected 127 fields, saw 128\\nSkipping line 1831157: expected 127 fields, saw 128\\nSkipping line 1831385: expected 127 fields, saw 128\\nSkipping line 1831636: expected 127 fields, saw 132\\nSkipping line 1834837: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1836199: expected 127 fields, saw 130\\nSkipping line 1838327: expected 127 fields, saw 128\\nSkipping line 1841124: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1846039: expected 127 fields, saw 128\\nSkipping line 1846040: expected 127 fields, saw 128\\nSkipping line 1846041: expected 127 fields, saw 128\\nSkipping line 1848973: expected 127 fields, saw 128\\nSkipping line 1848974: expected 127 fields, saw 128\\nSkipping line 1848975: expected 127 fields, saw 128\\nSkipping line 1848976: expected 127 fields, saw 128\\nSkipping line 1850631: expected 127 fields, saw 143\\n'\n",
      "b'Skipping line 1854695: expected 127 fields, saw 128\\nSkipping line 1857982: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1861407: expected 127 fields, saw 128\\nSkipping line 1862572: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 1872231: expected 127 fields, saw 130\\nSkipping line 1876334: expected 127 fields, saw 128\\nSkipping line 1876899: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1878212: expected 127 fields, saw 130\\nSkipping line 1878675: expected 127 fields, saw 131\\nSkipping line 1878711: expected 127 fields, saw 130\\nSkipping line 1878751: expected 127 fields, saw 130\\nSkipping line 1880272: expected 127 fields, saw 134\\nSkipping line 1882915: expected 127 fields, saw 133\\nSkipping line 1884429: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1886502: expected 127 fields, saw 130\\nSkipping line 1886715: expected 127 fields, saw 130\\nSkipping line 1889041: expected 127 fields, saw 146\\nSkipping line 1889698: expected 127 fields, saw 133\\nSkipping line 1891693: expected 127 fields, saw 128\\nSkipping line 1893155: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 1893783: expected 127 fields, saw 132\\nSkipping line 1894132: expected 127 fields, saw 130\\nSkipping line 1894485: expected 127 fields, saw 132\\nSkipping line 1895901: expected 127 fields, saw 132\\nSkipping line 1899902: expected 127 fields, saw 129\\nSkipping line 1901032: expected 127 fields, saw 128\\nSkipping line 1901199: expected 127 fields, saw 128\\nSkipping line 1901202: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 1904149: expected 127 fields, saw 128\\nSkipping line 1904612: expected 127 fields, saw 128\\nSkipping line 1904706: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1910499: expected 127 fields, saw 128\\nSkipping line 1910500: expected 127 fields, saw 128\\nSkipping line 1911938: expected 127 fields, saw 128\\nSkipping line 1916561: expected 127 fields, saw 128\\nSkipping line 1916597: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1919051: expected 127 fields, saw 132\\nSkipping line 1921069: expected 127 fields, saw 129\\nSkipping line 1922676: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1928190: expected 127 fields, saw 130\\nSkipping line 1931201: expected 127 fields, saw 143\\nSkipping line 1933430: expected 127 fields, saw 132\\nSkipping line 1934009: expected 127 fields, saw 128\\nSkipping line 1934010: expected 127 fields, saw 128\\nSkipping line 1934011: expected 127 fields, saw 128\\nSkipping line 1934012: expected 127 fields, saw 128\\nSkipping line 1934013: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1935351: expected 127 fields, saw 130\\nSkipping line 1937437: expected 127 fields, saw 131\\nSkipping line 1938605: expected 127 fields, saw 128\\nSkipping line 1941327: expected 127 fields, saw 128\\nSkipping line 1942553: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 1945204: expected 127 fields, saw 133\\nSkipping line 1948363: expected 127 fields, saw 130\\nSkipping line 1948421: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1952822: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1960225: expected 127 fields, saw 130\\nSkipping line 1960991: expected 127 fields, saw 157\\nSkipping line 1966076: expected 127 fields, saw 128\\nSkipping line 1966423: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 1977580: expected 127 fields, saw 137\\nSkipping line 1980525: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1989415: expected 127 fields, saw 130\\nSkipping line 1990804: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 1994562: expected 127 fields, saw 128\\nSkipping line 1998858: expected 127 fields, saw 131\\nSkipping line 1999073: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 2005359: expected 127 fields, saw 130\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 2008931: expected 127 fields, saw 130\\nSkipping line 2010181: expected 127 fields, saw 128\\nSkipping line 2011005: expected 127 fields, saw 129\\nSkipping line 2011192: expected 127 fields, saw 134\\nSkipping line 2012724: expected 127 fields, saw 132\\nSkipping line 2013700: expected 127 fields, saw 128\\nSkipping line 2014254: expected 127 fields, saw 130\\nSkipping line 2014420: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2016602: expected 127 fields, saw 128\\nSkipping line 2019017: expected 127 fields, saw 128\\nSkipping line 2019018: expected 127 fields, saw 128\\nSkipping line 2019019: expected 127 fields, saw 128\\nSkipping line 2019020: expected 127 fields, saw 128\\nSkipping line 2020212: expected 127 fields, saw 128\\nSkipping line 2020213: expected 127 fields, saw 128\\nSkipping line 2020214: expected 127 fields, saw 128\\nSkipping line 2020977: expected 127 fields, saw 146\\n'\n",
      "b'Skipping line 2026787: expected 127 fields, saw 128\\nSkipping line 2032171: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2033667: expected 127 fields, saw 130\\nSkipping line 2035036: expected 127 fields, saw 133\\nSkipping line 2035362: expected 127 fields, saw 130\\nSkipping line 2035967: expected 127 fields, saw 150\\nSkipping line 2038089: expected 127 fields, saw 130\\nSkipping line 2040247: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2043810: expected 127 fields, saw 128\\nSkipping line 2047649: expected 127 fields, saw 130\\nSkipping line 2048458: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2049835: expected 127 fields, saw 128\\nSkipping line 2049836: expected 127 fields, saw 130\\nSkipping line 2055149: expected 127 fields, saw 132\\nSkipping line 2055230: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2060945: expected 127 fields, saw 143\\nSkipping line 2062724: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2065803: expected 127 fields, saw 130\\nSkipping line 2072015: expected 127 fields, saw 128\\nSkipping line 2072016: expected 127 fields, saw 128\\nSkipping line 2072017: expected 127 fields, saw 128\\nSkipping line 2072018: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2073926: expected 127 fields, saw 128\\nSkipping line 2077774: expected 127 fields, saw 128\\nSkipping line 2081790: expected 127 fields, saw 136\\n'\n",
      "b'Skipping line 2081958: expected 127 fields, saw 132\\nSkipping line 2083287: expected 127 fields, saw 128\\nSkipping line 2083288: expected 127 fields, saw 128\\nSkipping line 2083289: expected 127 fields, saw 128\\nSkipping line 2083291: expected 127 fields, saw 128\\nSkipping line 2085370: expected 127 fields, saw 128\\nSkipping line 2089194: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 2091230: expected 127 fields, saw 128\\nSkipping line 2091231: expected 127 fields, saw 128\\nSkipping line 2093438: expected 127 fields, saw 140\\nSkipping line 2094072: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 2098509: expected 127 fields, saw 130\\nSkipping line 2098652: expected 127 fields, saw 131\\nSkipping line 2099219: expected 127 fields, saw 130\\nSkipping line 2102308: expected 127 fields, saw 128\\nSkipping line 2102885: expected 127 fields, saw 128\\nSkipping line 2105232: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2114373: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2115240: expected 127 fields, saw 128\\nSkipping line 2115375: expected 127 fields, saw 128\\nSkipping line 2115480: expected 127 fields, saw 133\\nSkipping line 2116478: expected 127 fields, saw 129\\nSkipping line 2117161: expected 127 fields, saw 132\\nSkipping line 2118791: expected 127 fields, saw 128\\nSkipping line 2119036: expected 127 fields, saw 128\\nSkipping line 2119048: expected 127 fields, saw 128\\nSkipping line 2119049: expected 127 fields, saw 128\\nSkipping line 2119773: expected 127 fields, saw 129\\nSkipping line 2120185: expected 127 fields, saw 144\\nSkipping line 2120206: expected 127 fields, saw 130\\nSkipping line 2121648: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 2123230: expected 127 fields, saw 130\\nSkipping line 2124313: expected 127 fields, saw 130\\nSkipping line 2124315: expected 127 fields, saw 128\\nSkipping line 2125680: expected 127 fields, saw 128\\nSkipping line 2125881: expected 127 fields, saw 130\\nSkipping line 2128464: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2132250: expected 127 fields, saw 132\\nSkipping line 2132612: expected 127 fields, saw 129\\nSkipping line 2137081: expected 127 fields, saw 128\\nSkipping line 2137173: expected 127 fields, saw 128\\nSkipping line 2137619: expected 127 fields, saw 132\\nSkipping line 2137820: expected 127 fields, saw 133\\nSkipping line 2137989: expected 127 fields, saw 154\\nSkipping line 2139247: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2144483: expected 127 fields, saw 130\\nSkipping line 2144937: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2152026: expected 127 fields, saw 138\\nSkipping line 2152924: expected 127 fields, saw 130\\nSkipping line 2152956: expected 127 fields, saw 129\\nSkipping line 2153851: expected 127 fields, saw 130\\nSkipping line 2153883: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 2156879: expected 127 fields, saw 138\\nSkipping line 2157855: expected 127 fields, saw 133\\nSkipping line 2163654: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 2166324: expected 127 fields, saw 130\\nSkipping line 2169789: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 2173637: expected 127 fields, saw 128\\nSkipping line 2174530: expected 127 fields, saw 128\\nSkipping line 2174531: expected 127 fields, saw 128\\nSkipping line 2174532: expected 127 fields, saw 128\\nSkipping line 2174534: expected 127 fields, saw 128\\nSkipping line 2177383: expected 127 fields, saw 144\\nSkipping line 2177534: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2181538: expected 127 fields, saw 130\\nSkipping line 2186108: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2192072: expected 127 fields, saw 131\\nSkipping line 2192367: expected 127 fields, saw 130\\nSkipping line 2194826: expected 127 fields, saw 130\\nSkipping line 2196417: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2201779: expected 127 fields, saw 130\\nSkipping line 2203058: expected 127 fields, saw 128\\nSkipping line 2204038: expected 127 fields, saw 128\\nSkipping line 2204193: expected 127 fields, saw 133\\nSkipping line 2204538: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 2205274: expected 127 fields, saw 131\\nSkipping line 2206923: expected 127 fields, saw 130\\nSkipping line 2211241: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2213764: expected 127 fields, saw 130\\nSkipping line 2217036: expected 127 fields, saw 132\\nSkipping line 2219277: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2223922: expected 127 fields, saw 128\\nSkipping line 2225461: expected 127 fields, saw 128\\nSkipping line 2225920: expected 127 fields, saw 132\\nSkipping line 2226624: expected 127 fields, saw 129\\nSkipping line 2227814: expected 127 fields, saw 128\\nSkipping line 2227817: expected 127 fields, saw 128\\nSkipping line 2227819: expected 127 fields, saw 128\\nSkipping line 2227820: expected 127 fields, saw 128\\nSkipping line 2227821: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2229514: expected 127 fields, saw 130\\nSkipping line 2230885: expected 127 fields, saw 132\\nSkipping line 2231755: expected 127 fields, saw 129\\nSkipping line 2231862: expected 127 fields, saw 128\\nSkipping line 2235566: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2240075: expected 127 fields, saw 130\\nSkipping line 2241118: expected 127 fields, saw 128\\nSkipping line 2242434: expected 127 fields, saw 128\\nSkipping line 2244116: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2253854: expected 127 fields, saw 133\\nSkipping line 2253907: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2254980: expected 127 fields, saw 129\\nSkipping line 2256302: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2264564: expected 127 fields, saw 133\\nSkipping line 2265721: expected 127 fields, saw 142\\n'\n",
      "b'Skipping line 2271266: expected 127 fields, saw 128\\nSkipping line 2271372: expected 127 fields, saw 134\\nSkipping line 2272356: expected 127 fields, saw 128\\nSkipping line 2273256: expected 127 fields, saw 128\\nSkipping line 2273257: expected 127 fields, saw 128\\nSkipping line 2273259: expected 127 fields, saw 128\\nSkipping line 2273260: expected 127 fields, saw 128\\nSkipping line 2274792: expected 127 fields, saw 129\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 2280427: expected 127 fields, saw 128\\nSkipping line 2280428: expected 127 fields, saw 128\\nSkipping line 2280429: expected 127 fields, saw 128\\nSkipping line 2281041: expected 127 fields, saw 130\\nSkipping line 2282551: expected 127 fields, saw 128\\nSkipping line 2282552: expected 127 fields, saw 128\\nSkipping line 2282556: expected 127 fields, saw 128\\nSkipping line 2282668: expected 127 fields, saw 128\\nSkipping line 2282670: expected 127 fields, saw 128\\nSkipping line 2282671: expected 127 fields, saw 128\\nSkipping line 2284235: expected 127 fields, saw 132\\nSkipping line 2284294: expected 127 fields, saw 128\\nSkipping line 2284972: expected 127 fields, saw 130\\nSkipping line 2285291: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 2287700: expected 127 fields, saw 129\\nSkipping line 2290998: expected 127 fields, saw 128\\nSkipping line 2291745: expected 127 fields, saw 128\\nSkipping line 2292079: expected 127 fields, saw 144\\n'\n",
      "b'Skipping line 2296407: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 2303428: expected 127 fields, saw 134\\nSkipping line 2309310: expected 127 fields, saw 128\\nSkipping line 2309382: expected 127 fields, saw 151\\n'\n",
      "b'Skipping line 2312483: expected 127 fields, saw 128\\nSkipping line 2312484: expected 127 fields, saw 128\\nSkipping line 2312485: expected 127 fields, saw 128\\nSkipping line 2313675: expected 127 fields, saw 133\\nSkipping line 2316809: expected 127 fields, saw 128\\nSkipping line 2317913: expected 127 fields, saw 132\\nSkipping line 2319562: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2324118: expected 127 fields, saw 132\\nSkipping line 2324522: expected 127 fields, saw 136\\nSkipping line 2324909: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2328812: expected 127 fields, saw 128\\nSkipping line 2329812: expected 127 fields, saw 134\\nSkipping line 2329813: expected 127 fields, saw 138\\n'\n",
      "b'Skipping line 2336944: expected 127 fields, saw 146\\nSkipping line 2343343: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 2345334: expected 127 fields, saw 130\\nSkipping line 2347427: expected 127 fields, saw 128\\nSkipping line 2350498: expected 127 fields, saw 128\\nSkipping line 2351937: expected 127 fields, saw 137\\n'\n",
      "b'Skipping line 2354100: expected 127 fields, saw 128\\nSkipping line 2354101: expected 127 fields, saw 128\\nSkipping line 2354104: expected 127 fields, saw 128\\nSkipping line 2354896: expected 127 fields, saw 128\\nSkipping line 2355242: expected 127 fields, saw 132\\nSkipping line 2355507: expected 127 fields, saw 128\\nSkipping line 2358914: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2369293: expected 127 fields, saw 128\\nSkipping line 2370422: expected 127 fields, saw 131\\nSkipping line 2370454: expected 127 fields, saw 145\\nSkipping line 2370455: expected 127 fields, saw 128\\nSkipping line 2370666: expected 127 fields, saw 128\\nSkipping line 2373514: expected 127 fields, saw 130\\nSkipping line 2375218: expected 127 fields, saw 145\\n'\n",
      "b'Skipping line 2378158: expected 127 fields, saw 130\\nSkipping line 2378672: expected 127 fields, saw 132\\nSkipping line 2384085: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 2386614: expected 127 fields, saw 144\\nSkipping line 2388537: expected 127 fields, saw 132\\nSkipping line 2392097: expected 127 fields, saw 133\\nSkipping line 2392559: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 2394481: expected 127 fields, saw 128\\nSkipping line 2395545: expected 127 fields, saw 130\\nSkipping line 2399534: expected 127 fields, saw 132\\nSkipping line 2400450: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2404647: expected 127 fields, saw 132\\nSkipping line 2408962: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2411626: expected 127 fields, saw 128\\nSkipping line 2412681: expected 127 fields, saw 132\\nSkipping line 2415499: expected 127 fields, saw 130\\nSkipping line 2417839: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2419168: expected 127 fields, saw 130\\nSkipping line 2419183: expected 127 fields, saw 143\\nSkipping line 2421976: expected 127 fields, saw 132\\nSkipping line 2422017: expected 127 fields, saw 130\\nSkipping line 2422024: expected 127 fields, saw 128\\nSkipping line 2422850: expected 127 fields, saw 128\\nSkipping line 2422851: expected 127 fields, saw 128\\nSkipping line 2422852: expected 127 fields, saw 128\\nSkipping line 2422853: expected 127 fields, saw 128\\nSkipping line 2424072: expected 127 fields, saw 130\\nSkipping line 2425605: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 2430485: expected 127 fields, saw 132\\nSkipping line 2431791: expected 127 fields, saw 128\\nSkipping line 2432948: expected 127 fields, saw 128\\nSkipping line 2432949: expected 127 fields, saw 128\\nSkipping line 2432950: expected 127 fields, saw 128\\nSkipping line 2432951: expected 127 fields, saw 128\\nSkipping line 2434146: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2437196: expected 127 fields, saw 131\\nSkipping line 2440105: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 2442998: expected 127 fields, saw 131\\nSkipping line 2443455: expected 127 fields, saw 128\\nSkipping line 2443521: expected 127 fields, saw 131\\nSkipping line 2446768: expected 127 fields, saw 128\\nSkipping line 2449674: expected 127 fields, saw 132\\nSkipping line 2449764: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2451101: expected 127 fields, saw 128\\nSkipping line 2453967: expected 127 fields, saw 130\\nSkipping line 2455475: expected 127 fields, saw 130\\nSkipping line 2457441: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2460786: expected 127 fields, saw 130\\nSkipping line 2465803: expected 127 fields, saw 137\\n'\n",
      "b'Skipping line 2467685: expected 127 fields, saw 128\\nSkipping line 2468885: expected 127 fields, saw 128\\nSkipping line 2468886: expected 127 fields, saw 128\\nSkipping line 2470156: expected 127 fields, saw 128\\nSkipping line 2471383: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2475895: expected 127 fields, saw 130\\nSkipping line 2476864: expected 127 fields, saw 128\\nSkipping line 2477162: expected 127 fields, saw 128\\nSkipping line 2477252: expected 127 fields, saw 128\\nSkipping line 2477957: expected 127 fields, saw 130\\nSkipping line 2478608: expected 127 fields, saw 128\\nSkipping line 2479602: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2483734: expected 127 fields, saw 133\\nSkipping line 2488930: expected 127 fields, saw 128\\nSkipping line 2489020: expected 127 fields, saw 130\\nSkipping line 2490186: expected 127 fields, saw 128\\nSkipping line 2490188: expected 127 fields, saw 128\\nSkipping line 2490189: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2497892: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2500122: expected 127 fields, saw 130\\nSkipping line 2500190: expected 127 fields, saw 141\\nSkipping line 2501414: expected 127 fields, saw 136\\nSkipping line 2504190: expected 127 fields, saw 132\\nSkipping line 2504279: expected 127 fields, saw 128\\nSkipping line 2504807: expected 127 fields, saw 130\\nSkipping line 2508042: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2509200: expected 127 fields, saw 128\\nSkipping line 2509408: expected 127 fields, saw 130\\nSkipping line 2510048: expected 127 fields, saw 128\\nSkipping line 2516128: expected 127 fields, saw 132\\nSkipping line 2516158: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 2518933: expected 127 fields, saw 130\\nSkipping line 2520411: expected 127 fields, saw 128\\nSkipping line 2520413: expected 127 fields, saw 128\\nSkipping line 2521308: expected 127 fields, saw 131\\nSkipping line 2522542: expected 127 fields, saw 140\\nSkipping line 2523012: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2524731: expected 127 fields, saw 132\\nSkipping line 2526069: expected 127 fields, saw 130\\nSkipping line 2528825: expected 127 fields, saw 128\\nSkipping line 2531724: expected 127 fields, saw 128\\nSkipping line 2531725: expected 127 fields, saw 128\\nSkipping line 2531727: expected 127 fields, saw 128\\nSkipping line 2532639: expected 127 fields, saw 143\\n'\n",
      "b'Skipping line 2536546: expected 127 fields, saw 129\\nSkipping line 2537808: expected 127 fields, saw 130\\nSkipping line 2538749: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 2542489: expected 127 fields, saw 128\\nSkipping line 2542663: expected 127 fields, saw 146\\nSkipping line 2547525: expected 127 fields, saw 130\\nSkipping line 2548639: expected 127 fields, saw 128\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 2551401: expected 127 fields, saw 130\\nSkipping line 2552742: expected 127 fields, saw 131\\nSkipping line 2556787: expected 127 fields, saw 128\\nSkipping line 2556788: expected 127 fields, saw 128\\nSkipping line 2556900: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 2559847: expected 127 fields, saw 128\\nSkipping line 2559848: expected 127 fields, saw 128\\nSkipping line 2559849: expected 127 fields, saw 128\\nSkipping line 2559850: expected 127 fields, saw 128\\nSkipping line 2559851: expected 127 fields, saw 128\\nSkipping line 2561384: expected 127 fields, saw 128\\nSkipping line 2561413: expected 127 fields, saw 130\\nSkipping line 2565166: expected 127 fields, saw 130\\nSkipping line 2565198: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 2566270: expected 127 fields, saw 132\\nSkipping line 2567912: expected 127 fields, saw 130\\nSkipping line 2568102: expected 127 fields, saw 144\\nSkipping line 2569523: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2574653: expected 127 fields, saw 143\\nSkipping line 2575429: expected 127 fields, saw 130\\nSkipping line 2577944: expected 127 fields, saw 128\\nSkipping line 2579050: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2585041: expected 127 fields, saw 138\\nSkipping line 2585513: expected 127 fields, saw 130\\nSkipping line 2585955: expected 127 fields, saw 128\\nSkipping line 2586274: expected 127 fields, saw 134\\nSkipping line 2586416: expected 127 fields, saw 151\\nSkipping line 2587052: expected 127 fields, saw 130\\nSkipping line 2589807: expected 127 fields, saw 130\\nSkipping line 2589815: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2591189: expected 127 fields, saw 133\\nSkipping line 2592749: expected 127 fields, saw 132\\nSkipping line 2592775: expected 127 fields, saw 132\\nSkipping line 2594980: expected 127 fields, saw 128\\nSkipping line 2594981: expected 127 fields, saw 128\\nSkipping line 2594982: expected 127 fields, saw 128\\nSkipping line 2595528: expected 127 fields, saw 128\\nSkipping line 2597944: expected 127 fields, saw 145\\n'\n",
      "b'Skipping line 2600679: expected 127 fields, saw 139\\nSkipping line 2601028: expected 127 fields, saw 128\\nSkipping line 2601366: expected 127 fields, saw 128\\nSkipping line 2606325: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2606665: expected 127 fields, saw 128\\nSkipping line 2607440: expected 127 fields, saw 130\\nSkipping line 2611533: expected 127 fields, saw 128\\nSkipping line 2611775: expected 127 fields, saw 129\\nSkipping line 2612218: expected 127 fields, saw 128\\nSkipping line 2613930: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2616453: expected 127 fields, saw 133\\nSkipping line 2617134: expected 127 fields, saw 138\\nSkipping line 2619721: expected 127 fields, saw 134\\nSkipping line 2621040: expected 127 fields, saw 132\\nSkipping line 2622625: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 2625156: expected 127 fields, saw 128\\nSkipping line 2625159: expected 127 fields, saw 128\\nSkipping line 2626001: expected 127 fields, saw 128\\nSkipping line 2626024: expected 127 fields, saw 135\\nSkipping line 2627392: expected 127 fields, saw 128\\nSkipping line 2627949: expected 127 fields, saw 130\\nSkipping line 2627993: expected 127 fields, saw 130\\nSkipping line 2628293: expected 127 fields, saw 130\\nSkipping line 2630286: expected 127 fields, saw 133\\n'\n",
      "b'Skipping line 2632938: expected 127 fields, saw 141\\nSkipping line 2632939: expected 127 fields, saw 134\\nSkipping line 2638211: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2643702: expected 127 fields, saw 132\\nSkipping line 2646315: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2648998: expected 127 fields, saw 130\\nSkipping line 2650017: expected 127 fields, saw 142\\nSkipping line 2652744: expected 127 fields, saw 143\\nSkipping line 2652773: expected 127 fields, saw 128\\nSkipping line 2652860: expected 127 fields, saw 129\\nSkipping line 2653721: expected 127 fields, saw 129\\nSkipping line 2654469: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2657220: expected 127 fields, saw 128\\nSkipping line 2657593: expected 127 fields, saw 128\\nSkipping line 2657605: expected 127 fields, saw 128\\nSkipping line 2658097: expected 127 fields, saw 130\\nSkipping line 2658424: expected 127 fields, saw 130\\nSkipping line 2659359: expected 127 fields, saw 130\\nSkipping line 2663031: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2667141: expected 127 fields, saw 148\\nSkipping line 2668211: expected 127 fields, saw 129\\n'\n",
      "b'Skipping line 2675911: expected 127 fields, saw 129\\nSkipping line 2676230: expected 127 fields, saw 128\\nSkipping line 2678904: expected 127 fields, saw 128\\nSkipping line 2679472: expected 127 fields, saw 132\\n'\n",
      "b'Skipping line 2682240: expected 127 fields, saw 129\\nSkipping line 2687439: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2689068: expected 127 fields, saw 128\\nSkipping line 2689482: expected 127 fields, saw 130\\nSkipping line 2690339: expected 127 fields, saw 128\\nSkipping line 2690392: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2697243: expected 127 fields, saw 128\\nSkipping line 2698226: expected 127 fields, saw 130\\nSkipping line 2701456: expected 127 fields, saw 146\\nSkipping line 2702095: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2707207: expected 127 fields, saw 128\\nSkipping line 2707208: expected 127 fields, saw 128\\nSkipping line 2710672: expected 127 fields, saw 136\\nSkipping line 2711263: expected 127 fields, saw 130\\nSkipping line 2712886: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2717396: expected 127 fields, saw 130\\nSkipping line 2719875: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2721663: expected 127 fields, saw 128\\nSkipping line 2722912: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2730815: expected 127 fields, saw 129\\nSkipping line 2732023: expected 127 fields, saw 129\\nSkipping line 2736750: expected 127 fields, saw 145\\n'\n",
      "b'Skipping line 2738873: expected 127 fields, saw 139\\nSkipping line 2744146: expected 127 fields, saw 128\\nSkipping line 2744147: expected 127 fields, saw 128\\nSkipping line 2744149: expected 127 fields, saw 128\\nSkipping line 2745495: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2748325: expected 127 fields, saw 131\\nSkipping line 2749568: expected 127 fields, saw 142\\nSkipping line 2749569: expected 127 fields, saw 134\\nSkipping line 2751220: expected 127 fields, saw 128\\nSkipping line 2751221: expected 127 fields, saw 128\\nSkipping line 2752912: expected 127 fields, saw 146\\n'\n",
      "b'Skipping line 2762881: expected 127 fields, saw 128\\nSkipping line 2764370: expected 127 fields, saw 136\\nSkipping line 2764473: expected 127 fields, saw 132\\nSkipping line 2764477: expected 127 fields, saw 128\\nSkipping line 2765066: expected 127 fields, saw 128\\nSkipping line 2765961: expected 127 fields, saw 128\\nSkipping line 2766054: expected 127 fields, saw 130\\nSkipping line 2768016: expected 127 fields, saw 128\\nSkipping line 2768017: expected 127 fields, saw 133\\nSkipping line 2768982: expected 127 fields, saw 128\\nSkipping line 2769441: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2771309: expected 127 fields, saw 129\\nSkipping line 2777615: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2780388: expected 127 fields, saw 128\\nSkipping line 2780391: expected 127 fields, saw 128\\nSkipping line 2780994: expected 127 fields, saw 128\\nSkipping line 2780996: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2788372: expected 127 fields, saw 128\\nSkipping line 2790804: expected 127 fields, saw 132\\nSkipping line 2792735: expected 127 fields, saw 128\\nSkipping line 2793074: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2797434: expected 127 fields, saw 128\\nSkipping line 2798106: expected 127 fields, saw 132\\nSkipping line 2800724: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2803748: expected 127 fields, saw 130\\nSkipping line 2807123: expected 127 fields, saw 130\\nSkipping line 2807885: expected 127 fields, saw 132\\nSkipping line 2810268: expected 127 fields, saw 130\\nSkipping line 2810296: expected 127 fields, saw 128\\nSkipping line 2810297: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2813503: expected 127 fields, saw 132\\nSkipping line 2814221: expected 127 fields, saw 130\\nSkipping line 2814809: expected 127 fields, saw 128\\nSkipping line 2816374: expected 127 fields, saw 130\\n'\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "b'Skipping line 2820314: expected 127 fields, saw 131\\nSkipping line 2823853: expected 127 fields, saw 128\\nSkipping line 2824079: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2831076: expected 127 fields, saw 129\\nSkipping line 2832000: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2836429: expected 127 fields, saw 130\\nSkipping line 2836540: expected 127 fields, saw 135\\nSkipping line 2838034: expected 127 fields, saw 128\\nSkipping line 2842677: expected 127 fields, saw 128\\nSkipping line 2842695: expected 127 fields, saw 131\\nSkipping line 2843575: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2845399: expected 127 fields, saw 128\\nSkipping line 2845400: expected 127 fields, saw 128\\nSkipping line 2845401: expected 127 fields, saw 128\\nSkipping line 2845796: expected 127 fields, saw 128\\nSkipping line 2847493: expected 127 fields, saw 130\\nSkipping line 2848922: expected 127 fields, saw 128\\nSkipping line 2849326: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2852507: expected 127 fields, saw 128\\nSkipping line 2854649: expected 127 fields, saw 131\\nSkipping line 2858257: expected 127 fields, saw 130\\nSkipping line 2858882: expected 127 fields, saw 130\\n'\n",
      "b'Skipping line 2861895: expected 127 fields, saw 130\\nSkipping line 2864322: expected 127 fields, saw 128\\nSkipping line 2864328: expected 127 fields, saw 128\\nSkipping line 2868424: expected 127 fields, saw 128\\n'\n",
      "b'Skipping line 2873548: expected 127 fields, saw 128\\nSkipping line 2874146: expected 127 fields, saw 136\\nSkipping line 2874575: expected 127 fields, saw 128\\nSkipping line 2875027: expected 127 fields, saw 128\\nSkipping line 2876078: expected 127 fields, saw 131\\n'\n",
      "b'Skipping line 2880032: expected 127 fields, saw 129\\nSkipping line 2882199: expected 127 fields, saw 129\\nSkipping line 2883429: expected 127 fields, saw 129\\nSkipping line 2883920: expected 127 fields, saw 134\\n'\n",
      "b'Skipping line 2886960: expected 127 fields, saw 130\\nSkipping line 2889404: expected 127 fields, saw 132\\nSkipping line 2890739: expected 127 fields, saw 128\\nSkipping line 2891057: expected 127 fields, saw 130\\nSkipping line 2892796: expected 127 fields, saw 130\\nSkipping line 2892895: expected 127 fields, saw 128\\n'\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(canonicalize) Operations: fill n/a + dehyphenate + replace_values + trim_tail + fill others (non-target classes)\n",
      "> dim(ts_ctrl): (2891340, 127)\n",
      "(save_data) Saved dataframe (dim=(2891340, 127)) to:\n",
      "/Users/barnett/Documents/work/loinc_predictor/data/andromeda-pond-hepatitis-c-ctrl.csv\n",
      "\n",
      "> Adding control data | N: 71224 -> 142448\n"
     ]
    }
   ],
   "source": [
    "# patients not in given domain/disease\n",
    "from cohort_search import filter_by_diagnosis\n",
    "from analyzer import save_data\n",
    "from transformer import resolve_duplicate\n",
    "\n",
    "######################################\n",
    "# params\n",
    "tFilter = False\n",
    "tSave = True\n",
    "tAddCtrl = True\n",
    "output_dir = 'data'\n",
    "######################################\n",
    "\n",
    "N0 = ts0.shape[0]\n",
    "\n",
    "# load source data\n",
    "ts_ctrl = load_src_data(input_file='andromeda_pond-10p.csv', canonicalized=True)\n",
    "# ts_ctrl = ts_ctrl.drop_duplicates(keep='last')  # drop duplicates\n",
    "# ts_ctrl = resolve_duplicate(ts_ctrl)\n",
    "# ... generate a new variable: 'count'\n",
    "# ts_ctrl = lc.canonicalize(ts_ctrl, col_target=col_target, token_missing=token_default, target_labels=loinc_set)\n",
    "# ... now included by default within load_data()\n",
    "\n",
    "print(\"> dim(ts_ctrl): {}\".format(ts_ctrl.shape))  \n",
    "\n",
    "if tFilter: \n",
    "    ts_ctrl = filter_by_diagnosis(ts_ctrl, condition='hepatitis c')\n",
    "    nctrl = ts_ctrl.shape[0]\n",
    "    print(\"> sample size | orig: {}, filtered(control): {}\".format(n0, nctrl))\n",
    "\n",
    "# summary(df_ctrl, n=1)\n",
    "# df_ctrl.info()   \n",
    "\n",
    "# save \n",
    "if tSave: \n",
    "    output_file = f\"andromeda-pond-{cohort}-ctrl.csv\" \n",
    "    save_data(ts_ctrl,  output_file=output_file, sep=',')\n",
    "    \n",
    "# mix-in \n",
    "if tAddCtrl: \n",
    "    Nctrl = ts_ctrl.shape[0]\n",
    "    assert Nctrl > N0, f\"Control data is too small | n({cohort})={N0} > n(ctrl)={Nctrl}\"\n",
    "    ts_ctrl = ts_ctrl.sample(n=N0, replace=False)\n",
    "    \n",
    "    ts0 = pd.concat([ts0, ts_ctrl], ignore_index=True)\n",
    "    \n",
    "assert np.sum(ts0[col_target].isnull()) == 0\n",
    "print(\"> Adding control data | N: {} -> {}\".format(N0, ts0.shape[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define Feature Set \n",
    "\n",
    "Note: Subsetting columns could potentially reduce the size of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Memo\n",
    "----\n",
    "1. medivo_test_result_type is a function of the following attributes: \n",
    "      \"meta_sender_name\",\n",
    "      \"receiving_organization_id\",\n",
    "      \"test_order_code\",\n",
    "      \"test_order_name\",\n",
    "      \"test_result_code\",\n",
    "      \"test_result_name\",\n",
    "      \"test_result_loinc_code\",\n",
    "      \"test_result_units_of_measure\"\n",
    "      \n",
    "\"\"\"\n",
    "from transformer import to_age\n",
    "from analyzer import sample_col_values\n",
    "from loinc import FeatureSet\n",
    "\n",
    "cat_cols = FeatureSet.cat_cols\n",
    "\n",
    "# ['patient_gender',\n",
    "#  'patient_state',\n",
    "#  'patient_bill_type',\n",
    "#  'fasting',\n",
    "#  'performing_organization_id',\n",
    "#  'receiving_organization_id',\n",
    "#  'test_result_status',\n",
    "#  'test_order_code',\n",
    "#  'test_order_name',\n",
    "#  'test_result_code',\n",
    "#  'test_result_name',\n",
    "#  'test_result_value',\n",
    "#  'test_result_range',\n",
    "#  'test_result_abnormal_flag',\n",
    "#  'test_result_reference_range',\n",
    "#  'test_result_units_of_measure',\n",
    "#  'test_result_comments',\n",
    "#  'test_cpt_code',\n",
    "#  'panel_order_code',\n",
    "#  'panel_order_name',\n",
    "#  'meta_sender_name',\n",
    "#  'medivo_test_result_type']\n",
    "\n",
    "cont_cols = FeatureSet.cont_cols  \n",
    "\n",
    "target_cols = FeatureSet.target_cols # ['test_result_loinc_code', ]\n",
    "\n",
    "derived_cols = FeatureSet.derived_cols\n",
    "# ['count',  # due to resolve_duplicate()\n",
    "# ]\n",
    "\n",
    "# cardinality < 100\n",
    "low_card_cols = ['patient_gender', 'fasting', 'meta_sender_name' ]\n",
    "high_card_cols = list(set(cat_cols)-set(low_card_cols))\n",
    "\n",
    "representative_cols = [\"meta_sender_name\",\n",
    "      # \"receiving_organization_id\",\n",
    "      # \"test_order_code\",\n",
    "      \"test_order_name\",\n",
    "      # \"test_result_code\",\n",
    "      \"test_result_name\",\n",
    "      \"test_result_loinc_code\",\n",
    "      \"test_result_units_of_measure\"]\n",
    "\n",
    "target_columns = cat_cols + cont_cols + target_cols\n",
    "\n",
    "# feature transformation\n",
    "#####################################################\n",
    "# to_age(ts0)\n",
    "# values = sample_col_values(ts0, col='age', n=10)\n",
    "# print(\"> age: {}\".format(values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Balance Classes \n",
    "\n",
    "note: balance classes from external dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(balance_classes) data size distribution:\n",
      "[('unknown', 12005), ('67686', 5703), ('17426', 4418), ('19752', 4408), ('17517', 4392), ('21600', 4167), ('7187', 3826), ('7773', 3783), ('178616', 3756), ('486431', 3735), ('30973', 3686), ('19208', 3589), ('66902', 3347), ('7310', 3045), ('28233', 3005), ('7112', 2791), ('7138', 2723), ('110114', 2559), ('7708', 2463), ('45443', 2413), ('7518', 2376), ('25718', 2241), ('134577', 2175), ('20933', 2165), ('20859', 2115)]\n",
      "\n",
      "(balance_classes) We have n=1506 with low sample size (< 5703)\n",
      "[load] Loading default input data: andromeda-pond-hepatitis-c-balanced.csv\n",
      "(canonicalize) Operations: fill n/a + dehyphenate + replace_values + trim_tail + fill others (non-target classes)\n",
      "(is_canonicalized) Found n=1 codes not in target set:\n",
      "{'nan'}\n",
      "\n",
      "(canonicalize) Operations: fill n/a + dehyphenate + replace_values + trim_tail + fill others (non-target classes)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\nOutput\\n------\\n   ts: balanced training data; well... as balaned as possible because external data may not be able to supply sufficient \\n       data for a subset of the classes/loincs\\n'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from analyzer import balance_data_incr, load_data_incr, stratify\n",
    "from transformer import resolve_duplicate\n",
    "import loinc as lc\n",
    "from loinc import LoincTSet\n",
    "\n",
    "# run here or just load the curated data generated by analyzer.t_stratify()\n",
    "tLoad = True\n",
    "tSave = not tLoad\n",
    "\n",
    "col_target = LoincTSet.col_target\n",
    "token_default = token_missing = 'unknown'\n",
    "\n",
    "ts = ts0\n",
    "ds = stratify(ts, col='test_result_loinc_code', ascending=False)\n",
    "print(\"(balance_classes) data size distribution:\\n{}\\n\".format(ds[:25]))\n",
    "ds = [(code, size) for code, size in ds if not code in lc.LoincTSet.non_codes]\n",
    "max_size = ds[0][1] # ds[2][1]\n",
    "codes_low_sz = set([code for code, size in ds if size < max_size])\n",
    "print(\"(balance_classes) We have n={} with low sample size (< {})\".format(len(codes_low_sz), max_size))\n",
    "\n",
    "if tLoad: \n",
    "    input_file=f\"andromeda-pond-{cohort}-balanced.csv\"\n",
    "    ts = load_src_data(input_file=input_file, warn_bad_lines=False)\n",
    "    ts[col_target] = ts[col_target].astype(str)\n",
    "    # ts = ts.drop_duplicates(keep='last')  # drop duplicates \n",
    "    # ts = lc.canonicalize(ts, col_target=col_target, token_missing=token_default, target_labels=loinc_set)\n",
    "    if not lc.is_canonicalized(ts, col_target=col_target, token_missing=token_default, target_labels=loinc_set): \n",
    "        ts = ts.drop_duplicates(keep='last')  # drop duplicates \n",
    "        ts = lc.canonicalize(ts, col_target=col_target, token_missing=token_default) # noisy_values/[]\n",
    "    \n",
    "else: \n",
    "    # max_size = 1000\n",
    "    input_file = f\"andromeda-pond-{cohort}-loinc.csv\"\n",
    "    codes = set(loinc_set)\n",
    "    codes_hit = set([])\n",
    "    for i, tsi in enumerate(load_data_incr(input_file=input_file, chunksize=1000000, warn_bad_lines=False)): \n",
    "        N0 = ts.shape[0]\n",
    "        # tsi = tsi.drop_duplicates(keep='last')  # drop duplicates \n",
    "        tsi = resolve_duplicate(tsi)\n",
    "        tsi = lc.canonicalize(tsi, col_target=col_target, token_missing=token_default, target_labels=loinc_set)\n",
    "        print(\"[{}] Processing chunk #{} | n(ts): {}, n(tsi): {} ...\".format(i, i+1, N0, tsi.shape[0]))\n",
    "\n",
    "        ts_incr, hit, missed = balance_data_incr(df=ts, df_extern=tsi, n_samples=max_size, col=col_target)\n",
    "        if not ts_incr.empty: ts = pd.concat([ts, ts_incr])\n",
    "            \n",
    "        # --- analysis --- \n",
    "        N = ts.shape[0]\n",
    "        codes_hit.union(hit) # still has this many codes without a match \n",
    "\n",
    "        print(f\"[{i}] size: {N0} -> {N}\")\n",
    "        ds = stratify(ts, col='test_result_loinc_code', ascending=False)\n",
    "        ds = [(code, size) for code, size in ds if size < max_size]\n",
    "        print(\"[{}] size(codes) < {} (n={}): \\n{}\\n\".format(i, max_size, len(ds), ds[:20]))\n",
    "\n",
    "    codes_missed = codes_low_sz - codes_hit  \n",
    "    print(\"(t_stratify) At last, we could not find a match for n={} codes among nl={} low-sample-size labels:\\n{}\\n\".format(\n",
    "        len(codes_missed), len(codes_low_sz), codes_missed))  \n",
    "    \n",
    "    # down sample control data (very often the negative examples are too huge)\n",
    "\n",
    "    if tSave: \n",
    "        \n",
    "        # should not have duplicates at this point (due to resolve_duplicate() call)\n",
    "        # ts = ts.drop_duplicates(keep='last')  # drop duplicates \n",
    "        \n",
    "        output_file = f\"andromeda-pond-{cohort}-balanced.csv\"\n",
    "        save_data(ts, output_file=output_file)\n",
    "\n",
    "\"\"\"\n",
    "Output\n",
    "------\n",
    "   ts: balanced training data; well... as balaned as possible because external data may not be able to supply sufficient \n",
    "       data for a subset of the classes/loincs\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Transformation\n",
    "\n",
    "note: patient_date_of_birth => age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> age: [60 34 21 84 69 78 45 56 43 24]\n"
     ]
    }
   ],
   "source": [
    "from transformer import to_age\n",
    "from analyzer import col_values\n",
    "# from loinc import FeatureSet\n",
    "\n",
    "to_age(ts)\n",
    "values = col_values(ts, col='age', n=10)\n",
    "print(\"> age: {}\".format(values))\n",
    "\n",
    "# resolve_duplicate() call adds a new column: count (of duplicates)\n",
    "# assert 'count' in ts.columns\n",
    "\n",
    "# datatime columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Subset Features and Handling missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Given features set:\n",
      "['age', 'patient_gender', 'patient_state', 'patient_bill_type', 'fasting', 'performing_organization_id', 'receiving_organization_id', 'test_result_status', 'test_order_code', 'test_order_name', 'test_result_code', 'test_result_name', 'test_result_value', 'test_result_range', 'test_result_abnormal_flag', 'test_result_reference_range', 'test_result_units_of_measure', 'test_result_comments', 'test_cpt_code', 'panel_order_code', 'panel_order_name', 'meta_sender_name', 'medivo_test_result_type']\n",
      "\n",
      "> Final feature set (nf=23):\n",
      "{'test_result_name', 'fasting', 'test_result_comments', 'test_result_units_of_measure', 'test_result_range', 'age', 'test_cpt_code', 'patient_state', 'test_result_status', 'test_result_reference_range', 'patient_bill_type', 'performing_organization_id', 'patient_gender', 'test_result_abnormal_flag', 'test_order_code', 'panel_order_code', 'receiving_organization_id', 'meta_sender_name', 'test_order_name', 'test_result_value', 'medivo_test_result_type', 'panel_order_name', 'test_result_code'}\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\nOutput\\n------\\n   dfX \\n   dfy\\n'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tCategorify = False\n",
    "tDropHighMissing = False # drop columns with high rate of missing values\n",
    "p_null = 0.9\n",
    "token_default = token_missing = 'unknown'\n",
    "\n",
    "df = ts # ... :)\n",
    "\n",
    "# V = list(feature_lookup.keys())\n",
    "V = cont_cols + cat_cols # + derived_cols\n",
    "L = target_cols\n",
    "dfX = df[V]\n",
    "dfy = df[L]\n",
    "\n",
    "print(\"> Given features set:\\n{}\\n\".format(V))\n",
    "\n",
    "assert np.sum(dfy[target_cols[0]].isnull()) == 0\n",
    "\n",
    "# drop columns/vars with too many missing values \n",
    "N = dfX.shape[0]\n",
    "n_thresh = int(N * p_null)\n",
    "nf0 = nf = dfX.shape[1]\n",
    "fset0 = set(dfX.columns.values)\n",
    "\n",
    "if tDropHighMissing: \n",
    "    dfX = dfX[dfX.columns[dfX.isnull().mean() < p_null]]\n",
    "    fset = set(dfX.columns.values)\n",
    "    nf = dfX.shape[1]\n",
    "    print(\"> Dropped n={} features:\\n{}\\n\".format(nf-nf0, fset0-fset))\n",
    "    \n",
    "fset = set(dfX.columns.values)\n",
    "print(\"> Final feature set (nf={}):\\n{}\\n\".format(nf, fset))\n",
    "\n",
    "# fill in missing values (also see default_values)\n",
    "dfX.fillna(value=token_default, inplace=True)\n",
    "#################################################\n",
    "# Convert our three categorical columns to category dtypes.\n",
    "\n",
    "cat_cols = [cat for cat in cat_cols if cat in dfX.columns]\n",
    "cont_cols = [c for c in cont_cols if c in dfX.columns]\n",
    "\n",
    "\"\"\"\n",
    "Output\n",
    "------\n",
    "   dfX \n",
    "   dfy\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode Variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(encoder_vars2) low card vars (n=['meta_sender_name', 'patient_gender', 'fasting']):\n",
      "3\n",
      " ... high card vars (n=['patient_state', 'test_result_name', 'test_result_status', 'test_result_reference_range', 'patient_bill_type', 'panel_order_name', 'performing_organization_id', 'test_result_comments', 'test_result_abnormal_flag', 'test_order_code', 'panel_order_code', 'receiving_organization_id', 'test_result_units_of_measure', 'test_order_name', 'test_result_range', 'test_result_value', 'medivo_test_result_type', 'test_cpt_code', 'test_result_code']):\n",
      "19\n",
      "\n",
      "... transforming var: patient_gender ...\n",
      "... transforming var: patient_state ...\n",
      "... transforming var: patient_bill_type ...\n",
      "... transforming var: fasting ...\n",
      "... transforming var: performing_organization_id ...\n",
      "... transforming var: receiving_organization_id ...\n",
      "... transforming var: test_result_status ...\n",
      "... transforming var: test_order_code ...\n",
      "... transforming var: test_order_name ...\n",
      "... transforming var: test_result_code ...\n",
      "... transforming var: test_result_name ...\n",
      "... transforming var: test_result_value ...\n",
      "... transforming var: test_result_range ...\n",
      "... transforming var: test_result_abnormal_flag ...\n",
      "... transforming var: test_result_reference_range ...\n",
      "... transforming var: test_result_units_of_measure ...\n",
      "... transforming var: test_result_comments ...\n",
      "... transforming var: test_cpt_code ...\n",
      "... transforming var: panel_order_code ...\n",
      "... transforming var: panel_order_name ...\n",
      "... transforming var: meta_sender_name ...\n",
      "... transforming var: medivo_test_result_type ...\n",
      "> After variable encoding we have dim(dfX): (1123499, 243) | nf: 23 -> 243\n",
      "> New feature set:\n",
      "Index(['age', 'patient_gender_1', 'patient_gender_2', 'patient_gender_3',\n",
      "       'patient_state_0', 'patient_state_1', 'patient_state_2',\n",
      "       'patient_state_3', 'patient_state_4', 'patient_state_5',\n",
      "       ...\n",
      "       'medivo_test_result_type_2', 'medivo_test_result_type_3',\n",
      "       'medivo_test_result_type_4', 'medivo_test_result_type_5',\n",
      "       'medivo_test_result_type_6', 'medivo_test_result_type_7',\n",
      "       'medivo_test_result_type_8', 'medivo_test_result_type_9',\n",
      "       'medivo_test_result_type_10', 'medivo_test_result_type_11'],\n",
      "      dtype='object', length=243)\n",
      "\n",
      "meta_sender_name                 test_order_name    test_result_name test_result_loinc_code test_result_units_of_measure\n",
      "          Athena      Comp. Metabolic Panel (14)          ALT (SGPT)                  17426                          NaN\n",
      "          Athena                   HCV FibroSure      ALT (SGPT) P5P                  17434                          NaN\n",
      "          Saturn             HEPATIC LIVER PANEL     TOTAL BILIRUBIN                  19752                        MG/DL\n",
      "          Saturn            Complete Blood Count  Basophils Absolute                   7047                     1000/cmm\n",
      "          Saturn   COMPREHENSIVE METABOLIC PANEL      CALC BUN/CREAT                  30973                        RATIO\n",
      "          Apollo           BASIC METABOLIC PANEL           POTASSIUM                  28233                       mmol/L\n",
      "          Athena  CBC With Differential/Platelet              Lymphs                   7369                          NaN\n",
      "          Athena      Hepatic Function Panel (7)          ALT (SGPT)                  17426                          NaN\n",
      "          Athena                   HCV FibroSure            Comment:                 772020                          NaN\n",
      "          Apollo         CBC (Includes Diff/Plt)                 RBC                   7898                     Mill/mcL\n"
     ]
    }
   ],
   "source": [
    "from transformer import encode_vars \n",
    "from loinc import FeatureSet\n",
    "# high_card_cols = FeatureSet.high_card_cols\n",
    "nf0 = dfX.shape[1]\n",
    "dfX, encoder = encode_vars(dfX, fset=cat_cols, high_card_cols=high_card_cols)\n",
    "print(\"> After variable encoding we have dim(dfX): {} | nf: {} -> {}\".format(dfX.shape, nf0, dfX.shape[1]))\n",
    "print(\"> New feature set:\\n{}\\n\".format(dfX.columns))\n",
    "print(df[representative_cols].head(10).to_string(index=False))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode Labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Low sample size classes | n=698 (< 1000)\n",
      "> Extreme low sample size classes | n=395 (< 10)\n",
      "> n_matched: 2 | max_size: 5703\n",
      "[96107] -> 1\n",
      "[772079] -> 1\n",
      "[356188] -> 1\n",
      "[160119] -> 1\n",
      "[448134] -> 1\n",
      "[159756] -> 1\n",
      "[206649] -> 1\n",
      "[80100] -> 1\n",
      "[223123] -> 1\n",
      "[71001] -> 1\n",
      "[156679] -> 1\n",
      "[530170] -> 1\n",
      "[264663] -> 1\n",
      "[52092] -> 1\n",
      "[148049] -> 1\n",
      "> sizes: [('nan', 980139), ('67686', 5872), ('17426', 4647), ('17517', 4602), ('19752', 4562), ('21600', 4379), ('unknown', 4267), ('178616', 4001), ('7773', 3938), ('7187', 3917), ('486431', 3861), ('30973', 3851), ('19208', 3726), ('66902', 3405), ('110114', 3253), ('28233', 3142), ('7310', 3104), ('7112', 2848), ('7138', 2827), ('7708', 2503)]\n",
      "> Sample sizes | Top N=5 codes:\n",
      "[('nan', 980139), ('67686', 5872), ('17426', 4647), ('17517', 4602), ('19752', 4562)]\n",
      "\n",
      "> Sample sizss | Last N=5 codes:\n",
      "[('76919', 1), ('206060', 1), ('62372', 1), ('162339', 1), ('64204', 1)]\n",
      "\n",
      "(encode_labels) sample size: Counter({1: 980139, 0: 143360})\n"
     ]
    }
   ],
   "source": [
    "from analyzer import encode_labels, summarize_dict, get_sample_sizes\n",
    "import collections, operator\n",
    "\n",
    "# verify\n",
    "assert dfX.shape[0] == dfy.shape[0], \"> dim(dfX): {} | dfy.cols: {}\".format(dfX.shape, dfy.columns.values)\n",
    "\n",
    "codebook={'pos': 1, 'neg': 0, '+': 1, '-': 0}\n",
    "\n",
    "# choose the one with a large sample size as 'positive'\n",
    "col_label = 'test_result_loinc_code' # strings\n",
    "\n",
    "topn = 5\n",
    "sizes = get_sample_sizes(dfy[col_label])\n",
    "# ... sizes: (loinc) label -> sample size\n",
    "# print(\"> n(sizes): {}\".format(len(sizes)))  # 734 for cohort='hepatitis-c'\n",
    "\n",
    "# Q: How many classes/codes have less than N instances? \n",
    "N_low = 1000\n",
    "n_low = sum(1 for l, c in sizes.items() if c < N_low)\n",
    "print(\"> Low sample size classes | n={} (< {})\".format(n_low, N_low))\n",
    "\n",
    "N_elow = 10\n",
    "n_elow = sum(1 for l, c in sizes.items() if c < N_elow)\n",
    "print(\"> Extreme low sample size classes | n={} (< {})\".format(n_elow, N_elow))\n",
    "\n",
    "# Q: How many classes/codes were able to match the most enriched class in terms of sample size (e.g. 5707)? \n",
    "eps = 10\n",
    "n_matched = sum(1 for l, c in sizes.items() if c >= max_size-eps)\n",
    "print(\"> n_matched: {} | max_size: {}\".format(n_matched, max_size))\n",
    "\n",
    "# sort by values\n",
    "sizes_sorted = sorted(sizes.items(), key=operator.itemgetter(1))\n",
    "summarize_dict(sizes, topn=15, sort_=True)\n",
    "\n",
    "print(\"> sizes: {}\".format(sizes.most_common(20)))\n",
    "most_sample_sizes = sizes.most_common(topn)  # take(topn, sizes.items())\n",
    "print(\"> Sample sizes | Top N={} codes:\\n{}\\n\".format(topn, most_sample_sizes))\n",
    "least_sample_sizes = sizes.most_common()[:-topn-1:-1]\n",
    "print(\"> Sample sizss | Last N={} codes:\\n{}\\n\".format(topn, least_sample_sizes))\n",
    "\n",
    "# test\n",
    "target = most_sample_sizes[0][0]\n",
    "y = encode_labels(dfy, pos_label=target, codebook=codebook, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initial Model Training\n",
    "\n",
    "### 1. Feature Selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Ref\n",
    "---\n",
    "1. pip install feature-selector\n",
    "\n",
    "   https://github.com/WillKoehrsen/feature-selector\n",
    "   \n",
    "   possible dependency \n",
    "      brew install libomp\n",
    "      \n",
    "   <debug> \n",
    "       + RuntimeError: Python is not installed as a framework.\n",
    "          > https://stackoverflow.com/questions/34977388/matplotlib-runtimeerror-python-is-not-installed-as-a-framework\n",
    "   \n",
    "\"\"\"\n",
    "import feature_selector "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Save a copy of encoded training set (optional)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(save_data) Saved dataframe (dim=(1123499, 129)) to:\n",
      "/Users/barnett/Documents/work/loinc_predictor/data/andromeda-pond-hepatitis-c-encoded.csv\n",
      "\n"
     ]
    }
   ],
   "source": [
    "tSaveEncodedTSet = True \n",
    "\n",
    "if tSaveEncodedTSet: \n",
    "    output_file = f\"andromeda-pond-{cohort}-encoded.csv\"\n",
    "    save_data(ts, output_file=output_file)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> dim(X): (1123499, 243), sample(y): ['56853' '7708' '103317' '295980' '25007' '339440' '115725' '66902'\n",
      " '169151' '10058' '39347' '160119' '177915' '21600' '505610' '28886'\n",
      " '204545' '7096' '143149' '139907']\n",
      "(encode_labels) sample size: Counter({0: 1118852, 1: 4647})\n",
      "> sample size | n(+): 4647, n(-): 1118852 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 4647, n(-): 1118852\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9983879634605051 p_th: 0.92472648938096 | F1: 0.9973161567364466, AUC: 0.9999791584731806\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.998389694041868 p_th: 0.6156882545750916 | F1: 0.998389694041868, AUC: 0.9999911294179356\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9978517722878624 p_th: 0.9223685692664245 | F1: 0.9967845659163987, AUC: 0.999987658320606\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9115691417877637 | F1: 0.9989247311827958, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9648448333342553 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.998925885958047, std: 0.0008986684050367221\n",
      "(encode_labels) sample size: Counter({0: 1123217, 1: 282})\n",
      "> sample size | n(+): 282, n(-): 1123217 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 282, n(-): 1123217\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9618841888156429 | F1: 0.9824561403508771, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8394386843915141 | F1: 0.9913043478260869, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9247958620198066 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9308082419595071 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9802557379600548 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1118937, 1: 4562})\n",
      "> sample size | n(+): 4562, n(-): 1118937 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 4562, n(-): 1118937\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9994514536478333 p_th: 0.9137708485303293 | F1: 0.9956284153005464, AUC: 0.9999959952840465\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9994520547945205 p_th: 0.8657914001419295 | F1: 0.9989059080962801, AUC: 0.9999991996417597\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9983579638752053 p_th: 0.9187823994469201 | F1: 0.9961811238406983, AUC: 0.9999947976714377\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9994520547945205 p_th: 0.7381061002249572 | F1: 0.9989047097480832, AUC: 0.9999991987641739\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9994514536478333 p_th: 0.8311466894667595 | F1: 0.9989035087719298, AUC: 0.9999935901133908\n",
      "> average: 0.9992329961519827, std: 0.0004375162209861296\n",
      "(encode_labels) sample size: Counter({0: 1119232, 1: 4267})\n",
      "> sample size | n(+): 4267, n(-): 1119232 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 4267, n(-): 1119232\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9988290398126464 p_th: 0.7978639894073463 | F1: 0.9947521865889213, AUC: 0.9999844360327813\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.997067448680352 p_th: 0.7679666330010699 | F1: 0.9947337624341721, AUC: 0.9999396223653395\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9953325554259044 p_th: 0.6920076010688471 | F1: 0.9924374636416521, AUC: 0.9997507135538641\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9964912280701754 p_th: 0.675770411331242 | F1: 0.9959088252483929, AUC: 0.999664786049238\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9982425307557118 p_th: 0.6791708326039418 | F1: 0.9976580796252927, AUC: 0.9999903832063306\n",
      "> average: 0.9971925605489581, std: 0.0012448004738752098\n",
      "(encode_labels) sample size: Counter({0: 1123455, 1: 44})\n",
      "> sample size | n(+): 44, n(-): 1123455 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 44, n(-): 1123455\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4942295170038211 | F1: 0.9411764705882353, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8997457926491962 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9 p_th: 0.39584415686903174 | F1: 0.7777777777777778, AUC: 0.982905982905983\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9550065792575473 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9473684210526316 p_th: 0.21630883530689982 | F1: 0.9411764705882353, AUC: 0.9957264957264957\n",
      "> average: 0.9694736842105263, std: 0.04027605572570313\n",
      "(encode_labels) sample size: Counter({0: 1119648, 1: 3851})\n",
      "> sample size | n(+): 3851, n(-): 1119648 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3851, n(-): 1119648\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9295215442295166 | F1: 0.9980557355800389, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9980532121998702 p_th: 0.8949785033314963 | F1: 0.9967595593000649, AUC: 0.9999617863745947\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8896399038375508 | F1: 0.9980557355800389, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9993519118600129 p_th: 0.8888921186936468 | F1: 0.9954809554551323, AUC: 0.9999994385208393\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9987029831387808 p_th: 0.8510202248862722 | F1: 0.9980557355800389, AUC: 0.9999977511665823\n",
      "> average: 0.9992216214397327, std: 0.0007566885487166725\n",
      "(encode_labels) sample size: Counter({0: 1120357, 1: 3142})\n",
      "> sample size | n(+): 3142, n(-): 1120357 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3142, n(-): 1120357\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9984101748807631 p_th: 0.972137698945468 | F1: 0.9976171564733916, AUC: 0.9996639671464563\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9992057188244639 p_th: 0.8960441026111932 | F1: 0.9984126984126983, AUC: 0.9999949395486921\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9968253968253968 p_th: 0.703998013155487 | F1: 0.9944576405384006, AUC: 0.9999763845605631\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9976171564733916 p_th: 0.74600547599538 | F1: 0.9960348929421095, AUC: 0.9998851137880349\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9984076433121019 p_th: 0.9086454853522418 | F1: 0.9968253968253968, AUC: 0.9998504789741337\n",
      "> average: 0.9980932180632234, std: 0.0008088248430253697\n",
      "(encode_labels) sample size: Counter({0: 1122017, 1: 1482})\n",
      "> sample size | n(+): 1482, n(-): 1122017 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1482, n(-): 1122017\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9983079526226735 p_th: 0.8878174242373822 | F1: 0.9983079526226735, AUC: 0.9999810203461889\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9072380586249716 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7016273060348664 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8265860183003341 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7525341332430697 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9996615905245347, std: 0.0006768189509306132\n",
      "(encode_labels) sample size: Counter({0: 1123097, 1: 402})\n",
      "> sample size | n(+): 402, n(-): 1123097 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 402, n(-): 1123097\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9544767090985622 | F1: 0.9937888198757764, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9889174507093824 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.973716918978765 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9649418953610587 | F1: 0.9937888198757764, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9329340265885047 | F1: 0.9876543209876543, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1122526, 1: 973})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size | n(+): 973, n(-): 1122526 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 973, n(-): 1122526\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9974293059125964 p_th: 0.7930443192916324 | F1: 0.9948717948717949, AUC: 0.9998682824025289\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.841096526684512 | F1: 0.9974424552429668, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9948717948717948 p_th: 0.8630448451505258 | F1: 0.9948717948717948, AUC: 0.9999735206891682\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7245684662283568 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.956380427368394 | F1: 0.9873417721518987, AUC: 1.0\n",
      "> average: 0.9984602201568784, std: 0.0020519427916753322\n",
      "(encode_labels) sample size: Counter({0: 1119120, 1: 4379})\n",
      "> sample size | n(+): 4379, n(-): 1119120 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 4379, n(-): 1119120\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9087737646845393 | F1: 0.9994295493439818, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9988571428571429 p_th: 0.89289814336246 | F1: 0.9982866933181039, AUC: 0.9999686888454012\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9982866933181039 p_th: 0.9155614897761071 | F1: 0.9977194982896237, AUC: 0.9999956545393551\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9988597491448118 p_th: 0.4065373722366523 | F1: 0.9982886480319452, AUC: 0.9999982618157421\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9994295493439818 p_th: 0.47158169028735664 | F1: 0.9988584474885844, AUC: 0.9999995654539355\n",
      "> average: 0.999086626932808, std: 0.0005823875684731572\n",
      "(encode_labels) sample size: Counter({0: 1120996, 1: 2503})\n",
      "> sample size | n(+): 2503, n(-): 1120996 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2503, n(-): 1120996\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7973058065866827 | F1: 0.99800796812749, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.998003992015968 p_th: 0.873740865099562 | F1: 0.9930486593843098, AUC: 0.9999880398988973\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8183653329184835 | F1: 0.998003992015968, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9990009990009989 p_th: 0.5405257190722219 | F1: 0.9990009990009989, AUC: 0.9999986684420772\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7521746949328946 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9994009982033933, std: 0.0007985029876756681\n",
      "(encode_labels) sample size: Counter({0: 1121176, 1: 2323})\n",
      "> sample size | n(+): 2323, n(-): 1121176 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2323, n(-): 1121176\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9989258861439313 p_th: 0.8133022631648165 | F1: 0.9935897435897436, AUC: 0.9999953718702272\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9989258861439313 p_th: 0.8886515644547401 | F1: 0.9967845659163987, AUC: 0.9998426435877262\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.818261146574607 | F1: 0.9978494623655914, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9989212513484358 p_th: 0.8893851190385056 | F1: 0.9957081545064378, AUC: 0.9999969079305397\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9978494623655914 p_th: 0.919609077173522 | F1: 0.993576017130621, AUC: 0.9999768427390409\n",
      "> average: 0.998924497200378, std: 0.0006800618445233827\n",
      "(encode_labels) sample size: Counter({0: 1121527, 1: 1972})\n",
      "> sample size | n(+): 1972, n(-): 1121527 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1972, n(-): 1121527\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9961783439490446 p_th: 0.9658207767759964 | F1: 0.991128010139417, AUC: 0.9999442653313212\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9261587094869066 | F1: 0.9949622166246851, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9987325728770595 p_th: 0.9141808043553958 | F1: 0.9949622166246851, AUC: 0.9999957199567716\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9987293519695045 p_th: 0.8782613994439581 | F1: 0.9962073324905183, AUC: 0.9999935636405766\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9369722595650589 | F1: 0.9974683544303797, AUC: 1.0\n",
      "> average: 0.9987280537591218, std: 0.0013954738766992008\n",
      "(encode_labels) sample size: Counter({0: 1117627, 1: 5872})\n",
      "> sample size | n(+): 5872, n(-): 1117627 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5872, n(-): 1117627\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9995739241585002 p_th: 0.5756205443361265 | F1: 0.9982978723404257, AUC: 0.9999990331569166\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9995746490854955 p_th: 0.8753732474456644 | F1: 0.9991496598639457, AUC: 0.9999997584263896\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9995746490854955 p_th: 0.9399249435855889 | F1: 0.9987250318742031, AUC: 0.9999997584263896\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6438759471198743 | F1: 0.9995742869306087, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8880877908160103 | F1: 0.9995742869306087, AUC: 1.0\n",
      "> average: 0.9997446444658984, std: 0.00020849708854916212\n",
      "(encode_labels) sample size: Counter({0: 1121126, 1: 2373})\n",
      "> sample size | n(+): 2373, n(-): 1121126 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2373, n(-): 1121126\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9978991596638656 p_th: 0.6912128252793742 | F1: 0.9958071278825995, AUC: 0.9999955647545831\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9989484752891693 p_th: 0.9263393348868524 | F1: 0.9947643979057591, AUC: 0.999998521584861\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.996832101372756 p_th: 0.7145711061280303 | F1: 0.9936974789915967, AUC: 0.9999733323851514\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9978858350951374 p_th: 0.9194611438394414 | F1: 0.9947312961011593, AUC: 0.9999762954534679\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9989462592202318 p_th: 0.8076922964321561 | F1: 0.9968454258675079, AUC: 0.9999689314642897\n",
      "> average: 0.998102366128232, std: 0.0007911772705790847\n",
      "(encode_labels) sample size: Counter({0: 1119561, 1: 3938})\n",
      "> sample size | n(+): 3938, n(-): 1119561 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3938, n(-): 1119561\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9980976537729866 p_th: 0.6230124597257478 | F1: 0.9962073324905183, AUC: 0.9999951666018633\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9974554707379135 p_th: 0.9605391542983021 | F1: 0.9949302915082383, AUC: 0.999973684832367\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9993642720915448 p_th: 0.9096879851296402 | F1: 0.9961977186311788, AUC: 0.9999935472804207\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9993642720915448 p_th: 0.9037699328093656 | F1: 0.9968294229549778, AUC: 0.9999973113668419\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9993650793650793 p_th: 0.8917347905930835 | F1: 0.9974651457541192, AUC: 0.9999957018271532\n",
      "> average: 0.9987293496118138, std: 0.0008040164734485862\n",
      "(encode_labels) sample size: Counter({0: 1120651, 1: 2848})\n",
      "> sample size | n(+): 2848, n(-): 1120651 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2848, n(-): 1120651\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9991235758106924 p_th: 0.8583526628785925 | F1: 0.9982486865148862, AUC: 0.9999979468859392\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8758787468300364 | F1: 0.9982486865148862, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9010939326878629 | F1: 0.9964973730297723, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9991204925241866 p_th: 0.880748238853115 | F1: 0.9973707274320771, AUC: 0.9999979432776545\n",
      "> 4 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 0.9991235758106924 p_th: 0.7138506700874838 | F1: 0.9965034965034965, AUC: 0.999998972841941\n",
      "> average: 0.9994735288291142, std: 0.00042986338535625304\n",
      "(encode_labels) sample size: Counter({0: 1119582, 1: 3917})\n",
      "> sample size | n(+): 3917, n(-): 1119582 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3917, n(-): 1119582\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9993618379068284 p_th: 0.6684909275431046 | F1: 0.9993618379068284, AUC: 0.9999994567676699\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9993618379068284 p_th: 0.9525800480928899 | F1: 0.996817313812858, AUC: 0.999997286148502\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9594603514607686 | F1: 0.9974554707379135, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9993618379068284 p_th: 0.916647674724642 | F1: 0.9974522292993631, AUC: 0.9999994565365072\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9974457215836526 p_th: 0.8307176172145435 | F1: 0.9955499046408137, AUC: 0.9999907611206217\n",
      "> average: 0.9991062470608275, std: 0.0008662700754735441\n",
      "(encode_labels) sample size: Counter({0: 1120672, 1: 2827})\n",
      "> sample size | n(+): 2827, n(-): 1120672 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2827, n(-): 1120672\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9586259236716234 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9982300884955753 p_th: 0.9233019633385344 | F1: 0.9964726631393298, AUC: 0.9999812487499167\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9655711161574063 | F1: 0.9973568281938325, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9182211456987317 | F1: 0.9991158267020336, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9991158267020336 p_th: 0.8426900628403469 | F1: 0.997352162400706, AUC: 0.9999979128402071\n",
      "> average: 0.9994691830395217, std: 0.0007078864212110737\n",
      "(encode_labels) sample size: Counter({0: 1123028, 1: 471})\n",
      "> sample size | n(+): 471, n(-): 1123028 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 471, n(-): 1123028\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8904430132315472 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8102202648776691 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9946524064171123 p_th: 0.8504562367883001 | F1: 0.9946524064171123, AUC: 0.9999248176828809\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7781276216577028 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9789473684210526 p_th: 0.8451477645969679 | F1: 0.9738219895287958, AUC: 0.9988305417232534\n",
      "> average: 0.9947199549676331, std: 0.00815371909998766\n",
      "(encode_labels) sample size: Counter({0: 1123135, 1: 364})\n",
      "> sample size | n(+): 364, n(-): 1123135 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 364, n(-): 1123135\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6539651211707579 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9303169649313664 | F1: 0.9863013698630138, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8713154106679109 | F1: 0.9864864864864865, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9931972789115647 p_th: 0.8734270622998096 | F1: 0.9798657718120806, AUC: 0.9997486489883122\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9931972789115647 p_th: 0.7702173231577013 | F1: 0.9864864864864865, AUC: 0.9995601357295464\n",
      "> average: 0.9972789115646259, std: 0.0033326391058274025\n",
      "(encode_labels) sample size: Counter({0: 1123094, 1: 405})\n",
      "> sample size | n(+): 405, n(-): 1123094 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 405, n(-): 1123094\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9937888198757764 p_th: 0.9543010313531358 | F1: 0.9937888198757764, AUC: 0.9998475842097241\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.705237402328778 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9938650306748467 p_th: 0.6536699246339577 | F1: 0.9938650306748467, AUC: 0.9999491947365747\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9938650306748467 p_th: 0.7491022774687676 | F1: 0.9938650306748467, AUC: 0.9999491947365747\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9938650306748467 p_th: 0.9243583496783071 | F1: 0.9878048780487805, AUC: 0.9998983894731495\n",
      "> average: 0.9950767823800633, std: 0.0024617857636679316\n",
      "(encode_labels) sample size: Counter({0: 1120094, 1: 3405})\n",
      "> sample size | n(+): 3405, n(-): 1120094 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3405, n(-): 1120094\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9985315712187959 p_th: 0.7778595084013243 | F1: 0.9978021978021978, AUC: 0.9999956874338292\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9610009635566079 | F1: 0.9941605839416059, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9977957384276267 p_th: 0.7264960767704988 | F1: 0.995601173020528, AUC: 0.9999690932757749\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8910756309013131 | F1: 0.9970717423133236, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9977924944812362 p_th: 0.8704650607354936 | F1: 0.9941605839416059, AUC: 0.9999856247794302\n",
      "> average: 0.9988239608255318, std: 0.0009972754966269992\n",
      "(encode_labels) sample size: Counter({0: 1119498, 1: 4001})\n",
      "> sample size | n(+): 4001, n(-): 1119498 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 4001, n(-): 1119498\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8836564633565758 | F1: 0.9981285090455396, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.99875 p_th: 0.913803854744525 | F1: 0.9956440572495334, AUC: 0.9999963556851311\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9987515605493134 p_th: 0.7536610243458103 | F1: 0.9975062344139651, AUC: 0.9999947938359017\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9968886123210952 p_th: 0.6815945969906834 | F1: 0.995031055900621, AUC: 0.9999490220557636\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9975062344139651 p_th: 0.8035476764062428 | F1: 0.9919404835709857, AUC: 0.9999208333333333\n",
      "> average: 0.9983792814568748, std: 0.0010850860028968191\n",
      "(encode_labels) sample size: Counter({0: 1119773, 1: 3726})\n",
      "> sample size | n(+): 3726, n(-): 1119773 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3726, n(-): 1119773\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9993284083277367 p_th: 0.9213888413969088 | F1: 0.9953177257525083, AUC: 0.9999963981702705\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9986577181208054 p_th: 0.6835596478537639 | F1: 0.9979879275653923, AUC: 0.9999723859720737\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9993293091884642 p_th: 0.9538618650130705 | F1: 0.9959893048128343, AUC: 0.9999945972554056\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8900639514971823 | F1: 0.999330207635633, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9993293091884642 p_th: 0.7340956391233038 | F1: 0.9979906229068988, AUC: 0.9999993994264522\n",
      "> average: 0.999328948965094, std: 0.0004244669338319098\n",
      "(encode_labels) sample size: Counter({0: 1120246, 1: 3253})\n",
      "> sample size | n(+): 3253, n(-): 1120246 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3253, n(-): 1120246\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9977011494252873 p_th: 0.8027209110718578 | F1: 0.9977011494252873, AUC: 0.9996639784946236\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9938837920489297 p_th: 0.7194010553515626 | F1: 0.9938837920489297, AUC: 0.9994302586185189\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.995398773006135 p_th: 0.8822913208790564 | F1: 0.993114001530222, AUC: 0.9994829760403531\n",
      "> 3 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 0.995405819295559 p_th: 0.9094634473634876 | F1: 0.9938837920489296, AUC: 0.9993608133669609\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.995398773006135 p_th: 0.8309598426667172 | F1: 0.9946360153256705, AUC: 0.9993913869841847\n",
      "> average: 0.9955576613564092, std: 0.0012222869210699707\n",
      "(encode_labels) sample size: Counter({0: 1122425, 1: 1074})\n",
      "> sample size | n(+): 1074, n(-): 1122425 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1074, n(-): 1122425\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9907407407407408 p_th: 0.8469155127278413 | F1: 0.9884526558891455, AUC: 0.9990769785469623\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9907407407407407 p_th: 0.5032524542905642 | F1: 0.9907407407407407, AUC: 0.9986452220531767\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9976798143851509 p_th: 0.8647429553903304 | F1: 0.9953703703703703, AUC: 0.9999205546728297\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9953703703703703 p_th: 0.877065943109637 | F1: 0.9885057471264368, AUC: 0.9992488805431172\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9953488372093023 p_th: 0.8758299802714143 | F1: 0.9885057471264368, AUC: 0.9992922143579372\n",
      "> average: 0.993976100689261, std: 0.0027742024666997864\n",
      "(encode_labels) sample size: Counter({0: 1122967, 1: 532})\n",
      "> sample size | n(+): 532, n(-): 1122967 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 532, n(-): 1122967\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9037169601198959 | F1: 0.9953051643192489, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8627472092273423 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9953488372093023 p_th: 0.945136138746371 | F1: 0.9953488372093023, AUC: 0.9998242170333694\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.94916034106727 | F1: 0.9953051643192489, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9904761904761905 p_th: 0.9292904195621768 | F1: 0.9859154929577464, AUC: 0.9996155438575738\n",
      "> average: 0.9971650055370986, std: 0.003798691751852628\n",
      "(encode_labels) sample size: Counter({0: 1123413, 1: 86})\n",
      "> sample size | n(+): 86, n(-): 1123413 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 86, n(-): 1123413\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9786732828562739 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8654844532204371 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9796190820120372 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.972972972972973 p_th: 0.6305028331140453 | F1: 0.972972972972973, AUC: 0.9989106753812635\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.966134140539454 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9945945945945945, std: 0.010810810810810789\n",
      "(encode_labels) sample size: Counter({0: 1123285, 1: 214})\n",
      "> sample size | n(+): 214, n(-): 1123285 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 214, n(-): 1123285\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9183069936520855 | F1: 0.9662921348314606, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9879518072289156 p_th: 0.8638234120172794 | F1: 0.9647058823529412, AUC: 0.9994462901439645\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9767441860465116 p_th: 0.6471499260586675 | F1: 0.9767441860465116, AUC: 0.9990915697674418\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7323002478181835 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9772727272727273 p_th: 0.9050252692779366 | F1: 0.9662921348314606, AUC: 0.9990915697674418\n",
      "> average: 0.9883937441096309, std: 0.010285862903467778\n",
      "(encode_labels) sample size: Counter({0: 1122864, 1: 635})\n",
      "> sample size | n(+): 635, n(-): 1122864 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 635, n(-): 1122864\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9936880994828314 | F1: 0.996078431372549, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.984641252775717 | F1: 0.996078431372549, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9919997950870969 | F1: 0.996078431372549, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9838676943308389 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9782941999244213 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1122634, 1: 865})\n",
      "> sample size | n(+): 865, n(-): 1122634 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 865, n(-): 1122634\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9971181556195965 p_th: 0.8133931792456511 | F1: 0.9914040114613181, AUC: 0.9999331751812623\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8957847776509312 | F1: 0.9971181556195965, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9971014492753623 p_th: 0.8789912816975467 | F1: 0.9885057471264367, AUC: 0.999944312651052\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9971181556195965 p_th: 0.6989826350721762 | F1: 0.9942528735632185, AUC: 0.9999888625302104\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9971181556195965 p_th: 0.7859614059362525 | F1: 0.9942528735632185, AUC: 0.9999777250604207\n",
      "> average: 0.9976911832268304, std: 0.0011544265192328324\n",
      "(encode_labels) sample size: Counter({0: 1122967, 1: 532})\n",
      "> sample size | n(+): 532, n(-): 1122967 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 532, n(-): 1122967\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.986046511627907 p_th: 0.6054109418770894 | F1: 0.9814814814814815, AUC: 0.9995283018867924\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.939160253268113 | F1: 0.9907407407407407, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7998202402689463 | F1: 0.9907407407407407, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9953051643192489 p_th: 0.3500467282788782 | F1: 0.9905660377358491, AUC: 0.9999704264505825\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9905660377358491 p_th: 0.9108274345520503 | F1: 0.9767441860465116, AUC: 0.9993198083633998\n",
      "> average: 0.9943835427366011, std: 0.005440919092632685\n",
      "(encode_labels) sample size: Counter({0: 1122671, 1: 828})\n",
      "> sample size | n(+): 828, n(-): 1122671 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 828, n(-): 1122671\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.988095238095238 p_th: 0.9132050795369727 | F1: 0.9851632047477744, AUC: 0.9997939443892269\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.996996996996997 p_th: 0.5635835979179857 | F1: 0.996996996996997, AUC: 0.9999878790817193\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9969788519637462 p_th: 0.7745149673271191 | F1: 0.9939759036144578, AUC: 0.999914639351259\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9969788519637462 p_th: 0.8795270493751578 | F1: 0.9939759036144578, AUC: 0.9987439790256691\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.996996996996997 p_th: 0.5051341598741763 | F1: 0.9940119760479043, AUC: 0.9999878546443839\n",
      "> average: 0.9952093872033447, std: 0.003557083810024827\n",
      "(encode_labels) sample size: Counter({0: 1121028, 1: 2471})\n",
      "> sample size | n(+): 2471, n(-): 1121028 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2471, n(-): 1121028\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9989868287740628 p_th: 0.9408505751422668 | F1: 0.995967741935484, AUC: 0.9999945400094458\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9989868287740628 p_th: 0.8435972984992233 | F1: 0.9989868287740628, AUC: 0.9999781600377832\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.886640628805968 | F1: 0.9989888776541962, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9989909182643794 p_th: 0.7111962418417928 | F1: 0.9989909182643794, AUC: 0.9999972736814843\n",
      "> 4 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.922054958210193 | F1: 0.9989888776541962, AUC: 1.0\n",
      "> average: 0.999392915162501, std: 0.0004956849434250422\n",
      "(encode_labels) sample size: Counter({0: 1123485, 1: 14})\n",
      "> sample size | n(+): 14, n(-): 1123485 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 14, n(-): 1123485\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9696668131916392 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9724800618723248 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9369812336284101 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9371025361818622 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.1962643615339869 | F1: 0.8, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1121047, 1: 2452})\n",
      "> sample size | n(+): 2452, n(-): 1121047 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2452, n(-): 1121047\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8779490726084858 | F1: 0.9979633401221996, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9348460763747487 | F1: 0.9959432048681541, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9280597555846252 | F1: 0.9979674796747968, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8666134642753609 | F1: 0.9939148073022313, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9979633401221996 p_th: 0.8315550603241088 | F1: 0.9959349593495934, AUC: 0.9999958379000818\n",
      "> average: 0.9995926680244398, std: 0.0008146639511201759\n",
      "(encode_labels) sample size: Counter({0: 1123016, 1: 483})\n",
      "> sample size | n(+): 483, n(-): 1123016 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 483, n(-): 1123016\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9948186528497409 p_th: 0.8935139614587633 | F1: 0.989795918367347, AUC: 0.9999289015286171\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4365006633066816 | F1: 0.9895833333333333, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5260230741340238 | F1: 0.9948186528497409, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9896907216494846 p_th: 0.9313090561995161 | F1: 0.9846153846153847, AUC: 0.9998563218390805\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9469760863145638 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.996901874899845, std: 0.004126395102722349\n",
      "(encode_labels) sample size: Counter({0: 1123261, 1: 238})\n",
      "> sample size | n(+): 238, n(-): 1123261 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 238, n(-): 1123261\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9817786924807524 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9487874170878187 | F1: 0.9795918367346939, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8852496813025778 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9361999133998664 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9075686283957224 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1121237, 1: 2262})\n",
      "> sample size | n(+): 2262, n(-): 1121237 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2262, n(-): 1121237\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9977827050997783 p_th: 0.9341422541378036 | F1: 0.9955947136563876, AUC: 0.9999885959310282\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9977973568281938 p_th: 0.8797040218436409 | F1: 0.9956043956043955, AUC: 0.9999772254404844\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9175861010697378 | F1: 0.9988974641675854, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9977924944812362 p_th: 0.8322380944083716 | F1: 0.9966923925027563, AUC: 0.9999934785869402\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9115873872054593 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9986745112818417, std: 0.0010822672980739167\n",
      "(encode_labels) sample size: Counter({0: 1122616, 1: 883})\n",
      "> sample size | n(+): 883, n(-): 1122616 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 883, n(-): 1122616\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9943820224719102 p_th: 0.5906599988667453 | F1: 0.9943820224719102, AUC: 0.9999680204669011\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9860724233983287 p_th: 0.5757530727479838 | F1: 0.9860724233983287, AUC: 0.9994350282485875\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9106560202695619 | F1: 0.9971671388101983, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9971671388101983 p_th: 0.6441601145907722 | F1: 0.9971671388101983, AUC: 0.9999892795883362\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9081248070755599 | F1: 0.9860724233983287, AUC: 1.0\n",
      "> average: 0.9955243169360873, std: 0.005165374518449948\n",
      "(encode_labels) sample size: Counter({0: 1123328, 1: 171})\n",
      "> sample size | n(+): 171, n(-): 1123328 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 171, n(-): 1123328\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9816164727628774 | F1: 0.9855072463768115, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9897582877628033 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8847978762795818 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9479955897552939 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.980205609852008 | F1: 0.9855072463768115, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1121260, 1: 2239})\n",
      "> sample size | n(+): 2239, n(-): 1121260 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2239, n(-): 1121260\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9966555183946487 p_th: 0.9409543881869145 | F1: 0.9922480620155039, AUC: 0.9999601403061225\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9988826815642458 p_th: 0.7961318127087271 | F1: 0.9944382647385984, AUC: 0.9999966709278789\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5407479657140034 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9177759526170426 | F1: 0.9966629588431591, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9988851727982163 p_th: 0.5898826612514456 | F1: 0.9988851727982163, AUC: 0.9999966758855441\n",
      "> average: 0.9988846745514222, std: 0.001221232418831027\n",
      "(encode_labels) sample size: Counter({0: 1123460, 1: 39})\n",
      "> sample size | n(+): 39, n(-): 1123460 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 39, n(-): 1123460\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9761081435019121 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9883435863744365 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9809504017318115 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.979160559561869 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9817575169026578 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123242, 1: 257})\n",
      "> sample size | n(+): 257, n(-): 1123242 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 257, n(-): 1123242\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9514991601275219 | F1: 0.9902912621359222, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8741366340696394 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9631711893826034 | F1: 0.9904761904761905, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9639857497610334 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9110286153903633 | F1: 1.0, AUC: 1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123131, 1: 368})\n",
      "> sample size | n(+): 368, n(-): 1123131 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 368, n(-): 1123131\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9277032185269851 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9726027397260274 p_th: 0.9710756226797478 | F1: 0.9419354838709678, AUC: 0.9987159104806164\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.828316423160265 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9460271110288854 | F1: 0.9931972789115647, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7620954482498857 | F1: 0.9932885906040269, AUC: 1.0\n",
      "> average: 0.9945205479452055, std: 0.010958904109589041\n",
      "(encode_labels) sample size: Counter({0: 1123319, 1: 180})\n",
      "> sample size | n(+): 180, n(-): 1123319 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 180, n(-): 1123319\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8637471604534392 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9753950071002921 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.958241698979446 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9903546111262637 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9859154929577464 p_th: 0.9740627805286366 | F1: 0.9859154929577464, AUC: 0.999485596707819\n",
      "> average: 0.9971830985915492, std: 0.005633802816901445\n",
      "(encode_labels) sample size: Counter({0: 1118897, 1: 4602})\n",
      "> sample size | n(+): 4602, n(-): 1118897 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 4602, n(-): 1118897\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.998914223669924 p_th: 0.47635807359442517 | F1: 0.9983704508419337, AUC: 0.999751282939269\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9972929074174337 p_th: 0.6090897104827797 | F1: 0.9972929074174337, AUC: 0.999552869363529\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9972929074174337 p_th: 0.7713791724855162 | F1: 0.9956756756756756, AUC: 0.9997447776754005\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9983704508419337 p_th: 0.8663907772745335 | F1: 0.9978308026030369, AUC: 0.9998507944506558\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9983686786296899 p_th: 0.8308753638688677 | F1: 0.9967462039045553, AUC: 0.9998374092562556\n",
      "> average: 0.9980478335952829, std: 0.0006476854565884195\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123116, 1: 383})\n",
      "> sample size | n(+): 383, n(-): 1123116 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 383, n(-): 1123116\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9454750771987651 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9871794871794872 p_th: 0.4719460953277321 | F1: 0.9806451612903225, AUC: 0.9993788819875777\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.616164579796708 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8282584377631241 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9935483870967742 p_th: 0.2856758798792813 | F1: 0.9934640522875817, AUC: 0.9999432881528951\n",
      "> average: 0.9961455748552522, std: 0.005132365771292708\n",
      "(encode_labels) sample size: Counter({0: 1122859, 1: 640})\n",
      "> sample size | n(+): 640, n(-): 1122859 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 640, n(-): 1122859\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9961089494163424 p_th: 0.8766892652377224 | F1: 0.9884169884169884, AUC: 0.9999593098958334\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7205679699906973 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9921259842519685 p_th: 0.8397709666760085 | F1: 0.988235294117647, AUC: 0.9997965494791666\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9961089494163424 p_th: 0.6511739680514221 | F1: 0.9884169884169884, AUC: 0.9999796549479167\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.996078431372549 p_th: 0.9393495427733224 | F1: 0.996078431372549, AUC: 0.9998372395833333\n",
      "> average: 0.9960844628914405, std: 0.0024900690571422364\n",
      "(encode_labels) sample size: Counter({0: 1123079, 1: 420})\n",
      "> sample size | n(+): 420, n(-): 1123079 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 420, n(-): 1123079\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7252348042542518 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.933895755834521 | F1: 0.9824561403508771, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9302541946691291 | F1: 0.9940828402366864, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7725641927337333 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9463596871456686 | F1: 0.9940828402366864, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123390, 1: 109})\n",
      "> sample size | n(+): 109, n(-): 1123390 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 109, n(-): 1123390\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9503234331542862 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9073168129296181 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9475427314584857 | F1: 0.9777777777777777, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8153928369273298 | F1: 0.9777777777777777, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5531386704007343 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1119638, 1: 3861})\n",
      "> sample size | n(+): 3861, n(-): 1119638 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3861, n(-): 1119638\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9987063389391979 p_th: 0.7389046997973798 | F1: 0.9967721110393803, AUC: 0.9999955275380662\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.880865520337033 | F1: 0.9980607627666451, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9993527508090615 p_th: 0.7092854603175829 | F1: 0.9993527508090615, AUC: 0.9999972047112915\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9993535875888817 p_th: 0.43709029557185813 | F1: 0.9987063389391979, AUC: 0.9999994414244124\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.848934291164483 | F1: 0.9980607627666451, AUC: 1.0\n",
      "> average: 0.9994825354674284, std: 0.00048404374767271945\n",
      "(encode_labels) sample size: Counter({0: 1123426, 1: 73})\n",
      "> sample size | n(+): 73, n(-): 1123426 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 73, n(-): 1123426\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9745197845496772 | F1: 0.967741935483871, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.967741935483871 p_th: 0.5766592494084143 | F1: 0.967741935483871, AUC: 0.9954545454545455\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9085412123830726 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9621740953889959 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6786855933918722 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9935483870967742, std: 0.012903225806451599\n",
      "(encode_labels) sample size: Counter({0: 1123277, 1: 222})\n",
      "> sample size | n(+): 222, n(-): 1123277 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 222, n(-): 1123277\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8207329313845154 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 0.967741935483871 p_th: 0.576792058496042 | F1: 0.967741935483871, AUC: 0.9988304093567252\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.989010989010989 p_th: 0.5442581687145859 | F1: 0.989010989010989, AUC: 0.9998329156223893\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7951939051604533 | F1: 0.9777777777777777, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9355695305124856 | F1: 0.9777777777777777, AUC: 1.0\n",
      "> average: 0.991350584898972, std: 0.012548140745894505\n",
      "(encode_labels) sample size: Counter({0: 1123247, 1: 252})\n",
      "> sample size | n(+): 252, n(-): 1123247 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 252, n(-): 1123247\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.500599516932866 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9902912621359222 p_th: 0.6352632144413509 | F1: 0.9902912621359222, AUC: 0.9998701467341903\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9902912621359222 p_th: 0.5572694205783376 | F1: 0.9902912621359222, AUC: 0.9998701467341903\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.99009900990099 p_th: 0.49686095252338847 | F1: 0.98, AUC: 0.9950993377483444\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8327404142955613 | F1: 0.99009900990099, AUC: 1.0\n",
      "> average: 0.994136306834567, std: 0.00478820006005836\n",
      "(encode_labels) sample size: Counter({0: 1120395, 1: 3104})\n",
      "> sample size | n(+): 3104, n(-): 1120395 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 3104, n(-): 1120395\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7961119290756393 | F1: 0.999195494770716, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8307894181839223 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.999195494770716 p_th: 0.5214796158797924 | F1: 0.999195494770716, AUC: 0.9999982703480578\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.998389694041868 p_th: 0.8846314459720191 | F1: 0.9975903614457832, AUC: 0.9999161118808062\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9983870967741936 p_th: 0.8707804695657636 | F1: 0.9975864843121481, AUC: 0.9999143822288641\n",
      "> average: 0.9991944571173554, std: 0.0007207321388229676\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1122860, 1: 639})\n",
      "> sample size | n(+): 639, n(-): 1122860 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 639, n(-): 1122860\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9961089494163424 p_th: 0.9258886413587163 | F1: 0.9922480620155039, AUC: 0.9999593098958334\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9921259842519685 p_th: 0.7730969085029639 | F1: 0.9843749999999999, AUC: 0.9998974737532809\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.996078431372549 p_th: 0.8858009163298057 | F1: 0.996078431372549, AUC: 0.9999388054830287\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9922480620155039 p_th: 0.7511288286672665 | F1: 0.9884169884169884, AUC: 0.9999184073107049\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8866937963892534 | F1: 0.9961089494163424, AUC: 1.0\n",
      "> average: 0.9953122854112728, std: 0.002923638190107689\n",
      "(encode_labels) sample size: Counter({0: 1123427, 1: 72})\n",
      "> sample size | n(+): 72, n(-): 1123427 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 72, n(-): 1123427\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9587540873043028 | F1: 0.9655172413793104, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.955019996534267 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.972446378487499 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9911689141237284 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9600658803675166 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1122679, 1: 820})\n",
      "> sample size | n(+): 820, n(-): 1122679 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 820, n(-): 1122679\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9539558648314816 | F1: 0.993939393939394, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9311894591412986 | F1: 0.9969604863221885, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9969418960244648 p_th: 0.883572300147866 | F1: 0.9969418960244648, AUC: 0.9998884592504461\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.993939393939394 p_th: 0.8567295727612971 | F1: 0.993939393939394, AUC: 0.9999256395002974\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9969604863221885 p_th: 0.9028104294276377 | F1: 0.993939393939394, AUC: 0.9999876065833828\n",
      "> average: 0.9975683552572094, std: 0.0022696743541282365\n",
      "(encode_labels) sample size: Counter({0: 1122893, 1: 606})\n",
      "> sample size | n(+): 606, n(-): 1122893 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 606, n(-): 1122893\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.960608555801897 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.976449378250019 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9771650771935092 | F1: 0.9958847736625513, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9824208130961798 | F1: 0.9959183673469388, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9468828397152803 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123044, 1: 455})\n",
      "> sample size | n(+): 455, n(-): 1123044 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 455, n(-): 1123044\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9891304347826088 p_th: 0.8080531258397962 | F1: 0.9837837837837838, AUC: 0.9998792416374834\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.994535519125683 p_th: 0.9597365887751103 | F1: 0.9891304347826088, AUC: 0.9999597472124945\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9370652441070387 | F1: 0.994535519125683, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8786279948375402 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7706310476025308 | F1: 0.994535519125683, AUC: 1.0\n",
      "> average: 0.9967331907816582, std: 0.004350811117382314\n",
      "(encode_labels) sample size: Counter({0: 1121127, 1: 2372})\n",
      "> sample size | n(+): 2372, n(-): 1121127 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 2372, n(-): 1121127\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8783008656922714 | F1: 0.9978947368421053, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9271834075287358 | F1: 0.9947643979057591, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9968387776606955 p_th: 0.8491391392436259 | F1: 0.9926854754440961, AUC: 0.9999807670969412\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9989440337909187 p_th: 0.7486440907979878 | F1: 0.995798319327731, AUC: 0.9999940696988296\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9989440337909187 p_th: 0.803692690677032 | F1: 0.9968454258675079, AUC: 0.9999955522741222\n",
      "> average: 0.9989453690485066, std: 0.001154315704538517\n",
      "(encode_labels) sample size: Counter({0: 1122941, 1: 558})\n",
      "> sample size | n(+): 558, n(-): 1122941 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 558, n(-): 1122941\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9315522030200761 | F1: 0.9911504424778761, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9955555555555555 p_th: 0.6541765373879846 | F1: 0.9911504424778761, AUC: 0.9999466950959488\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9954751131221719 p_th: 0.6757168768850468 | F1: 0.990990990990991, AUC: 0.999892429743176\n",
      "> 3 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 0.9955156950672646 p_th: 0.1828212030464557 | F1: 0.9954751131221719, AUC: 0.9999731074357939\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9698686954596438 | F1: 0.9955555555555555, AUC: 1.0\n",
      "> average: 0.9973092727489984, std: 0.0021971168703661006\n",
      "(encode_labels) sample size: Counter({0: 1123057, 1: 442})\n",
      "> sample size | n(+): 442, n(-): 1123057 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 442, n(-): 1123057\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.801708079028407 | F1: 0.9723756906077349, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7802231032503405 | F1: 0.9944134078212291, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9887640449438202 p_th: 0.2966390138243345 | F1: 0.9886363636363636, AUC: 0.9991520033919864\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9832402234636871 p_th: 0.31222412226179447 | F1: 0.9714285714285714, AUC: 0.9996140651801029\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9943502824858756 p_th: 0.558429272556484 | F1: 0.9943502824858756, AUC: 0.9998713550600343\n",
      "> average: 0.9932709101786765, std: 0.00652154666384974\n",
      "(encode_labels) sample size: Counter({0: 1123454, 1: 45})\n",
      "> sample size | n(+): 45, n(-): 1123454 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 45, n(-): 1123454\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.963093165379487 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7758335648107347 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.948698482319397 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7528084195851013 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7144183221150203 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1121926, 1: 1573})\n",
      "> sample size | n(+): 1573, n(-): 1121926 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1573, n(-): 1121926\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9081075720488524 | F1: 0.9984152139461173, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9984101748807631 p_th: 0.963215989151159 | F1: 0.9936708860759493, AUC: 0.999986548291633\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9984051036682616 p_th: 0.905245065966965 | F1: 0.9936507936507937, AUC: 0.9999898790888481\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9597597474612642 | F1: 0.9952456418383517, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.947860049600115 | F1: 0.9968354430379747, AUC: 1.0\n",
      "> average: 0.9993630557098049, std: 0.0007800959011158441\n",
      "(encode_labels) sample size: Counter({0: 1122307, 1: 1192})\n",
      "> sample size | n(+): 1192, n(-): 1122307 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1192, n(-): 1122307\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9979035639412999 p_th: 0.8821414149666261 | F1: 0.9916666666666667, AUC: 0.9999941317309047\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9958333333333333 p_th: 0.5074975190523113 | F1: 0.9958333333333333, AUC: 0.9999707405565147\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9719108853635072 | F1: 0.9979123173277662, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9979035639412999 p_th: 0.9150960103199764 | F1: 0.99581589958159, AUC: 0.9999706176176765\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9238129351348638 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9983280922431866, std: 0.0015604372253367636\n",
      "(encode_labels) sample size: Counter({0: 1122663, 1: 836})\n",
      "> sample size | n(+): 836, n(-): 1122663 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 836, n(-): 1122663\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9970149253731343 p_th: 0.9223348304228411 | F1: 0.9910979228486646, AUC: 0.9999761433308682\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9500537279363684 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9497701618168284 | F1: 0.9940476190476192, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9970326409495549 p_th: 0.7988709982667926 | F1: 0.9970326409495549, AUC: 0.9999524760003802\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9970149253731343 p_th: 0.9083708060241861 | F1: 0.9910979228486646, AUC: 0.9997370528404269\n",
      "> average: 0.9982124983391648, std: 0.0014595033300906484\n",
      "(encode_labels) sample size: Counter({0: 1122264, 1: 1235})\n",
      "> sample size | n(+): 1235, n(-): 1122264 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1235, n(-): 1122264\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9532750134203716 | F1: 0.997979797979798, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.995967741935484 p_th: 0.8652416662498928 | F1: 0.9919678714859438, AUC: 0.9999726816262081\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5262709177778011 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9407983696955726 | F1: 0.993963782696177, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9701047976296374 | F1: 0.997979797979798, AUC: 1.0\n",
      "> average: 0.9991935483870968, std: 0.0016129032258064056\n",
      "(encode_labels) sample size: Counter({0: 1123317, 1: 182})\n",
      "> sample size | n(+): 182, n(-): 1123317 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 182, n(-): 1123317\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.972972972972973 p_th: 0.9473728095437816 | F1: 0.972972972972973, AUC: 0.9987373737373737\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9726426586262631 | F1: 0.9866666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6373727491163002 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9732776229275133 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7759543246332941 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9945945945945945, std: 0.010810810810810787\n",
      "(encode_labels) sample size: Counter({0: 1123289, 1: 210})\n",
      "> sample size | n(+): 210, n(-): 1123289 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 210, n(-): 1123289\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9889479999957096 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9823546461987808 | F1: 0.988235294117647, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9705828903287073 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9656947127024529 | F1: 0.988235294117647, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9668987863769695 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123312, 1: 187})\n",
      "> sample size | n(+): 187, n(-): 1123312 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 187, n(-): 1123312\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9829789046746396 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2507000434461381 | F1: 0.9866666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5204885775124041 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9843980637301345 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.982424367924127 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123481, 1: 18})\n",
      "> sample size | n(+): 18, n(-): 1123481 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 18, n(-): 1123481\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9565459157695645 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9300471519911612 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.89762317754459 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9448953704566705 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.888888888888889 p_th: 0.15086993486331693 | F1: 0.8571428571428571, AUC: 0.9750000000000001\n",
      "> average: 0.9777777777777779, std: 0.04444444444444442\n",
      "(encode_labels) sample size: Counter({0: 1123226, 1: 273})\n",
      "> sample size | n(+): 273, n(-): 1123226 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 273, n(-): 1123226\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.972972972972973 p_th: 0.9372283925349528 | F1: 0.9557522123893805, AUC: 0.9983370288248337\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9649122807017544 p_th: 0.1480033012557228 | F1: 0.9549549549549549, AUC: 0.9980044345898004\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9908256880733944 p_th: 0.4415498760927347 | F1: 0.9906542056074767, AUC: 0.9998870822041553\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9277294575944808 | F1: 0.9908256880733944, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9818181818181818 p_th: 0.7382862377120385 | F1: 0.9818181818181818, AUC: 0.9988845510317903\n",
      "> average: 0.9821058247132607, std: 0.012452688073124805\n",
      "(encode_labels) sample size: Counter({0: 1123046, 1: 453})\n",
      "> sample size | n(+): 453, n(-): 1123046 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 453, n(-): 1123046\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9199331545689249 | F1: 0.9891304347826088, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.994475138121547 p_th: 0.9867953382839746 | F1: 0.994475138121547, AUC: 0.9998787976729153\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9944134078212291 p_th: 0.5810042763141888 | F1: 0.9888888888888889, AUC: 0.9999183006535948\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9363297863639569 | F1: 0.989010989010989, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8004599363663295 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9977777091885554, std: 0.002721809276739573\n",
      "(encode_labels) sample size: Counter({0: 1122305, 1: 1194})\n",
      "> sample size | n(+): 1194, n(-): 1122305 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1194, n(-): 1122305\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9958333333333333 p_th: 0.9307189680826958 | F1: 0.9855670103092783, AUC: 0.9988737358706372\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9896049896049895 p_th: 0.18904832955619283 | F1: 0.9895615866388309, AUC: 0.9987869624837383\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9937629937629938 p_th: 0.8487068428942044 | F1: 0.989648033126294, AUC: 0.9995032841682054\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9958333333333333 p_th: 0.8675451925510511 | F1: 0.9958333333333333, AUC: 0.9988137257193613\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9979123173277662 p_th: 0.8727786504115392 | F1: 0.9979123173277662, AUC: 0.9999707814216592\n",
      "> average: 0.9945893934724832, std: 0.0028165160783124384\n",
      "(encode_labels) sample size: Counter({0: 1123241, 1: 258})\n",
      "> sample size | n(+): 258, n(-): 1123241 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 258, n(-): 1123241\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7827235444628946 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9610046999720221 | F1: 0.9904761904761905, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.28574161398420145 | F1: 0.99009900990099, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9894004738747765 | F1: 0.9807692307692307, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9904761904761905 p_th: 0.7090017509715444 | F1: 0.9904761904761905, AUC: 0.9996253746253747\n",
      "> average: 0.9980952380952381, std: 0.003809523809523796\n",
      "(encode_labels) sample size: Counter({0: 1123327, 1: 172})\n",
      "> sample size | n(+): 172, n(-): 1123327 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 172, n(-): 1123327\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9352194032944919 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9582247517585539 | F1: 0.9859154929577464, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9510159939681085 | F1: 0.9859154929577464, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9897054249287154 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.751737188177412 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6549566579000694 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6978784711945882 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7018804982248127 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8301258509761665 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9514229982485466 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123319, 1: 180})\n",
      "> sample size | n(+): 180, n(-): 1123319 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 180, n(-): 1123319\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9631594329189002 | F1: 0.9863013698630138, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9863013698630138 p_th: 0.5967957796642519 | F1: 0.972972972972973, AUC: 0.9997427983539094\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.972972972972973 p_th: 0.7879098200511145 | F1: 0.972972972972973, AUC: 0.9925411522633745\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8902297922726876 | F1: 0.9863013698630138, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9058179635952689 | F1: 0.9863013698630138, AUC: 1.0\n",
      "> average: 0.9918548685671974, std: 0.010829559799561994\n",
      "(encode_labels) sample size: Counter({0: 1123438, 1: 61})\n",
      "> sample size | n(+): 61, n(-): 1123438 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 61, n(-): 1123438\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9600000000000001 p_th: 0.9164139616390168 | F1: 0.9600000000000001, AUC: 0.9977477477477478\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9223173746928278 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5791857435509409 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9436670532024107 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9006533036220435 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.992, std: 0.01599999999999997\n",
      "(encode_labels) sample size: Counter({0: 1123349, 1: 150})\n",
      "> sample size | n(+): 150, n(-): 1123349 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 150, n(-): 1123349\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7475487535588401 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.763140547750322 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9186568273916903 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9015881387076662 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9836065573770492 p_th: 0.9459980258765776 | F1: 0.9836065573770492, AUC: 0.9951851851851852\n",
      "> average: 0.9967213114754099, std: 0.006557377049180336\n",
      "(encode_labels) sample size: Counter({0: 1123379, 1: 120})\n",
      "> sample size | n(+): 120, n(-): 1123379 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 120, n(-): 1123379\n",
      "> 0 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9201803510976984 | F1: 0.9600000000000001, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3352033302381314 | F1: 0.9787234042553191, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9314988274221527 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7837029243043688 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9600000000000001 p_th: 0.8796973570114667 | F1: 0.9600000000000001, AUC: 0.9976851851851851\n",
      "> average: 0.992, std: 0.01599999999999997\n",
      "(encode_labels) sample size: Counter({0: 1123428, 1: 71})\n",
      "> sample size | n(+): 71, n(-): 1123428 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 71, n(-): 1123428\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9868876225385528 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9638362236229499 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9509320970532065 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9838758551211851 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9846135609199813 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1122810, 1: 689})\n",
      "> sample size | n(+): 689, n(-): 1122810 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 689, n(-): 1122810\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.209006939423681 | F1: 0.9927007299270074, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9149471497421607 | F1: 0.9963636363636363, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8989623631469793 | F1: 0.996389891696751, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.996389891696751 p_th: 0.9554185591715293 | F1: 0.989247311827957, AUC: 0.999157806084851\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9963636363636363 p_th: 0.9715829197296129 | F1: 0.9928057553956835, AUC: 0.9999649085868687\n",
      "> average: 0.9985507056120774, std: 0.001775035286564002\n",
      "(encode_labels) sample size: Counter({0: 1123257, 1: 242})\n",
      "> sample size | n(+): 242, n(-): 1123257 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 242, n(-): 1123257\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9353842326062054 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.98989898989899 p_th: 0.7401650252450772 | F1: 0.98989898989899, AUC: 0.9935256861365236\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9791666666666666 p_th: 0.8226817555020228 | F1: 0.9791666666666666, AUC: 0.998311048557354\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8942417440200308 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7650065049337246 | F1: 0.9896907216494846, AUC: 1.0\n",
      "> average: 0.9938131313131313, std: 0.008302667789279603\n",
      "(encode_labels) sample size: Counter({0: 1123392, 1: 107})\n",
      "> sample size | n(+): 107, n(-): 1123392 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 107, n(-): 1123392\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8195355507549745 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9635477082650771 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9610721200873203 | F1: 0.9777777777777777, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.975609756097561 p_th: 0.974830365609121 | F1: 0.975609756097561, AUC: 0.9962797619047619\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9225970088361979 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9951219512195122, std: 0.009756097560975618\n",
      "(encode_labels) sample size: Counter({0: 1123034, 1: 465})\n",
      "> sample size | n(+): 465, n(-): 1123034 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 465, n(-): 1123034\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9945945945945946 p_th: 0.8688630650667342 | F1: 0.989247311827957, AUC: 0.9998072994951247\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9946524064171123 p_th: 0.9420029450018486 | F1: 0.9946524064171123, AUC: 0.9995760588892743\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8528842786114426 | F1: 0.9893617021276596, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9946524064171123 p_th: 0.588045456571712 | F1: 0.9893617021276596, AUC: 0.9999614598990249\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9945945945945946 p_th: 0.9414561215161265 | F1: 0.9787234042553192, AUC: 0.9998843796970748\n",
      "> average: 0.9956988004046827, std: 0.002150755200168963\n",
      "(encode_labels) sample size: Counter({0: 1123484, 1: 15})\n",
      "> sample size | n(+): 15, n(-): 1123484 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 15, n(-): 1123484\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9434679362594889 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8611910898925454 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.370742622072181 | F1: 0.5, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9720140267957589 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7664200259758139 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123446, 1: 53})\n",
      "> sample size | n(+): 53, n(-): 1123446 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 53, n(-): 1123446\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5387365234283211 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5649032856143151 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7078122403650468 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9111044128120721 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9565217391304348 p_th: 0.5933684141440434 | F1: 0.9565217391304348, AUC: 0.9853372434017595\n",
      "> average: 0.9913043478260869, std: 0.017391304347826077\n",
      "(encode_labels) sample size: Counter({0: 1122396, 1: 1103})\n",
      "> sample size | n(+): 1103, n(-): 1122396 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1103, n(-): 1122396\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9866071428571429 p_th: 0.8770410743682718 | F1: 0.9866071428571429, AUC: 0.9953179040614619\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9932584269662922 p_th: 0.8756050001873311 | F1: 0.9932584269662922, AUC: 0.9981955133901108\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9865470852017937 p_th: 0.7896769323560823 | F1: 0.9865470852017937, AUC: 0.9963952210931063\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9932279909706546 p_th: 0.5023097080667787 | F1: 0.9932279909706546, AUC: 0.9973290304861302\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9954954954954954 p_th: 0.8721559317417762 | F1: 0.9954954954954954, AUC: 0.9990347820729595\n",
      "> average: 0.9910272282982758, std: 0.003725475890511556\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9764191708391751 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8016868377973085 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9228687557607411 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9638173497159805 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4671925238781515 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123454, 1: 45})\n",
      "> sample size | n(+): 45, n(-): 1123454 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 45, n(-): 1123454\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7641008968683658 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.859116419224149 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8923962011327887 | F1: 0.9, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2363390211404617 | F1: 0.9411764705882353, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7957978821478014 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123444, 1: 55})\n",
      "> sample size | n(+): 55, n(-): 1123444 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 55, n(-): 1123444\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9588858829338344 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9152969515188225 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8502044758556543 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.624296288476664 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8595935041150125 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123234, 1: 265})\n",
      "> sample size | n(+): 265, n(-): 1123234 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 265, n(-): 1123234\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9613158731898843 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9550990891432628 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9906542056074767 p_th: 0.8746277431848501 | F1: 0.9814814814814815, AUC: 0.9998813338079982\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.982359048386143 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9403435943007423 | F1: 0.9906542056074767, AUC: 1.0\n",
      "> average: 0.9981308411214954, std: 0.0037383177570093017\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9662302975125862 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9651345096785124 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9752473885450677 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9687445624869834 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9568139384730234 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123477, 1: 22})\n",
      "> sample size | n(+): 22, n(-): 1123477 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 22, n(-): 1123477\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.8571428571428571 p_th: 0.9898894682213148 | F1: 0.8571428571428571, AUC: 0.875\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8609319803819552 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5679855625719671 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9887968871957133 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9715706468858976 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9714285714285715, std: 0.05714285714285716\n",
      "(encode_labels) sample size: Counter({0: 1122516, 1: 983})\n",
      "> sample size | n(+): 983, n(-): 1122516 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 983, n(-): 1122516\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9974683544303797 p_th: 0.9076532714733162 | F1: 0.9974683544303797, AUC: 0.9999827927385357\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9974554707379135 p_th: 0.9084620465423814 | F1: 0.9924433249370278, AUC: 0.9999741891078034\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9241316228089836 | F1: 0.9974554707379135, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9949238578680203 p_th: 0.660736755876393 | F1: 0.9924050632911393, AUC: 0.9998270494638534\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9974554707379135 p_th: 0.9171088423726742 | F1: 0.9949494949494949, AUC: 0.9999827635241698\n",
      "> average: 0.9974606307548454, std: 0.00160522433007236\n",
      "(encode_labels) sample size: Counter({0: 1122203, 1: 1296})\n",
      "> sample size | n(+): 1296, n(-): 1122203 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 1296, n(-): 1122203\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9961538461538463 p_th: 0.8799738433095893 | F1: 0.9923371647509579, AUC: 0.9999702236206092\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9923371647509579 p_th: 0.852554435040063 | F1: 0.9923371647509579, AUC: 0.9993349941936059\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9904397705544934 p_th: 0.822028720875477 | F1: 0.9866666666666666, AUC: 0.9983871127829996\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9923664122137404 p_th: 0.7460014530897114 | F1: 0.9904761904761905, AUC: 0.9979605979605979\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9923371647509579 p_th: 0.9495404322823183 | F1: 0.9904397705544934, AUC: 0.9970334371878773\n",
      "> average: 0.9927268716847992, std: 0.0018659399567290764\n",
      "(encode_labels) sample size: Counter({0: 1123258, 1: 241})\n",
      "> sample size | n(+): 241, n(-): 1123258 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 241, n(-): 1123258\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9896907216494846 p_th: 0.7283094808842329 | F1: 0.9896907216494846, AUC: 0.9998563218390805\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.93479554763037 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9795918367346939 p_th: 0.9398309487001354 | F1: 0.9600000000000001, AUC: 0.9988505747126436\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3708396860700026 | F1: 0.9896907216494846, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9896907216494846 p_th: 0.7545545412718848 | F1: 0.9896907216494846, AUC: 0.9998553240740741\n",
      "> average: 0.9917946560067327, std: 0.007647446865993335\n",
      "(encode_labels) sample size: Counter({0: 1122589, 1: 910})\n",
      "> sample size | n(+): 910, n(-): 1122589 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 910, n(-): 1122589\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8870861783326772 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.994475138121547 p_th: 0.94971707340516 | F1: 0.994475138121547, AUC: 0.9995874089280683\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.924679262659921 | F1: 0.9918256130790191, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9972451790633609 p_th: 0.8300552103658785 | F1: 0.9945054945054945, AUC: 0.9999798736062473\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.17208423747549587 | F1: 0.9972451790633609, AUC: 1.0\n",
      "> average: 0.9983440634369817, std: 0.0022091857171918504\n",
      "(encode_labels) sample size: Counter({0: 1123165, 1: 334})\n",
      "> sample size | n(+): 334, n(-): 1123165 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 334, n(-): 1123165\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8414312820725502 | F1: 0.9925925925925926, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6791869594789457 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8443872844778163 | F1: 0.9852941176470589, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.41229324036199155 | F1: 0.9924812030075187, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8973672561705353 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9674725491547584 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4751563653529249 | F1: 0.8, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.972196261619922 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.38488791373619513 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9688364564665993 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1122965, 1: 534})\n",
      "> sample size | n(+): 534, n(-): 1122965 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 534, n(-): 1122965\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9747656594604365 | F1: 0.9907407407407407, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9772159430908104 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8066353780094925 | F1: 0.9907407407407407, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9726056560395517 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9447666227889844 | F1: 0.9953488372093023, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123412, 1: 87})\n",
      "> sample size | n(+): 87, n(-): 1123412 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 87, n(-): 1123412\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9405583531809597 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8660503862910677 | F1: 0.972972972972973, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6593621863877038 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9295401237273215 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9411764705882353 p_th: 0.7998399084907767 | F1: 0.9411764705882353, AUC: 0.9841628959276018\n",
      "> average: 0.9882352941176471, std: 0.023529411764705885\n",
      "(encode_labels) sample size: Counter({0: 1123472, 1: 27})\n",
      "> sample size | n(+): 27, n(-): 1123472 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 27, n(-): 1123472\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9090909090909091 p_th: 0.10038581253022127 | F1: 0.888888888888889, AUC: 0.9882352941176471\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5722875629890989 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9523794597404741 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9535050791887908 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9662383355691823 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9818181818181818, std: 0.036363636363636376\n",
      "(encode_labels) sample size: Counter({0: 1122740, 1: 759})\n",
      "> sample size | n(+): 759, n(-): 1122740 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 759, n(-): 1122740\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8784700236478402 | F1: 0.9967213114754099, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9759091684863088 | F1: 0.993421052631579, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9967213114754099 p_th: 0.8387326494861914 | F1: 0.9967213114754099, AUC: 0.9998554077501446\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9900990099009901 p_th: 0.9257140314049738 | F1: 0.9836065573770492, AUC: 0.9998409485251591\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9966996699669968 p_th: 0.9522177335311297 | F1: 0.9870129870129869, AUC: 0.9999710815500289\n",
      "> average: 0.9967039982686794, std: 0.003615340776313365\n",
      "(encode_labels) sample size: Counter({0: 1123225, 1: 274})\n",
      "> sample size | n(+): 274, n(-): 1123225 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 274, n(-): 1123225\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9909909909909909 p_th: 0.6101622435106389 | F1: 0.9909909909909909, AUC: 0.9994490358126722\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6522075197730159 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9909909909909909 p_th: 0.9568440365474611 | F1: 0.9909909909909909, AUC: 0.9995565410199556\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9166259632293605 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8371749591788396 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9963963963963962, std: 0.004413495032041826\n",
      "(encode_labels) sample size: Counter({0: 1123074, 1: 425})\n",
      "> sample size | n(+): 425, n(-): 1123074 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 425, n(-): 1123074\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7976442981441907 | F1: 0.9883720930232558, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9354445840387177 | F1: 0.9883720930232558, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9883720930232558 p_th: 0.6921652986028205 | F1: 0.9770114942528736, AUC: 0.9994002306805075\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9940828402366864 p_th: 0.886022703652319 | F1: 0.9826589595375723, AUC: 0.9999077277970012\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9941520467836257 p_th: 0.43560782357695976 | F1: 0.9882352941176471, AUC: 0.9997231833910035\n",
      "> average: 0.9953213960087135, std: 0.004358276737631298\n",
      "(encode_labels) sample size: Counter({0: 1123391, 1: 108})\n",
      "> sample size | n(+): 108, n(-): 1123391 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 108, n(-): 1123391\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9678804470647686 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9255560302586773 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9767441860465117 p_th: 0.9178692718720547 | F1: 0.9767441860465117, AUC: 0.9956043956043956\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4074860290169154 | F1: 0.975609756097561, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8568763715599674 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9953488372093023, std: 0.009302325581395321\n",
      "(encode_labels) sample size: Counter({0: 1123431, 1: 68})\n",
      "> sample size | n(+): 68, n(-): 1123431 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 68, n(-): 1123431\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.365325733875354 | F1: 0.962962962962963, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5724414120120604 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.962962962962963 p_th: 0.794187078458649 | F1: 0.962962962962963, AUC: 0.9962476547842402\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.962962962962963 p_th: 0.9254791372067178 | F1: 0.962962962962963, AUC: 0.9887429643527205\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7703213187867337 | F1: 0.9655172413793104, AUC: 1.0\n",
      "> average: 0.9851851851851852, std: 0.01814436846506055\n",
      "(encode_labels) sample size: Counter({0: 1123400, 1: 99})\n",
      "> sample size | n(+): 99, n(-): 1123400 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 99, n(-): 1123400\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9412239660739086 | F1: 0.975609756097561, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8509634204709875 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9523809523809523 p_th: 0.4688005009293167 | F1: 0.9268292682926829, AUC: 0.9915254237288136\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5344698831762436 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6887508799362678 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9904761904761905, std: 0.019047619047619067\n",
      "(encode_labels) sample size: Counter({0: 1123444, 1: 55})\n",
      "> sample size | n(+): 55, n(-): 1123444 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size (after downsampling majority) | n(+): 55, n(-): 1123444\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9593574933252628 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6802348575924403 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8869538209447378 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8943993378877916 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8317928987454651 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123327, 1: 172})\n",
      "> sample size | n(+): 172, n(-): 1123327 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 172, n(-): 1123327\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9883835058715597 | F1: 0.9855072463768115, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9848722894769096 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9894723617878748 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9767770487343161 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.980521847417563 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1122646, 1: 853})\n",
      "> sample size | n(+): 853, n(-): 1122646 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 853, n(-): 1122646\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9883720930232557 p_th: 0.9361989851250565 | F1: 0.9883720930232557, AUC: 0.9990976790935673\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9913043478260869 p_th: 0.8915800821690805 | F1: 0.9913043478260869, AUC: 0.9978641264619883\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9855072463768115 p_th: 0.8105407464016777 | F1: 0.9855072463768115, AUC: 0.9959099264705882\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9970674486803519 p_th: 0.888456041354167 | F1: 0.9941520467836257, AUC: 0.9985983455882353\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9884393063583815 p_th: 0.8549740116648006 | F1: 0.9884393063583815, AUC: 0.9971446882045296\n",
      "> average: 0.9901380884529776, std: 0.0039198339986073714\n",
      "(encode_labels) sample size: Counter({0: 1123334, 1: 165})\n",
      "> sample size | n(+): 165, n(-): 1123334 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 165, n(-): 1123334\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.922856918150015 | F1: 0.9705882352941176, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9799576583221934 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9865687895321446 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9419104196271781 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9134057882350559 | F1: 0.9850746268656716, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123451, 1: 48})\n",
      "> sample size | n(+): 48, n(-): 1123451 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 48, n(-): 1123451\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9531266610899178 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8498577583668302 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9569604781634423 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9088357841075759 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8997940527148016 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123122, 1: 377})\n",
      "> sample size | n(+): 377, n(-): 1123122 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 377, n(-): 1123122\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9933774834437086 p_th: 0.8037152593954339 | F1: 0.9933774834437086, AUC: 0.9999412628487518\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9706174594330422 | F1: 0.9934640522875817, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9201001456265558 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9933774834437086 p_th: 0.9095283652166832 | F1: 0.9933774834437086, AUC: 0.9999410029498526\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9541743477681977 | F1: 0.9868421052631579, AUC: 1.0\n",
      "> average: 0.9973509933774836, std: 0.003244357275209524\n",
      "(encode_labels) sample size: Counter({0: 1123474, 1: 25})\n",
      "> sample size | n(+): 25, n(-): 1123474 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 25, n(-): 1123474\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6233131435199311 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9763822013627732 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9890326442410894 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9871006869724097 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8458923952010686 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9712508809636491 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8925568591883161 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.15642817659489713 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9259004219422619 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9056823364405447 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123302, 1: 197})\n",
      "> sample size | n(+): 197, n(-): 1123302 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 197, n(-): 1123302\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9834702058025611 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9771297982126786 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8548460615781869 | F1: 0.975609756097561, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9912083703089958 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9843901848611735 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123431, 1: 68})\n",
      "> sample size | n(+): 68, n(-): 1123431 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 68, n(-): 1123431\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.709208798450339 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8634883324939023 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9124057059819881 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9467897830332701 | F1: 0.962962962962963, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9306797434439896 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123331, 1: 168})\n",
      "> sample size | n(+): 168, n(-): 1123331 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 168, n(-): 1123331\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.894797903463925 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9850746268656716 p_th: 0.9771251265789633 | F1: 0.9444444444444444, AUC: 0.9994175888177053\n",
      "> 2 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 0.9850746268656716 p_th: 0.8678292071260791 | F1: 0.9850746268656716, AUC: 0.9921992199219922\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9850746268656716 p_th: 0.9497194358641522 | F1: 0.9565217391304348, AUC: 0.9924992499249925\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7314901354035438 | F1: 0.9855072463768115, AUC: 1.0\n",
      "> average: 0.991044776119403, std: 0.007311909679949807\n",
      "(encode_labels) sample size: Counter({0: 1123353, 1: 146})\n",
      "> sample size | n(+): 146, n(-): 1123353 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 146, n(-): 1123353\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9043902353307235 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6936316652180383 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9217749144753113 | F1: 0.983050847457627, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5833620336584757 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.983050847457627 p_th: 0.887667010246288 | F1: 0.983050847457627, AUC: 0.9964328180737217\n",
      "> average: 0.9966101694915255, std: 0.006779661016949178\n",
      "(encode_labels) sample size: Counter({0: 1123366, 1: 133})\n",
      "> sample size | n(+): 133, n(-): 1123366 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 133, n(-): 1123366\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9660130683024031 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9375636179303576 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.942401287209729 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9900160842250892 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9657101402335918 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123298, 1: 201})\n",
      "> sample size | n(+): 201, n(-): 1123298 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 201, n(-): 1123298\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.975609756097561 p_th: 0.7566746247431868 | F1: 0.975609756097561, AUC: 0.9948347107438017\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9876543209876543 p_th: 0.7571607113180578 | F1: 0.9876543209876543, AUC: 0.9989669421487604\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.890415510430654 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9096623038531221 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8745698434628709 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9926528154170431, std: 0.009771327252898011\n",
      "(encode_labels) sample size: Counter({0: 1123228, 1: 271})\n",
      "> sample size | n(+): 271, n(-): 1123228 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 271, n(-): 1123228\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9908256880733944 p_th: 0.8812584244437865 | F1: 0.9818181818181818, AUC: 0.9997727789138833\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5014160338908734 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9818181818181818 p_th: 0.3946882555289967 | F1: 0.9724770642201834, AUC: 0.9995455578277663\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9821428571428572 p_th: 0.6922915736182171 | F1: 0.9821428571428572, AUC: 0.9993265993265993\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9618006233778245 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9909573454068866, std: 0.008059479540168724\n",
      "(encode_labels) sample size: Counter({0: 1123458, 1: 41})\n",
      "> sample size | n(+): 41, n(-): 1123458 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 41, n(-): 1123458\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.987649602966762 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9730118580874834 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9681022400335303 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9706724100198635 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9779607860000203 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9791984360706318 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9623485491863185 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.961275382999758 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9135158049621728 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9826710534915706 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123342, 1: 157})\n",
      "> sample size | n(+): 157, n(-): 1123342 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 157, n(-): 1123342\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7805672108482693 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9696969696969697 p_th: 0.8987001601264177 | F1: 0.9696969696969697, AUC: 0.9990026595744681\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8032911650089091 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9590725426140907 | F1: 0.96875, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9559704048050627 | F1: 0.96875, AUC: 1.0\n",
      "> average: 0.9939393939393939, std: 0.01212121212121211\n",
      "(encode_labels) sample size: Counter({0: 1123338, 1: 161})\n",
      "> sample size | n(+): 161, n(-): 1123338 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 161, n(-): 1123338\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8722618610423984 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7720984511431057 | F1: 0.9846153846153847, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9513587061365014 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9705882352941176 p_th: 0.9003878124111447 | F1: 0.9705882352941176, AUC: 0.9974747474747474\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9268925496311876 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9941176470588236, std: 0.011764705882352944\n",
      "(encode_labels) sample size: Counter({0: 1123383, 1: 116})\n",
      "> sample size | n(+): 116, n(-): 1123383 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 116, n(-): 1123383\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.932400555570185 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9352675699993216 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9408211137511601 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9789906650567471 | F1: 0.9795918367346939, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9688384712851033 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123282, 1: 217})\n",
      "> sample size | n(+): 217, n(-): 1123282 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 217, n(-): 1123282\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9555555555555556 p_th: 0.29610402241397543 | F1: 0.942528735632184, AUC: 0.9982247470264513\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9885057471264368 p_th: 0.7701917882473921 | F1: 0.9772727272727273, AUC: 0.9996503496503497\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9484762700138635 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 0.9772727272727273 p_th: 0.44377199539199075 | F1: 0.9655172413793104, AUC: 0.9974955277280859\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9418076957494982 | F1: 0.9885057471264368, AUC: 1.0\n",
      "> average: 0.9842668059909438, std: 0.016651056711713925\n",
      "(encode_labels) sample size: Counter({0: 1123470, 1: 29})\n",
      "> sample size | n(+): 29, n(-): 1123470 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 29, n(-): 1123470\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.727960769420226 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.48761568892154833 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6346828909292911 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9525973160516844 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7682169342337752 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8812352353501611 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8733631313319704 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8926878127427232 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6712398472230892 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8633085189236427 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123298, 1: 201})\n",
      "> sample size | n(+): 201, n(-): 1123298 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 201, n(-): 1123298\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.607275742304374 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9876543209876543 p_th: 0.8585037554963716 | F1: 0.9876543209876543, AUC: 0.9997933884297521\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9713198811083182 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6245064694173755 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8224758105662026 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9975308641975309, std: 0.004938271604938294\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9751272157341158 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9811469030281397 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.983326732697817 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9741726730095157 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9201355261322999 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123314, 1: 185})\n",
      "> sample size | n(+): 185, n(-): 1123314 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 185, n(-): 1123314\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8246236626512731 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9866666666666666 p_th: 0.7985400317172433 | F1: 0.9866666666666666, AUC: 0.9997565132700268\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9283667189424696 | F1: 0.9866666666666666, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9642801993132922 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9827249938358663 | F1: 0.9866666666666666, AUC: 1.0\n",
      "> average: 0.9973333333333333, std: 0.005333333333333368\n",
      "(encode_labels) sample size: Counter({0: 1123432, 1: 67})\n",
      "> sample size | n(+): 67, n(-): 1123432 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 67, n(-): 1123432\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9117196640662484 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9680762769875897 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.43121513830151226 | F1: 0.962962962962963, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9785556175098901 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9867565686118358 | F1: 0.962962962962963, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123466, 1: 33})\n",
      "> sample size | n(+): 33, n(-): 1123466 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 33, n(-): 1123466\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8554997239831355 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9260884375047128 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9055351511125492 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9571341517254965 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.783734036401023 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123442, 1: 57})\n",
      "> sample size | n(+): 57, n(-): 1123442 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 57, n(-): 1123442\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9512355499885881 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.904885994616705 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8800725826160213 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9565217391304348 p_th: 0.8380824717759902 | F1: 0.9565217391304348, AUC: 0.9946524064171123\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9648434351505739 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9913043478260869, std: 0.017391304347826077\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8423494846251514 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9514724201405612 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9546232856762858 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6667638876891944 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.895514528296601 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123474, 1: 25})\n",
      "> sample size | n(+): 25, n(-): 1123474 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 25, n(-): 1123474\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5472538006037301 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9876180565567287 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9090909090909091 p_th: 0.0875759228661024 | F1: 0.888888888888889, AUC: 0.9866666666666667\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.18687718323483918 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9797638068905005 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9818181818181818, std: 0.036363636363636376\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6907530657200245 | F1: 1.0, AUC: 1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8811369717455244 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9477899095624867 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.941262030461344 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9412303926866725 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123450, 1: 49})\n",
      "> sample size | n(+): 49, n(-): 1123450 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 49, n(-): 1123450\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8088356159445788 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8247170220095912 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6431411805390943 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6422957190229328 | F1: 0.9523809523809523, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5153567665437675 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123451, 1: 48})\n",
      "> sample size | n(+): 48, n(-): 1123451 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 48, n(-): 1123451\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9865067655535008 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.750811123367372 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5977579551288023 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9695584958671064 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.981438101014895 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9716760434411907 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9544595885523722 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9760130801796164 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9742671751080594 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8779436561195207 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123459, 1: 40})\n",
      "> sample size | n(+): 40, n(-): 1123459 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 40, n(-): 1123459\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9411764705882353 p_th: 0.33840109513549194 | F1: 0.7999999999999999, AUC: 0.9583333333333334\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5487746756916769 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6892918488835481 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9411764705882353 p_th: 0.9293300325521594 | F1: 0.9411764705882353, AUC: 0.9583333333333334\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.16648972939003878 | F1: 0.9333333333333333, AUC: 1.0\n",
      "> average: 0.9764705882352942, std: 0.028817526385684456\n",
      "(encode_labels) sample size: Counter({0: 1123250, 1: 249})\n",
      "> sample size | n(+): 249, n(-): 1123250 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 249, n(-): 1123250\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.99009900990099 p_th: 0.1894928313858769 | F1: 0.98989898989899, AUC: 0.9998666666666666\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2733225147697365 | F1: 0.9896907216494846, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9166298796372292 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.99009900990099 p_th: 0.8767052025659643 | F1: 0.9803921568627451, AUC: 0.9993288590604027\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.99009900990099 p_th: 0.9261160416993612 | F1: 0.99009900990099, AUC: 0.9993288590604027\n",
      "> average: 0.994059405940594, std: 0.0048504747381845695\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123271, 1: 228})\n",
      "> sample size | n(+): 228, n(-): 1123271 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 228, n(-): 1123271\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9703999543278399 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9810610947858611 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9818174868901306 | F1: 0.989010989010989, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9327933933593051 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.975456590554925 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5536911211012875 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9755521597567974 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9500471032588954 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.733861120401915 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6747202638387968 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1122920, 1: 579})\n",
      "> sample size | n(+): 579, n(-): 1122920 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 579, n(-): 1122920\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.994076983991848 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9915540040424737 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9896908242978969 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.99494676097594 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9928334268009607 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123480, 1: 19})\n",
      "> sample size | n(+): 19, n(-): 1123480 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 19, n(-): 1123480\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9773605118049848 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9651718736847251 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9657808226895707 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.8571428571428571 p_th: 0.955760058234591 | F1: 0.75, AUC: 0.8863636363636362\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9744668509655513 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9714285714285715, std: 0.05714285714285716\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8190297410233001 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9303454444513388 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.47747519874423067 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3314511740396533 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7041230295100063 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9857393223156976 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.987497906899585 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9610128150320529 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9845216025398026 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9288163148112962 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123410, 1: 89})\n",
      "> sample size | n(+): 89, n(-): 1123410 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 89, n(-): 1123410\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9436982301499443 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9209303043291166 | F1: 0.9444444444444444, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8668230690671428 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8494113420290503 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9611840038613281 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9415539878832141 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7456766710750081 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4905894957682087 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8091783483966712 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8213278024040113 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123415, 1: 84})\n",
      "> sample size | n(+): 84, n(-): 1123415 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 84, n(-): 1123415\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9714285714285714 p_th: 0.8287392700577219 | F1: 0.9714285714285714, AUC: 0.9930795847750865\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9696969696969697 p_th: 0.4612781447549311 | F1: 0.9375, AUC: 0.9987745098039216\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9591028528666204 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9057629955016664 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9379849993498366 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9882251082251082, std: 0.01443163050113567\n",
      "(encode_labels) sample size: Counter({0: 1123481, 1: 18})\n",
      "> sample size | n(+): 18, n(-): 1123481 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 18, n(-): 1123481\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8022183122633886 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5392022953999781 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.93182154507501 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7990661503681743 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.888888888888889 p_th: 0.6372071098020731 | F1: 0.888888888888889, AUC: 0.9\n",
      "> average: 0.9777777777777779, std: 0.04444444444444442\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9659835729766343 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.09801045711842554 | F1: 0.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.5206330521038457 | F1: 0.8, AUC: 0.875\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9425017233470422 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5383222970118805 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123161, 1: 338})\n",
      "> sample size | n(+): 338, n(-): 1123161 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 338, n(-): 1123161\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5310817795307159 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9352009343684602 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9925925925925926 p_th: 0.7235837418913218 | F1: 0.9925925925925926, AUC: 0.9999264759944122\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9366041804075442 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7006148901443618 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9985185185185184, std: 0.002962962962962967\n",
      "(encode_labels) sample size: Counter({0: 1123447, 1: 52})\n",
      "> sample size | n(+): 52, n(-): 1123447 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 52, n(-): 1123447\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9727926754561763 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9640114379745188 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8858825921569821 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9596742768588882 | F1: 0.9523809523809523, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9344456582439283 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123480, 1: 19})\n",
      "> sample size | n(+): 19, n(-): 1123480 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 19, n(-): 1123480\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9508571581607075 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9854900712358922 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9573212281499186 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9359660303180544 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5028396317336381 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123320, 1: 179})\n",
      "> sample size | n(+): 179, n(-): 1123320 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 179, n(-): 1123320\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.937600318720366 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9914610005866286 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9859154929577464 p_th: 0.9828773995346213 | F1: 0.9722222222222222, AUC: 0.9994807892004154\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9622968633604826 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9772245381670628 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9971830985915492, std: 0.005633802816901445\n",
      "(encode_labels) sample size: Counter({0: 1123437, 1: 62})\n",
      "> sample size | n(+): 62, n(-): 1123437 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 62, n(-): 1123437\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9450960667172825 | F1: 0.9600000000000001, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7888565589432004 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8262443341251237 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.963909831721384 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9680136826423865 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123402, 1: 97})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size | n(+): 97, n(-): 1123402 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 97, n(-): 1123402\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9192120408701439 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.1865583579327853 | F1: 0.9743589743589743, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9255571281767255 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9679255452442659 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9835453510263054 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123473, 1: 26})\n",
      "> sample size | n(+): 26, n(-): 1123473 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 26, n(-): 1123473\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9459888208278137 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9808222500174302 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.982585274953951 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9898064266221293 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.985188079027246 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123314, 1: 185})\n",
      "> sample size | n(+): 185, n(-): 1123314 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 185, n(-): 1123314\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9547454106365333 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9887764911021539 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9879235069737402 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9230311835713302 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9936496240992175 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.979934523381977 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9614219816572416 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9586748358145762 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9752093112157635 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9743512598708108 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123320, 1: 179})\n",
      "> sample size | n(+): 179, n(-): 1123320 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 179, n(-): 1123320\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9002848393049961 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9611973569457366 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9843149648517806 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9924842324108987 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9729783457413468 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9377176713362438 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9745841651983959 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9737558958613591 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6575738444378554 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9719171396361517 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123452, 1: 47})\n",
      "> sample size | n(+): 47, n(-): 1123452 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 47, n(-): 1123452\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9079570779805034 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9839546816323682 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9727978210087335 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9122827520624676 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9627375743785588 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123461, 1: 38})\n",
      "> sample size | n(+): 38, n(-): 1123461 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 38, n(-): 1123461\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7506001447210762 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9780679068592713 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9704301330946576 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8745627779189354 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9619712405268744 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123446, 1: 53})\n",
      "> sample size | n(+): 53, n(-): 1123446 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 53, n(-): 1123446\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9667300652993949 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6535490834332951 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.937170710890102 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9146455908682318 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7625310303874085 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123469, 1: 30})\n",
      "> sample size | n(+): 30, n(-): 1123469 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 30, n(-): 1123469\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9640773112201626 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8900899066535608 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9382043894972 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.915437059187243 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9405514473653575 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123477, 1: 22})\n",
      "> sample size | n(+): 22, n(-): 1123477 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 22, n(-): 1123477\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9820864160460755 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9120069574414145 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9386638957487621 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.1530317357449426 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9819047452730343 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(encode_labels) sample size: Counter({0: 1123466, 1: 33})\n",
      "> sample size | n(+): 33, n(-): 1123466 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 33, n(-): 1123466\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3274422934484662 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.25575099490229014 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9327392558117354 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9448206475061953 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9373065857482205 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123468, 1: 31})\n",
      "> sample size | n(+): 31, n(-): 1123468 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 31, n(-): 1123468\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9408285742363176 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3269007155517452 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9417604914254227 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4095502983394346 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.923076923076923 p_th: 0.9439531474437777 | F1: 0.923076923076923, AUC: 0.9907407407407407\n",
      "> average: 0.9846153846153847, std: 0.030769230769230795\n",
      "(encode_labels) sample size: Counter({0: 1123396, 1: 103})\n",
      "> sample size | n(+): 103, n(-): 1123396 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 103, n(-): 1123396\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7949612120184884 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9767441860465117 p_th: 0.9461734156661293 | F1: 0.9767441860465117, AUC: 0.9969278033794163\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.975609756097561 p_th: 0.9392221683246256 | F1: 0.975609756097561, AUC: 0.9967741935483871\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9567119073500355 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.29220553201718974 | F1: 0.975609756097561, AUC: 1.0\n",
      "> average: 0.9904707884288145, std: 0.011676365140039524\n",
      "(encode_labels) sample size: Counter({0: 1123453, 1: 46})\n",
      "> sample size | n(+): 46, n(-): 1123453 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 46, n(-): 1123453\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9089195199737763 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9551748388475338 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9159482320537018 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9662605832094091 | F1: 0.9523809523809523, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.062466584583024135 | F1: 0.9411764705882353, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.06716212276103611 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9820824584611342 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9771188872319981 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9678271778413151 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8622551259966662 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123450, 1: 49})\n",
      "> sample size | n(+): 49, n(-): 1123450 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 49, n(-): 1123450\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9389322772030073 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9725788960712423 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9343438873296512 | F1: 0.9523809523809523, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9780055495963086 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8879536034144724 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123475, 1: 24})\n",
      "> sample size | n(+): 24, n(-): 1123475 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 24, n(-): 1123475\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9441982303758925 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9917540205496248 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9915588143511449 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.08549265511788633 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9390010959032093 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123477, 1: 22})\n",
      "> sample size | n(+): 22, n(-): 1123477 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 22, n(-): 1123477\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7849924869868761 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9612287736440008 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7992582538447613 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4197774183199471 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.865454030514704 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123444, 1: 55})\n",
      "> sample size | n(+): 55, n(-): 1123444 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 55, n(-): 1123444\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9783255792172163 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9755224939910139 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9748683188375187 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9696536656238164 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.958844259902906 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.892266915742273 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8636539660321078 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9660528005840566 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8479741550369917 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6825096082642349 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9805158287003675 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4699059177507024 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.968918746856453 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9762350655551923 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9819972085405477 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123452, 1: 47})\n",
      "> sample size | n(+): 47, n(-): 1123452 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 47, n(-): 1123452\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8381543003235148 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.22013633205723898 | F1: 0.9473684210526316, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2871689925774292 | F1: 0.9473684210526316, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9839628228949316 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9834980632347801 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9340329592518349 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.28256741720176604 | F1: 0.0, AUC: 0.75\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.20538816625604575 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9470951144123049 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9421164321722426 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9333333333333332, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9545193794747806 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9632558260205819 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9806307316673829 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9514003738005443 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9599019119458801 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123457, 1: 42})\n",
      "> sample size | n(+): 42, n(-): 1123457 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 42, n(-): 1123457\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4034866852689172 | F1: 0.9333333333333333, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5967528945696695 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7434577313145727 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8375487826384342 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.866065186990704 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8878284258365552 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9619192252292507 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9649966011155932 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.06300272822600574 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9729733239027787 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123484, 1: 15})\n",
      "> sample size | n(+): 15, n(-): 1123484 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 15, n(-): 1123484\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9874771374847228 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9859805696002416 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9866955986142444 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9823024349913453 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9614757604726413 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123475, 1: 24})\n",
      "> sample size | n(+): 24, n(-): 1123475 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 24, n(-): 1123475\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6568852599448741 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9691997543747093 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7732583901538302 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6893467866736377 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9502268197188597 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1122943, 1: 556})\n",
      "> sample size | n(+): 556, n(-): 1122943 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 556, n(-): 1122943\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9818773060322671 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9874432680068337 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.984001972830836 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9950309497150607 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9929769065295392 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123472, 1: 27})\n",
      "> sample size | n(+): 27, n(-): 1123472 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 27, n(-): 1123472\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8878822266690284 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9607376926921211 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7951370273792764 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9560544768070701 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8821285503153633 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.4 p_th: 0.02638450141430324 | F1: 0.0, AUC: 0.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8472028902736165 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9476572817033401 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9469164275242213 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9436052181893202 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.8800000000000001, std: 0.24\n",
      "(encode_labels) sample size: Counter({0: 1123420, 1: 79})\n",
      "> sample size | n(+): 79, n(-): 1123420 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 79, n(-): 1123420\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7727755473266967 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8983151595891259 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.7053606297698373 | F1: 0.9696969696969697, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7126823839571412 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9411764705882353 p_th: 0.7306167414025817 | F1: 0.9411764705882353, AUC: 0.9813829787234043\n",
      "> average: 0.9882352941176471, std: 0.023529411764705885\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7174925351276429 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2945941526927452 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8695314013893174 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7404799118014905 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3569653921937001 | F1: 0.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123476, 1: 23})\n",
      "> sample size | n(+): 23, n(-): 1123476 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 23, n(-): 1123476\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4989855460432445 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.479168010657365 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9334546108277417 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.43347386011711514 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8371933538160611 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123459, 1: 40})\n",
      "> sample size | n(+): 40, n(-): 1123459 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 40, n(-): 1123459\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.879642972168109 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9124582684682323 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9787751458330807 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6806809974005014 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8716329636115341 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123475, 1: 24})\n",
      "> sample size | n(+): 24, n(-): 1123475 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 24, n(-): 1123475\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7816534487885054 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6483889689590154 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3649016280811243 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7948889068184158 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9291658948401181 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123384, 1: 115})\n",
      "> sample size | n(+): 115, n(-): 1123384 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 115, n(-): 1123384\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9583333333333334 p_th: 0.37039130457092095 | F1: 0.9361702127659574, AUC: 0.9949590422180214\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9090026848748828 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9301838798913066 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.631995921375002 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9787234042553191 p_th: 0.8743341015885115 | F1: 0.9787234042553191, AUC: 0.9993698802772527\n",
      "> average: 0.9874113475177305, std: 0.016711874706309464\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9710631819338056 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.858530531952454 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9703186973432464 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9860883547997433 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9287698331999648 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123459, 1: 40})\n",
      "> sample size | n(+): 40, n(-): 1123459 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 40, n(-): 1123459\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9234725854035795 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9749772520566186 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9816173402272678 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5127432504430488 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9152431355265649 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.975687251624264 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9723699405176003 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9728798863916058 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9729820210984621 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9790333315556922 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9351547430283725 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7471247484615887 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9416709105320744 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8716061096622783 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9052664021759101 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123463, 1: 36})\n",
      "> sample size | n(+): 36, n(-): 1123463 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 36, n(-): 1123463\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6824224226336775 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.8750000000000001 p_th: 0.08356517440123121 | F1: 0.8571428571428571, AUC: 0.974025974025974\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9650368693210568 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9356047641311159 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9333333333333333 p_th: 0.8896850698058548 | F1: 0.9333333333333333, AUC: 0.9863945578231292\n",
      "> average: 0.9616666666666667, std: 0.05044248650140515\n",
      "(encode_labels) sample size: Counter({0: 1123462, 1: 37})\n",
      "> sample size | n(+): 37, n(-): 1123462 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size (after downsampling majority) | n(+): 37, n(-): 1123462\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9170401583687293 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7463159346702868 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6620163615064041 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8920685727516038 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7381714540423623 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123313, 1: 186})\n",
      "> sample size | n(+): 186, n(-): 1123313 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 186, n(-): 1123313\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7691036302962143 | F1: 0.9736842105263158, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8523448531823486 | F1: 0.9736842105263158, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7389439540242035 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9870129870129869 p_th: 0.5478560193075622 | F1: 0.9870129870129869, AUC: 0.9995258416311048\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.972972972972973 p_th: 0.8175498382770514 | F1: 0.9599999999999999, AUC: 0.9968346725103482\n",
      "> average: 0.9919971919971919, std: 0.010760093090633123\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8954286486647165 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.44020004189136436 | F1: 0.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7948242436252392 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7221725143824964 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.17561742578644238 | F1: 0.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123477, 1: 22})\n",
      "> sample size | n(+): 22, n(-): 1123477 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 22, n(-): 1123477\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9635713884370578 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.94587653329024 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9618510914292691 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9182086406764677 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9544414561954895 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9637485474346891 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9674085387909197 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.964752433382087 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.20164578703624547 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9526133773870279 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123474, 1: 25})\n",
      "> sample size | n(+): 25, n(-): 1123474 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 25, n(-): 1123474\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9090938380706879 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8836877039614368 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9689942162799146 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9566570718669535 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9774008904927723 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123351, 1: 148})\n",
      "> sample size | n(+): 148, n(-): 1123351 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 148, n(-): 1123351\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9745499016671799 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9836065573770492 p_th: 0.5618718706002064 | F1: 0.9836065573770492, AUC: 0.999625468164794\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7649444974597864 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.876239421511664 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9717401919798139 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9967213114754099, std: 0.006557377049180336\n",
      "(encode_labels) sample size: Counter({0: 1123432, 1: 67})\n",
      "> sample size | n(+): 67, n(-): 1123432 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 67, n(-): 1123432\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8212440548021067 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9648661749872943 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.47468499932347613 | F1: 0.962962962962963, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9503983713365523 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9719022417855571 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123458, 1: 41})\n",
      "> sample size | n(+): 41, n(-): 1123458 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 41, n(-): 1123458\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9333333333333333 p_th: 0.13101298753688875 | F1: 0.8571428571428571, AUC: 0.99\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9927817770647762 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9521861590727574 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9602266843092218 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.994443112656974 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9866666666666667, std: 0.02666666666666666\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9460826604216982 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.21562298687354642 | F1: 0.0, AUC: 0.9\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9695191196559002 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4531746921699326 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7313137847006287 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123451, 1: 48})\n",
      "> sample size | n(+): 48, n(-): 1123451 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 48, n(-): 1123451\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9774495815502499 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8723536092107856 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7055182579149563 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7883026301562518 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9926472675865339 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123473, 1: 26})\n",
      "> sample size | n(+): 26, n(-): 1123473 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 26, n(-): 1123473\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8169862487309625 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.947796166073319 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9593047350483093 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7570250574645498 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9636864418745451 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123473, 1: 26})\n",
      "> sample size | n(+): 26, n(-): 1123473 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 26, n(-): 1123473\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9296006353680613 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9535278909962464 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.21130378719038914 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7260775381211612 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9390160290828455 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123453, 1: 46})\n",
      "> sample size | n(+): 46, n(-): 1123453 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 46, n(-): 1123453\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8523290342848562 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7420276272780314 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9535903466930012 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9758552440778834 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9724781840520855 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9797803573141388 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8722511198638758 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9025190648527217 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9800243732097088 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9648650370445027 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123484, 1: 15})\n",
      "> sample size | n(+): 15, n(-): 1123484 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 15, n(-): 1123484\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9357201123801937 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9208152603502124 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9682264758201723 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9574280219663334 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8653866952718147 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123465, 1: 34})\n",
      "> sample size | n(+): 34, n(-): 1123465 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 34, n(-): 1123465\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9461521715795898 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9672151904816692 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.39054344762823323 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7172015575018205 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9290690173743839 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9718944346357169 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9765669041052358 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9630908892072091 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.926304817759762 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9731653090285767 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123454, 1: 45})\n",
      "> sample size | n(+): 45, n(-): 1123454 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 45, n(-): 1123454\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7990901859086359 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7962645225652343 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8734828633644116 | F1: 0.9473684210526316, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5520364135005374 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7802370246960283 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123286, 1: 213})\n",
      "> sample size | n(+): 213, n(-): 1123286 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 213, n(-): 1123286\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9885057471264368 p_th: 0.9697939628289494 | F1: 0.9885057471264368, AUC: 0.9998183139534884\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.36132893196433297 | F1: 0.988235294117647, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9166749904173482 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.988235294117647 p_th: 0.9275873656822456 | F1: 0.9767441860465117, AUC: 0.9996279761904762\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9690183477725878 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9953482082488169, std: 0.005697899983990684\n",
      "(encode_labels) sample size: Counter({0: 1123469, 1: 30})\n",
      "> sample size | n(+): 30, n(-): 1123469 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 30, n(-): 1123469\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9577103726404447 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9884899464236944 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5852799870125417 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.1696577807391152 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9856725584121727 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123466, 1: 33})\n",
      "> sample size | n(+): 33, n(-): 1123466 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 33, n(-): 1123466\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.29497667736733574 | F1: 0.8333333333333333, AUC: 1.0\n",
      "> 1 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9575245468676439 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.959101333626427 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8365275883482873 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3145309671258981 | F1: 0.923076923076923, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4925970055564295 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5793282131851517 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9326554032342204 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.23193310302916295 | F1: 0.8, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9290307431056262 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9109817391307479 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9798915154615154 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2786855651984538 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9836481760934999 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9754767006205757 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123438, 1: 61})\n",
      "> sample size | n(+): 61, n(-): 1123438 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 61, n(-): 1123438\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5919468311573265 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.547760311286874 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.961097164006033 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9482648112297434 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8515529864614375 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9759633790075661 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9759916616791923 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9844510635326975 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.16640794297899206 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9841613719101114 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123396, 1: 103})\n",
      "> sample size | n(+): 103, n(-): 1123396 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 103, n(-): 1123396\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.42386927190188434 | F1: 0.975609756097561, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7259111755548808 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.975609756097561 p_th: 0.5799433725248658 | F1: 0.9523809523809523, AUC: 0.9991935483870968\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7611285245191063 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9767441860465117 p_th: 0.6537424399653297 | F1: 0.9767441860465117, AUC: 0.994535519125683\n",
      "> average: 0.9904707884288145, std: 0.011676365140039523\n",
      "(encode_labels) sample size: Counter({0: 1123480, 1: 19})\n",
      "> sample size | n(+): 19, n(-): 1123480 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 19, n(-): 1123480\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9561577520656102 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.8571428571428571 p_th: 0.16140628330426515 | F1: 0.8, AUC: 0.9722222222222222\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8910738913201467 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9301450873787146 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.952248629660292 | F1: 0.888888888888889, AUC: 1.0\n",
      "> average: 0.9714285714285715, std: 0.05714285714285716\n",
      "(encode_labels) sample size: Counter({0: 1123484, 1: 15})\n",
      "> sample size | n(+): 15, n(-): 1123484 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 15, n(-): 1123484\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8952288753758307 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9564198260464013 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9773862999498488 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9850070495384816 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9094945843072739 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123408, 1: 91})\n",
      "> sample size | n(+): 91, n(-): 1123408 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 91, n(-): 1123408\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8300361508309243 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8410701422071011 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9720609183844617 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9743589743589743 p_th: 0.8314082249857151 | F1: 0.9268292682926829, AUC: 0.9980506822612086\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8421467816947116 | F1: 0.972972972972973, AUC: 1.0\n",
      "> average: 0.9948717948717949, std: 0.010256410256410265\n",
      "(encode_labels) sample size: Counter({0: 1123427, 1: 72})\n",
      "> sample size | n(+): 72, n(-): 1123427 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 72, n(-): 1123427\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9655172413793104 p_th: 0.7801018996852151 | F1: 0.9655172413793104, AUC: 0.9967532467532467\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.967741935483871 p_th: 0.9159446665948644 | F1: 0.9375, AUC: 0.9984496124031007\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.967741935483871 p_th: 0.8086701325684746 | F1: 0.967741935483871, AUC: 0.9968992248062015\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9533085701582401 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7978364987537069 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9802002224694105, std: 0.01618684739807006\n",
      "(encode_labels) sample size: Counter({0: 1123465, 1: 34})\n",
      "> sample size | n(+): 34, n(-): 1123465 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 34, n(-): 1123465\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.889998807305532 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9629668586474898 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9383531319822342 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9474029896314943 | F1: 1.0, AUC: 1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7241592392866785 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123465, 1: 34})\n",
      "> sample size | n(+): 34, n(-): 1123465 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 34, n(-): 1123465\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9834918339782857 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9650110024839116 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9644681668374455 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8822233165705382 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9838999923141383 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123390, 1: 109})\n",
      "> sample size | n(+): 109, n(-): 1123390 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 109, n(-): 1123390\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9380596208322393 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8352005715211692 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9777777777777777 p_th: 0.7340776088933283 | F1: 0.9565217391304348, AUC: 0.9993006993006993\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9777777777777777 p_th: 0.39094991363979026 | F1: 0.9545454545454546, AUC: 0.9993006993006993\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9575711298865429 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9911111111111112, std: 0.010886621079036363\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9067211186555346 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9746148083823212 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9739196486135049 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9619028229217895 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5627001227738293 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123455, 1: 44})\n",
      "> sample size | n(+): 44, n(-): 1123455 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 44, n(-): 1123455\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7989961957997626 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8538332778685449 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9413587770746467 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.957070027679551 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.417656074247083 | F1: 0.8750000000000001, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123479, 1: 20})\n",
      "> sample size | n(+): 20, n(-): 1123479 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 20, n(-): 1123479\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.09689269252355838 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.24339622869015023 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9792908880019041 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9659932841054012 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9696213998126861 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123449, 1: 50})\n",
      "> sample size | n(+): 50, n(-): 1123449 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 50, n(-): 1123449\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.969366576750315 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9732458454891066 | F1: 0.9523809523809523, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5146222023016811 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7425987208855659 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8553514536892567 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9814748162747811 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.97710628410943 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.614396190999918 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9620131443702816 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9676481128586089 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9655472200932297 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.1303320496233735 | F1: 0.6666666666666666, AUC: 0.875\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.41151458617612835 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9640574821878061 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7233178554184044 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8765742546315443 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9614352010010404 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9593385764029161 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9520429463504356 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9438644612463531 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9861735806214812 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9169940799350652 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9836879995011605 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7699450364230778 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9879037613486786 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9078999367918298 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.32154816798010954 | F1: 0.5, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9657162084204646 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.885138597563111 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9094137340530344 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123476, 1: 23})\n",
      "> sample size | n(+): 23, n(-): 1123476 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 23, n(-): 1123476\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9706353015349616 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9854237675482915 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8208688723886507 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.94855920583534 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9188191371247543 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123444, 1: 55})\n",
      "> sample size | n(+): 55, n(-): 1123444 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 55, n(-): 1123444\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9804603095508717 | F1: 0.9166666666666666, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8774081955683577 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9565217391304348 p_th: 0.7726833459724014 | F1: 0.9166666666666666, AUC: 0.9862258953168044\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9523809523809523 p_th: 0.7648632746663963 | F1: 0.9090909090909091, AUC: 0.9944903581267218\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9403407076986368 | F1: 0.9565217391304348, AUC: 1.0\n",
      "> average: 0.9817805383022774, std: 0.022352579008307717\n",
      "(encode_labels) sample size: Counter({0: 1123473, 1: 26})\n",
      "> sample size | n(+): 26, n(-): 1123473 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 26, n(-): 1123473\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8950842161764009 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8641848313910298 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8284434153380565 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7561204337223923 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.945992231803033 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9550349922376631 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6995596522275153 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7988281592040656 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8519902584145255 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9739658608852966 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.975709286088344 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9784809338306888 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9204846775847241 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8627972163595943 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9555758555121967 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8796060731349267 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.527379504531981 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9010944741086762 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.7414270721916403 | F1: 0.8, AUC: 0.9375\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.48433732584660555 | F1: 0.8, AUC: 1.0\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.821285234988691 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8058681839424554 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9362809596261618 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9482964018048223 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8049732710566745 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123481, 1: 18})\n",
      "> sample size | n(+): 18, n(-): 1123481 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 18, n(-): 1123481\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9528040247852633 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9530507779753898 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9816605093190484 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9851029079730913 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9771836566478593 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123436, 1: 63})\n",
      "> sample size | n(+): 63, n(-): 1123436 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 63, n(-): 1123436\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9671691937401369 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2203689804011 | F1: 0.9600000000000001, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9600000000000001 p_th: 0.936552768491131 | F1: 0.9600000000000001, AUC: 0.9956140350877193\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.33912083687051775 | F1: 0.9565217391304348, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7997861783178348 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.992, std: 0.01599999999999997\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.39050688274040096 | F1: 0.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8369377582434885 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9250181852400188 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9051543386497021 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.809781117837687 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9745120323496419 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9797388468217413 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9815946985503159 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9832995418184817 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8693317795820772 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123464, 1: 35})\n",
      "> sample size | n(+): 35, n(-): 1123464 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 35, n(-): 1123464\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9183515892303022 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9807911184525426 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9784032501168803 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5741535495084223 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9038387927663233 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9675986422054074 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9747754504162033 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9595050835975845 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8448715895470551 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9774293941589298 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123485, 1: 14})\n",
      "> sample size | n(+): 14, n(-): 1123485 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 14, n(-): 1123485\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9544364651232702 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9742339428744984 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.547757881140857 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5506191498668407 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9846896865837208 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7253447705333774 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9117408854450099 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9406971677895085 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9769088236328688 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9726103322233479 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123423, 1: 76})\n",
      "> sample size | n(+): 76, n(-): 1123423 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 76, n(-): 1123423\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9227634311815898 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9499670095948829 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.967741935483871 p_th: 0.9028193371721434 | F1: 0.967741935483871, AUC: 0.991304347826087\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6298903880962139 | F1: 0.9696969696969697, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7716062496780349 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9935483870967742, std: 0.0129032258064516\n",
      "(encode_labels) sample size: Counter({0: 1123482, 1: 17})\n",
      "> sample size | n(+): 17, n(-): 1123482 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 17, n(-): 1123482\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5958169345688072 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8252873747660895 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.952969457178029 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9218102474291271 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.913128679481541 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9769295095870121 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9129593607947081 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9715683200764547 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9139123155758001 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9709656123034893 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.970309737007338 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9740169705839596 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9679986832631773 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9748948342275685 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9752438469530542 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9805360131942112 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9815722195370635 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9248802593931181 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8546929176156914 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9846473737198169 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123450, 1: 49})\n",
      "> sample size | n(+): 49, n(-): 1123450 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 49, n(-): 1123450\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7138153565990354 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.653923681420922 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7491796036907947 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6637972020696149 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.742757891246406 | F1: 0.9523809523809523, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123339, 1: 160})\n",
      "> sample size | n(+): 160, n(-): 1123339 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 160, n(-): 1123339\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9696969696969697 p_th: 0.2722130425172304 | F1: 0.96875, AUC: 0.9990234375\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9846153846153847 p_th: 0.6475565319794515 | F1: 0.9846153846153847, AUC: 0.9983723958333334\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9846153846153847 p_th: 0.8871931073164826 | F1: 0.9846153846153847, AUC: 0.998046875\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.26045919318398963 | F1: 0.967741935483871, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9846153846153847 p_th: 0.8139998376411224 | F1: 0.9846153846153847, AUC: 0.9996744791666667\n",
      "> average: 0.9847086247086247, std: 0.009583339977644446\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9808514513831578 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9766313656532829 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.979598868502707 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9787194110260542 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9800793475812694 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9717489411957257 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9702733966969451 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9722070015927979 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9722749885511445 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9709951165808517 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.982049216144614 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9817086936031361 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9617958413793551 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9812830223679041 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9818838633501071 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.18356498303477825 | F1: 0.5, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.865885070717639 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7537066917568943 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9779683863453141 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.989290341044772 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9746023262577792 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9786075304276299 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9681342164596516 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8977606148525036 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9749198404800296 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123436, 1: 63})\n",
      "> sample size | n(+): 63, n(-): 1123436 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 63, n(-): 1123436\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.679011428397807 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8144242061874641 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9794415107456799 | F1: 0.9600000000000001, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9565217391304348 p_th: 0.4235758250177732 | F1: 0.9090909090909091, AUC: 0.9956140350877194\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9091589314843794 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9913043478260869, std: 0.017391304347826077\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.09632736142879945 | F1: 0.6666666666666666, AUC: 0.9\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9216428925476292 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9166616062891441 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9701343958242835 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9151524455905136 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123452, 1: 47})\n",
      "> sample size | n(+): 47, n(-): 1123452 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size (after downsampling majority) | n(+): 47, n(-): 1123452\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9301976655305011 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9523809523809523 p_th: 0.6826460098129701 | F1: 0.9523809523809523, AUC: 0.9964285714285714\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.37367812695905195 | F1: 0.9473684210526316, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9078942307963142 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9473684210526316 p_th: 0.7934781580230653 | F1: 0.9473684210526316, AUC: 0.9920634920634921\n",
      "> average: 0.9799498746867169, std: 0.024607393902022205\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9392011910921484 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.11657311411673019 | F1: 0.5, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8597115658288386 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5284704759043393 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9533560226518702 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123462, 1: 37})\n",
      "> sample size | n(+): 37, n(-): 1123462 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 37, n(-): 1123462\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7622722858798562 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5048368278251028 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9411764705882353 p_th: 0.9445271113075977 | F1: 0.9411764705882353, AUC: 0.9715909090909092\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8205842392431898 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9626183671020028 | F1: 0.9333333333333333, AUC: 1.0\n",
      "> average: 0.9882352941176471, std: 0.02352941176470589\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7231885127944652 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9015577430822653 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.23776890231248643 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9208392710758561 | F1: 0.8, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9759812799612959 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123468, 1: 31})\n",
      "> sample size | n(+): 31, n(-): 1123468 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 31, n(-): 1123468\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9690483160019169 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5741051499285658 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3312902033133921 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9333333333333333 p_th: 0.5095713193314361 | F1: 0.9333333333333333, AUC: 0.9920634920634921\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6381389012213303 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9866666666666667, std: 0.02666666666666666\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9214357286263672 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.967503633936479 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9732422432848896 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9776568053211612 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9866974417565793 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9704926032453177 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8703478502631067 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5649713675683465 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9729763094181734 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9609146323056583 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9831105614716014 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9754082031273927 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9133816720162397 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7671860260211387 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.975956197710116 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9820253319271105 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9863245181766388 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.988876823876112 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7198073017955497 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9851980116822293 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9622015575012622 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6722392872049288 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8425177971384358 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9641023841695403 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9002832990117969 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8663156539582278 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6614683990798041 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9565886494439279 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.48228759800434073 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7165887555938607 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123479, 1: 20})\n",
      "> sample size | n(+): 20, n(-): 1123479 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size (after downsampling majority) | n(+): 20, n(-): 1123479\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.15627756091349854 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9283606197828163 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.981280334272925 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.888888888888889 p_th: 0.045321566972579254 | F1: 0.8571428571428571, AUC: 0.9791666666666666\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9264068734132379 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9777777777777779, std: 0.04444444444444442\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123463, 1: 36})\n",
      "> sample size | n(+): 36, n(-): 1123463 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 36, n(-): 1123463\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9729708454016981 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9773236878157348 | F1: 0.9333333333333333, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9762524046318359 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.98452268226112 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9914124422182923 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123427, 1: 72})\n",
      "> sample size | n(+): 72, n(-): 1123427 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 72, n(-): 1123427\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9655172413793104 p_th: 0.9516213930639319 | F1: 0.9333333333333333, AUC: 0.9902597402597403\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8018495761261183 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.967741935483871 p_th: 0.9801194073959099 | F1: 0.967741935483871, AUC: 0.9813953488372092\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4074709849380241 | F1: 0.962962962962963, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9655172413793104 p_th: 0.1681327429230688 | F1: 0.962962962962963, AUC: 0.9983388704318936\n",
      "> average: 0.9797552836484984, std: 0.01654969069045363\n",
      "(encode_labels) sample size: Counter({0: 1123472, 1: 27})\n",
      "> sample size | n(+): 27, n(-): 1123472 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 27, n(-): 1123472\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9768469949245141 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3654821396228008 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9688524612769643 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9891633866770531 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.0425948427277304 | F1: 0.888888888888889, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123440, 1: 59})\n",
      "> sample size | n(+): 59, n(-): 1123440 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 59, n(-): 1123440\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8526197642467922 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9211437219932093 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9600000000000001 p_th: 0.6279119880969029 | F1: 0.9600000000000001, AUC: 0.9976190476190476\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9600000000000001 p_th: 0.5783420089692912 | F1: 0.9600000000000001, AUC: 0.9928571428571429\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8324652525394686 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.984, std: 0.019595917942265388\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.947610757367626 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.958956471187565 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9614215383694718 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8040620277217156 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9880189306002015 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9723001698510335 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.967938853126897 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9710961528683724 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9731304630207102 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9341816114534578 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123480, 1: 19})\n",
      "> sample size | n(+): 19, n(-): 1123480 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 19, n(-): 1123480\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8821312191980359 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7333505050045857 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8490700208614177 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8117786410191049 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.888888888888889 p_th: 0.418720824448004 | F1: 0.75, AUC: 0.9318181818181818\n",
      "> average: 0.9777777777777779, std: 0.04444444444444442\n",
      "(encode_labels) sample size: Counter({0: 1123473, 1: 26})\n",
      "> sample size | n(+): 26, n(-): 1123473 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 26, n(-): 1123473\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7413999952895486 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9769960030442326 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9745365350431879 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9542710143967225 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9396397672567861 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123489, 1: 10})\n",
      "> sample size | n(+): 10, n(-): 1123489 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 10, n(-): 1123489\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9694439803441276 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9729540799296061 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9757791040153991 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9696316869908321 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9065383633561118 | F1: 1.0, AUC: 1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9559300491379857 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9787641055199164 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9664196503904764 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9674979210741125 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.1583470052205944 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.922924025409387 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9600316301399677 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9609396155189349 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9741346255537172 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.911149263202081 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123458, 1: 41})\n",
      "> sample size | n(+): 41, n(-): 1123458 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 41, n(-): 1123458\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9544654422299358 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8330473520938737 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.976616314794334 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9874154480796099 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.21574795792841958 | F1: 0.9333333333333333, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9596351111339383 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.960743071237036 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9566381261963338 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.32528752457766047 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9611257751180504 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9718771262903624 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9637816346052384 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9775240260954379 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9838020934429995 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9777973882184023 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9881787999591063 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9699342156171076 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6488667755008486 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9222927703782562 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7011267409475789 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9339464705286262 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9280677857521132 | F1: 0.8, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.32662829118856007 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9291020517059174 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.83795362996924 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9443917358680142 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9589627231034931 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7948739339305256 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9622030534908489 | F1: 0.8, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9719957915249102 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123416, 1: 83})\n",
      "> sample size | n(+): 83, n(-): 1123416 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 83, n(-): 1123416\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.718752139616122 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9821619971376778 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9690959978338618 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9720845336308751 | F1: 0.9696969696969697, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9441682326939543 | F1: 0.9714285714285714, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9761292778222331 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9157724157842888 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9789315691907385 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7335501542359614 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9350805331277098 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123482, 1: 17})\n",
      "> sample size | n(+): 17, n(-): 1123482 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size (after downsampling majority) | n(+): 17, n(-): 1123482\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.909325843649936 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9624421071395381 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9580036686655786 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5178986778731617 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.48599028567682623 | F1: 0.8, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123489, 1: 10})\n",
      "> sample size | n(+): 10, n(-): 1123489 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 10, n(-): 1123489\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9800056285311545 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9498324937674094 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9096572353579668 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.875884238125143 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9786626502031164 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123482, 1: 17})\n",
      "> sample size | n(+): 17, n(-): 1123482 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 17, n(-): 1123482\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9740453148731839 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7415719272176229 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8589576373573262 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9431935936410959 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9843963966262935 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123441, 1: 58})\n",
      "> sample size | n(+): 58, n(-): 1123441 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 58, n(-): 1123441\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.759088373894889 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9115883902701037 | F1: 0.9600000000000001, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9387887941578035 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9546085037074659 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9565217391304348 p_th: 0.9460366891417882 | F1: 0.923076923076923, AUC: 0.9950980392156863\n",
      "> average: 0.9913043478260869, std: 0.017391304347826077\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.969757439645436 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9655731853917995 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9604301560635575 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9522270676092549 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9229114030467808 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123489, 1: 10})\n",
      "> sample size | n(+): 10, n(-): 1123489 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 10, n(-): 1123489\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9749104298205299 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9638469568643907 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.95524332994545 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9739592884369829 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9702889115770187 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.05870479654285961 | F1: 0.6666666666666666, AUC: 0.8571428571428572\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9525418063398133 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9480600402262979 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9418332176659131 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9694139255027451 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9333333333333332, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123462, 1: 37})\n",
      "> sample size | n(+): 37, n(-): 1123462 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 37, n(-): 1123462\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9846544621681366 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9853894027462865 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9889263084159766 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.986420200914556 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9796040566878917 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123385, 1: 114})\n",
      "> sample size | n(+): 114, n(-): 1123385 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 114, n(-): 1123385\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6965015824449949 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7283549140411167 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8336604067551473 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6515420803670061 | F1: 0.9787234042553191, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7697391907841995 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123484, 1: 15})\n",
      "> sample size | n(+): 15, n(-): 1123484 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 15, n(-): 1123484\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9031035243055576 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7377787665783836 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9790469039933922 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9785128517033939 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5235581341398852 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123157, 1: 342})\n",
      "> sample size | n(+): 342, n(-): 1123157 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size (after downsampling majority) | n(+): 342, n(-): 1123157\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9927007299270074 p_th: 0.9591770755767282 | F1: 0.9927007299270074, AUC: 0.9999286122215877\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6881770072553737 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.9857142857142858 p_th: 0.8252018121327569 | F1: 0.9857142857142858, AUC: 0.9959703075291623\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9927007299270074 p_th: 0.9303409596053545 | F1: 0.9927007299270074, AUC: 0.9994978479196557\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9855072463768115 p_th: 0.5419152709989812 | F1: 0.9855072463768115, AUC: 0.9964849354375896\n",
      "> average: 0.9913245983890224, std: 0.005373402309034959\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9504303523922197 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.808824547026026 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9452059698462468 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7078076125715679 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.36980654174594896 | F1: 0.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123476, 1: 23})\n",
      "> sample size | n(+): 23, n(-): 1123476 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 23, n(-): 1123476\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8915745242697846 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9406154247953291 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9877906701115239 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9904523117837329 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9913596757259114 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9595382897307702 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9615729814219036 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.05505600871946055 | F1: 0.6666666666666666, AUC: 0.875\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9779047942540887 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9802196901749923 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9333333333333332, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123472, 1: 27})\n",
      "> sample size | n(+): 27, n(-): 1123472 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 27, n(-): 1123472\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2732723354190379 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7773217424731136 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7705150692224316 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9524746640602134 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9578658889525394 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123482, 1: 17})\n",
      "> sample size | n(+): 17, n(-): 1123482 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 17, n(-): 1123482\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9868465744241995 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9842860499503323 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9897768093858571 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9844216098648332 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.988316793595082 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9528190898228198 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9030454964014606 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.42502155413795495 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9612904170107911 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9473798217323076 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7106543151345444 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.929804651316311 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9205059092622713 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9481322404118595 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9296172199151295 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9571926646874719 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8167514598188984 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9710985588826906 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.669980874332446 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.44879046990023247 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9008126827118991 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9618711963616382 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9785553472252199 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9788931039824256 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9603713389876255 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9505984353726247 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9058125875558656 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8839467067834662 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.975998069860399 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9574690134076391 | F1: 1.0, AUC: 1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.979734854401544 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9706506501411197 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9689868230298072 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9718917890230732 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9628039082151721 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9743259282870963 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7734460039954694 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9599343568697136 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.959072174612089 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9677228564502711 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.920735170142351 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9591316512767202 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.984745751080264 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9298321623179638 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9055221969973738 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123472, 1: 27})\n",
      "> sample size | n(+): 27, n(-): 1123472 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 27, n(-): 1123472\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9874430321372204 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5142641981598416 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.604014234599249 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.9090909090909091 p_th: 0.21260534586888785 | F1: 0.888888888888889, AUC: 0.9875\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9706811314338772 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9818181818181818, std: 0.036363636363636376\n",
      "(encode_labels) sample size: Counter({0: 1123478, 1: 21})\n",
      "> sample size | n(+): 21, n(-): 1123478 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 21, n(-): 1123478\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8203205611616323 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9860084451125388 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9876384559066512 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9840133814277957 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.8571428571428571 p_th: 0.9898569340974951 | F1: 0.8571428571428571, AUC: 0.875\n",
      "> average: 0.9714285714285713, std: 0.05714285714285716\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.972967805406193 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9744829738687156 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9651696613420794 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.584950667128533 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4118192825498631 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123485, 1: 14})\n",
      "> sample size | n(+): 14, n(-): 1123485 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 14, n(-): 1123485\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7965590499059428 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9719677906526439 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9812309475123949 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9800356241904514 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9850616314629396 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123459, 1: 40})\n",
      "> sample size | n(+): 40, n(-): 1123459 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 40, n(-): 1123459\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.9333333333333333 p_th: 0.9135668481925232 | F1: 0.9333333333333333, AUC: 0.984375\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9016363801289519 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.977323424805244 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9759192072661673 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8837372025586595 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9866666666666667, std: 0.02666666666666666\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8875652491187319 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8502739149945158 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9159379249216147 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9590293249117564 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8668551679229716 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.04051413801371374 | F1: 0.6666666666666666, AUC: 0.8333333333333334\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9748973548329587 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9492702008403238 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9645980562395438 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9727234967352667 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9333333333333332, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8452938651975156 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.985472899689741 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9712328933584796 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9723231437922859 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8279139244323003 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123481, 1: 18})\n",
      "> sample size | n(+): 18, n(-): 1123481 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 18, n(-): 1123481\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9710079634131336 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9795036970456154 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9870214858232267 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9836484934496728 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9716005619362521 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4480756703422819 | F1: 0.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5305548147032567 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8567336241193997 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.969371825735847 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9086557959575755 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123484, 1: 15})\n",
      "> sample size | n(+): 15, n(-): 1123484 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 15, n(-): 1123484\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9128253119764155 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9690916158567343 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.876891911910088 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.946396253220233 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9728532895673145 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9785516293710067 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4251684843781388 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9704310464367278 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9810017391935122 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9742413121409557 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6748302645300819 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9528580510764817 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7180819616978141 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9701788608026881 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.22218710296178548 | F1: 0.8, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9489436380494828 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9786431176705196 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9289137207246557 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9868774361272836 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8429319435433561 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9108930700312994 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9568887916927962 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9070799830767555 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9608038337252596 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9291161007842001 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9334892811999613 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.0627359923843788 | F1: 0.0, AUC: 0.6666666666666667\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9407661551784344 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9474098208946472 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9502497777076484 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9333333333333332, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9746926023864328 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9696434466765311 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9730596761847926 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9691342814932852 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9717093676919754 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9718745491181027 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9278368889801881 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.979595823640631 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9706735277878902 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9535303753751172 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123479, 1: 20})\n",
      "> sample size | n(+): 20, n(-): 1123479 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 20, n(-): 1123479\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9478826883786711 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9075431956723228 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.48154464877009406 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8328568956674284 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8970215095222615 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123440, 1: 59})\n",
      "> sample size | n(+): 59, n(-): 1123440 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 59, n(-): 1123440\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5743569832664368 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9166666666666666 p_th: 0.8113926879825856 | F1: 0.9166666666666666, AUC: 0.9873737373737373\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9642990387678877 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8696071649748771 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9344728314771986 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9833333333333332, std: 0.03333333333333335\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9793821768518007 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8697144720955398 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.245539577589173 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7688780035096326 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9755552844841623 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.21390936283398607 | F1: 0.6666666666666666, AUC: 0.9375\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2525980376711856 | F1: 0.5, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7730598704112982 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9701527350751008 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9609596752057973 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9603129952850845 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9478547627461004 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9458497753874616 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.13554137406925712 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9561960268417911 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.95453753367906 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.856334479147441 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9483532472678303 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8628630335328167 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.906871334874858 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9344643319588712 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.41260576455803694 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9761217111069899 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9731219407305299 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.28356389560093254 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.958927173752837 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9262093045821642 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9717774159736143 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.77448535299182 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9758438066557572 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.972167852438757 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9687830529525427 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9625982335775377 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9689890509603497 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9689661005524158 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.958187838603004 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8736104245885808 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9516892414471387 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9437889336717289 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6229651321381767 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123481, 1: 18})\n",
      "> sample size | n(+): 18, n(-): 1123481 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 18, n(-): 1123481\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7978404250262494 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8153520180195218 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9479255942771105 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8671989867740603 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9551295364361131 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8959412283112783 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9090692299439344 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8544893014217794 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9093168815446315 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.36854443199455417 | F1: 0.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.957663931753873 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9518367842132064 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4131691765982703 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6120823179338306 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9596243556295051 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.968624312910616 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2212021824995151 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.969990497599516 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4984614989941906 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8632645514246454 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9722284750971423 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9597467885284057 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9651185473864251 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.0644439832293634 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.965714076317435 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9800215511649272 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9754160284113838 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7888797133933901 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9851041366772371 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9897017533807225 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8896649831955605 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3071645574589783 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9172153702337944 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8263783872082021 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9386913885179402 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9572345374103195 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.14210661099759775 | F1: 0.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9474135688322357 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9343776773697989 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9488451575540587 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7641833258947553 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.38232433517939424 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9415181132470217 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9633592535814826 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9633671427872099 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123467, 1: 32})\n",
      "> sample size | n(+): 32, n(-): 1123467 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 32, n(-): 1123467\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8085460372925021 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2929876248677678 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9160409515549236 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5071674057406224 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5268578491386051 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9161189702793205 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9088295085543074 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9225402348561432 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8328353907055802 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8388168178040114 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123482, 1: 17})\n",
      "> sample size | n(+): 17, n(-): 1123482 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 17, n(-): 1123482\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9675253792801939 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4589375338086543 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6715768148444669 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.1436351997871816 | F1: 0.8, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9310883147130111 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123482, 1: 17})\n",
      "> sample size | n(+): 17, n(-): 1123482 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 17, n(-): 1123482\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8881312681406281 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6021677978591207 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8588355429355314 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9182332175388906 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9583696512372816 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9821448415332426 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9783630099077196 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9446209634189098 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8839494783643149 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9824111049605332 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9435108278215536 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6416444481734022 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8775749759508678 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9437836433399626 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.11220227662248221 | F1: 0.6666666666666666, AUC: 0.875\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9688543306581576 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.4533838378306051 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.273443127867973 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9706674561087019 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9767101216325818 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.985267092302603 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9816640034494846 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9799824862192257 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7805386076797483 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9851144388301729 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7704017157409906 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7648974913691128 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.45805111117825437 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7636881458543782 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6308859708367638 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9046065816964048 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9022582137736541 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7296248683735934 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8754749549093755 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.8061486618907252 | F1: 0.6666666666666666, AUC: 0.6666666666666667\n",
      "> average: 0.9333333333333333, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9689852478918357 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9339629732164245 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.687447247301467 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4202851041405499 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9698493397673721 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123470, 1: 29})\n",
      "> sample size | n(+): 29, n(-): 1123470 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 29, n(-): 1123470\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.952571340066252 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9851231127075545 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9229528365364913 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9894882275943949 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.923076923076923 p_th: 0.08167753764172185 | F1: 0.9090909090909091, AUC: 0.9901960784313726\n",
      "> average: 0.9846153846153847, std: 0.030769230769230795\n",
      "(encode_labels) sample size: Counter({0: 1123475, 1: 24})\n",
      "> sample size | n(+): 24, n(-): 1123475 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 24, n(-): 1123475\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7731541255202593 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.888888888888889 p_th: 0.48720546633159023 | F1: 0.75, AUC: 0.9666666666666667\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8155460381755395 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6685885259935384 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8136219880515727 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> average: 0.9777777777777779, std: 0.04444444444444442\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9575713696172347 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9599053119284086 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8539513701763177 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.11123627168341124 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9672722447547133 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9606720502954647 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9430677342895492 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.38852863822944333 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5765009339263231 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.28514345663648116 | F1: 0.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8993708620367323 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9168905862033966 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.945139362150517 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> Fmax: 1.0 p_th: 0.9051750161854194 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9608233943983274 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9847476413799946 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9868646357469788 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9807954995796655 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9301454402099139 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8950491371962113 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9666944619015658 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9819685218340318 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.985170479638628 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9619122608781862 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9846348144482611 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.12122955746166637 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9374862733947463 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.89918434284314 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9240575501883798 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9457643100974702 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9884011532518192 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9739421168494558 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9765562572902159 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9845104252468808 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7869991633075073 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.96198833838677 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.42750463881895334 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.745306515732515 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6276276303555515 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9521473793431384 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.908445376837008 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9201087747684855 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9596811776585983 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9671521652579635 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8052258992787857 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123479, 1: 20})\n",
      "> sample size | n(+): 20, n(-): 1123479 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 20, n(-): 1123479\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9241388246095956 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.31204756699874386 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9441161369900596 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8534569032896817 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9381418301803351 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8288005461275084 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8956895273351654 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9537638544483318 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9658981764638875 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.28195109377426913 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123489, 1: 10})\n",
      "> sample size | n(+): 10, n(-): 1123489 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 10, n(-): 1123489\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9444917986025138 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9595901946169736 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8273000031036141 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9341941756470702 | F1: 0.8, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7625800262325404 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8256335636505185 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9639120568669253 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.2540609070963435 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7691902177130951 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9729902230808904 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.972708719745873 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9207741461524388 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7007771900583618 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9837234493515594 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7917860823398272 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9797190321571978 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9413461536961938 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9746106667739937 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9703394458117817 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.043084087682124836 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123474, 1: 25})\n",
      "> sample size | n(+): 25, n(-): 1123474 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 25, n(-): 1123474\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7015411012076064 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9592203213402607 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9619303086974078 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.959183782163854 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.18381305371075812 | F1: 0.888888888888889, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9491305822490904 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.1405418963436556 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8756766664217399 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3637066776964706 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9727705122166246 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9731917362978302 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9630913374881095 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9736362144840061 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9773886133472426 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9717453202558607 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.86163302007274 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7029976827302316 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5548852524461361 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6897591040110521 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.28727523633787616 | F1: 0.0, AUC: 0.6666666666666667\n",
      "> average: 0.9333333333333333, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.33241420777769487 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.740607293083044 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7623028658689326 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9241824563508797 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.531126291044169 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123484, 1: 15})\n",
      "> sample size | n(+): 15, n(-): 1123484 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 15, n(-): 1123484\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9836525617806449 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9723397545368213 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9817609226076012 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9818353917979906 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9859117441865357 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123472, 1: 27})\n",
      "> sample size | n(+): 27, n(-): 1123472 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 27, n(-): 1123472\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8720922046377925 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3634553236309839 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.967652621031126 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9499831927576528 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8770994660283236 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.43495751682822176 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9822956911655775 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9668885781463495 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9757967218023358 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7638057487232577 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9801160826804796 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.976677539645999 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6730162997225827 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9720402645703486 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9768500247112245 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123485, 1: 14})\n",
      "> sample size | n(+): 14, n(-): 1123485 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 14, n(-): 1123485\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.13710121379060838 | F1: 0.5, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8092403733066456 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.48611230072068445 | F1: 0.8, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9030125744292391 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.9888706138301185 | F1: 0.8, AUC: 0.9166666666666666\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123491, 1: 8})\n",
      "> sample size | n(+): 8, n(-): 1123491 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 8, n(-): 1123491\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.9444182607199185 | F1: 0.6666666666666666, AUC: 0.7\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.937274905442958 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9152270671954531 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9699282678530106 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9561316151422213 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9333333333333332, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9157012825506657 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9109522721495487 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9635409260305131 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7886945421208776 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.496994864371553 | F1: 0.8, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9684709216631718 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.12621693277168652 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9483710262511158 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9562077104947789 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.951507777063207 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123481, 1: 18})\n",
      "> sample size | n(+): 18, n(-): 1123481 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 18, n(-): 1123481\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9202635664719878 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.955051204645977 | F1: 0.888888888888889, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9110232188787155 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9219487498211985 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9043040467790437 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123481, 1: 18})\n",
      "> sample size | n(+): 18, n(-): 1123481 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 18, n(-): 1123481\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9590507036682768 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6492372335223333 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8466581379880604 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9815471306597436 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6000318414460074 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.46740875044529534 | F1: 0.8, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9894150493185564 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9735528603454833 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9308559725617733 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7619804951617039 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8983584274781888 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.938581633029041 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.954812792108571 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3101666672408551 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5655635369010423 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.8571428571428571 p_th: 0.8046102688126218 | F1: 0.8571428571428571, AUC: 0.9166666666666666\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4513836028611149 | F1: 0.8, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9453434051563888 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9466282897783773 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.196843953213088 | F1: 0.0, AUC: 1.0\n",
      "> average: 0.9714285714285715, std: 0.05714285714285716\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.18172489726420404 | F1: 0.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9433338168493403 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3097575309841087 | F1: 0.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.43167884956340696 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.90848823637146 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9482998642473734 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.5 p_th: 0.17157438278774 | F1: 0.0, AUC: 0.6666666666666667\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.26480756066450456 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9048780427572614 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9525945981224796 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9, std: 0.20000000000000004\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.28900202889679644 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9336759866088847 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.781178963865155 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9546134471261053 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7750587707243045 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9697680480412231 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9147434507597384 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9333092417621922 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9777229028462994 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9777064461738552 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123485, 1: 14})\n",
      "> sample size | n(+): 14, n(-): 1123485 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 14, n(-): 1123485\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7023079876625108 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9711903670989892 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9433466217675539 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9764786093760913 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.8571428571428571 p_th: 0.2641982548808912 | F1: 0.8, AUC: 0.9583333333333334\n",
      "> average: 0.9714285714285713, std: 0.05714285714285716\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8523914676470234 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8673735247618682 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5415713332003695 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.929097497914815 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9396776159725142 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123377, 1: 122})\n",
      "> sample size | n(+): 122, n(-): 1123377 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 122, n(-): 1123377\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9713922323031013 | F1: 0.9795918367346939, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9719189664094009 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7375211256009819 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9438672008943043 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.9795918367346939 p_th: 0.8464177229104685 | F1: 0.9600000000000001, AUC: 0.9994292237442922\n",
      "> average: 0.9959183673469388, std: 0.008163265306122458\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9546258863885274 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9681783973240881 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9625127681378134 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9698646044713577 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9463525902278984 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123471, 1: 28})\n",
      "> sample size | n(+): 28, n(-): 1123471 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 28, n(-): 1123471\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9586382909183637 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9438049923783967 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9916799747821097 | F1: 0.9090909090909091, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5862508180054459 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.923076923076923 p_th: 0.08994167963852412 | F1: 0.9090909090909091, AUC: 0.9895833333333334\n",
      "> average: 0.9846153846153847, std: 0.030769230769230795\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123481, 1: 18})\n",
      "> sample size | n(+): 18, n(-): 1123481 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 18, n(-): 1123481\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9056918996234798 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.940934275993738 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.977991616244054 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9772511692873357 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9711866629354816 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9708513508694047 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.949805601128168 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9518441951977711 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9235380096637754 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9657098407996032 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.20029711494443284 | F1: 0.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9420820781825324 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7258347455288303 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9438000872844835 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4788245415960216 | F1: 0.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123489, 1: 10})\n",
      "> sample size | n(+): 10, n(-): 1123489 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 10, n(-): 1123489\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9673837379229734 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.29632103090157935 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8220224750979593 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6240898478757585 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9269036454428913 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9775080122285599 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9802092931883334 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.854670221661943 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9740230805793134 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9747143467467781 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8714281685122572 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9782703036139487 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6092947206223912 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9213652632824006 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9390490571634473 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9831942290113543 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9559907884975547 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9704302504064762 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.11755047246690616 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9858745151313824 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9846421174935066 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5209186895740525 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9164779678957834 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8384349436536213 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9843404318732314 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5726806898024087 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9038985502674145 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.41298017315396945 | F1: 0.5, AUC: 0.9285714285714286\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8006750584885524 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5993142433604921 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123489, 1: 10})\n",
      "> sample size | n(+): 10, n(-): 1123489 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 10, n(-): 1123489\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.857640981593104 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.966045110962037 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9746856637462917 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8733058761660725 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9158747074750972 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123490, 1: 9})\n",
      "> sample size | n(+): 9, n(-): 1123490 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 9, n(-): 1123490\n",
      "> 0 of KFold 5\n",
      "> Fmax: 0.8 p_th: 0.2697596864181291 | F1: 0.6666666666666666, AUC: 0.9166666666666667\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9347181639250378 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8722017935935268 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4100222575847519 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6880863081870784 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.96, std: 0.07999999999999999\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9548774027578996 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8268007899468788 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6377635777872037 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8517804818713384 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9593573729087531 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9617831318944291 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9044245642632662 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3409655131069167 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9407634641506408 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6828787966206776 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8768487978671037 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9620763002593439 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5260760834824039 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8452305954183189 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.968779752676633 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123463, 1: 36})\n",
      "> sample size | n(+): 36, n(-): 1123463 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 36, n(-): 1123463\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5917635022473119 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.49922740618169803 | F1: 0.923076923076923, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9645917201720802 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9726641238026053 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9811508942652277 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9134056128530108 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8647264082335822 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.914983081036682 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9342033427329425 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9777889307374245 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123483, 1: 16})\n",
      "> sample size | n(+): 16, n(-): 1123483 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 16, n(-): 1123483\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9332234387160195 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.906221583353968 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9793095307347476 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4154500576202656 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7014753667425165 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5265366843128176 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5046128777932429 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8631589790241024 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.806908884998512 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9260096895808598 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9540968538607308 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8633895832768065 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9268590121735977 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9525885016581666 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.18986868431715534 | F1: 0.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123413, 1: 86})\n",
      "> sample size | n(+): 86, n(-): 1123413 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 86, n(-): 1123413\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4239925018161171 | F1: 0.9696969696969697, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.9714285714285714 p_th: 0.9690401694529877 | F1: 0.9444444444444444, AUC: 0.9819004524886877\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6622593191321229 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9344452494357052 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.3871979770491109 | F1: 0.9696969696969697, AUC: 1.0\n",
      "> average: 0.9942857142857143, std: 0.011428571428571432\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123478, 1: 21})\n",
      "> sample size | n(+): 21, n(-): 1123478 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 21, n(-): 1123478\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.976820549421606 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9677036984938672 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9811612897302434 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9717609216010512 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9914763561696615 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6248300766937647 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9709028706463322 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.954457881291269 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.4811152877985234 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7397787953606485 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123487, 1: 12})\n",
      "> sample size | n(+): 12, n(-): 1123487 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 12, n(-): 1123487\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9729316785092583 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8951380063927971 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8320158914887106 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8774068446763696 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8262875770506514 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9672144778752615 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.33351352367192694 | F1: 0.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9219112156943716 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9433342661762779 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9650253228773477 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9179494224362799 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9417351319703774 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9149890711259093 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9785484847439883 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9193438069752307 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123484, 1: 15})\n",
      "> sample size | n(+): 15, n(-): 1123484 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 15, n(-): 1123484\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9696301843301777 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9634357932463414 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9812300966743465 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 0.8571428571428571 p_th: 0.048395883738555495 | F1: 0.8, AUC: 0.9629629629629629\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9653213180453151 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9714285714285715, std: 0.05714285714285716\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123493, 1: 6})\n",
      "> sample size | n(+): 6, n(-): 1123493 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 6, n(-): 1123493\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9669420283878873 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.973777710728026 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8151793689699811 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9559678143292105 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9735875192306296 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123477, 1: 22})\n",
      "> sample size | n(+): 22, n(-): 1123477 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 22, n(-): 1123477\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.985327685714551 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9316469566503405 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9513701948382776 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5405974058810391 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.25557887382541605 | F1: 0.8571428571428571, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123496, 1: 3})\n",
      "> sample size | n(+): 3, n(-): 1123496 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=3\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8600280846302669 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.6216474987248386 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9015926334179508 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.47873443362926454 | F1: 0.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7917548022837917 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9146809825620611 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9374607336073537 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9219356886130732 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9359183849092806 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.3663300292643386 | F1: 0.0, AUC: 0.6666666666666667\n",
      "> average: 0.9333333333333333, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123495, 1: 4})\n",
      "> sample size | n(+): 4, n(-): 1123495 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=4\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123488, 1: 11})\n",
      "> sample size | n(+): 11, n(-): 1123488 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 11, n(-): 1123488\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5668289201230106 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8063212057556262 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8949680017783036 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8002998927961137 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.717875048018121 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123444, 1: 55})\n",
      "> sample size | n(+): 55, n(-): 1123444 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 55, n(-): 1123444\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9396161824984984 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9260128775333714 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9762815953772742 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9401041797478578 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.907353651759412 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123486, 1: 13})\n",
      "> sample size | n(+): 13, n(-): 1123486 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 13, n(-): 1123486\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.984080639203067 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.840737007134913 | F1: 1.0, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.986698174919701 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9812632023161514 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9913815604624187 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123494, 1: 5})\n",
      "> sample size | n(+): 5, n(-): 1123494 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 5, n(-): 1123494\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.629571535812164 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8680348716076054 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.9042549567163651 | F1: 1.0, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8557239258548549 | F1: 1.0, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.8652532463694776 | F1: 1.0, AUC: 1.0\n",
      "> average: 1.0, std: 0.0\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123492, 1: 7})\n",
      "> sample size | n(+): 7, n(-): 1123492 | dim(y_encoded): (1123499,)\n",
      "(balance_by_downsampling) dim(X): (1123499, 243), nl: 2, labels: [0 1] | nf=243\n",
      "... After merging (X, y) => dim(X): (1123499, 244)\n",
      "> sample size (after downsampling majority) | n(+): 7, n(-): 1123492\n",
      "> 0 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.5759354643324918 | F1: 1.0, AUC: 1.0\n",
      "> 1 of KFold 5\n",
      "> Fmax: 0.6666666666666666 p_th: 0.147512730662943 | F1: 0.5, AUC: 0.75\n",
      "> 2 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.30149732939381974 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 3 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7282408238235103 | F1: 0.6666666666666666, AUC: 1.0\n",
      "> 4 of KFold 5\n",
      "> Fmax: 1.0 p_th: 0.7815808317027975 | F1: 1.0, AUC: 1.0\n",
      "> average: 0.9333333333333332, std: 0.13333333333333336\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123497, 1: 2})\n",
      "> sample size | n(+): 2, n(-): 1123497 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=2\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(encode_labels) sample size: Counter({0: 1123498, 1: 1})\n",
      "> sample size | n(+): 1, n(-): 1123498 | dim(y_encoded): (1123499,)\n",
      "> (positive) sample size too small, n=1\n",
      "(save) Saving performance dataframe to:\n",
      "/Users/barnett/Documents/work/loinc_predictor/result/performance-hepatitis-c.csv\n",
      " ... #\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'code'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2645\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2646\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2647\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'code'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-72796eb2a294>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0mdf_perf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msdict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcolumns\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mheader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m \u001b[0mdf_perf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdf_perf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msort_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mby\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'mean'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# ascending=False\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m \u001b[0manalyzer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave_performance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutput_dir\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'result'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# output_file/'' (performance-<cohort>.csv)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0mcohort\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'hepatitis-c'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Documents/work/loinc_predictor/analyzer.py\u001b[0m in \u001b[0;36msave_performance\u001b[0;34m(df, output_dir, output_file, **kargs)\u001b[0m\n\u001b[1;32m    507\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    508\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'(save) Saving performance dataframe to:\\n{}\\n ... #'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 509\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscore\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'code'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'mean'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    510\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"[{code}] -> {score}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    511\u001b[0m     \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2798\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2799\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2800\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2801\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2802\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2646\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2647\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2648\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2649\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2650\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mindexer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'code'"
     ]
    }
   ],
   "source": [
    "import utils_tree, utils_sys, analyzer\n",
    "import collections\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from analyzer import balance_by_downsampling\n",
    "\n",
    "# data transformation\n",
    "col = 'test_result_loinc_code'\n",
    "X, y = dfX.values, dfy[col].values\n",
    "print(\"> dim(X): {}, sample(y): {}\".format(X.shape, np.random.choice(np.unique(y),20) ))\n",
    "\n",
    "# feature scaling\n",
    "scaler = MinMaxScaler() # MinMaxScaler(), StandardScaler()\n",
    "X = scaler.fit_transform(X)\n",
    "\n",
    "n_fold = 5\n",
    "n_min = n_fold\n",
    "\n",
    "# to save performance data\n",
    "header = ['code', 'mean', 'std', 'n_pos']\n",
    "sdict = {h:[] for h in header}\n",
    "for code in loinc_set: \n",
    "    y_eff = analyzer.encode_labels(y, pos_label=code)\n",
    "    \n",
    "    counter = collections.Counter(y_eff)\n",
    "    n_pos, n_neg = counter[codebook['pos']], counter[codebook['neg']]\n",
    "    print(\"> sample size | n(+): {}, n(-): {} | dim(y_encoded): {}\".format(n_pos, n_neg, y_eff.shape))\n",
    "    \n",
    "    if n_pos >= n_min: \n",
    "        # downsampling majority classes (very often we have too many negative instances)\n",
    "        X_eff, y_eff = balance_by_downsampling(X, y_eff, method='multiple', majority_max=3)\n",
    "        n_pos, n_neg = counter[codebook['pos']], counter[codebook['neg']]\n",
    "        print(f\"> sample size (after downsampling majority) | n(+): {n_pos}, n(-): {n_neg}\")\n",
    "        \n",
    "        scores = analyzer.eval_performance(X_eff, y_eff, model=None, cv=n_fold, random_state=53, verbose=1)\n",
    "        mean_score = np.mean(scores)\n",
    "        std_score = np.std(scores)\n",
    "        print(\"> average: {}, std: {}\".format(mean_score, std_score))\n",
    "    else: \n",
    "        print(\"> (positive) sample size too small, n={}\".format(n_pos))\n",
    "        mean_score = -1 \n",
    "        std_score = -1\n",
    "    sdict['code'].append(code)\n",
    "    sdict['mean'].append(mean_score)\n",
    "    sdict['std'].append(std_score)\n",
    "    sdict['n_pos'].append(n_pos)\n",
    "\n",
    "# --------------------------------------------------\n",
    "# save performance dataframe\n",
    "df_perf = DataFrame(sdict, columns=header)\n",
    "df_perf = df_perf.sort_values(by=['mean', ]) # ascending=False\n",
    "analyzer.save_performance(df, output_dir='result') # output_file/'' (performance-<cohort>.csv)\n",
    "\n",
    "cohort = 'hepatitis-c'\n",
    "output_dir = os.path.join(os.getcwd(), 'result')\n",
    "output_file = f\"performance-{cohort}-0.csv\" \n",
    "output_path = os.path.join(output_dir, output_file)\n",
    "df_perf.to_csv(output_path, sep='|', index=False, header=True)\n",
    "\n",
    "for code, score in zip(df_perf['code'], df_perf['mean']):\n",
    "    print(f\"[{code}] -> {score}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualize Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "\n",
    "Memo\n",
    "---- \n",
    "1. performance plot\n",
    "\n",
    "   perplot: https://pypi.org/project/perfplot/\n",
    "\"\"\"\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from analyzer import load_performance\n",
    "\n",
    "sns.set(style=\"whitegrid\")\n",
    "\n",
    "# Initialize the matplotlib figure\n",
    "f, ax = plt.subplots(figsize=(6, 20))\n",
    "sns.set_color_codes(\"pastel\")\n",
    "\n",
    "#---------------------------------------------\n",
    "\n",
    "# load performance data\n",
    "cohort = 'hepatitis-c'\n",
    "df_perf = load_performance(input_dir='result', cohort=cohort)\n",
    "print(\"> dim(performance matrix): {}\".format(df_perf.shape))\n",
    "\n",
    "# sort ~ performance scores \n",
    "# df_perf = df_perf.sort_values(by=['mean', ], ascending=False)\n",
    "\n",
    "header = ['code', 'mean', 'std', 'n_pos']\n",
    "codes = df_perf['code']\n",
    "n_codes = len(codes)\n",
    "scores = df_perf['mean']\n",
    "\n",
    "# some statistics\n",
    "score_high = 0.90\n",
    "score_low = 0.50\n",
    "\n",
    "codes_low_sz = df_perf.loc[df_perf['mean'] < 0]['code']\n",
    "codes_scored = df_perf.loc[df_perf['mean'] >= 0]['code']\n",
    "codes_high_score = df_perf.loc[df_perf['mean'] >= score_high]['code']\n",
    "assert n_codes == len(codes_low_sz) + len(codes_scored)\n",
    "\n",
    "print(\"1. Total number of codes: {} | n(low_sample): {}, n(scored):{}, n(high scored):{}\".format(n_codes, \n",
    "   len(codes_low_sz), len(codes_scored), len(codes_high_score)))\n",
    "r_scored = len(codes_scored)/(n_codes+0.0)\n",
    "rh = len(codes_high_score)/(n_codes+0.0)\n",
    "print(\"2. Fraction of scored codes: {}\".format(r_scored))\n",
    "print(\"3. Fraction of highly scored codes: {}\".format(rh))\n",
    "\n",
    "# Effective performance dataframe, ruling out those codes without scores (due to low sample sizes)\n",
    "df_eff = df_perf.loc[df_perf['mean'] >= 0.0]\n",
    "\n",
    "n_offset = 25\n",
    "df_topn = df_eff.sort_values(['mean', ], ascending=False).head(n_offset)\n",
    "df_botn = df_eff.sort_values(['mean', ], ascending=True).head(n_offset)\n",
    "# print(df_botn)\n",
    "\n",
    "# codes = [str(c) for c in df_botn['code'].values]\n",
    "# print('lower codes: {}'.format(codes))\n",
    "# scores = df_botn['mean'].values\n",
    "# print('scores: {}'.format(scores))\n",
    "\n",
    "# top n + bottom n\n",
    "dfe = pd.concat([df_topn, df_botn], ignore_index=True)\n",
    "dfe.sort_values(by=['mean', ], ascending=False, inplace=True)\n",
    "codes = [str(c) for c in dfe['code'].values]\n",
    "scores = dfe['mean'].values\n",
    "# print('lower(n)+higher codes(n): {}'.format(codes))\n",
    "# print('scores: {}'.format(scores))\n",
    "print(dfe)\n",
    "\n",
    "# sns.barplot(x=\"total\", y=\"abbrev\", data=crashes,\n",
    "#             label=\"Total\", color=\"b\")\n",
    "\n",
    "# --------------------\n",
    "# ax = sns.barplot(x='mean', y='code', data=df_botn)\n",
    "# print(\"-------------------------\\n\\n\")\n",
    "# print(\"> dtype: {}\".format(df_botn.dtypes))\n",
    "# print(df_botn.head(10))\n",
    "\n",
    "# dfe = dfe[['mean', 'code']]\n",
    "# dfe.plot(kind='bar')\n",
    "\n",
    "sns.barplot(x='mean', y='code', data=dfe, order=dfe['code'], # order has to be specified; even if already sorted!!!\n",
    "            label=\"LOINC\", color=\"b\", orient='h')\n",
    "\n",
    "# ax = sns.barplot(x='mean', y='code', data=df)\n",
    "\n",
    "# ax.set_xlabel('Fmax Score')\n",
    "# ax.set_ylabel('LOINC')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}