combined.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parts of this code was used from Harper He Code"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### We can view .ipynb file contents right here\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import json\n",
    "# import os\n",
    "\n",
    "# # note -- we can look at a .ipynb file as a plain text file right here in Jupyter notebook\n",
    "# with open ('Lending_Club.ipynb', mode = 'r', encoding = 'utf-8') as f:\n",
    "#     a = json.load (f)\n",
    "\n",
    "# print(type(a))\n",
    "\n",
    "# print(a.keys())\n",
    "\n",
    "# #print(a)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generating list of notebooks to merge\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # generate list of files to be merged\n",
    "# notebooks_to_merge = [file for file in os.listdir(os.getcwd()) if file.endswith('.ipynb')]\n",
    "\n",
    "# # list should be in the order in which you want the notebooks to be merged\n",
    "# # my notebooks are labeled 1.ipynb, 2.ipynb... in the order I want\n",
    "# notebooks_to_merge.sort()\n",
    "\n",
    "# print(notebooks_to_merge)\n",
    "\n",
    "# del notebooks_to_merge[2:4]\n",
    "# print(notebooks_to_merge)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generate combined .ipynb file\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def combine_ipynb_files(list_of_notebooks, combined_file_name):\n",
    "    \n",
    "#     '''\n",
    "#     parameters:\n",
    "#     `list_of_notebooks` is an ordered list of your .ipynb files to be merged\n",
    "#     `combined_file_name` is the name of your combined .ipynb file which will be generated by this function\n",
    "    \n",
    "#     returns: the filepath of the new file\n",
    "#     '''\n",
    "    \n",
    "#     with open (notebooks_to_merge[0], mode = 'r', encoding = 'utf-8') as f:\n",
    "#         a = json.load (f)\n",
    "    \n",
    "#     for notebook in notebooks_to_merge[1:]:\n",
    "#         with open (notebook, mode = 'r', encoding = 'utf-8') as f:\n",
    "#             b = json.load(f)\n",
    "#             a['cells'].extend (b['cells']) \n",
    "#                 # extend here, not append, so that each dictionary in b['cells']\n",
    "#                 # is added to new dictionary in a['cells']\n",
    "\n",
    "#     with open(combined_file_name, mode='w', encoding='utf-8') as f:\n",
    "#         json.dump(a, f)\n",
    "    \n",
    "#     print('Generated file: \"{}\".'.format(combined_file_name))\n",
    "    \n",
    "#     return (os.path.realpath(combined_file_name))\n",
    "\n",
    "# combine_ipynb_files(notebooks_to_merge, \"combined.ipynb\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('spark.driver.host', 'DESKTOP-61D0FR3'),\n",
       " ('spark.executor.id', 'driver'),\n",
       " ('spark.app.name', 'pyspark-shell'),\n",
       " ('spark.cores.max', '3'),\n",
       " ('spark.app.id', 'local-1587483978916'),\n",
       " ('spark.driver.port', '60411'),\n",
       " ('spark.rdd.compress', 'True'),\n",
       " ('spark.driver.memory', '8g'),\n",
       " ('spark.serializer.objectStreamReset', '100'),\n",
       " ('spark.master', 'local[*]'),\n",
       " ('spark.executor.memory', '8g'),\n",
       " ('spark.submit.pyFiles', ''),\n",
       " ('spark.executor.cores', '3'),\n",
       " ('spark.submit.deployMode', 'client'),\n",
       " ('spark.ui.showConsoleProgress', 'true')]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import packages\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark import SparkConf,SparkContext\n",
    "from pyspark.ml import feature\n",
    "from pyspark.ml import classification\n",
    "from pyspark.sql import functions as fn\n",
    "from pyspark.sql.functions import isnan, when, count, col\n",
    "from pyspark.ml import Pipeline\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator, \\\n",
    "    MulticlassClassificationEvaluator, \\\n",
    "    RegressionEvaluator\n",
    "from pyspark.ml.tuning import CrossValidator, ParamGridBuilder\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.types import *\n",
    "\n",
    "from pyspark.sql.functions import row_number, monotonically_increasing_id\n",
    "from pyspark.sql import Window\n",
    "\n",
    "config = SparkConf().setAll([('spark.executor.memory', '8g'), ('spark.executor.cores', '3'), ('spark.cores.max', '3'), ('spark.driver.memory','8g')])\n",
    "# sc.stop()\n",
    "sc = SparkContext(conf=config)\n",
    "\n",
    "spark = SparkSession.builder.config(conf=config).getOrCreate()\n",
    "sc.getConf().getAll()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# expand the output display to see more columns of a pandas DataFrame\n",
    "pd.set_option('display.max_columns', None)  \n",
    "pd.set_option('display.expand_frame_repr', False)\n",
    "pd.set_option('max_colwidth', -1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is a function for checking the shape and size\n",
    "def size_shape(df):\n",
    "    print(\"rows:\",df.count())\n",
    "    print(\"columns:\",len(df.columns))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read the dataset\n",
    "df = spark.read.csv('loan.csv', header=True, inferSchema=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>member_id</th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_title</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>issue_d</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>url</th>\n",
       "      <th>desc</th>\n",
       "      <th>purpose</th>\n",
       "      <th>title</th>\n",
       "      <th>zip_code</th>\n",
       "      <th>addr_state</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>earliest_cr_line</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>mths_since_last_delinq</th>\n",
       "      <th>mths_since_last_record</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_d</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>next_pymnt_d</th>\n",
       "      <th>last_credit_pull_d</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>mths_since_last_major_derog</th>\n",
       "      <th>policy_code</th>\n",
       "      <th>application_type</th>\n",
       "      <th>annual_inc_joint</th>\n",
       "      <th>dti_joint</th>\n",
       "      <th>verification_status_joint</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "      <th>open_act_il</th>\n",
       "      <th>open_il_12m</th>\n",
       "      <th>open_il_24m</th>\n",
       "      <th>mths_since_rcnt_il</th>\n",
       "      <th>total_bal_il</th>\n",
       "      <th>il_util</th>\n",
       "      <th>open_rv_12m</th>\n",
       "      <th>open_rv_24m</th>\n",
       "      <th>max_bal_bc</th>\n",
       "      <th>all_util</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>inq_fi</th>\n",
       "      <th>total_cu_tl</th>\n",
       "      <th>inq_last_12m</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <th>mort_acc</th>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <th>mths_since_recent_bc_dlq</th>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <th>mths_since_recent_revol_delinq</th>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <th>num_bc_sats</th>\n",
       "      <th>num_bc_tl</th>\n",
       "      <th>num_il_tl</th>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <th>num_rev_accts</th>\n",
       "      <th>num_rev_tl_bal_gt_0</th>\n",
       "      <th>num_sats</th>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <th>num_tl_30dpd</th>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <th>num_tl_op_past_12m</th>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>revol_bal_joint</th>\n",
       "      <th>sec_app_earliest_cr_line</th>\n",
       "      <th>sec_app_inq_last_6mths</th>\n",
       "      <th>sec_app_mort_acc</th>\n",
       "      <th>sec_app_open_acc</th>\n",
       "      <th>sec_app_revol_util</th>\n",
       "      <th>sec_app_open_act_il</th>\n",
       "      <th>sec_app_num_rev_accts</th>\n",
       "      <th>sec_app_chargeoff_within_12_mths</th>\n",
       "      <th>sec_app_collections_12_mths_ex_med</th>\n",
       "      <th>sec_app_mths_since_last_major_derog</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>hardship_type</th>\n",
       "      <th>hardship_reason</th>\n",
       "      <th>hardship_status</th>\n",
       "      <th>deferral_term</th>\n",
       "      <th>hardship_amount</th>\n",
       "      <th>hardship_start_date</th>\n",
       "      <th>hardship_end_date</th>\n",
       "      <th>payment_plan_start_date</th>\n",
       "      <th>hardship_length</th>\n",
       "      <th>hardship_dpd</th>\n",
       "      <th>hardship_loan_status</th>\n",
       "      <th>orig_projected_additional_accrued_interest</th>\n",
       "      <th>hardship_payoff_balance_amount</th>\n",
       "      <th>hardship_last_payment_amount</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "      <th>debt_settlement_flag_date</th>\n",
       "      <th>settlement_status</th>\n",
       "      <th>settlement_date</th>\n",
       "      <th>settlement_amount</th>\n",
       "      <th>settlement_percentage</th>\n",
       "      <th>settlement_term</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2500</td>\n",
       "      <td>2500</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.56</td>\n",
       "      <td>84.92</td>\n",
       "      <td>C</td>\n",
       "      <td>C1</td>\n",
       "      <td>Chef</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>RENT</td>\n",
       "      <td>55000</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Current</td>\n",
       "      <td>n</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>109xx</td>\n",
       "      <td>NY</td>\n",
       "      <td>18.24</td>\n",
       "      <td>0</td>\n",
       "      <td>Apr-2001</td>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>45</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>4341</td>\n",
       "      <td>10.3</td>\n",
       "      <td>34</td>\n",
       "      <td>w</td>\n",
       "      <td>2386.02</td>\n",
       "      <td>2386.02</td>\n",
       "      <td>167.02</td>\n",
       "      <td>167.02</td>\n",
       "      <td>113.98</td>\n",
       "      <td>53.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>84.92</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16901</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>12560</td>\n",
       "      <td>69</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2137</td>\n",
       "      <td>28</td>\n",
       "      <td>42000</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>1878</td>\n",
       "      <td>34360.0</td>\n",
       "      <td>5.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>212</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>7</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>60124</td>\n",
       "      <td>16901</td>\n",
       "      <td>36500</td>\n",
       "      <td>18124</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>18.94</td>\n",
       "      <td>777.23</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>Postmaster</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>90000</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Current</td>\n",
       "      <td>n</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>713xx</td>\n",
       "      <td>LA</td>\n",
       "      <td>26.52</td>\n",
       "      <td>0</td>\n",
       "      <td>Jun-1987</td>\n",
       "      <td>0</td>\n",
       "      <td>71</td>\n",
       "      <td>75</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>12315</td>\n",
       "      <td>24.2</td>\n",
       "      <td>44</td>\n",
       "      <td>w</td>\n",
       "      <td>29387.75</td>\n",
       "      <td>29387.75</td>\n",
       "      <td>1507.11</td>\n",
       "      <td>1507.11</td>\n",
       "      <td>612.25</td>\n",
       "      <td>894.86</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>777.23</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>1208</td>\n",
       "      <td>321915</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>87153</td>\n",
       "      <td>88</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>998</td>\n",
       "      <td>57</td>\n",
       "      <td>50800</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>24763</td>\n",
       "      <td>13761.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>378</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>27</td>\n",
       "      <td>8</td>\n",
       "      <td>14</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>95.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>372872</td>\n",
       "      <td>99468</td>\n",
       "      <td>15000</td>\n",
       "      <td>94072</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>5000</td>\n",
       "      <td>5000</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>17.97</td>\n",
       "      <td>180.69</td>\n",
       "      <td>D</td>\n",
       "      <td>D1</td>\n",
       "      <td>Administrative</td>\n",
       "      <td>6 years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>59280</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Current</td>\n",
       "      <td>n</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>490xx</td>\n",
       "      <td>MI</td>\n",
       "      <td>10.51</td>\n",
       "      <td>0</td>\n",
       "      <td>Apr-2011</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>4599</td>\n",
       "      <td>19.1</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>4787.21</td>\n",
       "      <td>4787.21</td>\n",
       "      <td>353.89</td>\n",
       "      <td>353.89</td>\n",
       "      <td>212.79</td>\n",
       "      <td>141.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>180.69</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110299</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>7150</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>24100</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>18383</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>92</td>\n",
       "      <td>15</td>\n",
       "      <td>14</td>\n",
       "      <td>2</td>\n",
       "      <td>77.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>136927</td>\n",
       "      <td>11749</td>\n",
       "      <td>13800</td>\n",
       "      <td>10000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>18.94</td>\n",
       "      <td>146.51</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>IT Supervisor</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>92000</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Current</td>\n",
       "      <td>n</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>985xx</td>\n",
       "      <td>WA</td>\n",
       "      <td>16.74</td>\n",
       "      <td>0</td>\n",
       "      <td>Feb-2006</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>5468</td>\n",
       "      <td>78.1</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>3831.93</td>\n",
       "      <td>3831.93</td>\n",
       "      <td>286.71</td>\n",
       "      <td>286.71</td>\n",
       "      <td>168.07</td>\n",
       "      <td>118.64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>146.51</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>686</td>\n",
       "      <td>305049</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>30683</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3761</td>\n",
       "      <td>70</td>\n",
       "      <td>7000</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>30505</td>\n",
       "      <td>1239.0</td>\n",
       "      <td>75.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>154</td>\n",
       "      <td>64</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>64.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>385183</td>\n",
       "      <td>36151</td>\n",
       "      <td>5000</td>\n",
       "      <td>44984</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>16.14</td>\n",
       "      <td>731.78</td>\n",
       "      <td>C</td>\n",
       "      <td>C4</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>57250</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Current</td>\n",
       "      <td>n</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>212xx</td>\n",
       "      <td>MD</td>\n",
       "      <td>26.35</td>\n",
       "      <td>0</td>\n",
       "      <td>Dec-2000</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>829</td>\n",
       "      <td>3.6</td>\n",
       "      <td>26</td>\n",
       "      <td>w</td>\n",
       "      <td>29339.02</td>\n",
       "      <td>29339.02</td>\n",
       "      <td>1423.21</td>\n",
       "      <td>1423.21</td>\n",
       "      <td>660.98</td>\n",
       "      <td>762.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>731.78</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>116007</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>28845</td>\n",
       "      <td>89</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>516</td>\n",
       "      <td>54</td>\n",
       "      <td>23100</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>9667</td>\n",
       "      <td>8471.0</td>\n",
       "      <td>8.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>216</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>92.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>157548</td>\n",
       "      <td>29674</td>\n",
       "      <td>9300</td>\n",
       "      <td>32332</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     id member_id  loan_amnt  funded_amnt  funded_amnt_inv        term  int_rate  installment grade sub_grade       emp_title emp_length home_ownership annual_inc verification_status   issue_d loan_status pymnt_plan   url  desc             purpose               title zip_code addr_state    dti delinq_2yrs earliest_cr_line inq_last_6mths mths_since_last_delinq mths_since_last_record open_acc pub_rec revol_bal revol_util total_acc initial_list_status out_prncp out_prncp_inv total_pymnt total_pymnt_inv total_rec_prncp total_rec_int total_rec_late_fee recoveries collection_recovery_fee last_pymnt_d last_pymnt_amnt next_pymnt_d last_credit_pull_d collections_12_mths_ex_med mths_since_last_major_derog policy_code application_type annual_inc_joint dti_joint verification_status_joint acc_now_delinq tot_coll_amt tot_cur_bal open_acc_6m open_act_il open_il_12m open_il_24m mths_since_rcnt_il total_bal_il il_util open_rv_12m open_rv_24m max_bal_bc all_util total_rev_hi_lim inq_fi total_cu_tl inq_last_12m  acc_open_past_24mths avg_cur_bal  bc_open_to_buy bc_util  chargeoff_within_12_mths  delinq_amnt  mo_sin_old_il_acct  mo_sin_old_rev_tl_op  mo_sin_rcnt_rev_tl_op  mo_sin_rcnt_tl  mort_acc  mths_since_recent_bc  mths_since_recent_bc_dlq  mths_since_recent_inq  mths_since_recent_revol_delinq  num_accts_ever_120_pd  num_actv_bc_tl  num_actv_rev_tl  num_bc_sats  num_bc_tl  num_il_tl  num_op_rev_tl  num_rev_accts  num_rev_tl_bal_gt_0  num_sats  num_tl_120dpd_2m  num_tl_30dpd  num_tl_90g_dpd_24m  num_tl_op_past_12m  pct_tl_nvr_dlq  percent_bc_gt_75  pub_rec_bankruptcies  tax_liens  tot_hi_cred_lim  total_bal_ex_mort  total_bc_limit  total_il_high_credit_limit  revol_bal_joint sec_app_earliest_cr_line  sec_app_inq_last_6mths  sec_app_mort_acc  sec_app_open_acc  sec_app_revol_util  sec_app_open_act_il  sec_app_num_rev_accts  sec_app_chargeoff_within_12_mths  sec_app_collections_12_mths_ex_med  sec_app_mths_since_last_major_derog hardship_flag hardship_type hardship_reason hardship_status deferral_term hardship_amount hardship_start_date hardship_end_date payment_plan_start_date hardship_length hardship_dpd hardship_loan_status orig_projected_additional_accrued_interest hardship_payoff_balance_amount hardship_last_payment_amount disbursement_method debt_settlement_flag debt_settlement_flag_date settlement_status settlement_date settlement_amount settlement_percentage settlement_term\n",
       "0  None  None      2500       2500         2500.0            36 months  13.56     84.92        C     C1        Chef            10+ years  RENT           55000      Not Verified        Dec-2018  Current     n          None  None  debt_consolidation  Debt consolidation  109xx    NY         18.24  0           Apr-2001         1              None                   45                     9        1       4341      10.3       34        w                   2386.02   2386.02       167.02      167.02          113.98          53.04         0.0                0.0        0.0                     Feb-2019     84.92           Mar-2019     Feb-2019           0                          None                        1           Individual       None             None      None                      0              0            16901       2           2           1           2           2                  12560        69      2           7           2137       28       42000            1      11          2            9                     1878        34360.0         5.9     0.0                       0            140.0               212                   1                      1               0         1.0                  NaN                        2                     NaN                              0                      2               5                3            3          16         7              18             5                    9         0                 0             0                   3                   100.0           0.0               1                     0          60124            16901              36500           18124                      NaN               None                    NaN                     NaN               NaN               NaN                 NaN                  NaN                    NaN                               NaN                                 NaN                                   N             None          None            None            None          None            None                None              None                    None            None         None                 None                                       None                           None                         Cash                N                    None                      None              None            None              None                  None          \n",
       "1  None  None      30000      30000        30000.0           60 months  18.94     777.23       D     D2        Postmaster      10+ years  MORTGAGE       90000      Source Verified     Dec-2018  Current     n          None  None  debt_consolidation  Debt consolidation  713xx    LA         26.52  0           Jun-1987         0              71                     75                     13       1       12315     24.2       44        w                   29387.75  29387.75      1507.11     1507.11         612.25          894.86        0.0                0.0        0.0                     Feb-2019     777.23          Mar-2019     Feb-2019           0                          None                        1           Individual       None             None      None                      0              1208         321915      4           4           2           3           3                  87153        88      4           5           998        57       50800            2      15          2            10                    24763       13761.0         8.3     0.0                       0            163.0               378                   4                      3               3         4.0                  NaN                        4                     NaN                              0                      2               4                4            9          27         8              14             4                    13        0                 0             0                   6                   95.0            0.0               1                     0          372872           99468              15000           94072                      NaN               None                    NaN                     NaN               NaN               NaN                 NaN                  NaN                    NaN                               NaN                                 NaN                                   N             None          None            None            None          None            None                None              None                    None            None         None                 None                                       None                           None                         Cash                N                    None                      None              None            None              None                  None          \n",
       "2  None  None      5000       5000         5000.0            36 months  17.97     180.69       D     D1        Administrative  6 years    MORTGAGE       59280      Source Verified     Dec-2018  Current     n          None  None  debt_consolidation  Debt consolidation  490xx    MI         10.51  0           Apr-2011         0              None                   None                   8        0       4599      19.1       13        w                   4787.21   4787.21       353.89      353.89          212.79          141.10        0.0                0.0        0.0                     Feb-2019     180.69          Mar-2019     Feb-2019           0                          None                        1           Individual       None             None      None                      0              0            110299      0           1           0           2           14                 7150         72      0           2           0          35       24100            1      5           0            4                     18383       13800.0         0       0.0                       0            87.0                92                    15                     14              2         77.0                 NaN                        14                    NaN                              0                      0               3                3            3          4          6              7              3                    8         0                 0             0                   0                   100.0           0.0               0                     0          136927           11749              13800           10000                      NaN               None                    NaN                     NaN               NaN               NaN                 NaN                  NaN                    NaN                               NaN                                 NaN                                   N             None          None            None            None          None            None                None              None                    None            None         None                 None                                       None                           None                         Cash                N                    None                      None              None            None              None                  None          \n",
       "3  None  None      4000       4000         4000.0            36 months  18.94     146.51       D     D2        IT Supervisor   10+ years  MORTGAGE       92000      Source Verified     Dec-2018  Current     n          None  None  debt_consolidation  Debt consolidation  985xx    WA         16.74  0           Feb-2006         0              None                   None                   10       0       5468      78.1       13        w                   3831.93   3831.93       286.71      286.71          168.07          118.64        0.0                0.0        0.0                     Feb-2019     146.51          Mar-2019     Feb-2019           0                          None                        1           Individual       None             None      None                      0              686          305049      1           5           3           5           5                  30683        68      0           0           3761       70       7000             2      4           3            5                     30505       1239.0          75.2    0.0                       0            62.0                154                   64                     5               3         64.0                 NaN                        5                     NaN                              0                      1               2                1            2          7          2              3              2                    10        0                 0             0                   3                   100.0           100.0             0                     0          385183           36151              5000            44984                      NaN               None                    NaN                     NaN               NaN               NaN                 NaN                  NaN                    NaN                               NaN                                 NaN                                   N             None          None            None            None          None            None                None              None                    None            None         None                 None                                       None                           None                         Cash                N                    None                      None              None            None              None                  None          \n",
       "4  None  None      30000      30000        30000.0           60 months  16.14     731.78       C     C4        Mechanic        10+ years  MORTGAGE       57250      Not Verified        Dec-2018  Current     n          None  None  debt_consolidation  Debt consolidation  212xx    MD         26.35  0           Dec-2000         0              None                   None                   12       0       829       3.6        26        w                   29339.02  29339.02      1423.21     1423.21         660.98          762.23        0.0                0.0        0.0                     Feb-2019     731.78          Mar-2019     Feb-2019           0                          None                        1           Individual       None             None      None                      0              0            116007      3           5           3           5           4                  28845        89      2           4           516        54       23100            1      0           0            9                     9667        8471.0          8.9     0.0                       0            53.0                216                   2                      2               2         2.0                  NaN                        13                    NaN                              0                      2               2                3            8          9          6              15             2                    12        0                 0             0                   5                   92.3            0.0               0                     0          157548           29674              9300            32332                      NaN               None                    NaN                     NaN               NaN               NaN                 NaN                  NaN                    NaN                               NaN                                 NaN                                   N             None          None            None            None          None            None                None              None                    None            None         None                 None                                       None                           None                         Cash                N                    None                      None              None            None              None                  None          "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# take a look at the first 10 rows\n",
    "df_pd = df.limit(10).toPandas()\n",
    "display(df_pd.head())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-------+\n",
      "|         loan_status|  count|\n",
      "+--------------------+-------+\n",
      "|          Fully Paid|1041952|\n",
      "|             Default|     31|\n",
      "|     In Grace Period|   8952|\n",
      "|Does not meet the...|   1988|\n",
      "|         Charged Off| 261654|\n",
      "|            Oct-2015|      1|\n",
      "|  Late (31-120 days)|  21897|\n",
      "|             Current| 919695|\n",
      "|Does not meet the...|    761|\n",
      "|   Late (16-30 days)|   3737|\n",
      "+--------------------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# check how many records for each loan status\n",
    "df_bystatus=df.groupby(df.loan_status).count()\n",
    "df_bystatus.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# show all the column names\n",
    "# df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 2260668\n",
      "columns: 145\n"
     ]
    }
   ],
   "source": [
    "# check the shape of df\n",
    "# print(\"rows:\",df.count())\n",
    "# print(\"columns:\",len(df.columns))\n",
    "\n",
    "size_shape(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+---------------+------+\n",
      "|    loan_status| count|\n",
      "+---------------+------+\n",
      "|Not Charged Off|945329|\n",
      "|    Charged Off|261654|\n",
      "+---------------+------+\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# check the number of Nan or Null for each column\n",
    "#As per our objective, Filtering only the columns required\n",
    "current_df = df.filter(col('loan_status').isin(['Late (31-120 days)','Charged Off','Late (16-30 days)','Current']))\n",
    "#Renaming the Late values into a single late\n",
    "current_df = current_df.withColumn(\"loan_status\", \\\n",
    "              when(current_df[\"loan_status\"].isin(['Late (31-120 days)','Late (16-30 days)','Current']),'Not Charged Off').otherwise(current_df[\"loan_status\"]))\n",
    "\n",
    "#Our dataset with only the desired columns values (Charged off, Not Charged Off)\n",
    "print(current_df.groupby(current_df.loan_status).count().show())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>member_id</th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_title</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>issue_d</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>url</th>\n",
       "      <th>desc</th>\n",
       "      <th>purpose</th>\n",
       "      <th>title</th>\n",
       "      <th>zip_code</th>\n",
       "      <th>addr_state</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>earliest_cr_line</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>mths_since_last_delinq</th>\n",
       "      <th>mths_since_last_record</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_d</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>next_pymnt_d</th>\n",
       "      <th>last_credit_pull_d</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>mths_since_last_major_derog</th>\n",
       "      <th>policy_code</th>\n",
       "      <th>application_type</th>\n",
       "      <th>annual_inc_joint</th>\n",
       "      <th>dti_joint</th>\n",
       "      <th>verification_status_joint</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "      <th>open_act_il</th>\n",
       "      <th>open_il_12m</th>\n",
       "      <th>open_il_24m</th>\n",
       "      <th>mths_since_rcnt_il</th>\n",
       "      <th>total_bal_il</th>\n",
       "      <th>il_util</th>\n",
       "      <th>open_rv_12m</th>\n",
       "      <th>open_rv_24m</th>\n",
       "      <th>max_bal_bc</th>\n",
       "      <th>all_util</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>inq_fi</th>\n",
       "      <th>total_cu_tl</th>\n",
       "      <th>inq_last_12m</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <th>mort_acc</th>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <th>mths_since_recent_bc_dlq</th>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <th>mths_since_recent_revol_delinq</th>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <th>num_bc_sats</th>\n",
       "      <th>num_bc_tl</th>\n",
       "      <th>num_il_tl</th>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <th>num_rev_accts</th>\n",
       "      <th>num_rev_tl_bal_gt_0</th>\n",
       "      <th>num_sats</th>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <th>num_tl_30dpd</th>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <th>num_tl_op_past_12m</th>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>revol_bal_joint</th>\n",
       "      <th>sec_app_earliest_cr_line</th>\n",
       "      <th>sec_app_inq_last_6mths</th>\n",
       "      <th>sec_app_mort_acc</th>\n",
       "      <th>sec_app_open_acc</th>\n",
       "      <th>sec_app_revol_util</th>\n",
       "      <th>sec_app_open_act_il</th>\n",
       "      <th>sec_app_num_rev_accts</th>\n",
       "      <th>sec_app_chargeoff_within_12_mths</th>\n",
       "      <th>sec_app_collections_12_mths_ex_med</th>\n",
       "      <th>sec_app_mths_since_last_major_derog</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>hardship_type</th>\n",
       "      <th>hardship_reason</th>\n",
       "      <th>hardship_status</th>\n",
       "      <th>deferral_term</th>\n",
       "      <th>hardship_amount</th>\n",
       "      <th>hardship_start_date</th>\n",
       "      <th>hardship_end_date</th>\n",
       "      <th>payment_plan_start_date</th>\n",
       "      <th>hardship_length</th>\n",
       "      <th>hardship_dpd</th>\n",
       "      <th>hardship_loan_status</th>\n",
       "      <th>orig_projected_additional_accrued_interest</th>\n",
       "      <th>hardship_payoff_balance_amount</th>\n",
       "      <th>hardship_last_payment_amount</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "      <th>debt_settlement_flag_date</th>\n",
       "      <th>settlement_status</th>\n",
       "      <th>settlement_date</th>\n",
       "      <th>settlement_amount</th>\n",
       "      <th>settlement_percentage</th>\n",
       "      <th>settlement_term</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1206983</td>\n",
       "      <td>1206983</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>105247</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1206983</td>\n",
       "      <td>1187738</td>\n",
       "      <td>0</td>\n",
       "      <td>12520</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>622762</td>\n",
       "      <td>1021293</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>1120</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2414</td>\n",
       "      <td>1</td>\n",
       "      <td>261622</td>\n",
       "      <td>31</td>\n",
       "      <td>15</td>\n",
       "      <td>895784</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1104847</td>\n",
       "      <td>1104856</td>\n",
       "      <td>1109686</td>\n",
       "      <td>18</td>\n",
       "      <td>10302</td>\n",
       "      <td>10299</td>\n",
       "      <td>206300</td>\n",
       "      <td>206297</td>\n",
       "      <td>206304</td>\n",
       "      <td>206307</td>\n",
       "      <td>240368</td>\n",
       "      <td>206306</td>\n",
       "      <td>355256</td>\n",
       "      <td>206310</td>\n",
       "      <td>206308</td>\n",
       "      <td>206309</td>\n",
       "      <td>206487</td>\n",
       "      <td>10316</td>\n",
       "      <td>206310</td>\n",
       "      <td>206310</td>\n",
       "      <td>206313</td>\n",
       "      <td>6884</td>\n",
       "      <td>10370</td>\n",
       "      <td>21150</td>\n",
       "      <td>21696</td>\n",
       "      <td>39</td>\n",
       "      <td>28</td>\n",
       "      <td>50004</td>\n",
       "      <td>10310</td>\n",
       "      <td>10315</td>\n",
       "      <td>10314</td>\n",
       "      <td>6885</td>\n",
       "      <td>20333</td>\n",
       "      <td>933557</td>\n",
       "      <td>147619</td>\n",
       "      <td>815085</td>\n",
       "      <td>10318</td>\n",
       "      <td>10317</td>\n",
       "      <td>10319</td>\n",
       "      <td>8263</td>\n",
       "      <td>10317</td>\n",
       "      <td>10317</td>\n",
       "      <td>10317</td>\n",
       "      <td>10318</td>\n",
       "      <td>10317</td>\n",
       "      <td>8263</td>\n",
       "      <td>58317</td>\n",
       "      <td>10318</td>\n",
       "      <td>10316</td>\n",
       "      <td>10316</td>\n",
       "      <td>10341</td>\n",
       "      <td>21262</td>\n",
       "      <td>147</td>\n",
       "      <td>32</td>\n",
       "      <td>10309</td>\n",
       "      <td>6877</td>\n",
       "      <td>6882</td>\n",
       "      <td>10314</td>\n",
       "      <td>1112427</td>\n",
       "      <td>1112425</td>\n",
       "      <td>1112422</td>\n",
       "      <td>1112422</td>\n",
       "      <td>1112425</td>\n",
       "      <td>1114082</td>\n",
       "      <td>1112428</td>\n",
       "      <td>1112429</td>\n",
       "      <td>1112428</td>\n",
       "      <td>1112428</td>\n",
       "      <td>1175616</td>\n",
       "      <td>33</td>\n",
       "      <td>1198269</td>\n",
       "      <td>1198263</td>\n",
       "      <td>1198269</td>\n",
       "      <td>1198269</td>\n",
       "      <td>1198271</td>\n",
       "      <td>1198270</td>\n",
       "      <td>1198267</td>\n",
       "      <td>1198269</td>\n",
       "      <td>1198268</td>\n",
       "      <td>1198272</td>\n",
       "      <td>1198271</td>\n",
       "      <td>1200360</td>\n",
       "      <td>1198272</td>\n",
       "      <td>1198271</td>\n",
       "      <td>34</td>\n",
       "      <td>30</td>\n",
       "      <td>1173940</td>\n",
       "      <td>1173941</td>\n",
       "      <td>1173945</td>\n",
       "      <td>1173947</td>\n",
       "      <td>1173950</td>\n",
       "      <td>1173948</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  member_id  loan_amnt  funded_amnt  funded_amnt_inv  term  int_rate  installment  grade  sub_grade  emp_title  emp_length  home_ownership  annual_inc  verification_status  issue_d  loan_status  pymnt_plan      url     desc  purpose  title  zip_code  addr_state   dti  delinq_2yrs  earliest_cr_line  inq_last_6mths  mths_since_last_delinq  mths_since_last_record  open_acc  pub_rec  revol_bal  revol_util  total_acc  initial_list_status  out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_d  last_pymnt_amnt  next_pymnt_d  last_credit_pull_d  collections_12_mths_ex_med  mths_since_last_major_derog  policy_code  application_type  annual_inc_joint  dti_joint  verification_status_joint  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m  open_act_il  open_il_12m  open_il_24m  mths_since_rcnt_il  total_bal_il  il_util  open_rv_12m  open_rv_24m  max_bal_bc  all_util  total_rev_hi_lim  inq_fi  total_cu_tl  inq_last_12m  acc_open_past_24mths  avg_cur_bal  bc_open_to_buy  bc_util  chargeoff_within_12_mths  delinq_amnt  mo_sin_old_il_acct  mo_sin_old_rev_tl_op  mo_sin_rcnt_rev_tl_op  mo_sin_rcnt_tl  mort_acc  mths_since_recent_bc  mths_since_recent_bc_dlq  mths_since_recent_inq  mths_since_recent_revol_delinq  num_accts_ever_120_pd  num_actv_bc_tl  num_actv_rev_tl  num_bc_sats  num_bc_tl  num_il_tl  num_op_rev_tl  num_rev_accts  num_rev_tl_bal_gt_0  num_sats  num_tl_120dpd_2m  num_tl_30dpd  num_tl_90g_dpd_24m  num_tl_op_past_12m  pct_tl_nvr_dlq  percent_bc_gt_75  pub_rec_bankruptcies  tax_liens  tot_hi_cred_lim  total_bal_ex_mort  total_bc_limit  total_il_high_credit_limit  revol_bal_joint  sec_app_earliest_cr_line  sec_app_inq_last_6mths  sec_app_mort_acc  sec_app_open_acc  sec_app_revol_util  sec_app_open_act_il  sec_app_num_rev_accts  sec_app_chargeoff_within_12_mths  sec_app_collections_12_mths_ex_med  sec_app_mths_since_last_major_derog  hardship_flag  hardship_type  hardship_reason  hardship_status  deferral_term  hardship_amount  hardship_start_date  hardship_end_date  payment_plan_start_date  hardship_length  hardship_dpd  hardship_loan_status  orig_projected_additional_accrued_interest  hardship_payoff_balance_amount  hardship_last_payment_amount  disbursement_method  debt_settlement_flag  debt_settlement_flag_date  settlement_status  settlement_date  settlement_amount  settlement_percentage  settlement_term\n",
       "0  1206983  1206983    0          0            0                0     0         0            0      0          105247     0           0               0           0                    0        0            0           1206983  1187738  0        12520  1         1           1456  1            1                 1               622762                  1021293                 5         8        6          1120        2          3                    5          8              5            2                3                1              2                   2           2                        2414          1                261622        31                  15                          895784                       8            12                1104847           1104856    1109686                    18              10302         10299        206300       206297       206304       206307       240368              206306        355256   206310       206308       206309      206487    10316             206310  206310       206313        6884                  10370        21150           21696    39                        28           50004               10310                 10315                  10314           6885      20333                 933557                    147619                 815085                          10318                  10317           10319            8263         10317      10317      10317          10318          10317                8263      58317             10318         10316               10316               10341           21262             147                   32         10309            6877               6882            10314                       1112427          1112425                   1112422                 1112422           1112425           1114082             1112428              1112429                1112428                           1112428                             1175616                              33             1198269        1198263          1198269          1198269        1198271          1198270              1198267            1198269                  1198268          1198272       1198271               1200360                                     1198272                         1198271                       34                   30                    1173940                    1173941            1173945          1173947            1173950                1173948        "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Contains the nas present in all the columns in the dataset. We are planning to remove those coluomns with more than 50%Nas\n",
    "checkna_pd = current_df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in current_df.columns]).toPandas()\n",
    "checkna_pd\n",
    "\n",
    "# current_df.select([c for c in checkna_pd.columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#https://stackoverflow.com/questions/51322445/how-to-drop-all-columns-with-null-values-in-a-pyspark-dataframe - include in our references\n",
    "\n",
    "#Creating a dataframe which has the count of Nas of each column\n",
    "checkna = current_df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in current_df.columns])\n",
    "#Selecting the columns with NA values not greater than 50%\n",
    "No_of_rows = current_df.count()\n",
    "checkna_greater_than_50 = checkna.select([when(fn.col(c)<(No_of_rows*0.50),c).alias(c) for c in checkna.columns])\n",
    "\n",
    "\n",
    "def drop_null_columns(df):\n",
    "    \"\"\"\n",
    "    This function drops all columns which contain null values.\n",
    "    :param df: A PySpark DataFrame\n",
    "    \"\"\"\n",
    "    null_counts = df.select([fn.count(fn.when(fn.col(c).isNull()|isnan(fn.col(c)), c)).alias(c) for c in df.columns]).collect()[0].asDict()\n",
    "    to_drop = [k for k, v in null_counts.items() if v > 0]\n",
    "    df = df.drop(*to_drop)\n",
    "    return df\n",
    "\n",
    "# Using the function to find the list of columns with less than 50%Na values\n",
    "final_cols = drop_null_columns(checkna_greater_than_50)\n",
    "current_df1 = current_df.select([fn.col(c) for c in final_cols.columns])\n",
    "\n",
    "#selecting our dataframe with the final cols\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Benji Explore Columns 1-25"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_title</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>issue_d</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>purpose</th>\n",
       "      <th>title</th>\n",
       "      <th>zip_code</th>\n",
       "      <th>addr_state</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>earliest_cr_line</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2500</td>\n",
       "      <td>2500</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.56</td>\n",
       "      <td>84.92</td>\n",
       "      <td>C</td>\n",
       "      <td>C1</td>\n",
       "      <td>Chef</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>RENT</td>\n",
       "      <td>55000</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>109xx</td>\n",
       "      <td>NY</td>\n",
       "      <td>18.24</td>\n",
       "      <td>0</td>\n",
       "      <td>Apr-2001</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>18.94</td>\n",
       "      <td>777.23</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>Postmaster</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>90000</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>713xx</td>\n",
       "      <td>LA</td>\n",
       "      <td>26.52</td>\n",
       "      <td>0</td>\n",
       "      <td>Jun-1987</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5000</td>\n",
       "      <td>5000</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>17.97</td>\n",
       "      <td>180.69</td>\n",
       "      <td>D</td>\n",
       "      <td>D1</td>\n",
       "      <td>Administrative</td>\n",
       "      <td>6 years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>59280</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>490xx</td>\n",
       "      <td>MI</td>\n",
       "      <td>10.51</td>\n",
       "      <td>0</td>\n",
       "      <td>Apr-2011</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>18.94</td>\n",
       "      <td>146.51</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>IT Supervisor</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>92000</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>985xx</td>\n",
       "      <td>WA</td>\n",
       "      <td>16.74</td>\n",
       "      <td>0</td>\n",
       "      <td>Feb-2006</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>16.14</td>\n",
       "      <td>731.78</td>\n",
       "      <td>C</td>\n",
       "      <td>C4</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>57250</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Dec-2018</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>212xx</td>\n",
       "      <td>MD</td>\n",
       "      <td>26.35</td>\n",
       "      <td>0</td>\n",
       "      <td>Dec-2000</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv        term  int_rate  installment grade sub_grade       emp_title emp_length home_ownership annual_inc verification_status   issue_d      loan_status pymnt_plan             purpose               title zip_code addr_state    dti delinq_2yrs earliest_cr_line inq_last_6mths open_acc\n",
       "0  2500       2500         2500.0            36 months  13.56     84.92        C     C1        Chef            10+ years  RENT           55000      Not Verified        Dec-2018  Not Charged Off  n          debt_consolidation  Debt consolidation  109xx    NY         18.24  0           Apr-2001         1              9      \n",
       "1  30000      30000        30000.0           60 months  18.94     777.23       D     D2        Postmaster      10+ years  MORTGAGE       90000      Source Verified     Dec-2018  Not Charged Off  n          debt_consolidation  Debt consolidation  713xx    LA         26.52  0           Jun-1987         0              13     \n",
       "2  5000       5000         5000.0            36 months  17.97     180.69       D     D1        Administrative  6 years    MORTGAGE       59280      Source Verified     Dec-2018  Not Charged Off  n          debt_consolidation  Debt consolidation  490xx    MI         10.51  0           Apr-2011         0              8      \n",
       "3  4000       4000         4000.0            36 months  18.94     146.51       D     D2        IT Supervisor   10+ years  MORTGAGE       92000      Source Verified     Dec-2018  Not Charged Off  n          debt_consolidation  Debt consolidation  985xx    WA         16.74  0           Feb-2006         0              10     \n",
       "4  30000      30000        30000.0           60 months  16.14     731.78       C     C4        Mechanic        10+ years  MORTGAGE       57250      Not Verified        Dec-2018  Not Charged Off  n          debt_consolidation  Debt consolidation  212xx    MD         26.35  0           Dec-2000         0              12     "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part1=current_df1.select(current_df1.columns[0:25])\n",
    "#part1.columns\n",
    "part1.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206983\n",
      "columns: 22\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_title</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>purpose</th>\n",
       "      <th>title</th>\n",
       "      <th>addr_state</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2500</td>\n",
       "      <td>2500</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.56</td>\n",
       "      <td>84.92</td>\n",
       "      <td>C</td>\n",
       "      <td>C1</td>\n",
       "      <td>Chef</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>RENT</td>\n",
       "      <td>55000</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>NY</td>\n",
       "      <td>18.24</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>18.94</td>\n",
       "      <td>777.23</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>Postmaster</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>90000</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>LA</td>\n",
       "      <td>26.52</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5000</td>\n",
       "      <td>5000</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>17.97</td>\n",
       "      <td>180.69</td>\n",
       "      <td>D</td>\n",
       "      <td>D1</td>\n",
       "      <td>Administrative</td>\n",
       "      <td>6 years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>59280</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>MI</td>\n",
       "      <td>10.51</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>18.94</td>\n",
       "      <td>146.51</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>IT Supervisor</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>92000</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>WA</td>\n",
       "      <td>16.74</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>16.14</td>\n",
       "      <td>731.78</td>\n",
       "      <td>C</td>\n",
       "      <td>C4</td>\n",
       "      <td>Mechanic</td>\n",
       "      <td>10+ years</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>57250</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>debt_consolidation</td>\n",
       "      <td>Debt consolidation</td>\n",
       "      <td>MD</td>\n",
       "      <td>26.35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv        term  int_rate  installment grade sub_grade       emp_title emp_length home_ownership annual_inc verification_status      loan_status pymnt_plan             purpose               title addr_state    dti delinq_2yrs inq_last_6mths open_acc\n",
       "0  2500       2500         2500.0            36 months  13.56     84.92        C     C1        Chef            10+ years  RENT           55000      Not Verified        Not Charged Off  n          debt_consolidation  Debt consolidation  NY         18.24  0           1              9      \n",
       "1  30000      30000        30000.0           60 months  18.94     777.23       D     D2        Postmaster      10+ years  MORTGAGE       90000      Source Verified     Not Charged Off  n          debt_consolidation  Debt consolidation  LA         26.52  0           0              13     \n",
       "2  5000       5000         5000.0            36 months  17.97     180.69       D     D1        Administrative  6 years    MORTGAGE       59280      Source Verified     Not Charged Off  n          debt_consolidation  Debt consolidation  MI         10.51  0           0              8      \n",
       "3  4000       4000         4000.0            36 months  18.94     146.51       D     D2        IT Supervisor   10+ years  MORTGAGE       92000      Source Verified     Not Charged Off  n          debt_consolidation  Debt consolidation  WA         16.74  0           0              10     \n",
       "4  30000      30000        30000.0           60 months  16.14     731.78       C     C4        Mechanic        10+ years  MORTGAGE       57250      Not Verified        Not Charged Off  n          debt_consolidation  Debt consolidation  MD         26.35  0           0              12     "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Dropping columns that have date or will not use in analysis\n",
    "# drop the cols\n",
    "drop_var=['issue_d','zip_code','earliest_cr_line']\n",
    "part1=part1.drop(*drop_var)\n",
    "print(size_shape(part1))\n",
    "part1.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- loan_amnt: integer (nullable = true)\n",
      " |-- funded_amnt: integer (nullable = true)\n",
      " |-- funded_amnt_inv: double (nullable = true)\n",
      " |-- term: string (nullable = true)\n",
      " |-- int_rate: double (nullable = true)\n",
      " |-- installment: double (nullable = true)\n",
      " |-- grade: string (nullable = true)\n",
      " |-- sub_grade: string (nullable = true)\n",
      " |-- emp_title: string (nullable = true)\n",
      " |-- emp_length: string (nullable = true)\n",
      " |-- home_ownership: string (nullable = true)\n",
      " |-- annual_inc: string (nullable = true)\n",
      " |-- verification_status: string (nullable = true)\n",
      " |-- loan_status: string (nullable = true)\n",
      " |-- pymnt_plan: string (nullable = true)\n",
      " |-- purpose: string (nullable = true)\n",
      " |-- title: string (nullable = true)\n",
      " |-- addr_state: string (nullable = true)\n",
      " |-- dti: string (nullable = true)\n",
      " |-- delinq_2yrs: string (nullable = true)\n",
      " |-- inq_last_6mths: string (nullable = true)\n",
      " |-- open_acc: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part1.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>summary</td>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>loan_amnt</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15830.35548139452</td>\n",
       "      <td>9555.658605549792</td>\n",
       "      <td>900</td>\n",
       "      <td>40000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>funded_amnt</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15828.553219059424</td>\n",
       "      <td>9554.977988522249</td>\n",
       "      <td>900</td>\n",
       "      <td>40000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>funded_amnt_inv</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15819.40287733609</td>\n",
       "      <td>9554.938771141058</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>term</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>36 months</td>\n",
       "      <td>60 months</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>int_rate</td>\n",
       "      <td>1206983</td>\n",
       "      <td>13.464910922524924</td>\n",
       "      <td>5.05230831155585</td>\n",
       "      <td>5.31</td>\n",
       "      <td>30.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>installment</td>\n",
       "      <td>1206983</td>\n",
       "      <td>458.06588146641303</td>\n",
       "      <td>272.13968537730744</td>\n",
       "      <td>7.61</td>\n",
       "      <td>1719.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>grade</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>A</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>sub_grade</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>A1</td>\n",
       "      <td>G5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>emp_title</td>\n",
       "      <td>1101736</td>\n",
       "      <td>2835.36587628866</td>\n",
       "      <td>10391.035313133278</td>\n",
       "      <td>\\tCFO</td>\n",
       "      <td>👨‍🍳</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>emp_length</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>1 year</td>\n",
       "      <td>n/a</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>home_ownership</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ANY</td>\n",
       "      <td>RENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>annual_inc</td>\n",
       "      <td>1206983</td>\n",
       "      <td>78303.23068339817</td>\n",
       "      <td>139189.24715047498</td>\n",
       "      <td>0</td>\n",
       "      <td>99999.84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>verification_status</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>loan_status</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Charged Off</td>\n",
       "      <td>Not Charged Off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pymnt_plan</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>purpose</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Hilal Khalil Homaidan\"</td>\n",
       "      <td>wedding</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>title</td>\n",
       "      <td>1194463</td>\n",
       "      <td>Infinity</td>\n",
       "      <td>NaN</td>\n",
       "      <td>\\tother</td>\n",
       "      <td>~Life Reorganization~</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>addr_state</td>\n",
       "      <td>1206982</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Friday the 13th Part VII: The New Blood (1988). The movies director and production coordinator are moving this straight to DVD &amp; Blu-Ray as to open the market right up for the viewers. The movie will be produced by late 2009 and seeing income as soon as October. As for myself I am a very responsible person and will stake my name on this project</td>\n",
       "      <td>shopping. This past year though she has been showing \"\"her age\"\".....   even though she has faithfully been taken care of by a friend that owns a garage...... there is a time to let go</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dti</td>\n",
       "      <td>1205527</td>\n",
       "      <td>19.72749241389833</td>\n",
       "      <td>16.520479076307634</td>\n",
       "      <td>FUNCTIONAL</td>\n",
       "      <td>small_business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>delinq_2yrs</td>\n",
       "      <td>1206982</td>\n",
       "      <td>0.30476758810623783</td>\n",
       "      <td>0.8725759910741755</td>\n",
       "      <td>GOAL-ORIENTED</td>\n",
       "      <td>small_business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_last_6mths</td>\n",
       "      <td>1206982</td>\n",
       "      <td>0.52426437849173</td>\n",
       "      <td>0.825048392970046</td>\n",
       "      <td>and decided to see what kind of rate I could get to consolidate my debt.  &lt;br/&gt;I haven't had enough time to research the pros and cons of a home equity line of credit</td>\n",
       "      <td>medical</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_acc</td>\n",
       "      <td>1206978</td>\n",
       "      <td>11.699844236760125</td>\n",
       "      <td>5.8215632276588964</td>\n",
       "      <td>and have never had a problem since.  As you know</td>\n",
       "      <td>VA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           0                    1                   2                                                                                                                                                                                                                                                                                                                                                            3                                                                                                                                                                                         4\n",
       "summary              count    mean                 stddev              min                                                                                                                                                                                                                                                                                                                                                          max                                                                                                                                                                                     \n",
       "loan_amnt            1206983  15830.35548139452    9555.658605549792   900                                                                                                                                                                                                                                                                                                                                                          40000                                                                                                                                                                                   \n",
       "funded_amnt          1206983  15828.553219059424   9554.977988522249   900                                                                                                                                                                                                                                                                                                                                                          40000                                                                                                                                                                                   \n",
       "funded_amnt_inv      1206983  15819.40287733609    9554.938771141058   0.0                                                                                                                                                                                                                                                                                                                                                          40000.0                                                                                                                                                                                 \n",
       "term                 1206983  None                 None                 36 months                                                                                                                                                                                                                                                                                                                                                    60 months                                                                                                                                                                              \n",
       "int_rate             1206983  13.464910922524924   5.05230831155585    5.31                                                                                                                                                                                                                                                                                                                                                         30.99                                                                                                                                                                                   \n",
       "installment          1206983  458.06588146641303   272.13968537730744  7.61                                                                                                                                                                                                                                                                                                                                                         1719.83                                                                                                                                                                                 \n",
       "grade                1206983  None                 None                A                                                                                                                                                                                                                                                                                                                                                            G                                                                                                                                                                                       \n",
       "sub_grade            1206983  None                 None                A1                                                                                                                                                                                                                                                                                                                                                           G5                                                                                                                                                                                      \n",
       "emp_title            1101736  2835.36587628866     10391.035313133278  \\tCFO                                                                                                                                                                                                                                                                                                                                                        👨‍🍳                                                                                                                                                                                     \n",
       "emp_length           1206983  None                 None                1 year                                                                                                                                                                                                                                                                                                                                                       n/a                                                                                                                                                                                     \n",
       "home_ownership       1206983  None                 None                ANY                                                                                                                                                                                                                                                                                                                                                          RENT                                                                                                                                                                                    \n",
       "annual_inc           1206983  78303.23068339817    139189.24715047498  0                                                                                                                                                                                                                                                                                                                                                            99999.84                                                                                                                                                                                \n",
       "verification_status  1206983  None                 None                Not Verified                                                                                                                                                                                                                                                                                                                                                 Verified                                                                                                                                                                                \n",
       "loan_status          1206983  None                 None                Charged Off                                                                                                                                                                                                                                                                                                                                                  Not Charged Off                                                                                                                                                                         \n",
       "pymnt_plan           1206983  None                 None                n                                                                                                                                                                                                                                                                                                                                                            y                                                                                                                                                                                       \n",
       "purpose              1206983  None                 None                  Hilal Khalil Homaidan\"                                                                                                                                                                                                                                                                                                                                     wedding                                                                                                                                                                                 \n",
       "title                1194463  Infinity             NaN                 \\tother                                                                                                                                                                                                                                                                                                                                                      ~Life Reorganization~                                                                                                                                                                   \n",
       "addr_state           1206982  None                 None                 Friday the 13th Part VII: The New Blood (1988). The movies director and production coordinator are moving this straight to DVD & Blu-Ray as to open the market right up for the viewers. The movie will be produced by late 2009 and seeing income as soon as October. As for myself I am a very responsible person and will stake my name on this project  shopping. This past year though she has been showing \"\"her age\"\".....   even though she has faithfully been taken care of by a friend that owns a garage...... there is a time to let go\n",
       "dti                  1205527  19.72749241389833    16.520479076307634   FUNCTIONAL                                                                                                                                                                                                                                                                                                                                                  small_business                                                                                                                                                                          \n",
       "delinq_2yrs          1206982  0.30476758810623783  0.8725759910741755   GOAL-ORIENTED                                                                                                                                                                                                                                                                                                                                               small_business                                                                                                                                                                          \n",
       "inq_last_6mths       1206982  0.52426437849173     0.825048392970046    and decided to see what kind of rate I could get to consolidate my debt.  <br/>I haven't had enough time to research the pros and cons of a home equity line of credit                                                                                                                                                                                      medical                                                                                                                                                                                 \n",
       "open_acc             1206978  11.699844236760125   5.8215632276588964   and have never had a problem since.  As you know                                                                                                                                                                                                                                                                                                            VA                                                                                                                                                                                      "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the summary of each col\n",
    "part1.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>emp_title</th>\n",
       "      <th>emp_length</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>purpose</th>\n",
       "      <th>title</th>\n",
       "      <th>addr_state</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>105247</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12520</td>\n",
       "      <td>1</td>\n",
       "      <td>1456</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv  term  int_rate  installment  grade  sub_grade  emp_title  emp_length  home_ownership  annual_inc  verification_status  loan_status  pymnt_plan  purpose  title  addr_state   dti  delinq_2yrs  inq_last_6mths  open_acc\n",
       "0  0          0            0                0     0         0            0      0          105247     0           0               0           0                    0            0           0        12520  1           1456  1            1               5       "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the missing values\n",
    "checkna_part1 = part1.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part1.columns])\n",
    "checkna_part1.toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns with missing values | the number of missing values | explanation of the type of data\n",
    "emp_title      | 105,247 | The job title supplied by the Borrower when applying for the loan.*\n",
    "\n",
    "title          | 12,520 | The loan title provided by the borrower\n",
    "\n",
    "addr_state     | 1 | The state provided by the borrower in the loan application\n",
    "\n",
    "dti            | 1,456 | A ratio calculated using the borrower’s total monthly debt payments on the total debt obligations, excluding mortgage and the requested LC loan, divided by the borrower’s self-reported monthly income.\n",
    "\n",
    "delinq_2yrs    | 1 | The number of 30+ days past-due incidences of delinquency in the borrower's credit file for the past 2 years\n",
    "\n",
    "inq_last_6mths | 1 | The number of inquiries in past 6 months (excluding auto and mortgage inquiries)\n",
    "\n",
    "open_acc       | 5 | The number of open credit lines in the borrower's credit file.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets look at the columns thave null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|           emp_title|count|\n",
      "+--------------------+-----+\n",
      "|   Exhibits director|    2|\n",
      "|                 CSR|  645|\n",
      "| Pharmacist Director|    1|\n",
      "|Lead Sales Consul...|    2|\n",
      "|     Clinic Director|   29|\n",
      "|           Physician|  991|\n",
      "|Journeyman Electr...|  112|\n",
      "|Mortgage Loan off...|    6|\n",
      "|Digital Marketing...|   46|\n",
      "|  Corporate Director|   16|\n",
      "|Physical therapis...|   39|\n",
      "|    GES Coordinator |    2|\n",
      "|      Dispatch Clerk|    5|\n",
      "|           machinist|  413|\n",
      "|   Brand Coordinator|    5|\n",
      "|    Dedicated Driver|    5|\n",
      "|         Bank Teller|   53|\n",
      "|  English Instructor|   18|\n",
      "| Route sales manager|    2|\n",
      "|        library tech|    4|\n",
      "+--------------------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "None\n",
      "+--------------------+-----+\n",
      "|          addr_state|count|\n",
      "+--------------------+-----+\n",
      "|                  SC|14948|\n",
      "|                  AZ|28068|\n",
      "|                  LA|14102|\n",
      "|                  MN|20722|\n",
      "|Advocate business...|    1|\n",
      "|                  NJ|45687|\n",
      "| and anyone this ...|    1|\n",
      "|Loan To Eliminate...|    1|\n",
      "|             guiding|    1|\n",
      "|                  DC| 2392|\n",
      "|                  OR|13005|\n",
      "|shopping. This pa...|    1|\n",
      "|The Beginning of ...|    1|\n",
      "|                  VA|33129|\n",
      "|                null|    1|\n",
      "|                  RI| 5289|\n",
      "| and help expand ...|    1|\n",
      "|A Better Type Of ...|    1|\n",
      "|                  KY|11961|\n",
      "|                  WY| 2361|\n",
      "+--------------------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "None\n",
      "+------+-----+\n",
      "|   dti|count|\n",
      "+------+-----+\n",
      "| 13.87|  500|\n",
      "|  7.16|  262|\n",
      "| 17.42|  498|\n",
      "| 19.06|  483|\n",
      "| 23.97|  433|\n",
      "|  10.7|  396|\n",
      "| 31.47|  207|\n",
      "| 28.37|  301|\n",
      "|   8.5|  345|\n",
      "| 34.63|  141|\n",
      "| 36.65|   80|\n",
      "| 28.79|  188|\n",
      "| 26.63|  306|\n",
      "| 34.71|  155|\n",
      "| 37.21|   76|\n",
      "| 30.86|  204|\n",
      "| 20.64|  534|\n",
      "| 48.04|   11|\n",
      "| 43.67|    8|\n",
      "|131.57|    1|\n",
      "+------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "None\n",
      "+--------------------+------+\n",
      "|         delinq_2yrs| count|\n",
      "+--------------------+------+\n",
      "| and we have a gr...|     1|\n",
      "|                   7|  1113|\n",
      "|                  AZ|     1|\n",
      "|                  15|    51|\n",
      "|               12.24|     1|\n",
      "|                  11|   204|\n",
      "|                  42|     1|\n",
      "|               049xx|     1|\n",
      "|                   3| 15713|\n",
      "|                  30|     1|\n",
      "| some regulatory ...|     1|\n",
      "|                   8|   647|\n",
      "|Business Booster ...|     1|\n",
      "|                  22|     2|\n",
      "|                  28|     1|\n",
      "|                  16|    32|\n",
      "|                  35|     1|\n",
      "|                   0|984734|\n",
      "|MEDICAL CO-PAYS A...|     1|\n",
      "|                null|     1|\n",
      "+--------------------+------+\n",
      "only showing top 20 rows\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(part1.groupby(part1.emp_title).count().show())\n",
    "print(part1.groupby(part1.addr_state).count().show())\n",
    "print(part1.groupby(part1.dti).count().show())\n",
    "print(part1.groupby(part1.delinq_2yrs).count().show())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+------+\n",
      "|      inq_last_6mths| count|\n",
      "+--------------------+------+\n",
      "|                   7|    12|\n",
      "|            Aug-1995|     1|\n",
      "| let’s take this ...|     1|\n",
      "|               850xx|     1|\n",
      "| and decided to s...|     1|\n",
      "|                  NJ|     1|\n",
      "|                   3| 31322|\n",
      "| clean credit his...|     1|\n",
      "|                   8|     4|\n",
      "|                   0|765822|\n",
      "|                null|     1|\n",
      "| my FICO score is...|     1|\n",
      "| my intention is ...|     1|\n",
      "|                   5|  2314|\n",
      "|                  NV|     1|\n",
      "|               18.24|     1|\n",
      "|000 in taxable re...|     1|\n",
      "|                   6|   243|\n",
      "| and not with mor...|     1|\n",
      "| but I do not pra...|     1|\n",
      "+--------------------+------+\n",
      "only showing top 20 rows\n",
      "\n",
      "None\n",
      "+--------------------+-----+\n",
      "|            open_acc|count|\n",
      "+--------------------+-----+\n",
      "| and have never h...|    1|\n",
      "|Debt Smash & Capi...|    1|\n",
      "|                   7|90921|\n",
      "|                  51|   47|\n",
      "|                  15|53365|\n",
      "|                  54|   20|\n",
      "|            Oct-1999|    1|\n",
      "|                  11|91247|\n",
      "|                 101|    1|\n",
      "|                  29| 3370|\n",
      "|                  69|    3|\n",
      "|                  42|  251|\n",
      "|                  87|    1|\n",
      "|                  64|    3|\n",
      "|                   3|19363|\n",
      "|                  30| 2757|\n",
      "|                  34| 1110|\n",
      "|                  59|   12|\n",
      "|            Oct-1997|    1|\n",
      "|                   8|97952|\n",
      "+--------------------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "print(part1.groupby(part1.inq_last_6mths).count().show())\n",
    "print(part1.groupby(part1.open_acc).count().show())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|             purpose|count|\n",
      "+--------------------+-----+\n",
      "| MD\"\"... approx. ...|    1|\n",
      "| i wasn't even bo...|    1|\n",
      "| it is a historic...|    1|\n",
      "| expensive electr...|    1|\n",
      "| you may feel con...|    1|\n",
      "| so that our comp...|    1|\n",
      "|             wedding|  284|\n",
      "|,debt_consolidati...|    1|\n",
      "| at the time of t...|    1|\n",
      "|         educational|   55|\n",
      "| progressive mult...|    1|\n",
      "|          550 amount|    1|\n",
      "| meaning the metr...|    1|\n",
      "| it was too late ...|    1|\n",
      "|  and another one...|    1|\n",
      "|               proms|    1|\n",
      "| affordable famil...|    1|\n",
      "|               other|79362|\n",
      "|      small_business|13855|\n",
      "| which is their l...|    1|\n",
      "+--------------------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "None\n",
      "+----------+-------+\n",
      "|pymnt_plan|  count|\n",
      "+----------+-------+\n",
      "|         n|1206304|\n",
      "|         y|    679|\n",
      "+----------+-------+\n",
      "\n",
      "None\n",
      "+-------------------+------+\n",
      "|verification_status| count|\n",
      "+-------------------+------+\n",
      "|           Verified|316069|\n",
      "|    Source Verified|484749|\n",
      "|       Not Verified|406165|\n",
      "+-------------------+------+\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# CHeck the columns to see if they are categories\n",
    "print(part1.groupby(part1.purpose).count().show())\n",
    "print(part1.groupby(part1.pymnt_plan).count().show())\n",
    "print(part1.groupby(part1.verification_status).count().show())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------+------+\n",
      "|      term| count|\n",
      "+----------+------+\n",
      "| 36 months|772572|\n",
      "| 60 months|434411|\n",
      "+----------+------+\n",
      "\n",
      "None\n",
      "+-----+------+\n",
      "|grade| count|\n",
      "+-----+------+\n",
      "|    F| 24101|\n",
      "|    E| 78099|\n",
      "|    B|332113|\n",
      "|    D|185921|\n",
      "|    C|359555|\n",
      "|    A|219782|\n",
      "|    G|  7412|\n",
      "+-----+------+\n",
      "\n",
      "None\n",
      "+---------+-----+\n",
      "|sub_grade|count|\n",
      "+---------+-----+\n",
      "|       D5|28162|\n",
      "|       F2| 5351|\n",
      "|       B4|70645|\n",
      "|       A2|35334|\n",
      "|       E4|13337|\n",
      "|       B2|62685|\n",
      "|       C3|72139|\n",
      "|       D1|45204|\n",
      "|       C4|72228|\n",
      "|       F1| 7649|\n",
      "|       D3|37770|\n",
      "|       F5| 3132|\n",
      "|       G2| 1576|\n",
      "|       B1|63317|\n",
      "|       B3|62023|\n",
      "|       E5|14229|\n",
      "|       C5|67563|\n",
      "|       G3| 1276|\n",
      "|       A4|48675|\n",
      "|       F4| 3566|\n",
      "+---------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "None\n",
      "+--------------+------+\n",
      "|home_ownership| count|\n",
      "+--------------+------+\n",
      "|           OWN|140935|\n",
      "|          RENT|493054|\n",
      "|      MORTGAGE|572185|\n",
      "|           ANY|   774|\n",
      "|         OTHER|    27|\n",
      "|          NONE|     8|\n",
      "+--------------+------+\n",
      "\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# Check the columns to see if they are categories\n",
    "print(part1.groupby(part1.term).count().show())\n",
    "print(part1.groupby(part1.grade).count().show())\n",
    "print(part1.groupby(part1.sub_grade).count().show())\n",
    "print(part1.groupby(part1.home_ownership).count().show())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- term: string (nullable = true)\n",
      " |-- grade: string (nullable = true)\n",
      " |-- sub_grade: string (nullable = true)\n",
      " |-- home_ownership: string (nullable = true)\n",
      " |-- verification_status: string (nullable = true)\n",
      " |-- loan_status: string (nullable = true)\n",
      " |-- pymnt_plan: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Make the columns into categories\n",
    "\n",
    "cat_col=['term','grade','sub_grade','home_ownership','verification_status','loan_status','pymnt_plan']\n",
    "part1_cat_col=part1.select(cat_col)\n",
    "part1_cat_col.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now we had columns that had bad data but casting the correct type should solve issue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206983\n",
      "columns: 17\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2500</td>\n",
       "      <td>2500</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.56</td>\n",
       "      <td>84.92</td>\n",
       "      <td>C</td>\n",
       "      <td>C1</td>\n",
       "      <td>RENT</td>\n",
       "      <td>55000</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>18.24</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>18.94</td>\n",
       "      <td>777.23</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>90000</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>26.52</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5000</td>\n",
       "      <td>5000</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>17.97</td>\n",
       "      <td>180.69</td>\n",
       "      <td>D</td>\n",
       "      <td>D1</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>59280</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>10.51</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>18.94</td>\n",
       "      <td>146.51</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>92000</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>16.74</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>16.14</td>\n",
       "      <td>731.78</td>\n",
       "      <td>C</td>\n",
       "      <td>C4</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>57250</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>26.35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv        term  int_rate  installment grade sub_grade home_ownership annual_inc verification_status      loan_status pymnt_plan    dti delinq_2yrs inq_last_6mths open_acc\n",
       "0  2500       2500         2500.0            36 months  13.56     84.92        C     C1        RENT           55000      Not Verified        Not Charged Off  n          18.24  0           1              9      \n",
       "1  30000      30000        30000.0           60 months  18.94     777.23       D     D2        MORTGAGE       90000      Source Verified     Not Charged Off  n          26.52  0           0              13     \n",
       "2  5000       5000         5000.0            36 months  17.97     180.69       D     D1        MORTGAGE       59280      Source Verified     Not Charged Off  n          10.51  0           0              8      \n",
       "3  4000       4000         4000.0            36 months  18.94     146.51       D     D2        MORTGAGE       92000      Source Verified     Not Charged Off  n          16.74  0           0              10     \n",
       "4  30000      30000        30000.0           60 months  16.14     731.78       C     C4        MORTGAGE       57250      Not Verified        Not Charged Off  n          26.35  0           0              12     "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# After looking at the values, we have some more columns to drop\n",
    "drop_var_2=['emp_title',\"title\",'purpose','emp_length','addr_state']\n",
    "part1=part1.drop(*drop_var_2)\n",
    "print(size_shape(part1))\n",
    "part1.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at changing the numerical cols to the right type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "part1_num_col=part1.drop(*cat_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- loan_amnt: integer (nullable = true)\n",
      " |-- funded_amnt: integer (nullable = true)\n",
      " |-- funded_amnt_inv: double (nullable = true)\n",
      " |-- int_rate: double (nullable = true)\n",
      " |-- installment: double (nullable = true)\n",
      " |-- annual_inc: string (nullable = true)\n",
      " |-- dti: string (nullable = true)\n",
      " |-- delinq_2yrs: string (nullable = true)\n",
      " |-- inq_last_6mths: string (nullable = true)\n",
      " |-- open_acc: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part1_num_col.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "part1 = part1.withColumn('loan_amnt', part1['loan_amnt'].cast(FloatType()))\n",
    "part1 = part1.withColumn('funded_amnt', part1['funded_amnt'].cast(FloatType()))\n",
    "part1 = part1.withColumn('funded_amnt_inv', part1['funded_amnt_inv'].cast(FloatType()))\n",
    "part1 = part1.withColumn('int_rate',part1['int_rate'].cast(FloatType()))\n",
    "part1 = part1.withColumn('installment', part1['installment'].cast(FloatType()))\n",
    "part1 = part1.withColumn('annual_inc', part1['annual_inc'].cast(FloatType()))\n",
    "part1 = part1.withColumn('dti', part1['dti'].cast(FloatType()))\n",
    "part1 = part1.withColumn('delinq_2yrs', part1['delinq_2yrs'].cast(IntegerType()))\n",
    "part1 = part1.withColumn('inq_last_6mths', part1['inq_last_6mths'].cast(IntegerType()))\n",
    "part1 = part1.withColumn(\"open_acc\",part1[\"open_acc\"].cast(IntegerType()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- loan_amnt: float (nullable = true)\n",
      " |-- funded_amnt: float (nullable = true)\n",
      " |-- funded_amnt_inv: float (nullable = true)\n",
      " |-- term: string (nullable = true)\n",
      " |-- int_rate: float (nullable = true)\n",
      " |-- installment: float (nullable = true)\n",
      " |-- grade: string (nullable = true)\n",
      " |-- sub_grade: string (nullable = true)\n",
      " |-- home_ownership: string (nullable = true)\n",
      " |-- annual_inc: float (nullable = true)\n",
      " |-- verification_status: string (nullable = true)\n",
      " |-- loan_status: string (nullable = true)\n",
      " |-- pymnt_plan: string (nullable = true)\n",
      " |-- dti: float (nullable = true)\n",
      " |-- delinq_2yrs: integer (nullable = true)\n",
      " |-- inq_last_6mths: integer (nullable = true)\n",
      " |-- open_acc: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part1.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1489</td>\n",
       "      <td>31</td>\n",
       "      <td>24</td>\n",
       "      <td>23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv  term  int_rate  installment  grade  sub_grade  home_ownership  annual_inc  verification_status  loan_status  pymnt_plan   dti  delinq_2yrs  inq_last_6mths  open_acc\n",
       "0  0          0            0                0     0         0            0      0          0               0           0                    0            0           1489  31           24              23      "
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part1.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part1.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns with missing values | the number of missing values | explanation of the type of data\n",
    "\n",
    "dti            | 1,489 | A ratio calculated using the borrower’s total monthly debt payments on the total debt obligations, excluding mortgage and the requested LC loan, divided by the borrower’s self-reported monthly income.\n",
    "\n",
    "delinq_2yrs    | 31 | The number of 30+ days past-due incidences of delinquency in the borrower's credit file for the past 2 years\n",
    "\n",
    "inq_last_6mths | 24 | The number of inquiries in past 6 months (excluding auto and mortgage inquiries)\n",
    "\n",
    "open_acc       | 23 | The number of open credit lines in the borrower's credit file.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets fill in the values that are missing with the mean value\n",
    "\n",
    "dti_mean=part1.select(fn.avg('dti')).collect()\n",
    "part1 = part1.na.fill({'dti':dti_mean[0][0]})\n",
    "\n",
    "delinq_2yrs_mean=part1.select(fn.avg('delinq_2yrs')).collect()\n",
    "part1 = part1.na.fill({'delinq_2yrs':delinq_2yrs_mean[0][0]})\n",
    "\n",
    "inq_last_6mths_mean=part1.select(fn.avg('inq_last_6mths')).collect()\n",
    "part1 = part1.na.fill({'inq_last_6mths':inq_last_6mths_mean[0][0]})\n",
    "\n",
    "open_acc_mean=part1.select(fn.avg('open_acc')).collect()\n",
    "part1 = part1.na.fill({'open_acc':open_acc_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amnt  funded_amnt  funded_amnt_inv  term  int_rate  installment  grade  sub_grade  home_ownership  annual_inc  verification_status  loan_status  pymnt_plan  dti  delinq_2yrs  inq_last_6mths  open_acc\n",
       "0  0          0            0                0     0         0            0      0          0               0           0                    0            0           0    0            0               0       "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Recheck for missing values\n",
    "part1.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part1.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206983\n",
      "columns: 17\n"
     ]
    }
   ],
   "source": [
    "size_shape(part1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>summary</td>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>loan_amnt</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15830.35548139452</td>\n",
       "      <td>9555.658605549792</td>\n",
       "      <td>900.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>funded_amnt</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15828.553219059424</td>\n",
       "      <td>9554.977988522249</td>\n",
       "      <td>900.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>funded_amnt_inv</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15819.402877311093</td>\n",
       "      <td>9554.93877112251</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>term</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>36 months</td>\n",
       "      <td>60 months</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>int_rate</td>\n",
       "      <td>1206983</td>\n",
       "      <td>13.464910887388545</td>\n",
       "      <td>5.052308304807695</td>\n",
       "      <td>5.31</td>\n",
       "      <td>30.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>installment</td>\n",
       "      <td>1206983</td>\n",
       "      <td>458.065881507693</td>\n",
       "      <td>272.13968553916976</td>\n",
       "      <td>7.61</td>\n",
       "      <td>1719.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>grade</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>A</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>sub_grade</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>A1</td>\n",
       "      <td>G5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>home_ownership</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ANY</td>\n",
       "      <td>RENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>annual_inc</td>\n",
       "      <td>1206983</td>\n",
       "      <td>78303.23068381898</td>\n",
       "      <td>139189.24715062467</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.1E8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>verification_status</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>loan_status</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Charged Off</td>\n",
       "      <td>Not Charged Off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pymnt_plan</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dti</td>\n",
       "      <td>1206983</td>\n",
       "      <td>19.72749241566551</td>\n",
       "      <td>16.510285643285677</td>\n",
       "      <td>0.0</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>delinq_2yrs</td>\n",
       "      <td>1206983</td>\n",
       "      <td>0.30475822774637257</td>\n",
       "      <td>0.8725369152244395</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_last_6mths</td>\n",
       "      <td>1206983</td>\n",
       "      <td>0.5242534484744192</td>\n",
       "      <td>0.8250319563368279</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_acc</td>\n",
       "      <td>1206983</td>\n",
       "      <td>11.699830900683772</td>\n",
       "      <td>5.8215085617376126</td>\n",
       "      <td>0</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           0                    1                   2             3                4\n",
       "summary              count    mean                 stddev              min           max            \n",
       "loan_amnt            1206983  15830.35548139452    9555.658605549792   900.0         40000.0        \n",
       "funded_amnt          1206983  15828.553219059424   9554.977988522249   900.0         40000.0        \n",
       "funded_amnt_inv      1206983  15819.402877311093   9554.93877112251    0.0           40000.0        \n",
       "term                 1206983  None                 None                 36 months     60 months     \n",
       "int_rate             1206983  13.464910887388545   5.052308304807695   5.31          30.99          \n",
       "installment          1206983  458.065881507693     272.13968553916976  7.61          1719.83        \n",
       "grade                1206983  None                 None                A             G              \n",
       "sub_grade            1206983  None                 None                A1            G5             \n",
       "home_ownership       1206983  None                 None                ANY           RENT           \n",
       "annual_inc           1206983  78303.23068381898    139189.24715062467  0.0           1.1E8          \n",
       "verification_status  1206983  None                 None                Not Verified  Verified       \n",
       "loan_status          1206983  None                 None                Charged Off   Not Charged Off\n",
       "pymnt_plan           1206983  None                 None                n             y              \n",
       "dti                  1206983  19.72749241566551    16.510285643285677  0.0           999.0          \n",
       "delinq_2yrs          1206983  0.30475822774637257  0.8725369152244395  0             58             \n",
       "inq_last_6mths       1206983  0.5242534484744192   0.8250319563368279  0             20             \n",
       "open_acc             1206983  11.699830900683772   5.8215085617376126  0             101            "
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part1.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Convert string to index label using the StringIndexer\n",
    "# cat_col_list=['term','grade','sub_grade','home_ownership','verification_status','loan_status','pymnt_plan']\n",
    "# stringIndexer = feature.StringIndexer(inputCols=[cols for cols in cat_col_list],\n",
    "#                                       outputCols=[\"encoded_\" + cols for cols in cat_col_list])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets add the index column to be used later before we have to combine all the cleaned dataframes by index value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "part1 = part1.withColumn(\n",
    "    \"index\",\n",
    "    row_number().over(Window.orderBy(monotonically_increasing_id()))-1\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206983\n",
      "columns: 18\n"
     ]
    }
   ],
   "source": [
    "size_shape(part1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>summary</td>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>loan_amnt</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15830.35548139452</td>\n",
       "      <td>9555.658605549872</td>\n",
       "      <td>900.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>funded_amnt</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15828.553219059424</td>\n",
       "      <td>9554.977988521963</td>\n",
       "      <td>900.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>funded_amnt_inv</td>\n",
       "      <td>1206983</td>\n",
       "      <td>15819.402877311099</td>\n",
       "      <td>9554.9387711227</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>term</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>36 months</td>\n",
       "      <td>60 months</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>int_rate</td>\n",
       "      <td>1206983</td>\n",
       "      <td>13.464910887388545</td>\n",
       "      <td>5.052308304807768</td>\n",
       "      <td>5.31</td>\n",
       "      <td>30.99</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>installment</td>\n",
       "      <td>1206983</td>\n",
       "      <td>458.065881507693</td>\n",
       "      <td>272.1396855391722</td>\n",
       "      <td>7.61</td>\n",
       "      <td>1719.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>grade</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>A</td>\n",
       "      <td>G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>sub_grade</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>A1</td>\n",
       "      <td>G5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>home_ownership</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>ANY</td>\n",
       "      <td>RENT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>annual_inc</td>\n",
       "      <td>1206983</td>\n",
       "      <td>78303.23068381898</td>\n",
       "      <td>139189.24715062376</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.1E8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>verification_status</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Verified</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>loan_status</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Charged Off</td>\n",
       "      <td>Not Charged Off</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pymnt_plan</td>\n",
       "      <td>1206983</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>n</td>\n",
       "      <td>y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>dti</td>\n",
       "      <td>1206983</td>\n",
       "      <td>19.727492415665512</td>\n",
       "      <td>16.510285643285922</td>\n",
       "      <td>0.0</td>\n",
       "      <td>999.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>delinq_2yrs</td>\n",
       "      <td>1206983</td>\n",
       "      <td>0.30475822774637257</td>\n",
       "      <td>0.8725369152244312</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_last_6mths</td>\n",
       "      <td>1206983</td>\n",
       "      <td>0.5242534484744192</td>\n",
       "      <td>0.8250319563368264</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_acc</td>\n",
       "      <td>1206983</td>\n",
       "      <td>11.699830900683772</td>\n",
       "      <td>5.8215085617375815</td>\n",
       "      <td>0</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>index</td>\n",
       "      <td>1206983</td>\n",
       "      <td>603491.0</td>\n",
       "      <td>348426.1243161885</td>\n",
       "      <td>0</td>\n",
       "      <td>1206982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           0                    1                   2             3                4\n",
       "summary              count    mean                 stddev              min           max            \n",
       "loan_amnt            1206983  15830.35548139452    9555.658605549872   900.0         40000.0        \n",
       "funded_amnt          1206983  15828.553219059424   9554.977988521963   900.0         40000.0        \n",
       "funded_amnt_inv      1206983  15819.402877311099   9554.9387711227     0.0           40000.0        \n",
       "term                 1206983  None                 None                 36 months     60 months     \n",
       "int_rate             1206983  13.464910887388545   5.052308304807768   5.31          30.99          \n",
       "installment          1206983  458.065881507693     272.1396855391722   7.61          1719.83        \n",
       "grade                1206983  None                 None                A             G              \n",
       "sub_grade            1206983  None                 None                A1            G5             \n",
       "home_ownership       1206983  None                 None                ANY           RENT           \n",
       "annual_inc           1206983  78303.23068381898    139189.24715062376  0.0           1.1E8          \n",
       "verification_status  1206983  None                 None                Not Verified  Verified       \n",
       "loan_status          1206983  None                 None                Charged Off   Not Charged Off\n",
       "pymnt_plan           1206983  None                 None                n             y              \n",
       "dti                  1206983  19.727492415665512   16.510285643285922  0.0           999.0          \n",
       "delinq_2yrs          1206983  0.30475822774637257  0.8725369152244312  0             58             \n",
       "inq_last_6mths       1206983  0.5242534484744192   0.8250319563368264  0             20             \n",
       "open_acc             1206983  11.699830900683772   5.8215085617375815  0             101            \n",
       "index                1206983  603491.0             348426.1243161885   0             1206982        "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part1.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numeric_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the VectorAssembler object\n",
    "# va = feature.VectorAssembler(inputCols=[cols for cols in numeric_cols], outputCol=\"features\")\n",
    "# define the StandardScaler object\n",
    "# sScaler = feature.StandardScaler(withMean=True, withStd=True, inputCol=\"features\", outputCol=\"scaled_features\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the pipeline which contains 3 stages: stringindexer, VectorAssembler and StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part1_pipe=Pipeline(stages=[stringIndexer, va, sScaler])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fitted_part1 = part1_pipe.fit(part1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformed_part1 = fitted_part1.transform(part1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformed_part1.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Pyspark to read the dataset and check missing values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "26-50 Columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_d</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>next_pymnt_d</th>\n",
       "      <th>last_credit_pull_d</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>policy_code</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4341</td>\n",
       "      <td>10.3</td>\n",
       "      <td>34</td>\n",
       "      <td>w</td>\n",
       "      <td>2386.02</td>\n",
       "      <td>2386.02</td>\n",
       "      <td>167.02</td>\n",
       "      <td>167.02</td>\n",
       "      <td>113.98</td>\n",
       "      <td>53.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>84.92</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16901</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12315</td>\n",
       "      <td>24.2</td>\n",
       "      <td>44</td>\n",
       "      <td>w</td>\n",
       "      <td>29387.75</td>\n",
       "      <td>29387.75</td>\n",
       "      <td>1507.11</td>\n",
       "      <td>1507.11</td>\n",
       "      <td>612.25</td>\n",
       "      <td>894.86</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>777.23</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>1208</td>\n",
       "      <td>321915</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4599</td>\n",
       "      <td>19.1</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>4787.21</td>\n",
       "      <td>4787.21</td>\n",
       "      <td>353.89</td>\n",
       "      <td>353.89</td>\n",
       "      <td>212.79</td>\n",
       "      <td>141.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>180.69</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110299</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5468</td>\n",
       "      <td>78.1</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>3831.93</td>\n",
       "      <td>3831.93</td>\n",
       "      <td>286.71</td>\n",
       "      <td>286.71</td>\n",
       "      <td>168.07</td>\n",
       "      <td>118.64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>146.51</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>686</td>\n",
       "      <td>305049</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>829</td>\n",
       "      <td>3.6</td>\n",
       "      <td>26</td>\n",
       "      <td>w</td>\n",
       "      <td>29339.02</td>\n",
       "      <td>29339.02</td>\n",
       "      <td>1423.21</td>\n",
       "      <td>1423.21</td>\n",
       "      <td>660.98</td>\n",
       "      <td>762.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>731.78</td>\n",
       "      <td>Mar-2019</td>\n",
       "      <td>Feb-2019</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>116007</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pub_rec revol_bal revol_util total_acc initial_list_status out_prncp out_prncp_inv total_pymnt total_pymnt_inv total_rec_prncp total_rec_int total_rec_late_fee recoveries collection_recovery_fee last_pymnt_d last_pymnt_amnt next_pymnt_d last_credit_pull_d collections_12_mths_ex_med policy_code application_type acc_now_delinq tot_coll_amt tot_cur_bal open_acc_6m\n",
       "0  1       4341      10.3       34        w                   2386.02   2386.02       167.02      167.02          113.98          53.04         0.0                0.0        0.0                     Feb-2019     84.92           Mar-2019     Feb-2019           0                          1           Individual       0              0            16901       2         \n",
       "1  1       12315     24.2       44        w                   29387.75  29387.75      1507.11     1507.11         612.25          894.86        0.0                0.0        0.0                     Feb-2019     777.23          Mar-2019     Feb-2019           0                          1           Individual       0              1208         321915      4         \n",
       "2  0       4599      19.1       13        w                   4787.21   4787.21       353.89      353.89          212.79          141.10        0.0                0.0        0.0                     Feb-2019     180.69          Mar-2019     Feb-2019           0                          1           Individual       0              0            110299      0         \n",
       "3  0       5468      78.1       13        w                   3831.93   3831.93       286.71      286.71          168.07          118.64        0.0                0.0        0.0                     Feb-2019     146.51          Mar-2019     Feb-2019           0                          1           Individual       0              686          305049      1         \n",
       "4  0       829       3.6        26        w                   29339.02  29339.02      1423.21     1423.21         660.98          762.23        0.0                0.0        0.0                     Feb-2019     731.78          Mar-2019     Feb-2019           0                          1           Individual       0              0            116007      3         "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part2=current_df1.select(current_df1.columns[25:50])\n",
    "# part2.columns\n",
    "part2.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Drop date cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206983\n",
      "columns: 22\n",
      "None\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>policy_code</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>4341</td>\n",
       "      <td>10.3</td>\n",
       "      <td>34</td>\n",
       "      <td>w</td>\n",
       "      <td>2386.02</td>\n",
       "      <td>2386.02</td>\n",
       "      <td>167.02</td>\n",
       "      <td>167.02</td>\n",
       "      <td>113.98</td>\n",
       "      <td>53.04</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>84.92</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>16901</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>12315</td>\n",
       "      <td>24.2</td>\n",
       "      <td>44</td>\n",
       "      <td>w</td>\n",
       "      <td>29387.75</td>\n",
       "      <td>29387.75</td>\n",
       "      <td>1507.11</td>\n",
       "      <td>1507.11</td>\n",
       "      <td>612.25</td>\n",
       "      <td>894.86</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>777.23</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>1208</td>\n",
       "      <td>321915</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>4599</td>\n",
       "      <td>19.1</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>4787.21</td>\n",
       "      <td>4787.21</td>\n",
       "      <td>353.89</td>\n",
       "      <td>353.89</td>\n",
       "      <td>212.79</td>\n",
       "      <td>141.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180.69</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>110299</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>5468</td>\n",
       "      <td>78.1</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>3831.93</td>\n",
       "      <td>3831.93</td>\n",
       "      <td>286.71</td>\n",
       "      <td>286.71</td>\n",
       "      <td>168.07</td>\n",
       "      <td>118.64</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>146.51</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>686</td>\n",
       "      <td>305049</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>829</td>\n",
       "      <td>3.6</td>\n",
       "      <td>26</td>\n",
       "      <td>w</td>\n",
       "      <td>29339.02</td>\n",
       "      <td>29339.02</td>\n",
       "      <td>1423.21</td>\n",
       "      <td>1423.21</td>\n",
       "      <td>660.98</td>\n",
       "      <td>762.23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>731.78</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>116007</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  pub_rec revol_bal revol_util total_acc initial_list_status out_prncp out_prncp_inv total_pymnt total_pymnt_inv total_rec_prncp total_rec_int total_rec_late_fee recoveries collection_recovery_fee last_pymnt_amnt collections_12_mths_ex_med policy_code application_type acc_now_delinq tot_coll_amt tot_cur_bal open_acc_6m\n",
       "0  1       4341      10.3       34        w                   2386.02   2386.02       167.02      167.02          113.98          53.04         0.0                0.0        0.0                     84.92           0                          1           Individual       0              0            16901       2         \n",
       "1  1       12315     24.2       44        w                   29387.75  29387.75      1507.11     1507.11         612.25          894.86        0.0                0.0        0.0                     777.23          0                          1           Individual       0              1208         321915      4         \n",
       "2  0       4599      19.1       13        w                   4787.21   4787.21       353.89      353.89          212.79          141.10        0.0                0.0        0.0                     180.69          0                          1           Individual       0              0            110299      0         \n",
       "3  0       5468      78.1       13        w                   3831.93   3831.93       286.71      286.71          168.07          118.64        0.0                0.0        0.0                     146.51          0                          1           Individual       0              686          305049      1         \n",
       "4  0       829       3.6        26        w                   29339.02  29339.02      1423.21     1423.21         660.98          762.23        0.0                0.0        0.0                     731.78          0                          1           Individual       0              0            116007      3         "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the cols\n",
    "date_var=['last_pymnt_d','next_pymnt_d','last_credit_pull_d']\n",
    "part2=part2.drop(*date_var)\n",
    "print(size_shape(part2))\n",
    "part2.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- pub_rec: string (nullable = true)\n",
      " |-- revol_bal: string (nullable = true)\n",
      " |-- revol_util: string (nullable = true)\n",
      " |-- total_acc: string (nullable = true)\n",
      " |-- initial_list_status: string (nullable = true)\n",
      " |-- out_prncp: string (nullable = true)\n",
      " |-- out_prncp_inv: string (nullable = true)\n",
      " |-- total_pymnt: string (nullable = true)\n",
      " |-- total_pymnt_inv: string (nullable = true)\n",
      " |-- total_rec_prncp: string (nullable = true)\n",
      " |-- total_rec_int: string (nullable = true)\n",
      " |-- total_rec_late_fee: string (nullable = true)\n",
      " |-- recoveries: string (nullable = true)\n",
      " |-- collection_recovery_fee: string (nullable = true)\n",
      " |-- last_pymnt_amnt: string (nullable = true)\n",
      " |-- collections_12_mths_ex_med: string (nullable = true)\n",
      " |-- policy_code: string (nullable = true)\n",
      " |-- application_type: string (nullable = true)\n",
      " |-- acc_now_delinq: string (nullable = true)\n",
      " |-- tot_coll_amt: string (nullable = true)\n",
      " |-- tot_cur_bal: string (nullable = true)\n",
      " |-- open_acc_6m: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>summary</td>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec</td>\n",
       "      <td>1206975</td>\n",
       "      <td>0.1890756536458096</td>\n",
       "      <td>0.5624775092187297</td>\n",
       "      <td>PAYING DEBIT OFF.&lt;br/&gt;SLOW--BUT SURE.&lt;br/&gt; Borrower added on 06/02/10 &gt; JUST A MAN</td>\n",
       "      <td>debt_consolidation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>revol_bal</td>\n",
       "      <td>1206977</td>\n",
       "      <td>16807.967260173642</td>\n",
       "      <td>22685.863720159345</td>\n",
       "      <td>LOOKING FOR A HAND-UP</td>\n",
       "      <td>To be rewarded with reasonable terms...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>revol_util</td>\n",
       "      <td>1205863</td>\n",
       "      <td>49.61285204510645</td>\n",
       "      <td>25.884634838940773</td>\n",
       "      <td>and -- with the support of the members of the Lending Club -- we will prevail. Thank you for being a part of our future.&lt;br/&gt; Borrower added on 05/24/10 &gt; Here is some additional information about our company</td>\n",
       "      <td>Mar-1994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_acc</td>\n",
       "      <td>1206981</td>\n",
       "      <td>23.450838130462017</td>\n",
       "      <td>42.4613450933239</td>\n",
       "      <td>and the work we do.&lt;br/&gt;&lt;br/&gt;We are in the marketing research and consulting business. Our main task is to speak with customers</td>\n",
       "      <td>small_business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>initial_list_status</td>\n",
       "      <td>1206980</td>\n",
       "      <td>926.0172727272726</td>\n",
       "      <td>3263.110236056832</td>\n",
       "      <td>IN ME. .&lt;br/&gt;\"</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>out_prncp</td>\n",
       "      <td>1206978</td>\n",
       "      <td>8245.914858687509</td>\n",
       "      <td>8619.409939480662</td>\n",
       "      <td>but now I am completely back on track &amp; am doing whatever I can to raise my credit score back up</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>out_prncp_inv</td>\n",
       "      <td>1206975</td>\n",
       "      <td>8244.153263444356</td>\n",
       "      <td>8618.754491613367</td>\n",
       "      <td>but in a nutshell we provide direct-from-the-customer data</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_pymnt</td>\n",
       "      <td>1206978</td>\n",
       "      <td>7857.123649318276</td>\n",
       "      <td>7372.222174135592</td>\n",
       "      <td>and opinions</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_pymnt_inv</td>\n",
       "      <td>1206981</td>\n",
       "      <td>7852.0274051305105</td>\n",
       "      <td>7369.046643175955</td>\n",
       "      <td>extremely rare to miss work.&lt;br/&gt;&lt;br/&gt;It's extremely late</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_prncp</td>\n",
       "      <td>1206980</td>\n",
       "      <td>5160.8152297105835</td>\n",
       "      <td>5195.473326471696</td>\n",
       "      <td>I need to hit the hay. If this loan even funds for the original amount (slightly lower APR) in comment that I was considering</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_int</td>\n",
       "      <td>1206982</td>\n",
       "      <td>2440.0997156951144</td>\n",
       "      <td>2747.4575185324143</td>\n",
       "      <td>of $2</td>\n",
       "      <td>9999.70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_late_fee</td>\n",
       "      <td>1206981</td>\n",
       "      <td>1.9962355018726892</td>\n",
       "      <td>29.011852245782418</td>\n",
       "      <td>and continue to build upon them.&lt;br/&gt;\"</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>recoveries</td>\n",
       "      <td>1206981</td>\n",
       "      <td>254.5253475258523</td>\n",
       "      <td>977.1667932879081</td>\n",
       "      <td>0</td>\n",
       "      <td>small_business</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>collection_recovery_fee</td>\n",
       "      <td>1206981</td>\n",
       "      <td>42.2471526738202</td>\n",
       "      <td>171.24312905191488</td>\n",
       "      <td>0.0</td>\n",
       "      <td>xperienceddebtorqualifiedfor4xthisamount</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>last_pymnt_amnt</td>\n",
       "      <td>1206982</td>\n",
       "      <td>489.2054205197111</td>\n",
       "      <td>700.1138703432373</td>\n",
       "      <td>0.0</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>collections_12_mths_ex_med</td>\n",
       "      <td>1206968</td>\n",
       "      <td>0.037906233766018566</td>\n",
       "      <td>7.306349267798215</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>policy_code</td>\n",
       "      <td>1206975</td>\n",
       "      <td>1.0043449498414203</td>\n",
       "      <td>3.1299861759471175</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>application_type</td>\n",
       "      <td>1206971</td>\n",
       "      <td>1248.264375</td>\n",
       "      <td>4416.798379304432</td>\n",
       "      <td>0</td>\n",
       "      <td>Oct-2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>acc_now_delinq</td>\n",
       "      <td>1206965</td>\n",
       "      <td>0.008077810776460108</td>\n",
       "      <td>4.894703732813887</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_coll_amt</td>\n",
       "      <td>1196681</td>\n",
       "      <td>216.85500005849536</td>\n",
       "      <td>1699.2516375435855</td>\n",
       "      <td>0</td>\n",
       "      <td>Oct-2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_cur_bal</td>\n",
       "      <td>1196684</td>\n",
       "      <td>138926.9121216863</td>\n",
       "      <td>159336.07953433666</td>\n",
       "      <td>0</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_acc_6m</td>\n",
       "      <td>1000683</td>\n",
       "      <td>0.9026380214214762</td>\n",
       "      <td>1.4790538763597334</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  0                     1                   2                                                                                                                                                                                                                  3                                         4\n",
       "summary                     count    mean                  stddev              min                                                                                                                                                                                                                max                                     \n",
       "pub_rec                     1206975  0.1890756536458096    0.5624775092187297   PAYING DEBIT OFF.<br/>SLOW--BUT SURE.<br/> Borrower added on 06/02/10 > JUST A MAN                                                                                                                                debt_consolidation                      \n",
       "revol_bal                   1206977  16807.967260173642    22685.863720159345   LOOKING FOR A HAND-UP                                                                                                                                                                                             To be rewarded with reasonable terms... \n",
       "revol_util                  1205863  49.61285204510645     25.884634838940773   and -- with the support of the members of the Lending Club -- we will prevail. Thank you for being a part of our future.<br/> Borrower added on 05/24/10 > Here is some additional information about our company  Mar-1994                                \n",
       "total_acc                   1206981  23.450838130462017    42.4613450933239     and the work we do.<br/><br/>We are in the marketing research and consulting business. Our main task is to speak with customers                                                                                   small_business                          \n",
       "initial_list_status         1206980  926.0172727272726     3263.110236056832    IN ME. .<br/>\"                                                                                                                                                                                                    w                                       \n",
       "out_prncp                   1206978  8245.914858687509     8619.409939480662    but now I am completely back on track & am doing whatever I can to raise my credit score back up                                                                                                                  f                                       \n",
       "out_prncp_inv               1206975  8244.153263444356     8618.754491613367    but in a nutshell we provide direct-from-the-customer data                                                                                                                                                        f                                       \n",
       "total_pymnt                 1206978  7857.123649318276     7372.222174135592    and opinions                                                                                                                                                                                                      w                                       \n",
       "total_pymnt_inv             1206981  7852.0274051305105    7369.046643175955    extremely rare to miss work.<br/><br/>It's extremely late                                                                                                                                                         f                                       \n",
       "total_rec_prncp             1206980  5160.8152297105835    5195.473326471696    I need to hit the hay. If this loan even funds for the original amount (slightly lower APR) in comment that I was considering                                                                                     f                                       \n",
       "total_rec_int               1206982  2440.0997156951144    2747.4575185324143   of $2                                                                                                                                                                                                             9999.70                                 \n",
       "total_rec_late_fee          1206981  1.9962355018726892    29.011852245782418   and continue to build upon them.<br/>\"                                                                                                                                                                            f                                       \n",
       "recoveries                  1206981  254.5253475258523     977.1667932879081   0                                                                                                                                                                                                                  small_business                          \n",
       "collection_recovery_fee     1206981  42.2471526738202      171.24312905191488  0.0                                                                                                                                                                                                                xperienceddebtorqualifiedfor4xthisamount\n",
       "last_pymnt_amnt             1206982  489.2054205197111     700.1138703432373   0.0                                                                                                                                                                                                                f                                       \n",
       "collections_12_mths_ex_med  1206968  0.037906233766018566  7.306349267798215   0                                                                                                                                                                                                                  f                                       \n",
       "policy_code                 1206975  1.0043449498414203    3.1299861759471175  0                                                                                                                                                                                                                  f                                       \n",
       "application_type            1206971  1248.264375           4416.798379304432   0                                                                                                                                                                                                                  Oct-2016                                \n",
       "acc_now_delinq              1206965  0.008077810776460108  4.894703732813887   0                                                                                                                                                                                                                  Individual                              \n",
       "tot_coll_amt                1196681  216.85500005849536    1699.2516375435855  0                                                                                                                                                                                                                  Oct-2016                                \n",
       "tot_cur_bal                 1196684  138926.9121216863     159336.07953433666  0                                                                                                                                                                                                                  f                                       \n",
       "open_acc_6m                 1000683  0.9026380214214762    1.4790538763597334  0                                                                                                                                                                                                                  Individual                              "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the summary of each col\n",
    "part2.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>policy_code</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>6</td>\n",
       "      <td>1120</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>8</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>15</td>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>18</td>\n",
       "      <td>10302</td>\n",
       "      <td>10299</td>\n",
       "      <td>206300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pub_rec  revol_bal  revol_util  total_acc  initial_list_status  out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med  policy_code  application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  8        6          1120        2          3                    5          8              5            2                3                1              2                   2           2                        1                15                          8            12                18              10302         10299        206300     "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the missing values\n",
    "checkna_part2 = part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns])\n",
    "checkna_part2.toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cope with 3 categorical variables: 'initial_list_status', 'application_type', 'policy_code'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- initial_list_status: string (nullable = true)\n",
      " |-- application_type: string (nullable = true)\n",
      " |-- policy_code: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "cat_col=['initial_list_status', 'application_type','policy_code']\n",
    "part2_cat_col=part2.select(cat_col)\n",
    "part2_cat_col.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2=part2.filter(\"initial_list_status in ('w','f')\")\n",
    "part2=part2.filter(\"application_type in ('Individual','Joint App')\")\n",
    "part2=part2.filter(\"policy_code in ('1','2')\")\n",
    "# part2.select(fn.countDistinct(\"initial_list_status\")).show() returns 2\n",
    "# part2.select(\"application_type\").distinct().show() returns 'Joint App' and 'Individual'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+----------------+-------+\n",
      "|application_type|  count|\n",
      "+----------------+-------+\n",
      "|      Individual|1104844|\n",
      "|       Joint App| 102105|\n",
      "+----------------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.groupBy(\"application_type\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------------+------+\n",
      "|initial_list_status| count|\n",
      "+-------------------+------+\n",
      "|                  w|930185|\n",
      "|                  f|276764|\n",
      "+-------------------+------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.groupBy(\"initial_list_status\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------+-------+\n",
      "|policy_code|  count|\n",
      "+-----------+-------+\n",
      "|          1|1206949|\n",
      "+-----------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.groupBy(\"policy_code\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this case, the variable 'policy_code' should be dropped because all the samples belong to one category, which doesn't provide any useful information for classification."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+-------+\n",
      "|  policy_code|  count|\n",
      "+-------------+-------+\n",
      "|            1|1206949|\n",
      "|         null|      8|\n",
      "|            0|      8|\n",
      "|          0.0|      4|\n",
      "|30.2849999999|      1|\n",
      "|         1.76|      1|\n",
      "|         1926|      1|\n",
      "|       121.95|      1|\n",
      "|         3.84|      1|\n",
      "|89.1377999988|      1|\n",
      "|            f|      1|\n",
      "|     Oct-2016|      1|\n",
      "|         5.73|      1|\n",
      "|      2838.85|      1|\n",
      "|           49|      1|\n",
      "|       200.68|      1|\n",
      "|         0.00|      1|\n",
      "|     Jan-2018|      1|\n",
      "+-------------+-------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "current_df.groupBy(\"policy_code\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2=part2.drop(\"policy_code\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1115</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10285</td>\n",
       "      <td>10285</td>\n",
       "      <td>206280</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pub_rec  revol_bal  revol_util  total_acc  initial_list_status  out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med  application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  0        0          1115        0          0                    0          0              0            0                0                0              0                   0           0                        0                6                           0                 0               10285         10285        206280     "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cope with numerical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2_num_col=part2.drop(*cat_col)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- pub_rec: string (nullable = true)\n",
      " |-- revol_bal: string (nullable = true)\n",
      " |-- revol_util: string (nullable = true)\n",
      " |-- total_acc: string (nullable = true)\n",
      " |-- out_prncp: string (nullable = true)\n",
      " |-- out_prncp_inv: string (nullable = true)\n",
      " |-- total_pymnt: string (nullable = true)\n",
      " |-- total_pymnt_inv: string (nullable = true)\n",
      " |-- total_rec_prncp: string (nullable = true)\n",
      " |-- total_rec_int: string (nullable = true)\n",
      " |-- total_rec_late_fee: string (nullable = true)\n",
      " |-- recoveries: string (nullable = true)\n",
      " |-- collection_recovery_fee: string (nullable = true)\n",
      " |-- last_pymnt_amnt: string (nullable = true)\n",
      " |-- collections_12_mths_ex_med: string (nullable = true)\n",
      " |-- acc_now_delinq: string (nullable = true)\n",
      " |-- tot_coll_amt: string (nullable = true)\n",
      " |-- tot_cur_bal: string (nullable = true)\n",
      " |-- open_acc_6m: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2_num_col.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start with 'pub_rec' variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-----------------------+\n",
      "|count(DISTINCT pub_rec)|\n",
      "+-----------------------+\n",
      "|                     35|\n",
      "+-----------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# how many distinct values in pub_rec feature\n",
    "part2.select(fn.countDistinct(\"pub_rec\")).show()\n",
    "# number of samples that have pub_rec\n",
    "# part2.filter(fn.col('pub_rec')!=0).count() returns 185562"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print the distinct values \n",
    "# part2.select('pub_rec').distinct().show(52)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # REPLACE 11.59 15.31 etc\n",
    "# # [0-9]+\\.[0-9]+\n",
    "# part2=part2.withColumn('pub_rec',fn.regexp_replace(fn.col(\"pub_rec\"), \"[0-9]+\\.[0-9]+\", \"null\"))\n",
    "\n",
    "# # remove strings\n",
    "# # ^[\\D]+.*$\n",
    "# part2=part2.withColumn('pub_rec',fn.regexp_replace(fn.col(\"pub_rec\"), \"^[\\D]+.*$\", \"null\"))\n",
    "\n",
    "# # remove 120XX\n",
    "# # (^.*[\\D]+$)\n",
    "# part2=part2.withColumn('pub_rec',fn.regexp_replace(fn.col(\"pub_rec\"), \"(^.*[\\D]+$)\", \"null\"))\n",
    "\n",
    "# part2.select(fn.countDistinct(\"pub_rec\")).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-------------------+\n",
      "|summary|            pub_rec|\n",
      "+-------+-------------------+\n",
      "|  count|            1206949|\n",
      "|   mean|0.18891601882101067|\n",
      "| stddev| 0.5552736651970246|\n",
      "|    min|                  0|\n",
      "|    max|                  9|\n",
      "+-------+-------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.describe('pub_rec').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+------------------+\n",
      "|summary|         revol_bal|\n",
      "+-------+------------------+\n",
      "|  count|           1206949|\n",
      "|   mean|16808.217865046492|\n",
      "| stddev| 22685.94006979832|\n",
      "|    min|                 0|\n",
      "|    max|             99996|\n",
      "+-------+------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.describe('revol_bal').show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We conver the data type of each col to integer or float"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2 = part2.withColumn('pub_rec', part2['pub_rec'].cast(IntegerType()))\n",
    "part2 = part2.withColumn('revol_bal', part2['revol_bal'].cast(FloatType()))\n",
    "part2 = part2.withColumn('revol_util',part2['revol_util'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_acc', part2['total_acc'].cast(IntegerType()))\n",
    "part2 = part2.withColumn('out_prncp', part2['out_prncp'].cast(FloatType()))\n",
    "part2 = part2.withColumn('out_prncp_inv', part2['out_prncp_inv'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_pymnt', part2['total_pymnt'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_pymnt_inv', part2['total_pymnt_inv'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_rec_prncp', part2['total_rec_prncp'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_rec_int', part2['total_rec_int'].cast(FloatType()))\n",
    "part2 = part2.withColumn('total_rec_late_fee', part2['total_rec_late_fee'].cast(FloatType()))\n",
    "part2 = part2.withColumn('recoveries', part2['recoveries'].cast(FloatType()))\n",
    "part2 = part2.withColumn('collection_recovery_fee', part2['collection_recovery_fee'].cast(FloatType()))\n",
    "part2 = part2.withColumn('last_pymnt_amnt', part2['last_pymnt_amnt'].cast(FloatType()))\n",
    "part2 = part2.withColumn('collections_12_mths_ex_med', part2['collections_12_mths_ex_med'].cast(IntegerType()))\n",
    "part2 = part2.withColumn('acc_now_delinq', part2['acc_now_delinq'].cast(IntegerType()))\n",
    "part2 = part2.withColumn(\"open_acc_6m\",part2[\"open_acc_6m\"].cast(IntegerType()))\n",
    "part2 = part2.withColumn(\"tot_coll_amt\",part2[\"tot_coll_amt\"].cast(FloatType()))\n",
    "part2 = part2.withColumn(\"tot_cur_bal\",part2[\"tot_cur_bal\"].cast(FloatType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- pub_rec: integer (nullable = true)\n",
      " |-- revol_bal: float (nullable = true)\n",
      " |-- revol_util: float (nullable = true)\n",
      " |-- total_acc: integer (nullable = true)\n",
      " |-- initial_list_status: string (nullable = true)\n",
      " |-- out_prncp: float (nullable = true)\n",
      " |-- out_prncp_inv: float (nullable = true)\n",
      " |-- total_pymnt: float (nullable = true)\n",
      " |-- total_pymnt_inv: float (nullable = true)\n",
      " |-- total_rec_prncp: float (nullable = true)\n",
      " |-- total_rec_int: float (nullable = true)\n",
      " |-- total_rec_late_fee: float (nullable = true)\n",
      " |-- recoveries: float (nullable = true)\n",
      " |-- collection_recovery_fee: float (nullable = true)\n",
      " |-- last_pymnt_amnt: float (nullable = true)\n",
      " |-- collections_12_mths_ex_med: integer (nullable = true)\n",
      " |-- application_type: string (nullable = true)\n",
      " |-- acc_now_delinq: integer (nullable = true)\n",
      " |-- tot_coll_amt: float (nullable = true)\n",
      " |-- tot_cur_bal: float (nullable = true)\n",
      " |-- open_acc_6m: integer (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1115</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10285</td>\n",
       "      <td>10285</td>\n",
       "      <td>206280</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pub_rec  revol_bal  revol_util  total_acc  initial_list_status  out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med  application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  0        0          1115        0          0                    0          0              0            0                0                0              0                   0           0                        0                6                           0                 0               10285         10285        206280     "
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's cope with the columns with missing values:revol_util, collections_12_mths_ex_med, tot_coll_amt, tot_cur_bal, open_acc_6m <br>\n",
    "By replacing the missing values with the mean values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+-----------------+\n",
      "|summary|       revol_util|\n",
      "+-------+-----------------+\n",
      "|  count|          1205834|\n",
      "|   mean|49.60215702112277|\n",
      "| stddev|24.76317095036885|\n",
      "|    min|              0.0|\n",
      "|    max|            366.6|\n",
      "+-------+-----------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part2.describe('revol_util').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "49.60215702112277"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "revol_util_mean=part2.select(fn.avg('revol_util')).collect()\n",
    "revol_util_mean[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2 = part2.na.fill({'revol_util':revol_util_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------------------------------------------------------------+\n",
      "|count(CASE WHEN (isnan(revol_util) OR (revol_util IS NULL)) THEN true END)|\n",
      "+--------------------------------------------------------------------------+\n",
      "|                                                                         0|\n",
      "+--------------------------------------------------------------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# part2.select([count(when(isnan('revol_util'),True))]).show()\n",
    "# part2.select([count(when(col('revol_util').isNull(),True))]).show()\n",
    "part2.select([count(when(isnan('revol_util') | col('revol_util').isNull() , True))]).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAEICAYAAACqMQjAAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAej0lEQVR4nO3de5hdVX3/8ffHBMKdABkpJIEESauResGRxJ83SnwgoWiwD9KAlUhjUyl49ylBrCCiVVuhpSIWTSRcQsB4IbVQTLkU/f0gONwJETPlkowBMhgCQRQMfH9/rDWwc3LWTGbO5JxJ8nk9z3lmn+9ee+/vWeec/T1r733OKCIwMzOr51WtTsDMzIYuFwkzMytykTAzsyIXCTMzK3KRMDOzIhcJMzMrcpFoMUnflvQPg7SuAyQ9K2lYvn+zpI8Mxrrz+q6TNHOw1teP7Z4r6UlJjzd72yWSDpfUtZltt0i/1eYgaZmkwwd7O/0l6Z2SHqzcf0TSe/L05yR9dwtsc9DeR7ax4a1OYFsm6RFgX2AD8CLwAHApcHFEvAQQER/tx7o+EhH/XWoTESuB3RrL+uXtnQ0cHBF/VVn/tMFYdz/zGAt8BjgwItbUmX84cCPwHBDAauCrEfG9ZubZm2b1W0S8vhnbqSUpgAkR0Znz+BnwJ/XaRsRXBmF7Hya9F95RWe9mvY+aobY/tnYeSWx5742I3YEDga8CpwNzB3sjkrbVgn8g8Jt6BaJidUTsBuwBfAr4jqS6OymzRmzD77OyiPBtC92AR4D31MQOA14CDsn3LwHOzdOjgJ8A64C1wM9IhfyyvMzvgGeBvwfGkT45zwJWArdUYsPz+m4G/hG4HXgauAbYO887HOiqly8wFXgB+EPe3j2V9X0kT78K+DzwKLCGNELaM8/ryWNmzu1J4Mxe+mnPvHx3Xt/n8/rfkx/zSzmPS+osW+9xrAE+ULn/WmBJ7tMHgeNzfDLwODCs0vb9wL15egTwL6TRyeo8PaJ2u8AcYFFNDv8KXFCn3z4M/Bz4Z+Ap4GFgWmW58fm5XA/8N3AhcHmh3zZ67FReb8DZwNW5X9cDy4D2Stv9gR/kPn8Y+Hgvz8/L+VcfQ56+JT/Xv83P0V9uRl6X5+lv5mV6bhuAsyt9+r859weA9+f464Dfk0bmzwLrat9H+f7fAJ35OV8M7F+ZF8BHgRX5ObgQUOGxnw0sAi4HngE+QnoP30p6nz6WH8eOpf7I8WOAu/My/w94Q6v3T5t780iiySLidqALeGed2Z/J89pIh6k+lxaJD5F2tu+NiN0i4uuVZd5NeuMcVdjkScBfk3YKG4ALNiPH/wK+AlyVt/fGOs0+nG9/BhxEOsz1zZo27yAddpgCfEHS6wqb/DdSoTgoP56TgJMjHVqbRh4pRMSHe8tb0qskvY9UbDtzbFdSgVgAvBo4AfiWpNdHxG2kN/MRldWcmNsCnEkqJG8C3kjaOXy+zqavBI6WtEfe5jDg+Mp6ak0iFatRwNeBuZKU5y0gFfV9SDuoD/X2mPvwPmAhMJK0o/xmzu9VwH8A9wCjSc/PJyWVXkNFEfGuPPnG/Bxd1Y9lT8vL7EZ6rTxF+iADqUC8k/S6+CJwuaT9ImI5aQd/a152ZO16JR1B+nB0PLAf6YPHwppmxwBvJT2vx1N+/wBMJxWKkcAVpAL1KdLz9zZS//1dfkyb9IekQ4F5wN+Sntd/BxZLGtF3L7Wei0RrrAb2rhP/A+lFfWBE/CEifhb5Y0gvzo6I30bE7wrzL4uI+yPit8A/AMf3nNhu0AeB8yLioYh4FjgDmFEzHP9iRPwuIu4h7ZA2KTY5l78EzoiI9RHxCPAN+rdz3F/SOtKo40fApyPirjzvGOCRiPheRGyIiDtJn6CPy/OvJBUOJO0OHJ1jPY/xnIhYExHdpJ3VJnlFxKPAncCxOXQE8FwuQvU8GhHfiYgXgfmk53xfSQeQdlxfiIgXIuLnpJ37QP08Iq7N27mMV/r/rUBbRJyTt/MQ8B1gRgPbGjBJbcCPgY/1PG8R8f2IWB0RL+XCs4JUpDfHB4F5EXFnRDxPem2+TdK4SpuvRsS6SOfxbiJ9ECi5NSJ+nHP5XUTcERG35dfTI6Sd/rt7Wf5vgH+PiKUR8WJEzAeeJ30AGfJcJFpjNGkYXOufSJ+AfyrpIUlzNmNdq/ox/1FgB9InoEbtn9dXXfdw0gioR/VqpOeof1J9FLBjnXWN7kcuq/Mnyj1II6XqyOBAYJKkdT030k7kj/L8BcBf5E91fwHcmXf6UP8x7l/IYQG52LDxaKSel/slIp7Lk7vlda+txKDv57c3tf2/Uy7iB5ILa6VPPsfGz11TSNqB9Cl9QUQsrMRPknR3Jb9D2PzX7UbPW/4Q8xs2fk1tzmuzx0bPgaQ/lvQTSY9LeoY06u4ttwOBz9T091jKr6UhxUWiySS9lfRi/XntvPxJ+jMRcRDwXuDTkqb0zC6ssq+RxtjK9AGk0cqTpMMsu1TyGkY6zLW5611NevFX170BeKKP5Wo9mXOqXdev+7ke8qfG04E/ldTzqX4V8D8RMbJy2y0iTsnLPEDaoUxj0517vce4urD57wOHSxpDOq/RW5EoeQzYW9IuldjYUuMGrAIerumT3SPi6EL7jV4rvFJgB8O/kc47vHwYT9KBpJHNacA++QPA/UDPIbl+vTbzIcd9GMBrqrC9i4Bfkq5g2oNUYLXJUq9YBXy5pr93iYgre1lmyHCRaBJJe0g6hnRs9PKIuK9Om2MkHZyPTz9DOvb5Yp79BOmYfX/9laSJecdzDukE64vAr0ifLP88f5r7POlEbY8ngHH5+HU9VwKfkjRe0m68cg5jQ3+Sy7lcDXxZ0u55B/Fp0onCfouIF0iHq76QQz8B/ljShyTtkG9vrTk/sgD4OPAu0s6++hg/L6lN0qi8zrp55cNRNwPfI+2Alw8g90eBDuBsSTtKehvpw8Jgux14RtLpknaWNEzSIfkDTD13k0Zbu0g6mHSxRNWAXpuS/pZ0mObEyJeEZ7uSdszdud3JpJFEdXtjJO1YWPUC4GRJb8ojxK8AS/OhocGwO+n9+ayk1wKn1Myv7Y/vAB+VNEnJrvl9t/sg5bNFuUhsef8haT3p08SZwHnAyYW2E0hXtDxLunriWxFxc573j6Qd1jpJn+3H9i8jXfnxOLATaWdIRDxNOtn2XdInrN+STpr36NlZ/kbSnXXWOy+v+xbS1TG/Bz7Wj7yqPpa3/xBphLUgr3+g5gEHSHpvRKwHjiQdb19N6oevsXFBvJJ0Rc6NEfFkJX4uaad9L3Af6bzDub1sdwHpiqyBjCJ6fJB0MvQ3eVtXkY5fD5pcmN9LOg7/MGk0913SSeJ6zidd7fYE6RzKFTXzzwbm59fm8f1I5QTSznS10pdAn5X0uTy6+wbpPfAE8KfA/60sdyPpaq3HJT1Zu9KIuIF0/u0HpNHZaxjc8y2fJY0615MKQO3J+rOp9EdEdJDOS3yTdHK+k3TRx1ZBfZ8XNbNWkXQV8MuIOKvVudj2ySMJsyEkHwp7Tb6cdyrp8ssftzov235tf98eNBva/gj4IelEaxdwSuVyXrOm8+EmMzMr8uEmMzMr2uYON40aNSrGjRvX6jTMzLYqd9xxx5MR0VYb3+aKxLhx4+jo6Gh1GmZmWxVJj9aL+3CTmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRVtc9+43tYsWLqybvzESQc0ORMz2x55JGFmZkUuEmZmVtRnkZA0T9IaSfdXYv8k6ZeS7pX0I0kjK/POkNQp6UFJR1XiU3OsU9KcSny8pKWSVki6quefm0sake935vnjButBm5nZ5tmckcQlwNSa2BLgkIh4A/Ar4AwASRNJ/3D89XmZb0kaJmkYcCEwDZgInJDbQvqn9OdHxATSPwmfleOzgKci4mDSP2L/2oAeoZmZDVifRSIibgHW1sR+GhEb8t3bgDF5ejqwMCKej4iHgU7gsHzrjIiHIuIFYCEwXZKAI4BFefn5wLGVdc3P04uAKbm9mZk1yWCck/hr4Lo8PRpYVZnXlWOl+D7AukrB6YlvtK48/+ncfhOSZkvqkNTR3d3d8AMyM7OkoSIh6UxgA3BFT6hOsxhAvLd1bRqMuDgi2iOiva1tk3+sZGZmAzTg70lImgkcA0yJiJ6ddxcwttJsDLA6T9eLPwmMlDQ8jxaq7XvW1SVpOLAnNYe9tiWl70OYmbXSgIqEpKnA6cC7I+K5yqzFwAJJ5wH7AxOA20mjggmSxgO/Jp3cPjEiQtJNwHGk8xQzgWsq65oJ3Jrn31gpRts9f8nOzJqhzyIh6UrgcGCUpC7gLNLVTCOAJflc8m0R8dGIWCbpauAB0mGoUyPixbye04DrgWHAvIhYljdxOrBQ0rnAXcDcHJ8LXCapkzSCmDEIj9fMzPpB29qH8/b29ujo6Gh1Gv02WIebPJIws4GQdEdEtNfG/Y1rMzMrcpEwM7MiFwkzMytykTAzsyIXCTMzK3KRMDOzIhcJMzMr8r8vbTL//IaZbU08kjAzsyIXCTMzK/Lhpm2Mf/jPzAaTRxJmZlbkImFmZkUuEmZmVuQiYWZmRS4SZmZW5CJhZmZFLhJmZlbkImFmZkUuEmZmVuQiYWZmRS4SZmZW5CJhZmZFLhJmZlbkImFmZkV9FglJ8yStkXR/Jba3pCWSVuS/e+W4JF0gqVPSvZIOrSwzM7dfIWlmJf4WSfflZS6QpN62YWZmzbM5I4lLgKk1sTnADRExAbgh3weYBkzIt9nARZB2+MBZwCTgMOCsyk7/oty2Z7mpfWzDBmDB0pV1b2ZmvemzSETELcDamvB0YH6eng8cW4lfGsltwEhJ+wFHAUsiYm1EPAUsAabmeXtExK0REcClNeuqtw0zM2uSgZ6T2DciHgPIf1+d46OBVZV2XTnWW7yrTry3bWxC0mxJHZI6uru7B/iQzMys1mCfuFadWAwg3i8RcXFEtEdEe1tbW38XNzOzgoH+j+snJO0XEY/lQ0ZrcrwLGFtpNwZYneOH18RvzvExddr3to0hz8f6zWxbMdCRxGKg5wqlmcA1lfhJ+SqnycDT+VDR9cCRkvbKJ6yPBK7P89ZLmpyvajqpZl31tmFmZk3S50hC0pWkUcAoSV2kq5S+ClwtaRawEvhAbn4tcDTQCTwHnAwQEWslfQn4RW53TkT0nAw/hXQF1c7AdflGL9swM7Mm6bNIRMQJhVlT6rQN4NTCeuYB8+rEO4BD6sR/U28bZmbWPP7GtZmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWVGf/77Utm0Llq6sGz9x0gFNzsTMhiKPJMzMrMhFwszMilwkzMysyEXCzMyKXCTMzKyooSIh6VOSlkm6X9KVknaSNF7SUkkrJF0lacfcdkS+35nnj6us54wcf1DSUZX41BzrlDSnkVzNzKz/BlwkJI0GPg60R8QhwDBgBvA14PyImAA8BczKi8wCnoqIg4HzczskTczLvR6YCnxL0jBJw4ALgWnAROCE3NbMzJqk0cNNw4GdJQ0HdgEeA44AFuX584Fj8/T0fJ88f4ok5fjCiHg+Ih4GOoHD8q0zIh6KiBeAhbmtmZk1yYCLRET8GvhnYCWpODwN3AGsi4gNuVkXMDpPjwZW5WU35Pb7VOM1y5Tim5A0W1KHpI7u7u6BPiQzM6vRyOGmvUif7McD+wO7kg4N1YqeRQrz+hvfNBhxcUS0R0R7W1tbX6mbmdlmauRw03uAhyOiOyL+APwQ+D/AyHz4CWAMsDpPdwFjAfL8PYG11XjNMqW4mZk1SSO/3bQSmCxpF+B3wBSgA7gJOI50DmEmcE1uvzjfvzXPvzEiQtJiYIGk80gjkgnA7aSRxARJ44Ffk05un9hAvoOu9LtHZmbbigEXiYhYKmkRcCewAbgLuBj4T2ChpHNzbG5eZC5wmaRO0ghiRl7PMklXAw/k9ZwaES8CSDoNuJ505dS8iFg20HzNzKz/FFH3MP9Wq729PTo6OpqyrW15JOFfgTXbvki6IyLaa+P+xrWZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVlRIz8Vbtuw0o8X+of/zLYvHkmYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZUUNFQtJISYsk/VLScklvk7S3pCWSVuS/e+W2knSBpE5J90o6tLKembn9CkkzK/G3SLovL3OBJDWSr5mZ9U+jI4l/Bf4rIl4LvBFYDswBboiICcAN+T7ANGBCvs0GLgKQtDdwFjAJOAw4q6ew5DazK8tNbTBfMzPrhwEXCUl7AO8C5gJExAsRsQ6YDszPzeYDx+bp6cClkdwGjJS0H3AUsCQi1kbEU8ASYGqet0dE3BoRAVxaWZeZmTVBIyOJg4Bu4HuS7pL0XUm7AvtGxGMA+e+rc/vRwKrK8l051lu8q07czMyapJEiMRw4FLgoIt4M/JZXDi3VU+98QgwgvumKpdmSOiR1dHd39561mZlttkaKRBfQFRFL8/1FpKLxRD5URP67ptJ+bGX5McDqPuJj6sQ3EREXR0R7RLS3tbU18JDMzKxqwEUiIh4HVkn6kxyaAjwALAZ6rlCaCVyTpxcDJ+WrnCYDT+fDUdcDR0raK5+wPhK4Ps9bL2lyvqrppMq6zMysCRr996UfA66QtCPwEHAyqfBcLWkWsBL4QG57LXA00Ak8l9sSEWslfQn4RW53TkSszdOnAJcAOwPX5ZuZmTVJQ0UiIu4G2uvMmlKnbQCnFtYzD5hXJ94BHNJIjoOh9P+ezcy2df7GtZmZFblImJlZkYuEmZkVuUiYmVlRo1c32XamdBL/xEkHNDkTM2sGjyTMzKzIRcLMzIpcJMzMrMhFwszMilwkzMysyEXCzMyKXCTMzKzIRcLMzIpcJMzMrMjfuK7wT4KbmW3MIwkzMytykTAzsyIXCTMzK3KRMDOzIhcJMzMrcpEwM7MiFwkzMytykTAzsyIXCTMzK3KRMDOzooaLhKRhku6S9JN8f7ykpZJWSLpK0o45PiLf78zzx1XWcUaOPyjpqEp8ao51SprTaK5mZtY/gzGS+ASwvHL/a8D5ETEBeAqYleOzgKci4mDg/NwOSROBGcDrganAt3LhGQZcCEwDJgIn5LZmZtYkDRUJSWOAPwe+m+8LOAJYlJvMB47N09PzffL8Kbn9dGBhRDwfEQ8DncBh+dYZEQ9FxAvAwtzWzMyapNGRxL8Afw+8lO/vA6yLiA35fhcwOk+PBlYB5PlP5/Yvx2uWKcU3IWm2pA5JHd3d3Q0+JDMz6zHgIiHpGGBNRNxRDddpGn3M629802DExRHRHhHtbW1tvWRtZmb90cj/k3g78D5JRwM7AXuQRhYjJQ3Po4UxwOrcvgsYC3RJGg7sCaytxHtUlynFzcysCQY8koiIMyJiTESMI514vjEiPgjcBByXm80ErsnTi/N98vwbIyJyfEa++mk8MAG4HfgFMCFfLbVj3sbigeZrZmb9tyX+M93pwEJJ5wJ3AXNzfC5wmaRO0ghiBkBELJN0NfAAsAE4NSJeBJB0GnA9MAyYFxHLtkC+ZmZWMChFIiJuBm7O0w+RrkyqbfN74AOF5b8MfLlO/Frg2sHI0czM+s/fuDYzsyIXCTMzK3KRMDOzIhcJMzMrcpEwM7MiFwkzMytykTAzsyIXCTMzK3KRMDOzIhcJMzMrcpEwM7MiFwkzMytykTAzsyIXCTMzK3KRMDOzoi3xT4dsO7Rg6cq68RMnHdDkTMxsMHkkYWZmRS4SZmZW5CJhZmZFLhJmZlbkImFmZkUuEmZmVuQiYWZmRS4SZmZWNOAiIWmspJskLZe0TNIncnxvSUskrch/98pxSbpAUqekeyUdWlnXzNx+haSZlfhbJN2Xl7lAkhp5sGZm1j+NjCQ2AJ+JiNcBk4FTJU0E5gA3RMQE4IZ8H2AaMCHfZgMXQSoqwFnAJOAw4KyewpLbzK4sN7WBfM3MrJ8GXCQi4rGIuDNPrweWA6OB6cD83Gw+cGyeng5cGsltwEhJ+wFHAUsiYm1EPAUsAabmeXtExK0REcCllXWZmVkTDMo5CUnjgDcDS4F9I+IxSIUEeHVuNhpYVVmsK8d6i3fVidfb/mxJHZI6uru7G304ZmaWNVwkJO0G/AD4ZEQ801vTOrEYQHzTYMTFEdEeEe1tbW19pWxmZpupoSIhaQdSgbgiIn6Yw0/kQ0Xkv2tyvAsYW1l8DLC6j/iYOnEzM2uSRq5uEjAXWB4R51VmLQZ6rlCaCVxTiZ+Ur3KaDDydD0ddDxwpaa98wvpI4Po8b72kyXlbJ1XWZWZmTdDI/5N4O/Ah4D5Jd+fY54CvAldLmgWsBD6Q510LHA10As8BJwNExFpJXwJ+kdudExFr8/QpwCXAzsB1+WZmZk0y4CIRET+n/nkDgCl12gdwamFd84B5deIdwCEDzdHMzBrjb1ybmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVmRi4SZmRW5SJiZWZGLhJmZFblImJlZkYuEmZkVuUiYmVnRkC8SkqZKelBSp6Q5rc7HzGx7MqSLhKRhwIXANGAicIKkia3Nysxs+zG81Qn04TCgMyIeApC0EJgOPNDSrGyzLVi6sm78xEkHNDkTMxuIoV4kRgOrKve7gEm1jSTNBmbnu89KenCA2xsFPDnAZZtpa8iz1xw/2MRE+rDV9+UQsjXkuTXkCK3J88B6waFeJFQnFpsEIi4GLm54Y1JHRLQ3up4tbWvIc2vIEbaOPLeGHGHryHNryBGGVp5D+pwEaeQwtnJ/DLC6RbmYmW13hnqR+AUwQdJ4STsCM4DFLc7JzGy7MaQPN0XEBkmnAdcDw4B5EbFsC26y4UNWTbI15Lk15AhbR55bQ46wdeS5NeQIQyhPRWxyiN/MzAwY+oebzMyshVwkzMysyEUiG6o//yHpEUn3SbpbUkeO7S1piaQV+e9eLchrnqQ1ku6vxOrmpeSC3Lf3Sjq0hTmeLenXuT/vlnR0Zd4ZOccHJR3VjBzzdsdKuknScknLJH0ix4dMf/aS45DqT0k7Sbpd0j05zy/m+HhJS3NfXpUvhEHSiHy/M88f18IcL5H0cKUv35TjLXn/vCwitvsb6aT4/wIHATsC9wATW51Xzu0RYFRN7OvAnDw9B/haC/J6F3AocH9feQFHA9eRvvcyGVjawhzPBj5bp+3E/LyPAMbn18OwJuW5H3Bont4d+FXOZ8j0Zy85Dqn+zH2yW57eAVia++hqYEaOfxs4JU//HfDtPD0DuKqFOV4CHFenfUvePz03jySSl3/+IyJeAHp+/mOomg7Mz9PzgWObnUBE3AKsrQmX8poOXBrJbcBISfu1KMeS6cDCiHg+Ih4GOkmviy0uIh6LiDvz9HpgOenXBoZMf/aSY0lL+jP3ybP57g75FsARwKIcr+3Lnj5eBEyRVO9LvM3IsaQl758eLhJJvZ//6O0N0EwB/FTSHfnnRwD2jYjHIL15gVe3LLuNlfIaav17Wh62z6scqhsSOebDHW8mfbockv1ZkyMMsf6UNEzS3cAaYAlpFLMuIjbUyeXlPPP8p4F9mp1jRPT05ZdzX54vaURtjnXy3+JcJJLN+vmPFnl7RBxK+iXcUyW9q9UJDcBQ6t+LgNcAbwIeA76R4y3PUdJuwA+AT0bEM701rRNrSq51chxy/RkRL0bEm0i/0HAY8LpecmlJnrU5SjoEOAN4LfBWYG/g9Fbm2MNFIhmyP/8REavz3zXAj0gv+id6hpv575rWZbiRUl5Dpn8j4on8Bn0J+A6vHAJpaY6SdiDtfK+IiB/m8JDqz3o5DtX+zLmtA24mHccfKanny8PVXF7OM8/fk80/RDmYOU7Nh/QiIp4HvscQ6UsXiWRI/vyHpF0l7d4zDRwJ3E/KbWZuNhO4pjUZbqKU12LgpHyVxmTg6Z7DKM1Wcyz3/aT+hJTjjHy1y3hgAnB7k3ISMBdYHhHnVWYNmf4s5TjU+lNSm6SReXpn4D2k8yc3AcflZrV92dPHxwE3Rj5b3OQcf1n5QCDSOZNqX7bu/dPMs+RD+Ua6guBXpOOXZ7Y6n5zTQaQrRO4BlvXkRTpmegOwIv/duwW5XUk6vPAH0iedWaW8SMPlC3Pf3ge0tzDHy3IO95LefPtV2p+Zc3wQmNbEvnwH6fDBvcDd+Xb0UOrPXnIcUv0JvAG4K+dzP/CFHD+IVKQ6ge8DI3J8p3y/M88/qIU53pj78n7gcl65Aqol75+em3+Ww8zMiny4yczMilwkzMysyEXCzMyKXCTMzKzIRcLMzIpcJMzMrMhFwszMiv4/Ou0WMz6ncYcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(part2.select('revol_util').collect(),kde=False).set_title('Distribution of Revolving line utilization rate');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2.select('revol_util').toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZcAAAEICAYAAACTVrmbAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAdAklEQVR4nO3df5QV9Znn8fcngEIUww87DgEVHDFKiDGkFVyzGRKCto5JO+dg0LgRMzisETPGnd0BzRxlk3g0sxmZ8cQko4GAxAENyUQSYZj213jmrCCtEhUQ6VWjLUQQ/P0bffaP+l5y7b7d3G7q3tvdfF7n3NNVT32r6qnS62N963urFBGYmZnl6UO1TsDMzPoeFxczM8udi4uZmeXOxcXMzHLn4mJmZrlzcTEzs9y5uJiZWe5cXMxqRNLTkr64j9u4QNJ/5pWTWV5cXMzMLHcuLmY1IGkJcATwG0mvSfpbSZMk/V9JL0n6naTJRe0vkPSkpFclPSXpPEnHAT8BTk7beKlGh2PWjvz4F7PakPQ0cGFE3ClpJPAI8DXg34ApwDLgWOANYBtwYkRsljQCGBYRGyRdkLbx2Vocg1lHfOVi1jP8N2BlRKyMiPcjogloBs5Iy98HxksaFBHbImJDzTI1K4OLi1nPcCRwduoSeyl1cX0WGBERrwPTgYuAbZLukHRsLZM12xsXF7PaKe6TfhZYEhFDij4HRcS1ABGxOiKmAiOAx4GbSmzDrMdwcTGrneeBo9L0z4EvSTpNUj9JAyVNljRK0mGSvizpIOBt4DXgvaJtjJJ0QPXTN+uYi4tZ7VwD/F3qApsONAJXADvIrmT+F9l39EPA3wBbgV3AnwEXp23cDWwA/iDphapmb9YJjxYzM7Pc+crFzMxy5+JiZma5c3ExM7PcubiYmVnu+tc6gZ7i0EMPjdGjR9c6DTOzXuXBBx98ISLq2sZdXJLRo0fT3Nxc6zTMzHoVSb8vFXe3mJmZ5c7FxczMcufiYmZmufM9FzPr8959911aW1t56623ap1KrzVw4EBGjRrFgAEDymrv4mJmfV5rayuDBw9m9OjRSKp1Or1ORLBz505aW1sZM2ZMWeu4W8zM+ry33nqL4cOHu7B0kySGDx/epSs/Fxcz2y+4sOybrp4/FxczM8ud77mY2X5nftMTuW7vsqnH5Lq9vsBXLn3U/KYn2n3MrO8ZPXo0L7zQ9ffELVq0iK1bt+6Zv/DCC9m4ceM+bbOYi4uZWZVFBO+//35Nc2hbXH76058ybty43Lbv4mJmVgVPP/00xx13HBdffDETJkxgyZIlnHzyyUyYMIGzzz6b1157jVWrVvGVr3xlzzr33nsvX/rSlwBYunQpn/zkJxk/fjxz5swpe5/jx4/fM/+DH/yAefPmsXz5cpqbmznvvPM44YQTePPNN5k8eXKuz1esWHGRtFDSdkmPFcX+j6THJT0i6V8lDSladrmkFkmbJZ1WFG9IsRZJc4viYyStlbRF0q2SDkjxA9N8S1o+ulLHaGbWFZs3b+b888+nqamJBQsWcOedd/LQQw9RX1/Pddddx9SpU1mzZg2vv/46ALfeeivTp09n69atzJkzh7vvvpv169ezbt06fv3rX3c7j2nTplFfX88tt9zC+vXrGTRoUF6HuEclr1wWAQ1tYk3A+Ig4HngCuBxA0jjgHOATaZ0fSeonqR9wA3A6MA44N7UF+D4wPyLGAi8CM1N8JvBiRBwNzE/tzMxq7sgjj2TSpEmsWbOGjRs3csopp3DCCSewePFifv/739O/f38aGhr4zW9+w+7du7njjjtobGxk3bp1TJ48mbq6Ovr37895553HfffdV+vD6VTFRotFxH1trxoi4t+LZtcA09J0I7AsIt4GnpLUApyUlrVExJMAkpYBjZI2AV8AvpraLAbmAT9O25qX4suBH0pSRERuB2dm1g0HHXQQkN1zmTp1KkuXLm3XZvr06dxwww0MGzaME088kcGDB9Pd/3z179//A/d2qvn4m1oORf5L4NY0PZKs2BS0phjAs23iE4HhwEsRsbtE+5GFdSJit6SXU/t2Qx8kzQJmARxxxBH7eDhm1lvUeujwpEmTmD17Ni0tLRx99NG88cYbtLa2cswxxzB58mRmzpzJTTfdxPTp0wGYOHEil156KS+88AJDhw5l6dKlfPOb39zrfg477DC2b9/Ozp07Ofjgg/ntb39LQ0PWoTR48GBeffXVih1jTW7oS/o2sBu4pRAq0Sy6Ee9sW+2DETdGRH1E1NfVtXuRmplZRdTV1bFo0SLOPfdcjj/+eCZNmsTjjz8OQL9+/TjzzDNZtWoVZ555JgAjRozgmmuu4fOf/zyf+tSnmDBhAo2NjXvdz4ABA7jyyiuZOHEiZ555Jscee+yeZRdccAEXXXTRnhv6eVMle4tSt9hvI2J8UWwGcBEwJSLeSLHLASLimjS/mj92bc2LiNOK2wHXAjuAP0lXJycX2hXWjYj7JfUH/gDU7a1brL6+PnrjmyhL/X7lsqnHdBg32x9t2rSJ4447rtZp9HqlzqOkByOivm3bqnaLSWoA5gB/VigsyQrgXyRdB3wMGAs8QHYVMlbSGOA5spv+X42IkHQP2T2bZcAM4Paibc0A7k/L7/b9lj9y0TGzaqhYcZG0FJgMHCqpFbiKbHTYgUBTegjamoi4KCI2SLoN2EjWXTY7It5L27kEWA30AxZGxIa0iznAMknfAx4GFqT4AmBJGhSwi6wgmZn1WTt37mTKlCnt4nfddRfDhw+vQUaVHS12bonwghKxQvurgatLxFcCK0vEn+SPI8qK428BZ3cpWTPr8yKizz4Zefjw4axfv76i++hqB5B/oW9mfd7AgQPZuXNnt4f07u8KLwsbOHBg2ev4qchm1ueNGjWK1tZWduzYUetUeq3Ca47L5eJiZn3egAEDyn49r+XD3WJmZpY7FxczM8udi4uZmeXOxcXMzHLnG/q9hH9Zb2a9ia9czMwsdy4uZmaWO3eLGeBuNzPLl69czMwsdy4uZmaWOxcXMzPLnYuLmZnlzsXFzMxy5+JiZma5c3ExM7PcubiYmVnuXFzMzCx3Li5mZpY7FxczM8udi4uZmeXOxcXMzHLn4mJmZrmrWHGRtFDSdkmPFcWGSWqStCX9HZriknS9pBZJj0iaULTOjNR+i6QZRfHPSHo0rXO9JHW2DzMzq55Kvs9lEfBD4Oai2Fzgroi4VtLcND8HOB0Ymz4TgR8DEyUNA64C6oEAHpS0IiJeTG1mAWuAlUADsKqTfVg3lHrPC/hdL2bWuYpduUTEfcCuNuFGYHGaXgycVRS/OTJrgCGSRgCnAU0RsSsVlCagIS07JCLuj4ggK2Bn7WUfZmZWJdW+53JYRGwDSH8/muIjgWeL2rWmWGfx1hLxzvbRjqRZkpolNe/YsaPbB2VmZh/UU27oq0QsuhHvkoi4MSLqI6K+rq6uq6ubmVkHKnnPpZTnJY2IiG2pa2t7ircChxe1GwVsTfHJbeL3pvioEu0720eP5/sbZtZXVPvKZQVQGPE1A7i9KH5+GjU2CXg5dWmtBk6VNDSN+joVWJ2WvSppUholdn6bbZXah5mZVUnFrlwkLSW76jhUUivZqK9rgdskzQSeAc5OzVcCZwAtwBvA1wEiYpek7wLrUrvvRERhkMA3yEakDSIbJbYqxTvah5mZVUnFiktEnNvBoikl2gYwu4PtLAQWlog3A+NLxHeW2oeZmVVPT7mhb2ZmfYiLi5mZ5c7FxczMcufiYmZmuXNxMTOz3Lm4mJlZ7lxczMwsdy4uZmaWOxcXMzPLnYuLmZnlzsXFzMxy5+JiZma5c3ExM7PcubiYmVnuXFzMzCx3Li5mZpY7FxczM8udi4uZmeWuYq85tr5tftMTJeOXTT2mypmYWU/kKxczM8udi4uZmeXOxcXMzHLn4mJmZrlzcTEzs9zVpLhIukzSBkmPSVoqaaCkMZLWStoi6VZJB6S2B6b5lrR8dNF2Lk/xzZJOK4o3pFiLpLnVP0Izs/1b1YuLpJHAXwP1ETEe6AecA3wfmB8RY4EXgZlplZnAixFxNDA/tUPSuLTeJ4AG4EeS+knqB9wAnA6MA85Nbc3MrEpq1S3WHxgkqT/wYWAb8AVgeVq+GDgrTTemedLyKZKU4ssi4u2IeApoAU5Kn5aIeDIi3gGWpbZmZlYlVS8uEfEc8APgGbKi8jLwIPBSROxOzVqBkWl6JPBsWnd3aj+8ON5mnY7i7UiaJalZUvOOHTv2/eDMzAwos7hIGp/XDiUNJbuSGAN8DDiIrAurrSis0sGyrsbbByNujIj6iKivq6vbW+pmZlamcq9cfiLpAUkXSxqyj/v8IvBUROyIiHeBXwH/BRiSuskARgFb03QrcDhAWv4RYFdxvM06HcXNzKxKynq2WER8VtJY4C+BZkkPAD+LiKZu7PMZYJKkDwNvAlOAZuAeYBrZPZIZwO2p/Yo0f39afndEhKQVwL9Iuo7sCmgs8ADZlctYSWOA58hu+n+1G3lWTEfP5TIz6yvKfnBlRGyR9HdkheB64NPpxvoVEfGrLmxnraTlwEPAbuBh4EbgDmCZpO+l2IK0ygJgiaQWsiuWc9J2Nki6DdiYtjM7It4DkHQJsJpsJNrCiNhQbn5mZrbvyiouko4Hvg78OdAEfCkiHpL0MbIrirKLC0BEXAVc1Sb8JNlIr7Zt3wLO7mA7VwNXl4ivBFZ2JSczM8tPuVcuPwRuIrtKebMQjIit6WrGzMxsj3KLyxnAm0XdTh8CBkbEGxGxpGLZmZlZr1TuaLE7gUFF8x9OMTMzs3bKLS4DI+K1wkya/nBlUjIzs96u3OLyuqQJhRlJnyEbRmxmZtZOufdcvgX8QlLhx4gjgOmVScnMzHq7cn9EuU7SscDHyX6k+Hj6db2ZmVk7Zf+IEjgRGJ3W+bQkIuLmimRlZma9Wrk/olwC/CmwHngvhQNwcTEzs3bKvXKpB8ZFRMmnC5uZmRUrd7TYY8CfVDIRMzPrO8q9cjkU2Jiehvx2IRgRX65IVmZm1quVW1zmVTIJ6zs6ep3AZVOPqXImZlZL5Q5F/g9JRwJjI+LO9C6WfpVNzczMeqtyX3P8V8By4J9TaCTw60olZWZmvVu5N/RnA6cAr0D24jDgo5VKyszMerdyi8vbEfFOYSa9y97Dks3MrKRyi8t/SLoCGCRpKvAL4DeVS8vMzHqzcovLXGAH8Cjw38leIew3UJqZWUnljhZ7n+w1xzdVNh0zM+sLyn222FOUuMcSEUflnpGZmfV6XXm2WMFA4GxgWP7pmJlZX1DWPZeI2Fn0eS4i/hH4QoVzMzOzXqrcbrEJRbMfIruSGVyRjMzMrNcrd7TYPxR9rgE+A3yluzuVNETSckmPS9ok6WRJwyQ1SdqS/g5NbSXpekktkh4pLnSSZqT2WyTNKIp/RtKjaZ3rJam7uZqZWdeVO1rs8znv95+Af4uIaZIOAD4MXAHcFRHXSppLNvx5DnA6MDZ9JgI/BiZKGgZcRXYVFcCDklZExIupzSxgDdmw6QZgVc7HYGZmHSi3W+x/dLY8Iq4rd4eSDgE+B1yQ1n0HeEdSIzA5NVsM3EtWXBqBm9OLytakq54RqW1TROxK220CGiTdCxwSEfen+M3AWbi4mJlVTbndYvXAN8geWDkSuAgYR3bfpav3Xo4i+0HmzyQ9LOmnkg4CDouIbQDpb+HZZSOBZ4vWby3Ko6N4a4m4mZlVSVdeFjYhIl4FkDQP+EVEXNjNfU4AvhkRayX9E1kXWEdK3S+JbsTbb1iaRdZ9xhFHHNFZzmZm1gXlXrkcAbxTNP8OMLqb+2wFWiNibZpfTlZsnk/dXaS/24vaH160/ihg617io0rE24mIGyOiPiLq6+rqunk4ZmbWVrnFZQnwgKR5kq4C1gI3d2eHEfEH4FlJH0+hKcBGYAVQGPE1A7g9Ta8Azk+jxiYBL6dus9XAqZKGppFlpwKr07JXJU1Ko8TOL9qWmZlVQbmjxa6WtAr4ryn09Yh4eB/2+03gljRS7Eng62SF7jZJM4FnyJ4CANlorzOAFuCN1JaI2CXpu8C61O47hZv7ZPeHFgGDyG7k+2a+mVkVlXvPBbLhwq9ExM8k1UkaExFPdWenEbGeDz5SpmBKibZB9rKyUttZCCwsEW8Gxncntzx19D55M7O+rtzXHF9FNiz48hQaAPy8UkmZmVnvVu49l78Avgy8DhARW/HjX8zMrAPlFpd3UvdUAKTfpZiZmZVUbnG5TdI/A0Mk/RVwJ35xmJmZdaDc0WI/kDQVeAX4OHBlRDRVNDPrUzoa3HDZ1GOqnImZVcNei4ukfmS/H/ki4IJiZmZ7tddusYh4D3hD0keqkI+ZmfUB5f7O5S3g0fTk4dcLwYj464pkZWZmvVq5xeWO9DEzM9urTouLpCMi4pmIWFythMzMrPfb2z2XXxcmJP2ywrmYmVkfsbfiUvxulKMqmYiZmfUdeysu0cG0mZlZh/Z2Q/9Tkl4hu4IZlKZJ8xERh1Q0OzMz65U6LS4R0a9aifRmfrS+mdkHlftsMTMzs7K5uJiZWe5cXMzMLHcuLmZmljsXFzMzy52Li5mZ5c7FxczMcufiYmZmuXNxMTOz3Lm4mJlZ7mpWXCT1k/SwpN+m+TGS1kraIulWSQek+IFpviUtH120jctTfLOk04riDSnWImlutY/NzGx/V8srl0uBTUXz3wfmR8RY4EVgZorPBF6MiKOB+akdksYB5wCfABqAH6WC1Q+4ATgdGAecm9qamVmV1KS4SBoF/Dnw0zQv4AvA8tRkMXBWmm5M86TlU1L7RmBZRLwdEU8BLcBJ6dMSEU9GxDvAstTWzMyqpFZXLv8I/C3wfpofDrwUEbvTfCswMk2PBJ4FSMtfTu33xNus01G8HUmzJDVLat6xY8e+HpOZmSVVLy6SzgS2R8SDxeESTWMvy7oabx+MuDEi6iOivq6urpOszcysK/b2srBKOAX4sqQzgIHAIWRXMkMk9U9XJ6OAral9K3A40CqpP/ARYFdRvKB4nY7iZmZWBVW/comIyyNiVESMJrshf3dEnAfcA0xLzWYAt6fpFWmetPzuiIgUPyeNJhsDjAUeANYBY9PoswPSPlZU4dDMzCypxZVLR+YAyyR9D3gYWJDiC4AlklrIrljOAYiIDZJuAzYCu4HZEfEegKRLgNVAP2BhRGyo6pGYme3nalpcIuJe4N40/STZSK+2bd4Czu5g/auBq0vEVwIrc0zVzMy6wL/QNzOz3Lm4mJlZ7lxczMwsdy4uZmaWOxcXMzPLnYuLmZnlzsXFzMxy5+JiZma5c3ExM7PcubiYmVnuXFzMzCx3Li5mZpY7FxczM8udi4uZmeXOxcXMzHLXk14WZvuh+U1PlIxfNvWYKmdiZnnylYuZmeXOxcXMzHLn4mJmZrlzcTEzs9y5uJiZWe5cXMzMLHcuLmZmljsXFzMzy13Vi4ukwyXdI2mTpA2SLk3xYZKaJG1Jf4emuCRdL6lF0iOSJhRta0Zqv0XSjKL4ZyQ9mta5XpKqfZxmZvuzWly57Ab+JiKOAyYBsyWNA+YCd0XEWOCuNA9wOjA2fWYBP4asGAFXAROBk4CrCgUptZlVtF5DFY7LzMySqheXiNgWEQ+l6VeBTcBIoBFYnJotBs5K043AzZFZAwyRNAI4DWiKiF0R8SLQBDSkZYdExP0REcDNRdsyM7MqqOk9F0mjgU8Da4HDImIbZAUI+GhqNhJ4tmi11hTrLN5aIl5q/7MkNUtq3rFjx74ejpmZJTUrLpIOBn4JfCsiXumsaYlYdCPePhhxY0TUR0R9XV3d3lI2M7My1aS4SBpAVlhuiYhfpfDzqUuL9Hd7ircChxetPgrYupf4qBJxMzOrklqMFhOwANgUEdcVLVoBFEZ8zQBuL4qfn0aNTQJeTt1mq4FTJQ1NN/JPBVanZa9KmpT2dX7RtszMrApq8T6XU4CvAY9KWp9iVwDXArdJmgk8A5ydlq0EzgBagDeArwNExC5J3wXWpXbfiYhdafobwCJgELAqfczMrEqqXlwi4j8pfV8EYEqJ9gHM7mBbC4GFJeLNwPh9SNPMzPaBf6FvZma5c3ExM7PcubiYmVnuXFzMzCx3Li5mZpY7FxczM8udi4uZmeXOxcXMzHLn4mJmZrlzcTEzs9y5uJiZWe5cXMzMLHcuLmZmljsXFzMzy52Li5mZ5c7FxczMcufiYmZmuXNxMTOz3Lm4mJlZ7lxczMwsdy4uZmaWOxcXMzPLnYuLmZnlzsXFzMxy5+JiZma567PFRVKDpM2SWiTNrXU+Zmb7kz5ZXCT1A24ATgfGAedKGlfbrMzM9h/9a51AhZwEtETEkwCSlgGNwMaaZmVdMr/piXaxy6YeU4NMzKyrFBG1ziF3kqYBDRFxYZr/GjAxIi5p024WMCvNfhzY3M1dHgq80M11q6U35AjOM0+9IUfoHXn2hhyhNnkeGRF1bYN99cpFJWLtqmhE3AjcuM87k5ojon5ft1NJvSFHcJ556g05Qu/IszfkCD0rzz55zwVoBQ4vmh8FbK1RLmZm+52+WlzWAWMljZF0AHAOsKLGOZmZ7Tf6ZLdYROyWdAmwGugHLIyIDRXc5T53rVVBb8gRnGeeekOO0Dvy7A05Qg/Ks0/e0Dczs9rqq91iZmZWQy4uZmaWOxeXfdCTHzEj6WlJj0paL6k5xYZJapK0Jf0dWoO8FkraLumxoljJvJS5Pp3fRyRNqGGO8yQ9l87neklnFC27POW4WdJp1cgx7fdwSfdI2iRpg6RLU7zHnM9OcuxR51PSQEkPSPpdyvN/p/gYSWvTubw1DRBC0oFpviUtH13DHBdJeqroXJ6Q4jX5/uwREf5040M2UOD/AUcBBwC/A8bVOq+i/J4GDm0T+3tgbpqeC3y/Bnl9DpgAPLa3vIAzgFVkv1uaBKytYY7zgP9Zou249M/+QGBM+neiX5XyHAFMSNODgSdSPj3mfHaSY486n+mcHJymBwBr0zm6DTgnxX8CfCNNXwz8JE2fA9xawxwXAdNKtK/J96fw8ZVL9+15xExEvAMUHjHTkzUCi9P0YuCsaicQEfcBu9qEO8qrEbg5MmuAIZJG1CjHjjQCyyLi7Yh4Cmgh+3ej4iJiW0Q8lKZfBTYBI+lB57OTHDtSk/OZzslraXZA+gTwBWB5irc9l4VzvByYIqnUj7erkWNHavL9KXBx6b6RwLNF8610/qWptgD+XdKD6TE3AIdFxDbIvvTAR2uW3Qd1lFdPO8eXpO6FhUVdij0ix9Qt82my/5vtkeezTY7Qw86npH6S1gPbgSayq6aXImJ3iVz25JmWvwwMr3aOEVE4l1enczlf0oFtcyyRf8W5uHRfWY+YqaFTImIC2ZOhZ0v6XK0T6oaedI5/DPwpcAKwDfiHFK95jpIOBn4JfCsiXumsaYlYVXItkWOPO58R8V5EnED2RI+TgOM6yaUmebbNUdJ44HLgWOBEYBgwp5Y5Fri4dF+PfsRMRGxNf7cD/0r2ZXm+cFmc/m6vXYYf0FFePeYcR8Tz6Yv9PnATf+yqqWmOkgaQ/Uf7loj4VQr3qPNZKseeej5Tbi8B95LdpxgiqfBj8+Jc9uSZln+E8rtS88yxIXU9RkS8DfyMHnIuXVy6r8c+YkbSQZIGF6aBU4HHyPKbkZrNAG6vTYbtdJTXCuD8NOplEvByobun2tr0Vf8F2fmELMdz0uihMcBY4IEq5SRgAbApIq4rWtRjzmdHOfa08ympTtKQND0I+CLZ/aF7gGmpWdtzWTjH04C7I91Fr3KOjxf9j4TI7gkVn8vafX+qOXqgr33IRmM8QdY3++1a51OU11FkI25+B2wo5EbWJ3wXsCX9HVaD3JaSdYO8S/Z/VjM7yovssv6GdH4fBeprmOOSlMMjZF/aEUXtv51y3AycXsVz+Vmybo5HgPXpc0ZPOp+d5NijzidwPPBwyucx4MoUP4qsuLUAvwAOTPGBab4lLT+qhjnenc7lY8DP+eOIspp8fwofP/7FzMxy524xMzPLnYuLmZnlzsXFzMxy5+JiZma5c3ExM7PcubiYmVnuXFzMzCx3/x+lcA57ZcXf/QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "part2.select('revol_util').toPandas().plot.hist(bins=50, rwidth=0.9, alpha=0.5, title='test');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot_coll_amt_mean=part2.select(fn.avg('tot_coll_amt')).collect()\n",
    "part2 = part2.na.fill({'tot_coll_amt':tot_coll_amt_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot_cur_bal_mean=part2.select(fn.avg('tot_cur_bal')).collect()\n",
    "part2 = part2.na.fill({'tot_cur_bal':tot_cur_bal_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "open_acc_6m_mean=part2.select(fn.avg('open_acc_6m')).collect()\n",
    "part2 = part2.na.fill({'open_acc_6m':open_acc_6m_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "collections_12_mths_ex_med_mean=part2.select(fn.avg('collections_12_mths_ex_med')).collect()\n",
    "part2 = part2.na.fill({'collections_12_mths_ex_med':collections_12_mths_ex_med_mean[0][0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "#part2.describe('tot_coll_amt').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "#part2.describe('tot_cur_bal').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#part2.describe('open_acc_6m').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#part2.groupBy(\"open_acc_6m\").count().orderBy(\"count\",ascending=False).show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # check the missing values\n",
    "# checkna_part2 = part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns])\n",
    "# checkna_part2.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we do not have any missing values in these columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   pub_rec  revol_bal  revol_util  total_acc  initial_list_status  out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med  application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  0        0          0           0          0                    0          0              0            0                0                0              0                   0           0                        0                0                           0                 0               0             0            0          "
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part2.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in part2.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206949\n",
      "columns: 21\n"
     ]
    }
   ],
   "source": [
    "size_shape(part2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>summary</td>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec</td>\n",
       "      <td>1206949</td>\n",
       "      <td>0.18891601882101067</td>\n",
       "      <td>0.5552736651970246</td>\n",
       "      <td>0</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>revol_bal</td>\n",
       "      <td>1206949</td>\n",
       "      <td>16808.217865046492</td>\n",
       "      <td>22685.94006979832</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2559552.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>revol_util</td>\n",
       "      <td>1206949</td>\n",
       "      <td>49.60215702165087</td>\n",
       "      <td>24.751729978833588</td>\n",
       "      <td>0.0</td>\n",
       "      <td>366.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_acc</td>\n",
       "      <td>1206949</td>\n",
       "      <td>23.37079445776085</td>\n",
       "      <td>11.941722677905858</td>\n",
       "      <td>2</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>initial_list_status</td>\n",
       "      <td>1206949</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>f</td>\n",
       "      <td>w</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>out_prncp</td>\n",
       "      <td>1206949</td>\n",
       "      <td>8246.047815595506</td>\n",
       "      <td>8619.423099998028</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>out_prncp_inv</td>\n",
       "      <td>1206949</td>\n",
       "      <td>8244.196088209352</td>\n",
       "      <td>8618.689220779683</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_pymnt</td>\n",
       "      <td>1206949</td>\n",
       "      <td>7857.2732358664325</td>\n",
       "      <td>7372.212777604972</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61947.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_pymnt_inv</td>\n",
       "      <td>1206949</td>\n",
       "      <td>7852.147555765605</td>\n",
       "      <td>7369.027624445597</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61947.53</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_prncp</td>\n",
       "      <td>1206949</td>\n",
       "      <td>5160.822785302468</td>\n",
       "      <td>5195.404453009808</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_int</td>\n",
       "      <td>1206949</td>\n",
       "      <td>2440.0498857228645</td>\n",
       "      <td>2747.206620093818</td>\n",
       "      <td>0.0</td>\n",
       "      <td>27814.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rec_late_fee</td>\n",
       "      <td>1206949</td>\n",
       "      <td>1.9275032905808955</td>\n",
       "      <td>12.919605093795854</td>\n",
       "      <td>-9.5E-9</td>\n",
       "      <td>1427.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>recoveries</td>\n",
       "      <td>1206949</td>\n",
       "      <td>254.47306136312878</td>\n",
       "      <td>976.7144306346049</td>\n",
       "      <td>0.0</td>\n",
       "      <td>39859.55</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>collection_recovery_fee</td>\n",
       "      <td>1206949</td>\n",
       "      <td>42.23292951247622</td>\n",
       "      <td>170.9743693190562</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7174.719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>last_pymnt_amnt</td>\n",
       "      <td>1206949</td>\n",
       "      <td>489.148443251319</td>\n",
       "      <td>699.2390074303721</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40486.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>collections_12_mths_ex_med</td>\n",
       "      <td>1206949</td>\n",
       "      <td>0.020176494615762555</td>\n",
       "      <td>0.1577334168220779</td>\n",
       "      <td>0</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>application_type</td>\n",
       "      <td>1206949</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Individual</td>\n",
       "      <td>Joint App</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>acc_now_delinq</td>\n",
       "      <td>1206949</td>\n",
       "      <td>0.0035063619092438867</td>\n",
       "      <td>0.06363324149822101</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_coll_amt</td>\n",
       "      <td>1206949</td>\n",
       "      <td>216.8530439506285</td>\n",
       "      <td>1692.0028195280042</td>\n",
       "      <td>0.0</td>\n",
       "      <td>262740.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_cur_bal</td>\n",
       "      <td>1206949</td>\n",
       "      <td>138929.0007531387</td>\n",
       "      <td>158656.0233532199</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9971659.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_acc_6m</td>\n",
       "      <td>1206949</td>\n",
       "      <td>0.7472975245847173</td>\n",
       "      <td>1.0759859741062134</td>\n",
       "      <td>0</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  0                      1                    2           3          4\n",
       "summary                     count    mean                   stddev               min         max      \n",
       "pub_rec                     1206949  0.18891601882101067    0.5552736651970246   0           86       \n",
       "revol_bal                   1206949  16808.217865046492     22685.94006979832    0.0         2559552.0\n",
       "revol_util                  1206949  49.60215702165087      24.751729978833588   0.0         366.6    \n",
       "total_acc                   1206949  23.37079445776085      11.941722677905858   2           176      \n",
       "initial_list_status         1206949  None                   None                 f           w        \n",
       "out_prncp                   1206949  8246.047815595506      8619.423099998028    0.0         40000.0  \n",
       "out_prncp_inv               1206949  8244.196088209352      8618.689220779683    0.0         40000.0  \n",
       "total_pymnt                 1206949  7857.2732358664325     7372.212777604972    0.0         61947.53 \n",
       "total_pymnt_inv             1206949  7852.147555765605      7369.027624445597    0.0         61947.53 \n",
       "total_rec_prncp             1206949  5160.822785302468      5195.404453009808    0.0         40000.0  \n",
       "total_rec_int               1206949  2440.0498857228645     2747.206620093818    0.0         27814.3  \n",
       "total_rec_late_fee          1206949  1.9275032905808955     12.919605093795854   -9.5E-9     1427.25  \n",
       "recoveries                  1206949  254.47306136312878     976.7144306346049    0.0         39859.55 \n",
       "collection_recovery_fee     1206949  42.23292951247622      170.9743693190562    0.0         7174.719 \n",
       "last_pymnt_amnt             1206949  489.148443251319       699.2390074303721    0.0         40486.13 \n",
       "collections_12_mths_ex_med  1206949  0.020176494615762555   0.1577334168220779   0           20       \n",
       "application_type            1206949  None                   None                 Individual  Joint App\n",
       "acc_now_delinq              1206949  0.0035063619092438867  0.06363324149822101  0           7        \n",
       "tot_coll_amt                1206949  216.8530439506285      1692.0028195280042   0.0         262740.0 \n",
       "tot_cur_bal                 1206949  138929.0007531387      158656.0233532199    0.0         9971659.0\n",
       "open_acc_6m                 1206949  0.7472975245847173     1.0759859741062134   0           18       "
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part2.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now lets add the index to the part2 cleaned dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "part2 = part2.withColumn(\n",
    "    \"index\",\n",
    "    row_number().over(Window.orderBy(monotonically_increasing_id()))-1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206949\n",
      "columns: 22\n"
     ]
    }
   ],
   "source": [
    "size_shape(part2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>loan_amnt</th>\n",
       "      <th>funded_amnt</th>\n",
       "      <th>funded_amnt_inv</th>\n",
       "      <th>term</th>\n",
       "      <th>int_rate</th>\n",
       "      <th>installment</th>\n",
       "      <th>grade</th>\n",
       "      <th>sub_grade</th>\n",
       "      <th>home_ownership</th>\n",
       "      <th>annual_inc</th>\n",
       "      <th>verification_status</th>\n",
       "      <th>loan_status</th>\n",
       "      <th>pymnt_plan</th>\n",
       "      <th>dti</th>\n",
       "      <th>delinq_2yrs</th>\n",
       "      <th>inq_last_6mths</th>\n",
       "      <th>open_acc</th>\n",
       "      <th>pub_rec</th>\n",
       "      <th>revol_bal</th>\n",
       "      <th>revol_util</th>\n",
       "      <th>total_acc</th>\n",
       "      <th>initial_list_status</th>\n",
       "      <th>out_prncp</th>\n",
       "      <th>out_prncp_inv</th>\n",
       "      <th>total_pymnt</th>\n",
       "      <th>total_pymnt_inv</th>\n",
       "      <th>total_rec_prncp</th>\n",
       "      <th>total_rec_int</th>\n",
       "      <th>total_rec_late_fee</th>\n",
       "      <th>recoveries</th>\n",
       "      <th>collection_recovery_fee</th>\n",
       "      <th>last_pymnt_amnt</th>\n",
       "      <th>collections_12_mths_ex_med</th>\n",
       "      <th>application_type</th>\n",
       "      <th>acc_now_delinq</th>\n",
       "      <th>tot_coll_amt</th>\n",
       "      <th>tot_cur_bal</th>\n",
       "      <th>open_acc_6m</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>13.560000</td>\n",
       "      <td>84.919998</td>\n",
       "      <td>C</td>\n",
       "      <td>C1</td>\n",
       "      <td>RENT</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>18.24</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>4341.0</td>\n",
       "      <td>10.300000</td>\n",
       "      <td>34</td>\n",
       "      <td>w</td>\n",
       "      <td>2386.020020</td>\n",
       "      <td>2386.020020</td>\n",
       "      <td>167.020004</td>\n",
       "      <td>167.020004</td>\n",
       "      <td>113.980003</td>\n",
       "      <td>53.040001</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>84.919998</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>16901.0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>18.940001</td>\n",
       "      <td>777.229980</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>26.52</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "      <td>12315.0</td>\n",
       "      <td>24.200001</td>\n",
       "      <td>44</td>\n",
       "      <td>w</td>\n",
       "      <td>29387.750000</td>\n",
       "      <td>29387.750000</td>\n",
       "      <td>1507.109985</td>\n",
       "      <td>1507.109985</td>\n",
       "      <td>612.250000</td>\n",
       "      <td>894.859985</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>777.229980</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>1208.0</td>\n",
       "      <td>321915.0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>17.969999</td>\n",
       "      <td>180.690002</td>\n",
       "      <td>D</td>\n",
       "      <td>D1</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>59280.0</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>10.51</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>4599.0</td>\n",
       "      <td>19.100000</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>4787.209961</td>\n",
       "      <td>4787.209961</td>\n",
       "      <td>353.890015</td>\n",
       "      <td>353.890015</td>\n",
       "      <td>212.789993</td>\n",
       "      <td>141.100006</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>180.690002</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>110299.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>4000.0</td>\n",
       "      <td>36 months</td>\n",
       "      <td>18.940001</td>\n",
       "      <td>146.509995</td>\n",
       "      <td>D</td>\n",
       "      <td>D2</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>92000.0</td>\n",
       "      <td>Source Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>16.74</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>5468.0</td>\n",
       "      <td>78.099998</td>\n",
       "      <td>13</td>\n",
       "      <td>w</td>\n",
       "      <td>3831.929932</td>\n",
       "      <td>3831.929932</td>\n",
       "      <td>286.709991</td>\n",
       "      <td>286.709991</td>\n",
       "      <td>168.070007</td>\n",
       "      <td>118.639999</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>146.509995</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>686.0</td>\n",
       "      <td>305049.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>30000.0</td>\n",
       "      <td>60 months</td>\n",
       "      <td>16.139999</td>\n",
       "      <td>731.780029</td>\n",
       "      <td>C</td>\n",
       "      <td>C4</td>\n",
       "      <td>MORTGAGE</td>\n",
       "      <td>57250.0</td>\n",
       "      <td>Not Verified</td>\n",
       "      <td>Not Charged Off</td>\n",
       "      <td>n</td>\n",
       "      <td>26.35</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>829.0</td>\n",
       "      <td>3.600000</td>\n",
       "      <td>26</td>\n",
       "      <td>w</td>\n",
       "      <td>29339.019531</td>\n",
       "      <td>29339.019531</td>\n",
       "      <td>1423.209961</td>\n",
       "      <td>1423.209961</td>\n",
       "      <td>660.979980</td>\n",
       "      <td>762.229980</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>731.780029</td>\n",
       "      <td>0</td>\n",
       "      <td>Individual</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>116007.0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  loan_amnt  funded_amnt  funded_amnt_inv        term   int_rate  installment grade sub_grade home_ownership  annual_inc verification_status      loan_status pymnt_plan    dti  delinq_2yrs  inq_last_6mths  open_acc  pub_rec  revol_bal  revol_util  total_acc initial_list_status     out_prncp  out_prncp_inv  total_pymnt  total_pymnt_inv  total_rec_prncp  total_rec_int  total_rec_late_fee  recoveries  collection_recovery_fee  last_pymnt_amnt  collections_12_mths_ex_med application_type  acc_now_delinq  tot_coll_amt  tot_cur_bal  open_acc_6m\n",
       "0  0      2500.0     2500.0       2500.0            36 months  13.560000  84.919998    C     C1        RENT           55000.0     Not Verified        Not Charged Off  n          18.24  0            1               9         1        4341.0     10.300000   34         w                   2386.020020   2386.020020    167.020004   167.020004       113.980003       53.040001      0.0                 0.0         0.0                      84.919998        0                           Individual       0               0.0           16901.0      2          \n",
       "1  1      30000.0    30000.0      30000.0           60 months  18.940001  777.229980   D     D2        MORTGAGE       90000.0     Source Verified     Not Charged Off  n          26.52  0            0               13        1        12315.0    24.200001   44         w                   29387.750000  29387.750000   1507.109985  1507.109985      612.250000       894.859985     0.0                 0.0         0.0                      777.229980       0                           Individual       0               1208.0        321915.0     4          \n",
       "2  2      5000.0     5000.0       5000.0            36 months  17.969999  180.690002   D     D1        MORTGAGE       59280.0     Source Verified     Not Charged Off  n          10.51  0            0               8         0        4599.0     19.100000   13         w                   4787.209961   4787.209961    353.890015   353.890015       212.789993       141.100006     0.0                 0.0         0.0                      180.690002       0                           Individual       0               0.0           110299.0     0          \n",
       "3  3      4000.0     4000.0       4000.0            36 months  18.940001  146.509995   D     D2        MORTGAGE       92000.0     Source Verified     Not Charged Off  n          16.74  0            0               10        0        5468.0     78.099998   13         w                   3831.929932   3831.929932    286.709991   286.709991       168.070007       118.639999     0.0                 0.0         0.0                      146.509995       0                           Individual       0               686.0         305049.0     1          \n",
       "4  4      30000.0    30000.0      30000.0           60 months  16.139999  731.780029   C     C4        MORTGAGE       57250.0     Not Verified        Not Charged Off  n          26.35  0            0               12        0        829.0      3.600000    26         w                   29339.019531  29339.019531   1423.209961  1423.209961      660.979980       762.229980     0.0                 0.0         0.0                      731.780029       0                           Individual       0               0.0           116007.0     3          "
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3 = part1.join(part2, \"index\", \"outer\")\n",
    "df3.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's converts string to index (labels) using the StringIndexer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# categorical_cols = ['initial_list_status','application_type']\n",
    "# stringIndexer = feature.StringIndexer(inputCols=[cols for cols in categorical_cols],\n",
    "#                                       outputCols=[\"encoded_\" + cols for cols in categorical_cols])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The below codes are to check if this object works"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2_SI=stringIndexer.fit(part2).transform(part2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# part2_SI.describe('encoded_initial_list_status').show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2_SI.describe('encoded_application_type').show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's scale the numeric variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numeric_cols = [cols for cols in part2.columns if cols not in categorical_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numeric_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "# len(numeric_cols)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It turned out that StandardScaler can only take one column as input columns so we have to Assemble numeric columns to one column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # define the VectorAssembler object\n",
    "# va = feature.VectorAssembler(inputCols=[cols for cols in numeric_cols], outputCol=\"features\")\n",
    "# # define the StandardScaler object\n",
    "# sScaler = feature.StandardScaler(withMean=True, withStd=True, inputCol=\"features\", outputCol=\"scaled_features\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "# standardScaler = feature.StandardScaler(withMean=True, withStd=True, inputCols=[cols for cols in numeric_cols],\n",
    "#                                       outputCols=[\"scaled_\" + cols for cols in numeric_cols])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the pipeline which contains 3 stages: stringindexer, VectorAssembler and StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part2_pipe=Pipeline(stages=[stringIndexer, va, sScaler])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fitted_part2 = part2_pipe.fit(part2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformed_part2 = fitted_part2.transform(part2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# transformed_part2.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Pyspark to read the dataset and check missing values."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <th>num_bc_sats</th>\n",
       "      <th>num_bc_tl</th>\n",
       "      <th>num_il_tl</th>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <th>num_rev_accts</th>\n",
       "      <th>num_rev_tl_bal_gt_0</th>\n",
       "      <th>num_sats</th>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <th>num_tl_30dpd</th>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <th>num_tl_op_past_12m</th>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60124.0</td>\n",
       "      <td>16901.0</td>\n",
       "      <td>36500.0</td>\n",
       "      <td>18124.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>372872.0</td>\n",
       "      <td>99468.0</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>94072.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>136927.0</td>\n",
       "      <td>11749.0</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>385183.0</td>\n",
       "      <td>36151.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>44984.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>92.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157548.0</td>\n",
       "      <td>29674.0</td>\n",
       "      <td>9300.0</td>\n",
       "      <td>32332.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mths_since_recent_inq  num_accts_ever_120_pd  num_actv_bc_tl  num_actv_rev_tl  num_bc_sats  num_bc_tl  num_il_tl  num_op_rev_tl  num_rev_accts  num_rev_tl_bal_gt_0  num_sats  num_tl_120dpd_2m  num_tl_30dpd  num_tl_90g_dpd_24m  num_tl_op_past_12m  pct_tl_nvr_dlq  percent_bc_gt_75  pub_rec_bankruptcies  tax_liens  tot_hi_cred_lim  total_bal_ex_mort  total_bc_limit  total_il_high_credit_limit hardship_flag disbursement_method debt_settlement_flag\n",
       "0  2.0                    0.0                    2.0             5.0              3.0          3.0        16.0       7.0            18.0           5.0                  9.0       0.0               0.0           0.0                 3.0                 100.0           0.0               1.0                   0.0        60124.0          16901.0            36500.0         18124.0                     N             Cash                N                  \n",
       "1  4.0                    0.0                    2.0             4.0              4.0          9.0        27.0       8.0            14.0           4.0                  13.0      0.0               0.0           0.0                 6.0                 95.0            0.0               1.0                   0.0        372872.0         99468.0            15000.0         94072.0                     N             Cash                N                  \n",
       "2  14.0                   0.0                    0.0             3.0              3.0          3.0        4.0        6.0            7.0            3.0                  8.0       0.0               0.0           0.0                 0.0                 100.0           0.0               0.0                   0.0        136927.0         11749.0            13800.0         10000.0                     N             Cash                N                  \n",
       "3  5.0                    0.0                    1.0             2.0              1.0          2.0        7.0        2.0            3.0            2.0                  10.0      0.0               0.0           0.0                 3.0                 100.0           100.0             0.0                   0.0        385183.0         36151.0            5000.0          44984.0                     N             Cash                N                  \n",
       "4  13.0                   0.0                    2.0             2.0              3.0          8.0        9.0        6.0            15.0           2.0                  12.0      0.0               0.0           0.0                 5.0                 92.3            0.0               0.0                   0.0        157548.0         29674.0            9300.0          32332.0                     N             Cash                N                  "
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part4 = current_df1.select(current_df1.columns[76::1])\n",
    "# Displaying the dataframe in pandas\n",
    "# pd.set_option('display.max_columns', None)  \n",
    "# pd.set_option('display.max_colwidth',-1)\n",
    "part4.toPandas().head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- mths_since_recent_inq: integer (nullable = true)\n",
      " |-- num_accts_ever_120_pd: integer (nullable = true)\n",
      " |-- num_actv_bc_tl: integer (nullable = true)\n",
      " |-- num_actv_rev_tl: integer (nullable = true)\n",
      " |-- num_bc_sats: integer (nullable = true)\n",
      " |-- num_bc_tl: integer (nullable = true)\n",
      " |-- num_il_tl: integer (nullable = true)\n",
      " |-- num_op_rev_tl: integer (nullable = true)\n",
      " |-- num_rev_accts: integer (nullable = true)\n",
      " |-- num_rev_tl_bal_gt_0: integer (nullable = true)\n",
      " |-- num_sats: integer (nullable = true)\n",
      " |-- num_tl_120dpd_2m: integer (nullable = true)\n",
      " |-- num_tl_30dpd: integer (nullable = true)\n",
      " |-- num_tl_90g_dpd_24m: integer (nullable = true)\n",
      " |-- num_tl_op_past_12m: integer (nullable = true)\n",
      " |-- pct_tl_nvr_dlq: double (nullable = true)\n",
      " |-- percent_bc_gt_75: double (nullable = true)\n",
      " |-- pub_rec_bankruptcies: integer (nullable = true)\n",
      " |-- tax_liens: integer (nullable = true)\n",
      " |-- tot_hi_cred_lim: integer (nullable = true)\n",
      " |-- total_bal_ex_mort: integer (nullable = true)\n",
      " |-- total_bc_limit: integer (nullable = true)\n",
      " |-- total_il_high_credit_limit: integer (nullable = true)\n",
      " |-- hardship_flag: string (nullable = true)\n",
      " |-- disbursement_method: string (nullable = true)\n",
      " |-- debt_settlement_flag: string (nullable = true)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "part4.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_inq</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_accts_ever_120_pd</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_bc_tl</td>\n",
       "      <td>41</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_rev_tl</td>\n",
       "      <td>54</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_sats</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_tl</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_il_tl</td>\n",
       "      <td>118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_op_rev_tl</td>\n",
       "      <td>77</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_accts</td>\n",
       "      <td>111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_tl_bal_gt_0</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_sats</td>\n",
       "      <td>84</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_120dpd_2m</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_30dpd</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_90g_dpd_24m</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_op_past_12m</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pct_tl_nvr_dlq</td>\n",
       "      <td>662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>percent_bc_gt_75</td>\n",
       "      <td>261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec_bankruptcies</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tax_liens</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_hi_cred_lim</td>\n",
       "      <td>419380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_ex_mort</td>\n",
       "      <td>182485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bc_limit</td>\n",
       "      <td>10270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_il_high_credit_limit</td>\n",
       "      <td>168227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>hardship_flag</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>disbursement_method</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>debt_settlement_flag</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 0\n",
       "mths_since_recent_inq       26    \n",
       "num_accts_ever_120_pd       43    \n",
       "num_actv_bc_tl              41    \n",
       "num_actv_rev_tl             54    \n",
       "num_bc_sats                 58    \n",
       "num_bc_tl                   72    \n",
       "num_il_tl                   118   \n",
       "num_op_rev_tl               77    \n",
       "num_rev_accts               111   \n",
       "num_rev_tl_bal_gt_0         50    \n",
       "num_sats                    84    \n",
       "num_tl_120dpd_2m            6     \n",
       "num_tl_30dpd                6     \n",
       "num_tl_90g_dpd_24m          31    \n",
       "num_tl_op_past_12m          32    \n",
       "pct_tl_nvr_dlq              662   \n",
       "percent_bc_gt_75            261   \n",
       "pub_rec_bankruptcies        11    \n",
       "tax_liens                   34    \n",
       "tot_hi_cred_lim             419380\n",
       "total_bal_ex_mort           182485\n",
       "total_bc_limit              10270 \n",
       "total_il_high_credit_limit  168227\n",
       "hardship_flag               3     \n",
       "disbursement_method         2     \n",
       "debt_settlement_flag        3     "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Using count distinct to check about the columns and ascertain its type accordingly\n",
    "part4.select([fn.countDistinct(c).alias(c) for c in part4.columns]).toPandas().T\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>summary</td>\n",
       "      <td>count</td>\n",
       "      <td>mean</td>\n",
       "      <td>stddev</td>\n",
       "      <td>min</td>\n",
       "      <td>max</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_inq</td>\n",
       "      <td>1059364</td>\n",
       "      <td>7.156535430692378</td>\n",
       "      <td>6.026029657957612</td>\n",
       "      <td>0</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_accts_ever_120_pd</td>\n",
       "      <td>1196665</td>\n",
       "      <td>0.4976998575206929</td>\n",
       "      <td>1.3744526253635363</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_bc_tl</td>\n",
       "      <td>1196666</td>\n",
       "      <td>3.742203756102371</td>\n",
       "      <td>2.411370450471123</td>\n",
       "      <td>0</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_rev_tl</td>\n",
       "      <td>1196664</td>\n",
       "      <td>5.712285152724574</td>\n",
       "      <td>3.5063569925868396</td>\n",
       "      <td>0</td>\n",
       "      <td>72</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_sats</td>\n",
       "      <td>1198720</td>\n",
       "      <td>4.8267685531233315</td>\n",
       "      <td>3.124487039197658</td>\n",
       "      <td>0</td>\n",
       "      <td>71</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_tl</td>\n",
       "      <td>1196666</td>\n",
       "      <td>7.3691197042449605</td>\n",
       "      <td>4.579631632695831</td>\n",
       "      <td>0</td>\n",
       "      <td>86</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_il_tl</td>\n",
       "      <td>1196666</td>\n",
       "      <td>8.296971753187606</td>\n",
       "      <td>7.387660738960343</td>\n",
       "      <td>0</td>\n",
       "      <td>159</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_op_rev_tl</td>\n",
       "      <td>1196666</td>\n",
       "      <td>8.281238875341991</td>\n",
       "      <td>4.820853298189338</td>\n",
       "      <td>0</td>\n",
       "      <td>91</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_accts</td>\n",
       "      <td>1196665</td>\n",
       "      <td>13.469005110034972</td>\n",
       "      <td>7.940999256140997</td>\n",
       "      <td>0</td>\n",
       "      <td>151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_tl_bal_gt_0</td>\n",
       "      <td>1196666</td>\n",
       "      <td>5.655569724551379</td>\n",
       "      <td>3.410249287798023</td>\n",
       "      <td>0</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_sats</td>\n",
       "      <td>1198720</td>\n",
       "      <td>11.67925286972771</td>\n",
       "      <td>5.8111130138736</td>\n",
       "      <td>0</td>\n",
       "      <td>101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_120dpd_2m</td>\n",
       "      <td>1148666</td>\n",
       "      <td>5.084158493417582E-4</td>\n",
       "      <td>0.02558152744808453</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_30dpd</td>\n",
       "      <td>1196665</td>\n",
       "      <td>0.0023849615389436476</td>\n",
       "      <td>0.051510880442630115</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_90g_dpd_24m</td>\n",
       "      <td>1196667</td>\n",
       "      <td>0.08004064622823225</td>\n",
       "      <td>0.49073962771110646</td>\n",
       "      <td>0</td>\n",
       "      <td>58</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_op_past_12m</td>\n",
       "      <td>1196667</td>\n",
       "      <td>2.0555994274096303</td>\n",
       "      <td>1.8554281675042341</td>\n",
       "      <td>0</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pct_tl_nvr_dlq</td>\n",
       "      <td>1196642</td>\n",
       "      <td>94.03866561594971</td>\n",
       "      <td>9.284082240378515</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>percent_bc_gt_75</td>\n",
       "      <td>1185721</td>\n",
       "      <td>41.038419046300724</td>\n",
       "      <td>36.37784540991147</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec_bankruptcies</td>\n",
       "      <td>1206836</td>\n",
       "      <td>0.12671398599312583</td>\n",
       "      <td>0.35900163354812725</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tax_liens</td>\n",
       "      <td>1206951</td>\n",
       "      <td>0.044057298100751395</td>\n",
       "      <td>0.3824977523746807</td>\n",
       "      <td>0</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_hi_cred_lim</td>\n",
       "      <td>1196674</td>\n",
       "      <td>175710.3586808103</td>\n",
       "      <td>180251.52907170015</td>\n",
       "      <td>0</td>\n",
       "      <td>9999999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_ex_mort</td>\n",
       "      <td>1200106</td>\n",
       "      <td>52091.86321958227</td>\n",
       "      <td>51102.52462061506</td>\n",
       "      <td>0</td>\n",
       "      <td>2698600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bc_limit</td>\n",
       "      <td>1200101</td>\n",
       "      <td>23918.073182173834</td>\n",
       "      <td>23702.672881549708</td>\n",
       "      <td>0</td>\n",
       "      <td>1569000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_il_high_credit_limit</td>\n",
       "      <td>1196669</td>\n",
       "      <td>45046.47047262025</td>\n",
       "      <td>46191.09749514326</td>\n",
       "      <td>0</td>\n",
       "      <td>2118996</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>hardship_flag</td>\n",
       "      <td>1206950</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>disbursement_method</td>\n",
       "      <td>1206949</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Cash</td>\n",
       "      <td>DirectPay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>debt_settlement_flag</td>\n",
       "      <td>1206953</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>Cash</td>\n",
       "      <td>Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                  0                      1                     2     3          4\n",
       "summary                     count    mean                   stddev                min   max      \n",
       "mths_since_recent_inq       1059364  7.156535430692378      6.026029657957612     0     25       \n",
       "num_accts_ever_120_pd       1196665  0.4976998575206929     1.3744526253635363    0     58       \n",
       "num_actv_bc_tl              1196666  3.742203756102371      2.411370450471123     0     50       \n",
       "num_actv_rev_tl             1196664  5.712285152724574      3.5063569925868396    0     72       \n",
       "num_bc_sats                 1198720  4.8267685531233315     3.124487039197658     0     71       \n",
       "num_bc_tl                   1196666  7.3691197042449605     4.579631632695831     0     86       \n",
       "num_il_tl                   1196666  8.296971753187606      7.387660738960343     0     159      \n",
       "num_op_rev_tl               1196666  8.281238875341991      4.820853298189338     0     91       \n",
       "num_rev_accts               1196665  13.469005110034972     7.940999256140997     0     151      \n",
       "num_rev_tl_bal_gt_0         1196666  5.655569724551379      3.410249287798023     0     65       \n",
       "num_sats                    1198720  11.67925286972771      5.8111130138736       0     101      \n",
       "num_tl_120dpd_2m            1148666  5.084158493417582E-4   0.02558152744808453   0     7        \n",
       "num_tl_30dpd                1196665  0.0023849615389436476  0.051510880442630115  0     6        \n",
       "num_tl_90g_dpd_24m          1196667  0.08004064622823225    0.49073962771110646   0     58       \n",
       "num_tl_op_past_12m          1196667  2.0555994274096303     1.8554281675042341    0     32       \n",
       "pct_tl_nvr_dlq              1196642  94.03866561594971      9.284082240378515     0.0   100.0    \n",
       "percent_bc_gt_75            1185721  41.038419046300724     36.37784540991147     0.0   100.0    \n",
       "pub_rec_bankruptcies        1206836  0.12671398599312583    0.35900163354812725   0     11       \n",
       "tax_liens                   1206951  0.044057298100751395   0.3824977523746807    0     100      \n",
       "tot_hi_cred_lim             1196674  175710.3586808103      180251.52907170015    0     9999999  \n",
       "total_bal_ex_mort           1200106  52091.86321958227      51102.52462061506     0     2698600  \n",
       "total_bc_limit              1200101  23918.073182173834     23702.672881549708    0     1569000  \n",
       "total_il_high_credit_limit  1196669  45046.47047262025      46191.09749514326     0     2118996  \n",
       "hardship_flag               1206950  0.0                    NaN                   0     Y        \n",
       "disbursement_method         1206949  None                   None                  Cash  DirectPay\n",
       "debt_settlement_flag        1206953  None                   None                  Cash  Y        "
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# check the summary of each col\n",
    "part4.describe().toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Using count distinct to check about the columns and ascertain its type accordingly \n",
    "# - whether categorical,float ,integer..etc\n",
    "Distinctcount4 = part4.select([fn.countDistinct(c).alias(c) for c in part4.columns])\n",
    "\n",
    "# Distinct count Converting to Pandas(So it computes faster)\n",
    "DC4= Distinctcount4.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This is to check for categorical numerical and see if we have weird categories.()\n",
    "# for col in part4.columns:\n",
    "#     if DC4[col][0]<1000:\n",
    "#         print(part4.groupby(col).count().show())\n",
    "#     else:\n",
    "#         continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mths_since_recent_inq int\n",
      "mths_since_recent_inq  done\n",
      "mths_since_recent_inq winsorising done\n",
      "num_accts_ever_120_pd int\n",
      "num_accts_ever_120_pd  done\n",
      "num_accts_ever_120_pd winsorising done\n",
      "num_actv_bc_tl int\n",
      "num_actv_bc_tl  done\n",
      "num_actv_bc_tl winsorising done\n",
      "num_actv_rev_tl int\n",
      "num_actv_rev_tl  done\n",
      "num_actv_rev_tl winsorising done\n",
      "num_bc_sats int\n",
      "num_bc_sats  done\n",
      "num_bc_sats winsorising done\n",
      "num_bc_tl int\n",
      "num_bc_tl  done\n",
      "num_bc_tl winsorising done\n",
      "num_il_tl int\n",
      "num_il_tl  done\n",
      "num_il_tl winsorising done\n",
      "num_op_rev_tl int\n",
      "num_op_rev_tl  done\n",
      "num_op_rev_tl winsorising done\n",
      "num_rev_accts int\n",
      "num_rev_accts  done\n",
      "num_rev_accts winsorising done\n",
      "num_rev_tl_bal_gt_0 int\n",
      "num_rev_tl_bal_gt_0  done\n",
      "num_rev_tl_bal_gt_0 winsorising done\n",
      "num_sats int\n",
      "num_sats  done\n",
      "num_sats winsorising done\n",
      "num_tl_120dpd_2m int\n",
      "num_tl_120dpd_2m  done\n",
      "num_tl_120dpd_2m winsorising done\n",
      "num_tl_30dpd int\n",
      "num_tl_30dpd  done\n",
      "num_tl_30dpd winsorising done\n",
      "num_tl_90g_dpd_24m int\n",
      "num_tl_90g_dpd_24m  done\n",
      "num_tl_90g_dpd_24m winsorising done\n",
      "num_tl_op_past_12m int\n",
      "num_tl_op_past_12m  done\n",
      "num_tl_op_past_12m winsorising done\n",
      "pct_tl_nvr_dlq double\n",
      "pct_tl_nvr_dlq  done\n",
      "pct_tl_nvr_dlq winsorising done\n",
      "percent_bc_gt_75 double\n",
      "percent_bc_gt_75  done\n",
      "percent_bc_gt_75 winsorising done\n",
      "pub_rec_bankruptcies int\n",
      "pub_rec_bankruptcies  done\n",
      "pub_rec_bankruptcies winsorising done\n",
      "tax_liens int\n",
      "tax_liens  done\n",
      "tax_liens winsorising done\n",
      "tot_hi_cred_lim int\n",
      "tot_hi_cred_lim winsorising done\n",
      "total_bal_ex_mort int\n",
      "total_bal_ex_mort winsorising done\n",
      "total_bc_limit int\n",
      "total_bc_limit winsorising done\n",
      "total_il_high_credit_limit int\n",
      "total_il_high_credit_limit winsorising done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_inq</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_accts_ever_120_pd</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_bc_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_rev_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_sats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_il_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_op_rev_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_accts</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_tl_bal_gt_0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_sats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_120dpd_2m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_30dpd</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_90g_dpd_24m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_op_past_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pct_tl_nvr_dlq</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>percent_bc_gt_75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec_bankruptcies</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tax_liens</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_hi_cred_lim</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_ex_mort</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bc_limit</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_il_high_credit_limit</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>hardship_flag</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>disbursement_method</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>debt_settlement_flag</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             0\n",
       "mths_since_recent_inq       0 \n",
       "num_accts_ever_120_pd       0 \n",
       "num_actv_bc_tl              0 \n",
       "num_actv_rev_tl             0 \n",
       "num_bc_sats                 0 \n",
       "num_bc_tl                   0 \n",
       "num_il_tl                   0 \n",
       "num_op_rev_tl               0 \n",
       "num_rev_accts               0 \n",
       "num_rev_tl_bal_gt_0         0 \n",
       "num_sats                    0 \n",
       "num_tl_120dpd_2m            0 \n",
       "num_tl_30dpd                0 \n",
       "num_tl_90g_dpd_24m          0 \n",
       "num_tl_op_past_12m          0 \n",
       "pct_tl_nvr_dlq              0 \n",
       "percent_bc_gt_75            0 \n",
       "pub_rec_bankruptcies        0 \n",
       "tax_liens                   0 \n",
       "tot_hi_cred_lim             0 \n",
       "total_bal_ex_mort           0 \n",
       "total_bc_limit              0 \n",
       "total_il_high_credit_limit  0 \n",
       "hardship_flag               33\n",
       "disbursement_method         34\n",
       "debt_settlement_flag        30"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# All these values along with NAs should be replaced first - by the mode for categorical columns with less than 1000 \n",
    "# distinct values\n",
    "# These are taken from the above output\n",
    "cols = ['hardship_flag','disbursement_method','debt_settlement_flag']\n",
    "wrong_val = ['Feb-2019','Individual','Oct-2011','Oct-2016']\n",
    "cleaned_int_categorical4 = []\n",
    "\n",
    "numerical_cols4 = []\n",
    "d = {}\n",
    "\n",
    "for col,dtype in part4.dtypes:\n",
    "    \n",
    "    #Checking that the distinct values less than 1000 to be qualified as a categorical where we replace the values with mode\n",
    "    if DC4[col][0]<1000:\n",
    "        if dtype in ['double','int']:\n",
    "            print(col,dtype)\n",
    "            part4 = part4.withColumn(col,fn.when(fn.col(col).isin(wrong_val),'null').otherwise(fn.col(col)))\n",
    "            \n",
    "            #Converting them to integer, since in my case.. all are integer type category\n",
    "            part4 = part4.withColumn(col,part4[col].cast(IntegerType()))\n",
    "            \n",
    "            #Filling the NAs by the mode\n",
    "            temp = part4.groupby(fn.col(col)).count()\n",
    "            \n",
    "            #Making sure that the grouped mode value is not a null\n",
    "            mode_value = temp.orderBy(temp['count'].desc()).collect()[0][0]\n",
    "            if mode_value is None:\n",
    "                mode_value = temp.orderBy(temp['count'].desc()).collect()[1][0]\n",
    "            part4 = part4.fillna(mode_value,subset = col)\n",
    "            print(col,' done')\n",
    "            cleaned_int_categorical4.append(col)\n",
    "            \n",
    "            #Winsorizing the columns\n",
    "            d[col] = part4.approxQuantile(col,[0.01,0.99],0.25)\n",
    "            part4 = part4.withColumn(col,fn.when(fn.col(col)<d[col][0],d[col][0]).when(fn.col(col)>d[col][1],d[col][1]).otherwise(fn.col(col)))\n",
    "            print(col,'winsorising done')\n",
    "        else:\n",
    "            continue\n",
    "                    \n",
    "    else:\n",
    "        #This means that they have more than 1000 distinct values and can be treated as Float continous\n",
    "        print(col,dtype)\n",
    "        part4 = part4.withColumn(col,fn.when(part4[col].isin(wrong_val),'null').otherwise(part4[col]))\n",
    "        \n",
    "        #Casting to float type\n",
    "        part4 = part4.withColumn(col,part4[col].cast(FloatType()))\n",
    "        \n",
    "        #Replacing the float type value nas with its mean\n",
    "        mean_val = part4.select(fn.avg(fn.col(col))).collect()[0][0]\n",
    "        part4 = part4.fillna(mean_val,subset = col)\n",
    "        \n",
    "        #Winsorizing the values\n",
    "        d[col] = part4.approxQuantile(col,[0.01,0.99],0.25)\n",
    "        part4 = part4.withColumn(col,fn.when(fn.col(col)<d[col][0],d[col][0]).when(fn.col(col)>d[col][1],d[col][1]).otherwise(fn.col(col)))\n",
    "        print(col,'winsorising done')\n",
    "        numerical_cols4.append(col)\n",
    "   \n",
    "    \n",
    "#These are the count of Null values in the columns\n",
    "part4.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in part4.columns]).toPandas().T           \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_inq</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_accts_ever_120_pd</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_bc_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_actv_rev_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_sats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_bc_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_il_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_op_rev_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_accts</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_rev_tl_bal_gt_0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_sats</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_120dpd_2m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_30dpd</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_90g_dpd_24m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>num_tl_op_past_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pct_tl_nvr_dlq</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>percent_bc_gt_75</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>pub_rec_bankruptcies</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tax_liens</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>tot_hi_cred_lim</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_ex_mort</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bc_limit</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_il_high_credit_limit</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>hardship_flag</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>disbursement_method</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>debt_settlement_flag</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            0\n",
       "mths_since_recent_inq       0\n",
       "num_accts_ever_120_pd       0\n",
       "num_actv_bc_tl              0\n",
       "num_actv_rev_tl             0\n",
       "num_bc_sats                 0\n",
       "num_bc_tl                   0\n",
       "num_il_tl                   0\n",
       "num_op_rev_tl               0\n",
       "num_rev_accts               0\n",
       "num_rev_tl_bal_gt_0         0\n",
       "num_sats                    0\n",
       "num_tl_120dpd_2m            0\n",
       "num_tl_30dpd                0\n",
       "num_tl_90g_dpd_24m          0\n",
       "num_tl_op_past_12m          0\n",
       "pct_tl_nvr_dlq              0\n",
       "percent_bc_gt_75            0\n",
       "pub_rec_bankruptcies        0\n",
       "tax_liens                   0\n",
       "tot_hi_cred_lim             0\n",
       "total_bal_ex_mort           0\n",
       "total_bc_limit              0\n",
       "total_il_high_credit_limit  0\n",
       "hardship_flag               0\n",
       "disbursement_method         0\n",
       "debt_settlement_flag        0"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Now for the string Cols - Since those are categorical, I will replace it by the value 'unknown'. Since the fraction of Null is negligible\n",
    "string_cols4 = ['disbursement_method','debt_settlement_flag','hardship_flag']\n",
    "part4 = part4.fillna('unknown',subset = string_cols4)\n",
    "part4.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in part4.columns]).toPandas().T    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cleaned_dataframe_part4 = part4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets add the index to part4 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "part4 = part4.withColumn(\n",
    "    \"index\",\n",
    "    row_number().over(Window.orderBy(monotonically_increasing_id()))-1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# part4_pd = part4.describe().toPandas()\n",
    "# part4_pd "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_cols = []\n",
    "# # Now that cleaning is done ; Let us start with the pipeline\n",
    "# # Since all the categorical columns are numbers\n",
    "\n",
    "# cat_coded4_cols = []\n",
    "# string_cols4 = []\n",
    "# for c in cleaned_int_categorical4:\n",
    "#     print(c)\n",
    "#     part4 = Pipeline(stages = [feature.OneHotEncoder(inputCol = c,outputCol = c+'catcoded4')]).fit(part4).transform(part4)\n",
    "#     cat_coded4_cols.append(c+'catcoded4')\n",
    "# for c in string_cols:\n",
    "#     part4 = Pipeline(stages = [feature.StringIndexer(inputCol = c,outputCol = c+'index4'),\n",
    "#                               feature.OneHotEncoder(inputCol = c+'index4',outputCol = c+'stringcoded4')]).fit(part4).transform(part4)\n",
    "#     string_cols4.append(c+'stringcoded4')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "# total_cols4 = cat_coded4_cols + string_cols4 + numerical_cols4\n",
    "\n",
    "# #Use features4 as stages for your modelling needs\n",
    "# part4_pipe = Pipeline(stages = [feature.VectorAssembler(inputCols = total_cols4,outputCol = 'feature4')])\n",
    "# part4_pipe.fit(part4).transform(part4).first()\n",
    "# #For cleaned dataframe - use cleaned_dataframe_part4 ( which is just all columns cleaned )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>open_act_il</th>\n",
       "      <th>open_il_12m</th>\n",
       "      <th>open_il_24m</th>\n",
       "      <th>mths_since_rcnt_il</th>\n",
       "      <th>total_bal_il</th>\n",
       "      <th>il_util</th>\n",
       "      <th>open_rv_12m</th>\n",
       "      <th>open_rv_24m</th>\n",
       "      <th>max_bal_bc</th>\n",
       "      <th>all_util</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>inq_fi</th>\n",
       "      <th>total_cu_tl</th>\n",
       "      <th>inq_last_12m</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <th>mort_acc</th>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>12560</td>\n",
       "      <td>69</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>2137</td>\n",
       "      <td>28</td>\n",
       "      <td>42000</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>2</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1878</td>\n",
       "      <td>34360.0</td>\n",
       "      <td>5.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>87153</td>\n",
       "      <td>88</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>998</td>\n",
       "      <td>57</td>\n",
       "      <td>50800</td>\n",
       "      <td>2</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>24763</td>\n",
       "      <td>13761.0</td>\n",
       "      <td>8.3</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>378.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>7150</td>\n",
       "      <td>72</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>24100</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>18383</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>77.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>30683</td>\n",
       "      <td>68</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3761</td>\n",
       "      <td>70</td>\n",
       "      <td>7000</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "      <td>30505</td>\n",
       "      <td>1239.0</td>\n",
       "      <td>75.2</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>64.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>28845</td>\n",
       "      <td>89</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>516</td>\n",
       "      <td>54</td>\n",
       "      <td>23100</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9667</td>\n",
       "      <td>8471.0</td>\n",
       "      <td>8.9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  open_act_il open_il_12m open_il_24m mths_since_rcnt_il total_bal_il il_util open_rv_12m open_rv_24m max_bal_bc all_util total_rev_hi_lim inq_fi total_cu_tl inq_last_12m  acc_open_past_24mths avg_cur_bal  bc_open_to_buy bc_util  chargeoff_within_12_mths  delinq_amnt  mo_sin_old_il_acct  mo_sin_old_rev_tl_op  mo_sin_rcnt_rev_tl_op  mo_sin_rcnt_tl  mort_acc  mths_since_recent_bc\n",
       "0  2           1           2           2                  12560        69      2           7           2137       28       42000            1      11          2            9.0                   1878        34360.0         5.9     0.0                       0.0          140.0               212.0                 1.0                    1.0             0.0       1.0                 \n",
       "1  4           2           3           3                  87153        88      4           5           998        57       50800            2      15          2            10.0                  24763       13761.0         8.3     0.0                       0.0          163.0               378.0                 4.0                    3.0             3.0       4.0                 \n",
       "2  1           0           2           14                 7150         72      0           2           0          35       24100            1      5           0            4.0                   18383       13800.0         0       0.0                       0.0          87.0                92.0                  15.0                   14.0            2.0       77.0                \n",
       "3  5           3           5           5                  30683        68      0           0           3761       70       7000             2      4           3            5.0                   30505       1239.0          75.2    0.0                       0.0          62.0                154.0                 64.0                   5.0             3.0       64.0                \n",
       "4  5           3           5           4                  28845        89      2           4           516        54       23100            1      0           0            9.0                   9667        8471.0          8.9     0.0                       0.0          53.0                216.0                 2.0                    2.0             2.0       2.0                 "
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "part3 = current_df1.select(current_df1.columns[50:76:1])\n",
    "part3.toPandas().head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>open_act_il</td>\n",
       "      <td>206297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_il_12m</td>\n",
       "      <td>206304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_il_24m</td>\n",
       "      <td>206307</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_rcnt_il</td>\n",
       "      <td>240368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_il</td>\n",
       "      <td>206306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>il_util</td>\n",
       "      <td>355256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_rv_12m</td>\n",
       "      <td>206310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_rv_24m</td>\n",
       "      <td>206308</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max_bal_bc</td>\n",
       "      <td>206309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>all_util</td>\n",
       "      <td>206487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rev_hi_lim</td>\n",
       "      <td>10316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_fi</td>\n",
       "      <td>206310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_cu_tl</td>\n",
       "      <td>206310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_last_12m</td>\n",
       "      <td>206313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>acc_open_past_24mths</td>\n",
       "      <td>6884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>avg_cur_bal</td>\n",
       "      <td>10370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bc_open_to_buy</td>\n",
       "      <td>21150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bc_util</td>\n",
       "      <td>21696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>chargeoff_within_12_mths</td>\n",
       "      <td>39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>delinq_amnt</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_old_il_acct</td>\n",
       "      <td>50004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_old_rev_tl_op</td>\n",
       "      <td>10310</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_rcnt_rev_tl_op</td>\n",
       "      <td>10315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_rcnt_tl</td>\n",
       "      <td>10314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mort_acc</td>\n",
       "      <td>6885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_bc</td>\n",
       "      <td>20333</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               0\n",
       "open_act_il               206297\n",
       "open_il_12m               206304\n",
       "open_il_24m               206307\n",
       "mths_since_rcnt_il        240368\n",
       "total_bal_il              206306\n",
       "il_util                   355256\n",
       "open_rv_12m               206310\n",
       "open_rv_24m               206308\n",
       "max_bal_bc                206309\n",
       "all_util                  206487\n",
       "total_rev_hi_lim          10316 \n",
       "inq_fi                    206310\n",
       "total_cu_tl               206310\n",
       "inq_last_12m              206313\n",
       "acc_open_past_24mths      6884  \n",
       "avg_cur_bal               10370 \n",
       "bc_open_to_buy            21150 \n",
       "bc_util                   21696 \n",
       "chargeoff_within_12_mths  39    \n",
       "delinq_amnt               28    \n",
       "mo_sin_old_il_acct        50004 \n",
       "mo_sin_old_rev_tl_op      10310 \n",
       "mo_sin_rcnt_rev_tl_op     10315 \n",
       "mo_sin_rcnt_tl            10314 \n",
       "mort_acc                  6885  \n",
       "mths_since_recent_bc      20333 "
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#These are the count of Null values in the columns\n",
    "part3.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in part3.columns]).toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using count distinct to check about the columns and ascertain its type accordingly - whether categorical,float ,integer..etc\n",
    "Distinctcount3 = part3.select([fn.countDistinct(c).alias(c) for c in part3.columns])\n",
    "# Distinctcount Converting to Pandas(So it computes faster)\n",
    "DC3= Distinctcount3.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "#This is to check for wrong values & correct them.\n",
    "# for col in part3.columns:\n",
    "#     if Distinctcount3.select(fn.col(col)).collect()[0][0]<1000:\n",
    "#         print(part3.groupby(col).count().show())\n",
    "#     else:\n",
    "#         continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "open_act_il string\n",
      "open_act_il  done\n",
      "open_act_il winsorising done\n",
      "open_il_12m string\n",
      "open_il_12m  done\n",
      "open_il_12m winsorising done\n",
      "open_il_24m string\n",
      "open_il_24m  done\n",
      "open_il_24m winsorising done\n",
      "mths_since_rcnt_il string\n",
      "mths_since_rcnt_il  done\n",
      "mths_since_rcnt_il winsorising done\n",
      "total_bal_il string\n",
      "total_bal_il winsorising done\n",
      "il_util string\n",
      "il_util  done\n",
      "il_util winsorising done\n",
      "open_rv_12m string\n",
      "open_rv_12m  done\n",
      "open_rv_12m winsorising done\n",
      "open_rv_24m string\n",
      "open_rv_24m  done\n",
      "open_rv_24m winsorising done\n",
      "max_bal_bc string\n",
      "max_bal_bc winsorising done\n",
      "all_util string\n",
      "all_util  done\n",
      "all_util winsorising done\n",
      "total_rev_hi_lim string\n",
      "total_rev_hi_lim winsorising done\n",
      "inq_fi string\n",
      "inq_fi  done\n",
      "inq_fi winsorising done\n",
      "total_cu_tl string\n",
      "total_cu_tl  done\n",
      "total_cu_tl winsorising done\n",
      "inq_last_12m string\n",
      "inq_last_12m  done\n",
      "inq_last_12m winsorising done\n",
      "acc_open_past_24mths int\n",
      "acc_open_past_24mths  done\n",
      "acc_open_past_24mths winsorising done\n",
      "avg_cur_bal string\n",
      "avg_cur_bal winsorising done\n",
      "bc_open_to_buy int\n",
      "bc_open_to_buy winsorising done\n",
      "bc_util string\n",
      "bc_util winsorising done\n",
      "chargeoff_within_12_mths double\n",
      "chargeoff_within_12_mths  done\n",
      "chargeoff_within_12_mths winsorising done\n",
      "delinq_amnt int\n",
      "delinq_amnt winsorising done\n",
      "mo_sin_old_il_acct double\n",
      "mo_sin_old_il_acct  done\n",
      "mo_sin_old_il_acct winsorising done\n",
      "mo_sin_old_rev_tl_op int\n",
      "mo_sin_old_rev_tl_op  done\n",
      "mo_sin_old_rev_tl_op winsorising done\n",
      "mo_sin_rcnt_rev_tl_op int\n",
      "mo_sin_rcnt_rev_tl_op  done\n",
      "mo_sin_rcnt_rev_tl_op winsorising done\n",
      "mo_sin_rcnt_tl int\n",
      "mo_sin_rcnt_tl  done\n",
      "mo_sin_rcnt_tl winsorising done\n",
      "mort_acc int\n",
      "mort_acc  done\n",
      "mort_acc winsorising done\n",
      "mths_since_recent_bc int\n",
      "mths_since_recent_bc  done\n",
      "mths_since_recent_bc winsorising done\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>open_act_il</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_il_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_il_24m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_rcnt_il</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_bal_il</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>il_util</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_rv_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>open_rv_24m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>max_bal_bc</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>all_util</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_rev_hi_lim</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_fi</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>total_cu_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>inq_last_12m</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>acc_open_past_24mths</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>avg_cur_bal</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bc_open_to_buy</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>bc_util</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>chargeoff_within_12_mths</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>delinq_amnt</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_old_il_acct</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_old_rev_tl_op</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_rcnt_rev_tl_op</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mo_sin_rcnt_tl</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mort_acc</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>mths_since_recent_bc</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          0\n",
       "open_act_il               0\n",
       "open_il_12m               0\n",
       "open_il_24m               0\n",
       "mths_since_rcnt_il        0\n",
       "total_bal_il              0\n",
       "il_util                   0\n",
       "open_rv_12m               0\n",
       "open_rv_24m               0\n",
       "max_bal_bc                0\n",
       "all_util                  0\n",
       "total_rev_hi_lim          0\n",
       "inq_fi                    0\n",
       "total_cu_tl               0\n",
       "inq_last_12m              0\n",
       "acc_open_past_24mths      0\n",
       "avg_cur_bal               0\n",
       "bc_open_to_buy            0\n",
       "bc_util                   0\n",
       "chargeoff_within_12_mths  0\n",
       "delinq_amnt               0\n",
       "mo_sin_old_il_acct        0\n",
       "mo_sin_old_rev_tl_op      0\n",
       "mo_sin_rcnt_rev_tl_op     0\n",
       "mo_sin_rcnt_tl            0\n",
       "mort_acc                  0\n",
       "mths_since_recent_bc      0"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#All these values along with NAs should be replaced first - by the mode for categorical columns with less than 1000 distinct values\n",
    "#These are taken from the above output - BUT THE PROBLEM HERE IS (ONLY FROM THE TOP 20 ROWS)\n",
    "\n",
    "wrong_val = ['Feb-2019','Individual','Oct-2011','Oct-2016']\n",
    "cleaned_int_categorical3 = []\n",
    "\n",
    "numerical_cols3 = []\n",
    "d = {}\n",
    "\n",
    "for col,dtype in part3.dtypes:\n",
    "    \n",
    "    #Checking that the distinct values less than 1000 to be qualified as a categorical where we replace the values with mode\n",
    "    if DC3[col][0]<1000:\n",
    "        if dtype in ['string','double','int']:\n",
    "            print(col,dtype)\n",
    "            #COnverting those wrong values to null\n",
    "            part3 = part3.withColumn(col,fn.when(fn.col(col).isin(wrong_val),'null').otherwise(fn.col(col)))\n",
    "            \n",
    "            #Converting them to integer, since in my case.. all are integer type category\n",
    "            part3 = part3.withColumn(col,part3[col].cast(IntegerType()))\n",
    "            \n",
    "            #Filling the NAs by the mode\n",
    "            temp = part3.groupby(fn.col(col)).count()\n",
    "            \n",
    "            #Making sure that the grouped mode value is not a null\n",
    "            mode_value = temp.orderBy(temp['count'].desc()).collect()[0][0]\n",
    "            if mode_value is None:\n",
    "                mode_value = temp.orderBy(temp['count'].desc()).collect()[1][0]\n",
    "            part3 = part3.fillna(mode_value,subset = col)\n",
    "            print(col,' done')\n",
    "            cleaned_int_categorical3.append(col)\n",
    "            \n",
    "            #Winsorizing the columns\n",
    "            d[col] = part3.approxQuantile(col,[0.01,0.99],0.25)\n",
    "            part3 = part3.withColumn(col,fn.when(fn.col(col)<d[col][0],d[col][0]).when(fn.col(col)>d[col][1],d[col][1]).otherwise(fn.col(col)))\n",
    "            print(col,'winsorising done')\n",
    "    else:\n",
    "        #This means that they have more than 1000 distinct values and can be treated as continous\n",
    "        print(col,dtype)\n",
    "        part3 = part3.withColumn(col,fn.when(part3[col].isin(wrong_val),'null').otherwise(part3[col]))\n",
    "        #Casting to float type\n",
    "        part3 = part3.withColumn(col,part3[col].cast(FloatType()))\n",
    "        #Replacing the float type value nas with its mean\n",
    "        mean_val = part3.select(fn.avg(fn.col(col))).collect()[0][0]\n",
    "        part3 = part3.fillna(mean_val,subset = col)\n",
    "        #Winsorizing the values\n",
    "        d[col] = part3.approxQuantile(col,[0.01,0.99],0.25)\n",
    "        part3 = part3.withColumn(col,fn.when(fn.col(col)<d[col][0],d[col][0]).when(fn.col(col)>d[col][1],d[col][1]).otherwise(fn.col(col)))\n",
    "        print(col,'winsorising done')\n",
    "        numerical_cols3.append(col)\n",
    "   \n",
    "    \n",
    "#These are the count of Null values in the columns\n",
    "part3.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in part3.columns]).toPandas().T           \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets create an index for part 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "part3 = part3.withColumn(\n",
    "    \"index\",\n",
    "    row_number().over(Window.orderBy(monotonically_increasing_id()))-1\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets join part 3 and part 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>open_act_il</th>\n",
       "      <th>open_il_12m</th>\n",
       "      <th>open_il_24m</th>\n",
       "      <th>mths_since_rcnt_il</th>\n",
       "      <th>total_bal_il</th>\n",
       "      <th>il_util</th>\n",
       "      <th>open_rv_12m</th>\n",
       "      <th>open_rv_24m</th>\n",
       "      <th>max_bal_bc</th>\n",
       "      <th>all_util</th>\n",
       "      <th>total_rev_hi_lim</th>\n",
       "      <th>inq_fi</th>\n",
       "      <th>total_cu_tl</th>\n",
       "      <th>inq_last_12m</th>\n",
       "      <th>acc_open_past_24mths</th>\n",
       "      <th>avg_cur_bal</th>\n",
       "      <th>bc_open_to_buy</th>\n",
       "      <th>bc_util</th>\n",
       "      <th>chargeoff_within_12_mths</th>\n",
       "      <th>delinq_amnt</th>\n",
       "      <th>mo_sin_old_il_acct</th>\n",
       "      <th>mo_sin_old_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_rev_tl_op</th>\n",
       "      <th>mo_sin_rcnt_tl</th>\n",
       "      <th>mort_acc</th>\n",
       "      <th>mths_since_recent_bc</th>\n",
       "      <th>mths_since_recent_inq</th>\n",
       "      <th>num_accts_ever_120_pd</th>\n",
       "      <th>num_actv_bc_tl</th>\n",
       "      <th>num_actv_rev_tl</th>\n",
       "      <th>num_bc_sats</th>\n",
       "      <th>num_bc_tl</th>\n",
       "      <th>num_il_tl</th>\n",
       "      <th>num_op_rev_tl</th>\n",
       "      <th>num_rev_accts</th>\n",
       "      <th>num_rev_tl_bal_gt_0</th>\n",
       "      <th>num_sats</th>\n",
       "      <th>num_tl_120dpd_2m</th>\n",
       "      <th>num_tl_30dpd</th>\n",
       "      <th>num_tl_90g_dpd_24m</th>\n",
       "      <th>num_tl_op_past_12m</th>\n",
       "      <th>pct_tl_nvr_dlq</th>\n",
       "      <th>percent_bc_gt_75</th>\n",
       "      <th>pub_rec_bankruptcies</th>\n",
       "      <th>tax_liens</th>\n",
       "      <th>tot_hi_cred_lim</th>\n",
       "      <th>total_bal_ex_mort</th>\n",
       "      <th>total_bc_limit</th>\n",
       "      <th>total_il_high_credit_limit</th>\n",
       "      <th>hardship_flag</th>\n",
       "      <th>disbursement_method</th>\n",
       "      <th>debt_settlement_flag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12560.0</td>\n",
       "      <td>69.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2137.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>42000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1878.0</td>\n",
       "      <td>34360.0</td>\n",
       "      <td>5.900000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>140.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60124.0</td>\n",
       "      <td>16901.0</td>\n",
       "      <td>36500.0</td>\n",
       "      <td>18124.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>87153.0</td>\n",
       "      <td>88.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>998.0</td>\n",
       "      <td>57.0</td>\n",
       "      <td>50800.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>24763.0</td>\n",
       "      <td>13761.0</td>\n",
       "      <td>8.300000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>378.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>372872.0</td>\n",
       "      <td>99468.0</td>\n",
       "      <td>15000.0</td>\n",
       "      <td>94072.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7150.0</td>\n",
       "      <td>72.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>24100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>18383.0</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>136927.0</td>\n",
       "      <td>11749.0</td>\n",
       "      <td>13800.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>30683.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3761.0</td>\n",
       "      <td>70.0</td>\n",
       "      <td>7000.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>30505.0</td>\n",
       "      <td>1239.0</td>\n",
       "      <td>75.199997</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>64.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>100.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>385183.0</td>\n",
       "      <td>36151.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>44984.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>28845.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>516.0</td>\n",
       "      <td>54.0</td>\n",
       "      <td>23100.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9667.0</td>\n",
       "      <td>8471.0</td>\n",
       "      <td>8.900000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>92.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>157548.0</td>\n",
       "      <td>29674.0</td>\n",
       "      <td>9300.0</td>\n",
       "      <td>32332.0</td>\n",
       "      <td>N</td>\n",
       "      <td>Cash</td>\n",
       "      <td>N</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  open_act_il  open_il_12m  open_il_24m  mths_since_rcnt_il  total_bal_il  il_util  open_rv_12m  open_rv_24m  max_bal_bc  all_util  total_rev_hi_lim  inq_fi  total_cu_tl  inq_last_12m  acc_open_past_24mths  avg_cur_bal  bc_open_to_buy    bc_util  chargeoff_within_12_mths  delinq_amnt  mo_sin_old_il_acct  mo_sin_old_rev_tl_op  mo_sin_rcnt_rev_tl_op  mo_sin_rcnt_tl  mort_acc  mths_since_recent_bc  mths_since_recent_inq  num_accts_ever_120_pd  num_actv_bc_tl  num_actv_rev_tl  num_bc_sats  num_bc_tl  num_il_tl  num_op_rev_tl  num_rev_accts  num_rev_tl_bal_gt_0  num_sats  num_tl_120dpd_2m  num_tl_30dpd  num_tl_90g_dpd_24m  num_tl_op_past_12m  pct_tl_nvr_dlq  percent_bc_gt_75  pub_rec_bankruptcies  tax_liens  tot_hi_cred_lim  total_bal_ex_mort  total_bc_limit  total_il_high_credit_limit hardship_flag disbursement_method debt_settlement_flag\n",
       "0  0      2.0          1.0          2.0          2.0                 12560.0       69.0     2.0          7.0          2137.0      28.0      42000.0           1.0     11.0         2.0           9.0                   1878.0       34360.0         5.900000   0.0                       0.0          140.0               212.0                 1.0                    1.0             0.0       1.0                   2.0                    0.0                    2.0             5.0              3.0          3.0        16.0       7.0            18.0           5.0                  9.0       0.0               0.0           0.0                 3.0                 100.0           0.0               1.0                   0.0        60124.0          16901.0            36500.0         18124.0                     N             Cash                N                  \n",
       "1  1      4.0          2.0          3.0          3.0                 87153.0       88.0     4.0          5.0          998.0       57.0      50800.0           2.0     15.0         2.0           10.0                  24763.0      13761.0         8.300000   0.0                       0.0          163.0               378.0                 4.0                    3.0             3.0       4.0                   4.0                    0.0                    2.0             4.0              4.0          9.0        27.0       8.0            14.0           4.0                  13.0      0.0               0.0           0.0                 6.0                 95.0            0.0               1.0                   0.0        372872.0         99468.0            15000.0         94072.0                     N             Cash                N                  \n",
       "2  2      1.0          0.0          2.0          14.0                7150.0        72.0     0.0          2.0          0.0         35.0      24100.0           1.0     5.0          0.0           4.0                   18383.0      13800.0         0.000000   0.0                       0.0          87.0                92.0                  15.0                   14.0            2.0       77.0                  14.0                   0.0                    0.0             3.0              3.0          3.0        4.0        6.0            7.0            3.0                  8.0       0.0               0.0           0.0                 0.0                 100.0           0.0               0.0                   0.0        136927.0         11749.0            13800.0         10000.0                     N             Cash                N                  \n",
       "3  3      5.0          3.0          5.0          5.0                 30683.0       68.0     0.0          0.0          3761.0      70.0      7000.0            2.0     4.0          3.0           5.0                   30505.0      1239.0          75.199997  0.0                       0.0          62.0                154.0                 64.0                   5.0             3.0       64.0                  5.0                    0.0                    1.0             2.0              1.0          2.0        7.0        2.0            3.0            2.0                  10.0      0.0               0.0           0.0                 3.0                 100.0           100.0             0.0                   0.0        385183.0         36151.0            5000.0          44984.0                     N             Cash                N                  \n",
       "4  4      5.0          3.0          5.0          4.0                 28845.0       89.0     2.0          4.0          516.0       54.0      23100.0           1.0     0.0          0.0           9.0                   9667.0       8471.0          8.900000   0.0                       0.0          53.0                216.0                 2.0                    2.0             2.0       2.0                   13.0                   0.0                    2.0             2.0              3.0          8.0        9.0        6.0            15.0           2.0                  12.0      0.0               0.0           0.0                 5.0                 92.0            0.0               0.0                   0.0        157548.0         29674.0            9300.0          32332.0                     N             Cash                N                  "
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df4 = part3.join(part4, \"index\", \"outer\")\n",
    "df4.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lets join all parts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "allParts = df3.join(df4, \"index\", \"outer\").drop(\"index\")\n",
    "# allParts.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[loan_amnt: bigint, funded_amnt: bigint, funded_amnt_inv: bigint, term: bigint, int_rate: bigint, installment: bigint, grade: bigint, sub_grade: bigint, home_ownership: bigint, annual_inc: bigint, verification_status: bigint, loan_status: bigint, pymnt_plan: bigint, dti: bigint, delinq_2yrs: bigint, inq_last_6mths: bigint, open_acc: bigint, pub_rec: bigint, revol_bal: bigint, revol_util: bigint, total_acc: bigint, initial_list_status: bigint, out_prncp: bigint, out_prncp_inv: bigint, total_pymnt: bigint, total_pymnt_inv: bigint, total_rec_prncp: bigint, total_rec_int: bigint, total_rec_late_fee: bigint, recoveries: bigint, collection_recovery_fee: bigint, last_pymnt_amnt: bigint, collections_12_mths_ex_med: bigint, application_type: bigint, acc_now_delinq: bigint, tot_coll_amt: bigint, tot_cur_bal: bigint, open_acc_6m: bigint, open_act_il: bigint, open_il_12m: bigint, open_il_24m: bigint, mths_since_rcnt_il: bigint, total_bal_il: bigint, il_util: bigint, open_rv_12m: bigint, open_rv_24m: bigint, max_bal_bc: bigint, all_util: bigint, total_rev_hi_lim: bigint, inq_fi: bigint, total_cu_tl: bigint, inq_last_12m: bigint, acc_open_past_24mths: bigint, avg_cur_bal: bigint, bc_open_to_buy: bigint, bc_util: bigint, chargeoff_within_12_mths: bigint, delinq_amnt: bigint, mo_sin_old_il_acct: bigint, mo_sin_old_rev_tl_op: bigint, mo_sin_rcnt_rev_tl_op: bigint, mo_sin_rcnt_tl: bigint, mort_acc: bigint, mths_since_recent_bc: bigint, mths_since_recent_inq: bigint, num_accts_ever_120_pd: bigint, num_actv_bc_tl: bigint, num_actv_rev_tl: bigint, num_bc_sats: bigint, num_bc_tl: bigint, num_il_tl: bigint, num_op_rev_tl: bigint, num_rev_accts: bigint, num_rev_tl_bal_gt_0: bigint, num_sats: bigint, num_tl_120dpd_2m: bigint, num_tl_30dpd: bigint, num_tl_90g_dpd_24m: bigint, num_tl_op_past_12m: bigint, pct_tl_nvr_dlq: bigint, percent_bc_gt_75: bigint, pub_rec_bankruptcies: bigint, tax_liens: bigint, tot_hi_cred_lim: bigint, total_bal_ex_mort: bigint, total_bc_limit: bigint, total_il_high_credit_limit: bigint, hardship_flag: bigint, disbursement_method: bigint, debt_settlement_flag: bigint]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#These are the count of Null values in the columns\n",
    "allParts.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in allParts.columns])\n",
    "# allParts.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in allParts.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206983\n",
      "columns: 90\n"
     ]
    }
   ],
   "source": [
    "# Check the dimensions of the dataframe\n",
    "size_shape(allParts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop to rows where the Loan_Status is == \"Late\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop rows that have null values\n",
    "\n",
    "allParts = allParts.filter(allParts[\"recoveries\"]. isNotNull())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[loan_amnt: bigint, funded_amnt: bigint, funded_amnt_inv: bigint, term: bigint, int_rate: bigint, installment: bigint, grade: bigint, sub_grade: bigint, home_ownership: bigint, annual_inc: bigint, verification_status: bigint, loan_status: bigint, pymnt_plan: bigint, dti: bigint, delinq_2yrs: bigint, inq_last_6mths: bigint, open_acc: bigint, pub_rec: bigint, revol_bal: bigint, revol_util: bigint, total_acc: bigint, initial_list_status: bigint, out_prncp: bigint, out_prncp_inv: bigint, total_pymnt: bigint, total_pymnt_inv: bigint, total_rec_prncp: bigint, total_rec_int: bigint, total_rec_late_fee: bigint, recoveries: bigint, collection_recovery_fee: bigint, last_pymnt_amnt: bigint, collections_12_mths_ex_med: bigint, application_type: bigint, acc_now_delinq: bigint, tot_coll_amt: bigint, tot_cur_bal: bigint, open_acc_6m: bigint, open_act_il: bigint, open_il_12m: bigint, open_il_24m: bigint, mths_since_rcnt_il: bigint, total_bal_il: bigint, il_util: bigint, open_rv_12m: bigint, open_rv_24m: bigint, max_bal_bc: bigint, all_util: bigint, total_rev_hi_lim: bigint, inq_fi: bigint, total_cu_tl: bigint, inq_last_12m: bigint, acc_open_past_24mths: bigint, avg_cur_bal: bigint, bc_open_to_buy: bigint, bc_util: bigint, chargeoff_within_12_mths: bigint, delinq_amnt: bigint, mo_sin_old_il_acct: bigint, mo_sin_old_rev_tl_op: bigint, mo_sin_rcnt_rev_tl_op: bigint, mo_sin_rcnt_tl: bigint, mort_acc: bigint, mths_since_recent_bc: bigint, mths_since_recent_inq: bigint, num_accts_ever_120_pd: bigint, num_actv_bc_tl: bigint, num_actv_rev_tl: bigint, num_bc_sats: bigint, num_bc_tl: bigint, num_il_tl: bigint, num_op_rev_tl: bigint, num_rev_accts: bigint, num_rev_tl_bal_gt_0: bigint, num_sats: bigint, num_tl_120dpd_2m: bigint, num_tl_30dpd: bigint, num_tl_90g_dpd_24m: bigint, num_tl_op_past_12m: bigint, pct_tl_nvr_dlq: bigint, percent_bc_gt_75: bigint, pub_rec_bankruptcies: bigint, tax_liens: bigint, tot_hi_cred_lim: bigint, total_bal_ex_mort: bigint, total_bc_limit: bigint, total_il_high_credit_limit: bigint, hardship_flag: bigint, disbursement_method: bigint, debt_settlement_flag: bigint]"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#These are the count of Null values in the columns\n",
    "allParts.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in allParts.columns])\n",
    "\n",
    "#allParts.select([fn.count(fn.when(fn.isnan(c)|fn.isnull(c),c)).alias(c) for c in allParts.columns]).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "rows: 1206949\n",
      "columns: 90\n"
     ]
    }
   ],
   "source": [
    "# Check the dimensions of the dataframe\n",
    "size_shape(allParts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "# allParts.describe(string_cols4).show()\n",
    "# print(allParts.groupby(string_cols4[0]).count().show())\n",
    "# print(allParts.groupby(string_cols4[1]).count().show())\n",
    "# print(allParts.groupby(string_cols4[2]).count().show())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# allParts.describe(cleaned_int_categorical4).toPandas().T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numerical_cols4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From doing the correlation matrix, we notice that there are some columns that need to be removed from our "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "del_cols = ['policy_code','pymnt_plan','url','application_type','acc_now_delinq','emp_title','zip_code','title',\n",
    "        'installment',\n",
    "'total_rec_prncp',\n",
    "'total_rec_int',\n",
    "'recoveries',\n",
    "'inq_fi',\n",
    "'inq_last_12m',\n",
    "'chargeoff_within_12_mths',\n",
    "'num_actv_rev_tl',\n",
    "'num_bc_tl',\n",
    "'num_op_rev_tl',\n",
    "'num_tl_op_past_12m',\n",
    "'percent_bc_gt_75',\n",
    "'avg_cur_bal',\n",
    "'total_bc_limit',\n",
    "'total_il_high_credit_limit']\n",
    "allParts = allParts.drop(*cols)\n",
    "# size_shape(allParts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Here we will make a way to combine all of our work into a logical sequence so that categories can be changed using string indexer and numerical columns can be changed by using standard scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Categorical operations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "# The column that we are going to be predicting on needs to change to a numerical value\n",
    "en_pred = feature.StringIndexer(handleInvalid='skip', inputCol='loan_status', outputCol='encoded_loan_status')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert string to index label using the StringIndexer\n",
    "cat_col_list=['term','grade','verification_status','pymnt_plan']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['initial_list_status', 'term', 'grade', 'verification_status']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['initial_list_status', 'term', 'grade', 'verification_status']"
      ]
     },
     "execution_count": 271,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical_cols = ['initial_list_status','application_type']\n",
    "for i in cat_col_list:\n",
    "    if i in del_cols:\n",
    "        continue\n",
    "    else:\n",
    "        categorical_cols.append(i)\n",
    "categorical_cols_new = []\n",
    "for i in categorical_cols:\n",
    "    if i in del_cols:\n",
    "        continue\n",
    "    else:\n",
    "        categorical_cols_new.append(i)\n",
    "categorical_cols = categorical_cols_new\n",
    "\n",
    "print(categorical_cols)\n",
    "\n",
    "stringIndexer = feature.StringIndexer(handleInvalid='skip', inputCols=[cols for cols in categorical_cols],\n",
    "                                      outputCols=[\"encoded_\" + cols for cols in categorical_cols])\n",
    "\n",
    "categorical_cols_new = []\n",
    "for i in categorical_cols:\n",
    "    if i in del_cols:\n",
    "        continue\n",
    "    else:\n",
    "        categorical_cols_new.append(i)\n",
    "        \n",
    "        \n",
    "categorical_cols = categorical_cols_new\n",
    "#categorical_cols = categorical_cols[0:4]\n",
    "categorical_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use this feature on columns that are categorical and not ordinal\n",
    "one_hot_categorical_cols = []\n",
    "ones = ['sub_grade','home_ownership']\n",
    "\n",
    "for i in string_cols4:\n",
    "    one_hot_categorical_cols.append(i)\n",
    "\n",
    "\n",
    "string_hot = feature.StringIndexer(handleInvalid='skip', inputCols= [cols for cols in ones],\n",
    "                                      outputCols=[\"encoded_\" + cols for cols in ones])\n",
    "    \n",
    "onehot= feature.OneHotEncoder(inputCols=[\"encoded_\" + cols for cols in ones],\n",
    "                                      outputCols=[\"onehot_encoded_\" + cols for cols in ones])\n",
    "\n",
    "catHotVA = feature.VectorAssembler(inputCols=[\"onehot_encoded_\" + cols for cols in ones], outputCol=\"catHotfeatures\") # \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['loan_status',\n",
       " 'initial_list_status',\n",
       " 'term',\n",
       " 'grade',\n",
       " 'verification_status',\n",
       " 'disbursement_method',\n",
       " 'debt_settlement_flag',\n",
       " 'hardship_flag']"
      ]
     },
     "execution_count": 264,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "colsNonNumeric = [\"loan_status\"]\n",
    "    \n",
    "for i in categorical_cols:\n",
    "    if i == 'loan_status':\n",
    "        continue\n",
    "    else:\n",
    "        colsNonNumeric.append(i)\n",
    "    \n",
    "for i in one_hot_categorical_cols:\n",
    "    if i == 'loan_status':\n",
    "        continue\n",
    "    else:\n",
    "        colsNonNumeric.append(i)\n",
    "    \n",
    "\n",
    "colsNonNumeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {},
   "outputs": [],
   "source": [
    "# colsNonNumeric = [\"loan_status\"]\n",
    "    \n",
    "# for i in categorical_cols:\n",
    "#     colsNonNumeric.append(i)\n",
    "    \n",
    "# for i in one_hot_categorical_cols:\n",
    "#     colsNonNumeric.append(i)\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Need to get all the names of the encoded columns and put them into a list except for the enconded_loan_status\n",
    "# en_colsNonNumeric = []  # 'onehot_encoded_sub_grade','onehot_encoded_home_ownership'\n",
    "    \n",
    "# for i in categorical_cols:\n",
    "#     en_colsNonNumeric.append(\"encoded_\"+i)\n",
    "    \n",
    "# for i in one_hot_categorical_cols:\n",
    "#     en_colsNonNumeric.append(\"encoded_\"+i)\n",
    "    \n",
    "\n",
    "\n",
    "# # catCols  = [\"encoded_\" + cols for cols in one_hot_categorical_cols]\n",
    "\n",
    "# # oneCols = [\"encoded_\" + cols for cols in categorical_cols]\n",
    "\n",
    "# print(len(en_colsNonNumeric))\n",
    "# en_colsNonNumeric\n",
    "# del en_colsNonNumeric[6:8:1]\n",
    "# print(len(en_colsNonNumeric))\n",
    "# en_colsNonNumeric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Numerical Operations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now let's scale the numeric variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "numeric_cols = [cols for cols in allParts.columns if (cols not in colsNonNumeric) and (cols not in del_cols) and (cols not in ones)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [],
   "source": [
    "# numeric_cols = [cols for cols in allParts.columns if cols not in colsNonNumeric]\n",
    "# numeric_cols"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It turned out that StandardScaler can only take one column as input columns so we have to Assemble numeric columns to one column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62\n",
      "rows: 1206949\n",
      "columns: 87\n"
     ]
    }
   ],
   "source": [
    "print(len(numeric_cols))\n",
    "size_shape(allParts)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define the VectorAssembler object\n",
    "numericVA = feature.VectorAssembler(inputCols=[cols for cols in numeric_cols], outputCol=\"numfeatures\")\n",
    "\n",
    "#\n",
    "catVA = feature.VectorAssembler(inputCols=[\"encoded_\" + cols for cols in categorical_cols], outputCol=\"catfeatures\") # \n",
    "\n",
    "# define the StandardScaler object\n",
    "sScaler = feature.StandardScaler(withMean=True, withStd=True, inputCol=\"numfeatures\", outputCol=\"scaled_features\")\n",
    "\n",
    "# define a Vector assembler that handles all features that need to be scaled\n",
    "allVA = feature.VectorAssembler(inputCols=['scaled_features','catHotfeatures','catfeatures'], outputCol='final_features') # 'catfeatures'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the pipeline which contains 9 stages: stringindexer, VectorAssembler(numeric), StandardScaler, stringIndexer(one_hot_categorical),OneHotEnconder(one_hot_categroical),VectorAssembler(one_hot_categorical),VectorAssembler(categorical), stringIndexer(prediction_col), VectorAssembler(all columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 269,
   "metadata": {},
   "outputs": [],
   "source": [
    "allParts_pipe=Pipeline(stages=[stringIndexer, numericVA, sScaler, string_hot, onehot,catHotVA, catVA, en_pred, allVA])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "# allParts_pipe=Pipeline(stages=[stringIndexer, onehot, numericVA, sScaler, catVA, en_pred, allVA])\n",
    "# catVA = feature.VectorAssembler(inputCols=[cols for cols in en_colsNonNumeric], outputCol=\"catfeatures\")\n",
    "# allParts_pipe=Pipeline(stages=[stringIndexer, numericVA, sScaler, en_pred, allVA])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Fit the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stop here\n",
    "fitted_allParts = allParts_pipe.fit(allParts)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Apply the pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "transformed_allParts = fitted_allParts.transform(allParts)\n",
    "# transformed_allParts.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Already imported classification package\n",
    "\n",
    "train , test = allParts.randomSplit([0.8,0.2],0)\n",
    "lr = classification.LogisticRegression(featuresCol = 'final_features',labelCol = 'encoded_loan_status')\n",
    "lr_model = Pipeline(stages = [allParts_pipe,lr])\n",
    "fitted_lr_pipeline = lr_model.fit(train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Execute the classification pipeline defined by model_fitted using data as the input data.  Returns ROC AUC.\n",
    "# Uses model_pipeline to obtain information needed to execute the fitted model.\n",
    "# Args\n",
    "# model_pipeline:  The unfitted pipeline definition.\n",
    "# model_fitted: The fitted pipeline\n",
    "# data: The data on which to perform predictions\n",
    "def binary_evaluation(model_pipeline, model_fitted, data):\n",
    "    return BinaryClassificationEvaluator(labelCol=model_pipeline.getStages()[-1].getLabelCol(), \n",
    "                                rawPredictionCol=model_pipeline.getStages()[-1].getRawPredictionCol()).\\\n",
    "    evaluate(model_fitted.transform(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8838460391997753"
      ]
     },
     "execution_count": 275,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# binary_evaluation(model1_pipeline, model1_fitted, test)\n",
    "binary_evaluation(lr_model, fitted_lr_pipeline, test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#####  Interpretation of model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr_model_interpret = fitted_lr_pipeline.stages[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>funded_amnt_inv</td>\n",
       "      <td>5.526542</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>total_rec_int</td>\n",
       "      <td>3.490443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>pub_rec</td>\n",
       "      <td>2.640840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>term</td>\n",
       "      <td>1.195670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>68</td>\n",
       "      <td>num_il_tl</td>\n",
       "      <td>0.373852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>open_act_il</td>\n",
       "      <td>0.329462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>collection_recovery_fee</td>\n",
       "      <td>0.316748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>verification_status</td>\n",
       "      <td>0.270860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>64</td>\n",
       "      <td>num_actv_bc_tl</td>\n",
       "      <td>0.252404</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>last_pymnt_amnt</td>\n",
       "      <td>0.169501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>annual_inc</td>\n",
       "      <td>0.162395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>65</td>\n",
       "      <td>num_actv_rev_tl</td>\n",
       "      <td>0.144211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>delinq_2yrs</td>\n",
       "      <td>0.130055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>67</td>\n",
       "      <td>num_bc_tl</td>\n",
       "      <td>0.108863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>84</td>\n",
       "      <td>total_il_high_credit_limit</td>\n",
       "      <td>0.098809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>total_acc</td>\n",
       "      <td>0.096722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>dti</td>\n",
       "      <td>0.094946</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>51</td>\n",
       "      <td>avg_cur_bal</td>\n",
       "      <td>0.091266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>57</td>\n",
       "      <td>mo_sin_old_rev_tl_op</td>\n",
       "      <td>0.090181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>81</td>\n",
       "      <td>tot_hi_cred_lim</td>\n",
       "      <td>0.073962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        column    weight\n",
       "0   funded_amnt_inv             5.526542\n",
       "25  total_rec_int               3.490443\n",
       "15  pub_rec                     2.640840\n",
       "1   term                        1.195670\n",
       "68  num_il_tl                   0.373852\n",
       "36  open_act_il                 0.329462\n",
       "28  collection_recovery_fee     0.316748\n",
       "8   verification_status         0.270860\n",
       "64  num_actv_bc_tl              0.252404\n",
       "29  last_pymnt_amnt             0.169501\n",
       "7   annual_inc                  0.162395\n",
       "65  num_actv_rev_tl             0.144211\n",
       "12  delinq_2yrs                 0.130055\n",
       "67  num_bc_tl                   0.108863\n",
       "84  total_il_high_credit_limit  0.098809\n",
       "18  total_acc                   0.096722\n",
       "11  dti                         0.094946\n",
       "51  avg_cur_bal                 0.091266\n",
       "57  mo_sin_old_rev_tl_op        0.090181\n",
       "81  tot_hi_cred_lim             0.073962"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(list(zip(allParts.columns[2:], lr_model_interpret.coefficients.toArray())),\n",
    "            columns = ['column', 'weight']).sort_values('weight',ascending = False).head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Logistic Regression Using Different Parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define a grid search ParamGridBuilder object. The grid search object iterates over different combinations of paramters (elasticNetParam and regParam in this example).\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [],
   "source": [
    "paramGrid = ParamGridBuilder() \\\n",
    "    .addGrid(lr.elasticNetParam, [0., 0.01]) \\\n",
    "    .addGrid(lr.regParam, [0.1, 0.001]) \\\n",
    "    .build()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " #The length of the grid is 12 because it implements all combinations of elasticNetParam (3 options) and regParam (4 options).\n",
    "\n",
    "len(paramGrid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define an evaluator and a CrossValidator. The CrossValidator performs cross validation on each model in the grid defined above. The evaluator is used to evaluate the performance of each pass of the cross validation.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = BinaryClassificationEvaluator(labelCol=lr.getLabelCol(), rawPredictionCol=lr.getRawPredictionCol(), numBins=40)\n",
    "crossval = CrossValidator(estimator=lr_model, \n",
    "                          estimatorParamMaps=paramGrid, \n",
    "                          evaluator=evaluator, \n",
    "                          numFolds=2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The cell below will take a while to complete because we are training and testing 12 separate logistic regression models. And for each model, we train and evaluate the model twice (numFolds = 2). The total number of training and evaluation cycles is 12 * 2 = 24."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_model_fitted = crossval.fit(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print the AUC accuracy of the test data on the best model produced by the grid search.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.883456806508263"
      ]
     },
     "execution_count": 282,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.evaluate(final_model_fitted.transform(test))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Print the stages of the best model produced from the grid search.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[PipelineModel_c5a13eb1ee8b,\n",
       " LogisticRegressionModel: uid=LogisticRegression_e1738aa5a282, numClasses=2, numFeatures=105]"
      ]
     },
     "execution_count": 283,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_model_fitted.bestModel.stages\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DenseVector([0.1694, 0.0991, -0.038, -0.1548, -0.2218, -0.0326, 0.0554, 0.1658, 0.1183, 0.0147, 0.0091, 0.0938, 0.127, -0.7212, -0.7141, 0.2808, 0.2017, 0.0095, 0.101, -0.1121, -0.0048, -0.0087, 0.0307, -0.2147, -0.4226, 0.0557, 0.0328, -0.215, 0.2895, 0.1753, -0.3632, -0.0486, 0.0601, -0.0268, 0.0376, -0.1125, 0.2871, -0.1277, 0.05, 0.0117, 0.009, -0.0676, -0.1099, -0.0505, 0.0227, -0.0807, -0.0498, 0.0136, 0.0293, -0.0462, 0.0551, 0.0974, -0.1779, 0.0616, 0.0014, 0.0049, -0.0042, 0.0899, 0.0274, -0.0023, -0.094, -0.2933, 0.2619, -0.1391, 0.5319, 0.431, -0.2214, 0.4023, 0.6452, -0.5987, -0.4284, -0.2985, -0.9537, -1.1424, -1.9088, 0.0571, 0.1367, -1.4174, 0.2115, -1.4632, 0.2927, 0.3859, 0.2294, 0.3025, 0.4208, 0.7068, 0.5479, 0.7037, 0.7436, 0.9179, 1.1132, 1.0741, 0.891, 0.8882, 1.001, 1.1136, -0.1022, 0.1091, -0.0268, -0.6821, 4.6668, 0.2392, -0.8088, 0.2416, 0.1004])"
      ]
     },
     "execution_count": 284,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_model_fitted.bestModel.stages[-1].coefficients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "105"
      ]
     },
     "execution_count": 285,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(final_model_fitted.bestModel.stages[-1].coefficients)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Gradient Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbt = classification.GBTClassifier(featuresCol = 'final_features',labelCol = 'encoded_loan_status',maxBins = 40)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 287,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbt_model = Pipeline(stages = [allParts_pipe,gbt])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 288,
   "metadata": {},
   "outputs": [],
   "source": [
    "fitted_gbt_pipeline = gbt_model.fit(train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 289,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbt_model = fitted_gbt_pipeline.stages[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>column</th>\n",
       "      <th>weight</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>inq_last_6mths</td>\n",
       "      <td>0.311863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>pub_rec</td>\n",
       "      <td>0.074504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>open_acc</td>\n",
       "      <td>0.071860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>revol_bal</td>\n",
       "      <td>0.065716</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>initial_list_status</td>\n",
       "      <td>0.041302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>installment</td>\n",
       "      <td>0.035765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>36</td>\n",
       "      <td>open_act_il</td>\n",
       "      <td>0.035386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>recoveries</td>\n",
       "      <td>0.032603</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>50</td>\n",
       "      <td>acc_open_past_24mths</td>\n",
       "      <td>0.019920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>total_rec_int</td>\n",
       "      <td>0.018080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>int_rate</td>\n",
       "      <td>0.014960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>32</td>\n",
       "      <td>acc_now_delinq</td>\n",
       "      <td>0.013613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>funded_amnt_inv</td>\n",
       "      <td>0.013359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>total_pymnt</td>\n",
       "      <td>0.013111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>total_pymnt_inv</td>\n",
       "      <td>0.013009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>total_rec_prncp</td>\n",
       "      <td>0.012760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>31</td>\n",
       "      <td>application_type</td>\n",
       "      <td>0.008616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>33</td>\n",
       "      <td>tot_coll_amt</td>\n",
       "      <td>0.008353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>43</td>\n",
       "      <td>open_rv_24m</td>\n",
       "      <td>0.007912</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>38</td>\n",
       "      <td>open_il_24m</td>\n",
       "      <td>0.006750</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  column    weight\n",
       "13  inq_last_6mths        0.311863\n",
       "15  pub_rec               0.074504\n",
       "14  open_acc              0.071860\n",
       "16  revol_bal             0.065716\n",
       "19  initial_list_status   0.041302\n",
       "3   installment           0.035765\n",
       "36  open_act_il           0.035386\n",
       "27  recoveries            0.032603\n",
       "50  acc_open_past_24mths  0.019920\n",
       "25  total_rec_int         0.018080\n",
       "2   int_rate              0.014960\n",
       "32  acc_now_delinq        0.013613\n",
       "0   funded_amnt_inv       0.013359\n",
       "22  total_pymnt           0.013111\n",
       "23  total_pymnt_inv       0.013009\n",
       "24  total_rec_prncp       0.012760\n",
       "31  application_type      0.008616\n",
       "33  tot_coll_amt          0.008353\n",
       "43  open_rv_24m           0.007912\n",
       "38  open_il_24m           0.006750"
      ]
     },
     "execution_count": 290,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(list(zip(allParts.columns[2:], gbt_model.featureImportances.toArray())),\n",
    "            columns = ['column', 'weight']).sort_values('weight',ascending = False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 291,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9286414247164101"
      ]
     },
     "execution_count": 291,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bce = BinaryClassificationEvaluator(labelCol = 'encoded_loan_status')\n",
    "bce.evaluate(fitted_gbt_pipeline.transform(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}