Submission3.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Youn Hee is going to work in here :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data from https://collegescorecard.ed.gov/data/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2698: DtypeWarning: Columns (6,9,1725,1726,1727,1728,1729,1743) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "# load the data from csv\n",
    "big = pd.read_csv('/Users/yoni/documents/GA_prod/project-5/CollegeScorecard_Raw_Data/MERGED2015_16_PP.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['PCIP01', 'PCIP03', 'PCIP04', 'PCIP05', 'PCIP09', 'PCIP10',\n",
       "       'PCIP11', 'PCIP12', 'PCIP13', 'PCIP14', 'PCIP15', 'PCIP16',\n",
       "       'PCIP19', 'PCIP22', 'PCIP23', 'PCIP24', 'PCIP25', 'PCIP26',\n",
       "       'PCIP27', 'PCIP29', 'PCIP30', 'PCIP31', 'PCIP38', 'PCIP39',\n",
       "       'PCIP40', 'PCIP41', 'PCIP42', 'PCIP43', 'PCIP44', 'PCIP45',\n",
       "       'PCIP46', 'PCIP47', 'PCIP48', 'PCIP49', 'PCIP50', 'PCIP51',\n",
       "       'PCIP52', 'PCIP54', 'CIP01BACHL', 'CIP03BACHL', 'CIP04BACHL',\n",
       "       'CIP05BACHL', 'CIP09BACHL', 'CIP10BACHL', 'CIP11BACHL',\n",
       "       'CIP12BACHL', 'CIP13BACHL', 'CIP14BACHL', 'CIP15BACHL',\n",
       "       'CIP16BACHL', 'CIP19BACHL', 'CIP22BACHL', 'CIP23BACHL',\n",
       "       'CIP24BACHL', 'CIP25BACHL', 'CIP26BACHL', 'CIP27BACHL',\n",
       "       'CIP29BACHL', 'CIP30BACHL', 'CIP31BACHL', 'CIP38BACHL',\n",
       "       'CIP39BACHL', 'CIP40BACHL', 'CIP41BACHL', 'CIP42BACHL',\n",
       "       'CIP43BACHL', 'CIP44BACHL', 'CIP45BACHL', 'CIP46BACHL',\n",
       "       'CIP47BACHL', 'CIP48BACHL', 'CIP49BACHL', 'CIP50BACHL',\n",
       "       'CIP51BACHL', 'CIP52BACHL', 'CIP54BACHL', 'ADM_RATE', 'SATVRMID',\n",
       "       'SATMTMID', 'SATWRMID', 'ACTCMMID', 'ACTENMID', 'ACTMTMID',\n",
       "       'ACTWRMID', 'PCTFLOAN', 'GRAD_DEBT_MDN', 'WDRAW_DEBT_MDN',\n",
       "       'LO_INC_DEBT_MDN', 'MD_INC_DEBT_MDN', 'HI_INC_DEBT_MDN', 'C150_4',\n",
       "       'TRANS_4', 'COSTT4_A', 'COSTT4_P', 'TUITIONFEE_IN',\n",
       "       'TUITIONFEE_OUT', 'MD_EARN_WNE_P6', 'CONTROL', 'REGION', 'LOCALE',\n",
       "       'LOCALE2', nan, 'TUITFTE', 'INEXPFTE', 'AVGFACSAL', 'PFTFAC', 'UG',\n",
       "       'UG25ABV', 'PAR_ED_PCT_MS', 'PAR_ED_PCT_HS', 'PAR_ED_PCT_PS',\n",
       "       'AGE_ENTRY', 'FEMALE', 'D_PCTPELL_PCTFLOAN'], dtype=object)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ref = pd.read_csv('./college_data_variables.csv')\n",
    "ref['VARIABLE NAME'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target = big[['PCIP01', 'PCIP03', 'PCIP04', 'PCIP05', 'PCIP09', 'PCIP10',\n",
    "       'PCIP11', 'PCIP12', 'PCIP13', 'PCIP14', 'PCIP15', 'PCIP16',\n",
    "       'PCIP19', 'PCIP22', 'PCIP23', 'PCIP24', 'PCIP25', 'PCIP26',\n",
    "       'PCIP27', 'PCIP29', 'PCIP30', 'PCIP31', 'PCIP38', 'PCIP39',\n",
    "       'PCIP40', 'PCIP41', 'PCIP42', 'PCIP43', 'PCIP44', 'PCIP45',\n",
    "       'PCIP46', 'PCIP47', 'PCIP48', 'PCIP49', 'PCIP50', 'PCIP51',\n",
    "       'PCIP52', 'PCIP54', 'CIP01BACHL', 'CIP03BACHL', 'CIP04BACHL',\n",
    "       'CIP05BACHL', 'CIP09BACHL', 'CIP10BACHL', 'CIP11BACHL',\n",
    "       'CIP12BACHL', 'CIP13BACHL', 'CIP14BACHL', 'CIP15BACHL',\n",
    "       'CIP16BACHL', 'CIP19BACHL', 'CIP22BACHL', 'CIP23BACHL',\n",
    "       'CIP24BACHL', 'CIP25BACHL', 'CIP26BACHL', 'CIP27BACHL',\n",
    "       'CIP29BACHL', 'CIP30BACHL', 'CIP31BACHL', 'CIP38BACHL',\n",
    "       'CIP39BACHL', 'CIP40BACHL', 'CIP41BACHL', 'CIP42BACHL',\n",
    "       'CIP43BACHL', 'CIP44BACHL', 'CIP45BACHL', 'CIP46BACHL',\n",
    "       'CIP47BACHL', 'CIP48BACHL', 'CIP49BACHL', 'CIP50BACHL',\n",
    "       'CIP51BACHL', 'CIP52BACHL', 'CIP54BACHL', 'ADM_RATE', 'SATVRMID',\n",
    "       'SATMTMID', 'SATWRMID', 'ACTCMMID', 'ACTENMID', 'ACTMTMID',\n",
    "       'ACTWRMID', 'PCTFLOAN', 'GRAD_DEBT_MDN', 'WDRAW_DEBT_MDN',\n",
    "       'LO_INC_DEBT_MDN', 'MD_INC_DEBT_MDN', 'HI_INC_DEBT_MDN', 'C150_4',\n",
    "       'TRANS_4', 'COSTT4_A', 'COSTT4_P', 'TUITIONFEE_IN',\n",
    "       'TUITIONFEE_OUT', 'MD_EARN_WNE_P6', 'CONTROL', 'REGION', 'LOCALE',\n",
    "       'LOCALE2', 'TUITFTE', 'INEXPFTE', 'AVGFACSAL', 'PFTFAC', 'UG',\n",
    "       'UG25ABV', 'PAR_ED_PCT_MS', 'PAR_ED_PCT_HS', 'PAR_ED_PCT_PS',\n",
    "       'AGE_ENTRY', 'FEMALE', 'D_PCTPELL_PCTFLOAN']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7593, 113)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PCIP01</th>\n",
       "      <th>PCIP03</th>\n",
       "      <th>PCIP04</th>\n",
       "      <th>PCIP05</th>\n",
       "      <th>PCIP09</th>\n",
       "      <th>PCIP10</th>\n",
       "      <th>PCIP11</th>\n",
       "      <th>PCIP12</th>\n",
       "      <th>PCIP13</th>\n",
       "      <th>PCIP14</th>\n",
       "      <th>...</th>\n",
       "      <th>AVGFACSAL</th>\n",
       "      <th>PFTFAC</th>\n",
       "      <th>UG</th>\n",
       "      <th>UG25ABV</th>\n",
       "      <th>PAR_ED_PCT_MS</th>\n",
       "      <th>PAR_ED_PCT_HS</th>\n",
       "      <th>PAR_ED_PCT_PS</th>\n",
       "      <th>AGE_ENTRY</th>\n",
       "      <th>FEMALE</th>\n",
       "      <th>D_PCTPELL_PCTFLOAN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0446</td>\n",
       "      <td>0.0023</td>\n",
       "      <td>0.0094</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0164</td>\n",
       "      <td>0.0634</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1268</td>\n",
       "      <td>0.1432</td>\n",
       "      <td>...</td>\n",
       "      <td>7017.0</td>\n",
       "      <td>0.7096</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0877</td>\n",
       "      <td>0.0192989366</td>\n",
       "      <td>0.3694367861</td>\n",
       "      <td>0.6112642773</td>\n",
       "      <td>20.277133825</td>\n",
       "      <td>0.5472427116</td>\n",
       "      <td>4210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0009</td>\n",
       "      <td>0.0426</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0133</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0815</td>\n",
       "      <td>0.0577</td>\n",
       "      <td>...</td>\n",
       "      <td>10221.0</td>\n",
       "      <td>0.9081</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2363</td>\n",
       "      <td>0.0230535015</td>\n",
       "      <td>0.3334058286</td>\n",
       "      <td>0.6435406699</td>\n",
       "      <td>23.58415642</td>\n",
       "      <td>0.6369683532</td>\n",
       "      <td>11679.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>3217.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.8571</td>\n",
       "      <td>0.0619047619</td>\n",
       "      <td>0.4666666667</td>\n",
       "      <td>0.4714285714</td>\n",
       "      <td>33.065934066</td>\n",
       "      <td>0.6043956044</td>\n",
       "      <td>279.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0216</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0315</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0216</td>\n",
       "      <td>0.3027</td>\n",
       "      <td>...</td>\n",
       "      <td>9514.0</td>\n",
       "      <td>0.6173</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2255</td>\n",
       "      <td>0.0275862069</td>\n",
       "      <td>0.2862068966</td>\n",
       "      <td>0.6862068966</td>\n",
       "      <td>23.152713891</td>\n",
       "      <td>0.4779208832</td>\n",
       "      <td>5618.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0945</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0567</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.1531</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>...</td>\n",
       "      <td>7940.0</td>\n",
       "      <td>0.6395</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0974</td>\n",
       "      <td>0.0159163256</td>\n",
       "      <td>0.3533424284</td>\n",
       "      <td>0.630741246</td>\n",
       "      <td>20.201285657</td>\n",
       "      <td>0.612294094</td>\n",
       "      <td>4805.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 113 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   PCIP01  PCIP03  PCIP04  PCIP05  PCIP09  PCIP10  PCIP11  PCIP12  PCIP13  \\\n",
       "0  0.0446  0.0023  0.0094  0.0000  0.0000  0.0164  0.0634     0.0  0.1268   \n",
       "1  0.0000  0.0000  0.0000  0.0009  0.0426  0.0000  0.0133     0.0  0.0815   \n",
       "2  0.0000  0.0000  0.0000  0.0000  0.0000  0.0000  0.0000     0.0  0.0000   \n",
       "3  0.0000  0.0000  0.0000  0.0000  0.0216  0.0000  0.0315     0.0  0.0216   \n",
       "4  0.0000  0.0000  0.0000  0.0000  0.0945  0.0000  0.0567     0.0  0.1531   \n",
       "\n",
       "   PCIP14         ...          AVGFACSAL  PFTFAC  UG  UG25ABV  PAR_ED_PCT_MS  \\\n",
       "0  0.1432         ...             7017.0  0.7096 NaN   0.0877   0.0192989366   \n",
       "1  0.0577         ...            10221.0  0.9081 NaN   0.2363   0.0230535015   \n",
       "2  0.0000         ...             3217.0  1.0000 NaN   0.8571   0.0619047619   \n",
       "3  0.3027         ...             9514.0  0.6173 NaN   0.2255   0.0275862069   \n",
       "4  0.0000         ...             7940.0  0.6395 NaN   0.0974   0.0159163256   \n",
       "\n",
       "   PAR_ED_PCT_HS  PAR_ED_PCT_PS     AGE_ENTRY        FEMALE  \\\n",
       "0   0.3694367861   0.6112642773  20.277133825  0.5472427116   \n",
       "1   0.3334058286   0.6435406699   23.58415642  0.6369683532   \n",
       "2   0.4666666667   0.4714285714  33.065934066  0.6043956044   \n",
       "3   0.2862068966   0.6862068966  23.152713891  0.4779208832   \n",
       "4   0.3533424284    0.630741246  20.201285657   0.612294094   \n",
       "\n",
       "   D_PCTPELL_PCTFLOAN  \n",
       "0              4210.0  \n",
       "1             11679.0  \n",
       "2               279.0  \n",
       "3              5618.0  \n",
       "4              4805.0  \n",
       "\n",
       "[5 rows x 113 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PCIP01                 739\n",
       "PCIP03                 739\n",
       "PCIP04                 739\n",
       "PCIP05                 739\n",
       "PCIP09                 739\n",
       "PCIP10                 739\n",
       "PCIP11                 739\n",
       "PCIP12                 739\n",
       "PCIP13                 739\n",
       "PCIP14                 739\n",
       "PCIP15                 739\n",
       "PCIP16                 739\n",
       "PCIP19                 739\n",
       "PCIP22                 739\n",
       "PCIP23                 739\n",
       "PCIP24                 739\n",
       "PCIP25                 739\n",
       "PCIP26                 739\n",
       "PCIP27                 739\n",
       "PCIP29                 739\n",
       "PCIP30                 739\n",
       "PCIP31                 739\n",
       "PCIP38                 739\n",
       "PCIP39                 739\n",
       "PCIP40                 739\n",
       "PCIP41                 739\n",
       "PCIP42                 739\n",
       "PCIP43                 739\n",
       "PCIP44                 739\n",
       "PCIP45                 739\n",
       "                      ... \n",
       "ACTWRMID              7268\n",
       "PCTFLOAN               758\n",
       "GRAD_DEBT_MDN           29\n",
       "WDRAW_DEBT_MDN          29\n",
       "LO_INC_DEBT_MDN         29\n",
       "MD_INC_DEBT_MDN         29\n",
       "HI_INC_DEBT_MDN         29\n",
       "C150_4                5104\n",
       "TRANS_4               5104\n",
       "COSTT4_A              3673\n",
       "COSTT4_P              5093\n",
       "TUITIONFEE_IN         3290\n",
       "TUITIONFEE_OUT        3504\n",
       "MD_EARN_WNE_P6        7593\n",
       "CONTROL                  0\n",
       "REGION                   0\n",
       "LOCALE                 446\n",
       "LOCALE2               7593\n",
       "TUITFTE                460\n",
       "INEXPFTE               460\n",
       "AVGFACSAL             3139\n",
       "PFTFAC                3656\n",
       "UG                    7593\n",
       "UG25ABV                805\n",
       "PAR_ED_PCT_MS          218\n",
       "PAR_ED_PCT_HS          218\n",
       "PAR_ED_PCT_PS          218\n",
       "AGE_ENTRY              218\n",
       "FEMALE                 218\n",
       "D_PCTPELL_PCTFLOAN     758\n",
       "Length: 113, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# columns that Charles is not looking at for the moment\n",
    "other = target[['ADM_RATE', 'SATVRMID', 'SATMTMID', 'SATWRMID', 'ACTCMMID', 'ACTENMID', 'ACTMTMID','ACTWRMID', 'PCTFLOAN', \n",
    "'GRAD_DEBT_MDN', 'WDRAW_DEBT_MDN', 'LO_INC_DEBT_MDN', 'MD_INC_DEBT_MDN', 'HI_INC_DEBT_MDN', 'C150_4',\n",
    "'TRANS_4', 'COSTT4_A', 'COSTT4_P', 'TUITIONFEE_IN','TUITIONFEE_OUT', 'MD_EARN_WNE_P6', 'CONTROL', 'REGION', 'LOCALE',\n",
    "'LOCALE2', 'TUITFTE', 'INEXPFTE', 'AVGFACSAL', 'PFTFAC', 'UG', 'UG25ABV', 'PAR_ED_PCT_MS', 'PAR_ED_PCT_HS', \n",
    "'PAR_ED_PCT_PS','AGE_ENTRY', 'FEMALE', 'D_PCTPELL_PCTFLOAN']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7593, 37)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ADM_RATE</th>\n",
       "      <th>SATVRMID</th>\n",
       "      <th>SATMTMID</th>\n",
       "      <th>SATWRMID</th>\n",
       "      <th>ACTCMMID</th>\n",
       "      <th>ACTENMID</th>\n",
       "      <th>ACTMTMID</th>\n",
       "      <th>ACTWRMID</th>\n",
       "      <th>PCTFLOAN</th>\n",
       "      <th>GRAD_DEBT_MDN</th>\n",
       "      <th>...</th>\n",
       "      <th>AVGFACSAL</th>\n",
       "      <th>PFTFAC</th>\n",
       "      <th>UG</th>\n",
       "      <th>UG25ABV</th>\n",
       "      <th>PAR_ED_PCT_MS</th>\n",
       "      <th>PAR_ED_PCT_HS</th>\n",
       "      <th>PAR_ED_PCT_PS</th>\n",
       "      <th>AGE_ENTRY</th>\n",
       "      <th>FEMALE</th>\n",
       "      <th>D_PCTPELL_PCTFLOAN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.6538</td>\n",
       "      <td>427.0</td>\n",
       "      <td>420.0</td>\n",
       "      <td>414.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.8159</td>\n",
       "      <td>35000</td>\n",
       "      <td>...</td>\n",
       "      <td>7017.0</td>\n",
       "      <td>0.7096</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0877</td>\n",
       "      <td>0.0192989366</td>\n",
       "      <td>0.3694367861</td>\n",
       "      <td>0.6112642773</td>\n",
       "      <td>20.277133825</td>\n",
       "      <td>0.5472427116</td>\n",
       "      <td>4210.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.6043</td>\n",
       "      <td>575.0</td>\n",
       "      <td>594.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.5218</td>\n",
       "      <td>21500</td>\n",
       "      <td>...</td>\n",
       "      <td>10221.0</td>\n",
       "      <td>0.9081</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2363</td>\n",
       "      <td>0.0230535015</td>\n",
       "      <td>0.3334058286</td>\n",
       "      <td>0.6435406699</td>\n",
       "      <td>23.58415642</td>\n",
       "      <td>0.6369683532</td>\n",
       "      <td>11679.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.8781</td>\n",
       "      <td>23000</td>\n",
       "      <td>...</td>\n",
       "      <td>3217.0</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.8571</td>\n",
       "      <td>0.0619047619</td>\n",
       "      <td>0.4666666667</td>\n",
       "      <td>0.4714285714</td>\n",
       "      <td>33.065934066</td>\n",
       "      <td>0.6043956044</td>\n",
       "      <td>279.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.8120</td>\n",
       "      <td>585.0</td>\n",
       "      <td>615.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.4589</td>\n",
       "      <td>23500</td>\n",
       "      <td>...</td>\n",
       "      <td>9514.0</td>\n",
       "      <td>0.6173</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.2255</td>\n",
       "      <td>0.0275862069</td>\n",
       "      <td>0.2862068966</td>\n",
       "      <td>0.6862068966</td>\n",
       "      <td>23.152713891</td>\n",
       "      <td>0.4779208832</td>\n",
       "      <td>5618.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.4639</td>\n",
       "      <td>410.0</td>\n",
       "      <td>410.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>16.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.7692</td>\n",
       "      <td>32091</td>\n",
       "      <td>...</td>\n",
       "      <td>7940.0</td>\n",
       "      <td>0.6395</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0974</td>\n",
       "      <td>0.0159163256</td>\n",
       "      <td>0.3533424284</td>\n",
       "      <td>0.630741246</td>\n",
       "      <td>20.201285657</td>\n",
       "      <td>0.612294094</td>\n",
       "      <td>4805.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ADM_RATE  SATVRMID  SATMTMID  SATWRMID  ACTCMMID  ACTENMID  ACTMTMID  \\\n",
       "0    0.6538     427.0     420.0     414.0      18.0      17.0      17.0   \n",
       "1    0.6043     575.0     594.0       NaN      25.0      26.0      23.0   \n",
       "2       NaN       NaN       NaN       NaN       NaN       NaN       NaN   \n",
       "3    0.8120     585.0     615.0       NaN      27.0      28.0      27.0   \n",
       "4    0.4639     410.0     410.0       NaN      18.0      17.0      16.0   \n",
       "\n",
       "   ACTWRMID  PCTFLOAN GRAD_DEBT_MDN         ...         AVGFACSAL  PFTFAC  UG  \\\n",
       "0       NaN    0.8159         35000         ...            7017.0  0.7096 NaN   \n",
       "1       NaN    0.5218         21500         ...           10221.0  0.9081 NaN   \n",
       "2       NaN    0.8781         23000         ...            3217.0  1.0000 NaN   \n",
       "3       NaN    0.4589         23500         ...            9514.0  0.6173 NaN   \n",
       "4       NaN    0.7692         32091         ...            7940.0  0.6395 NaN   \n",
       "\n",
       "  UG25ABV  PAR_ED_PCT_MS  PAR_ED_PCT_HS  PAR_ED_PCT_PS     AGE_ENTRY  \\\n",
       "0  0.0877   0.0192989366   0.3694367861   0.6112642773  20.277133825   \n",
       "1  0.2363   0.0230535015   0.3334058286   0.6435406699   23.58415642   \n",
       "2  0.8571   0.0619047619   0.4666666667   0.4714285714  33.065934066   \n",
       "3  0.2255   0.0275862069   0.2862068966   0.6862068966  23.152713891   \n",
       "4  0.0974   0.0159163256   0.3533424284    0.630741246  20.201285657   \n",
       "\n",
       "         FEMALE  D_PCTPELL_PCTFLOAN  \n",
       "0  0.5472427116              4210.0  \n",
       "1  0.6369683532             11679.0  \n",
       "2  0.6043956044               279.0  \n",
       "3  0.4779208832              5618.0  \n",
       "4   0.612294094              4805.0  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ADM_RATE              5422\n",
       "SATVRMID              6400\n",
       "SATMTMID              6391\n",
       "SATWRMID              6860\n",
       "ACTCMMID              6359\n",
       "ACTENMID              6500\n",
       "ACTMTMID              6499\n",
       "ACTWRMID              7268\n",
       "PCTFLOAN               758\n",
       "GRAD_DEBT_MDN           29\n",
       "WDRAW_DEBT_MDN          29\n",
       "LO_INC_DEBT_MDN         29\n",
       "MD_INC_DEBT_MDN         29\n",
       "HI_INC_DEBT_MDN         29\n",
       "C150_4                5104\n",
       "TRANS_4               5104\n",
       "COSTT4_A              3673\n",
       "COSTT4_P              5093\n",
       "TUITIONFEE_IN         3290\n",
       "TUITIONFEE_OUT        3504\n",
       "MD_EARN_WNE_P6        7593\n",
       "CONTROL                  0\n",
       "REGION                   0\n",
       "LOCALE                 446\n",
       "LOCALE2               7593\n",
       "TUITFTE                460\n",
       "INEXPFTE               460\n",
       "AVGFACSAL             3139\n",
       "PFTFAC                3656\n",
       "UG                    7593\n",
       "UG25ABV                805\n",
       "PAR_ED_PCT_MS          218\n",
       "PAR_ED_PCT_HS          218\n",
       "PAR_ED_PCT_PS          218\n",
       "AGE_ENTRY              218\n",
       "FEMALE                 218\n",
       "D_PCTPELL_PCTFLOAN     758\n",
       "dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7593 entries, 0 to 7592\n",
      "Data columns (total 37 columns):\n",
      "ADM_RATE              2171 non-null float64\n",
      "SATVRMID              1193 non-null float64\n",
      "SATMTMID              1202 non-null float64\n",
      "SATWRMID              733 non-null float64\n",
      "ACTCMMID              1234 non-null float64\n",
      "ACTENMID              1093 non-null float64\n",
      "ACTMTMID              1094 non-null float64\n",
      "ACTWRMID              325 non-null float64\n",
      "PCTFLOAN              6835 non-null float64\n",
      "GRAD_DEBT_MDN         7564 non-null object\n",
      "WDRAW_DEBT_MDN        7564 non-null object\n",
      "LO_INC_DEBT_MDN       7564 non-null object\n",
      "MD_INC_DEBT_MDN       7564 non-null object\n",
      "HI_INC_DEBT_MDN       7564 non-null object\n",
      "C150_4                2489 non-null float64\n",
      "TRANS_4               2489 non-null float64\n",
      "COSTT4_A              3920 non-null float64\n",
      "COSTT4_P              2500 non-null float64\n",
      "TUITIONFEE_IN         4303 non-null float64\n",
      "TUITIONFEE_OUT        4089 non-null float64\n",
      "MD_EARN_WNE_P6        0 non-null float64\n",
      "CONTROL               7593 non-null int64\n",
      "REGION                7593 non-null int64\n",
      "LOCALE                7147 non-null float64\n",
      "LOCALE2               0 non-null float64\n",
      "TUITFTE               7133 non-null float64\n",
      "INEXPFTE              7133 non-null float64\n",
      "AVGFACSAL             4454 non-null float64\n",
      "PFTFAC                3937 non-null float64\n",
      "UG                    0 non-null float64\n",
      "UG25ABV               6788 non-null float64\n",
      "PAR_ED_PCT_MS         7375 non-null object\n",
      "PAR_ED_PCT_HS         7375 non-null object\n",
      "PAR_ED_PCT_PS         7375 non-null object\n",
      "AGE_ENTRY             7375 non-null object\n",
      "FEMALE                7375 non-null object\n",
      "D_PCTPELL_PCTFLOAN    6835 non-null float64\n",
      "dtypes: float64(25), int64(2), object(10)\n",
      "memory usage: 2.1+ MB\n"
     ]
    }
   ],
   "source": [
    "other.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### I suggest that we start with drop following columns, since they are 100% missing values:\n",
    "- MD_EARN_WNE_P6 (Median earnings of students working and not enrolled 6 years after entry)\n",
    "- LOCALE2 (Degree of urbanization of institution)\n",
    "- UG (Enrollment of all undergraduate students) - It says in the documentation that this is only available in 2000-2001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "other = other.drop(['MD_EARN_WNE_P6', 'LOCALE2', 'UG'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7593, 34)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 7593 entries, 0 to 7592\n",
      "Data columns (total 34 columns):\n",
      "ADM_RATE              2171 non-null float64\n",
      "SATVRMID              1193 non-null float64\n",
      "SATMTMID              1202 non-null float64\n",
      "SATWRMID              733 non-null float64\n",
      "ACTCMMID              1234 non-null float64\n",
      "ACTENMID              1093 non-null float64\n",
      "ACTMTMID              1094 non-null float64\n",
      "ACTWRMID              325 non-null float64\n",
      "PCTFLOAN              6835 non-null float64\n",
      "GRAD_DEBT_MDN         7564 non-null object\n",
      "WDRAW_DEBT_MDN        7564 non-null object\n",
      "LO_INC_DEBT_MDN       7564 non-null object\n",
      "MD_INC_DEBT_MDN       7564 non-null object\n",
      "HI_INC_DEBT_MDN       7564 non-null object\n",
      "C150_4                2489 non-null float64\n",
      "TRANS_4               2489 non-null float64\n",
      "COSTT4_A              3920 non-null float64\n",
      "COSTT4_P              2500 non-null float64\n",
      "TUITIONFEE_IN         4303 non-null float64\n",
      "TUITIONFEE_OUT        4089 non-null float64\n",
      "CONTROL               7593 non-null int64\n",
      "REGION                7593 non-null int64\n",
      "LOCALE                7147 non-null float64\n",
      "TUITFTE               7133 non-null float64\n",
      "INEXPFTE              7133 non-null float64\n",
      "AVGFACSAL             4454 non-null float64\n",
      "PFTFAC                3937 non-null float64\n",
      "UG25ABV               6788 non-null float64\n",
      "PAR_ED_PCT_MS         7375 non-null object\n",
      "PAR_ED_PCT_HS         7375 non-null object\n",
      "PAR_ED_PCT_PS         7375 non-null object\n",
      "AGE_ENTRY             7375 non-null object\n",
      "FEMALE                7375 non-null object\n",
      "D_PCTPELL_PCTFLOAN    6835 non-null float64\n",
      "dtypes: float64(22), int64(2), object(10)\n",
      "memory usage: 2.0+ MB\n"
     ]
    }
   ],
   "source": [
    "other.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FACTS ABOUT THE COLUMNS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**TEST SCORES**\n",
    "- ACTCMMID = Midpoint of the ACT cumulative score<BR /> \n",
    "(6359 NULL out of 7593 rows)\n",
    "- ACTENMID = Midpoint of the ACT English score<BR />\n",
    "(6500 NULL out of 7593 rows)\n",
    "- ACTMTMID = Midpoint of the ACT math score<BR /> \n",
    "(6490 NULL out of 7593 rows)\n",
    "- ACTWRMID = Midpoint of the ACT writing score<BR /> \n",
    "(7268 NULL out of 7593 rows)\n",
    "- SATMTMID = Midpoint of SAT scores at the institution (math)<BR /> \n",
    "(6391 NULL out of 7593 rows)\n",
    "- SATVRMID = Midpoint of SAT scores at the institution (critical reading)<BR /> \n",
    "(6400 NULL out of 7593 rows)\n",
    "- SATWRMID = Midpoint of SAT scores at the institution (writing)<BR /> \n",
    "(6860 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: I suggest that we do not use any of the columns since there are so much missing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**ADMISSION AND ENROLLMENT**\n",
    "- ADM_RATE = Admission rate<BR />\n",
    "(5422 NULL out of 7593 rows)\n",
    "- UG = Enrollment of all undergraduate students **(DROPPED)**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: One column I already dropepd due to 100% missing data. I suggest that we do not use the other column either since there are so much missing data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**AGE AND GENDER**\n",
    "- AGE_ENTRY = Average age of entry<BR />\n",
    "(218 NULL out of 7593 rows)\n",
    "- UG25ABV = Percentage of undergraduates aged 25 and above<BR />\n",
    "(805 NULL out of 7593 rows)\n",
    "- FEMALE = Share of female students<BR />\n",
    "(218 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: Some missing data, but we can work with this. We should go for mean/median/mode to replace. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**BACKGROUND**\n",
    "- PAR_ED_PCT_MS = Percent of students whose parents' highest educational level is middle school<BR />\n",
    "(218 NULL out of 7593 rows)\n",
    "- PAR_ED_PCT_HS = Percent of students whose parents' highest educational level is high school<BR />\n",
    "(218 NULL out of 7593 rows)\n",
    "- PAR_ED_PCT_PS = Percent of students whose parents' highest educational level was is some form of postsecondary education<BR />\n",
    "(218 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: Some missing data, but we can work with this. We should go for mean/median/mode to replace. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**TRANSFER**\n",
    "- TRANS_4 = Transfer rate for first-time, full-time students at four-year institutions (within 150% of expected time to completion/6 years)<BR />\n",
    "(5104 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: In my opinion too much missing values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**COMPLETION**\n",
    "- C150_4 = Completion rate for first-time, full-time students at four-year institutions (150% of expected time to completion) - **THIS IS OUR Y**<BR />\n",
    "(5104 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: If this is our Y we have a problem. We might want to re-think what we want to look at based on that data we have that is not superbad quality. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**COSTS AND FEES**\n",
    "- COSTT4_A = Average cost of attendance (academic year institutions)<BR />\n",
    "(3673 NULL out of 7593 rows)\n",
    "- COSTT4_P = Average cost of attendance (program-year institutions)<BR />\n",
    "(5093 NULL out of 7593 rows)\n",
    "- INEXPFTE = Instructional expenditures per full-time equivalent student<BR />\n",
    "(460 NULL out of 7593 rows)\n",
    "- TUITFTE = Net tuition revenue per full-time equivalent student<BR />\n",
    "(460 NULL out of 7593 rows)\n",
    "- TUITIONFEE_IN = In-state tuition and fees<BR />\n",
    "(3290 NULL out of 7593 rows)\n",
    "- TUITIONFEE_OUT = Out-of-state tuition and fees<BR />\n",
    "(3504 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: INEXPFTE and TUITFTE we can use, we have to talk about the rest and how to fill the missng values."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**STAFF**\n",
    "- AVGFACSAL = Average faculty salary<BR />\n",
    "(3139 NULL out of 7593 rows)\n",
    "- PFTFAC = Proportion of faculty that is full-time<BR />\n",
    "(3656 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: We need to talk about these as well"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**LOANS/DEBTS**\n",
    "- D_PCTPELL_PCTFLOAN = Number of undergraduate students (denominator percent receiving a pell grant or federal student loan)<BR />\n",
    "(758 NULL out of 7593 rows)\n",
    "- PCTFLOAN = Percent of all undergraduate students receiving a federal student loan<BR />\n",
    "(758 NULL out of 7593 rows)\n",
    "- GRAD_DEBT_MDN = The median debt for students who have completed<BR />\n",
    "(29 NULL out of 7593 rows)\n",
    "- WDRAW_DEBT_MDN = The median debt for students who have not completed<BR />\n",
    "(29 NULL out of 7593 rows)\n",
    "- LO_INC_DEBT_MDN = The median debt for students with family income between 0-30,000<BR />\n",
    "(29 NULL out of 7593 rows)\n",
    "- MD_INC_DEBT_MDN = The median debt for students with family income between 30,001-75,000<BR />\n",
    "(29 NULL out of 7593 rows)\n",
    "- HI_INC_DEBT_MDN = The median debt for students with family income 75,001+<BR />\n",
    "(29 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: Few missing values. Should we us mean/median/mode to replace?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**INCOME**\n",
    "- MD_EARN_WNE_P6 = Median earnings of students working and not enrolled 6 years after entry **(DROPPED)**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: Column dropped due to 100% missing values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**OTHER**\n",
    "- CONTROL = Control of institution *(I do not understad fromt he documentation what this is)*<BR />\n",
    "(0 NULL out of 7593 rows)\n",
    "- LOCALE = Locale of institution *(how populated is the area)*<BR />\n",
    "(446 NULL out of 7593 rows)\n",
    "- LOCALE2 = Degree of urbanization of institution **(DROPPED)**\n",
    "- REGION = Region (IPEDS)<BR />\n",
    "(0 NULL out of 7593 rows)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "COMMENT: We can use the columns that I did not drop. We have to discuss how to handle the missing values of LOCALE. I also need help to understand that CONTROL and LOCAL actually give us for info. We also need to find what number correspond to what kind of region (could not find in documentation)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([5, 3, 8, 6, 4, 7, 1, 2, 0, 9])"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "other['REGION'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7593, 1805)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "big.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['UNITID' 'OPEID' 'OPEID6' 'INSTNM' 'CITY' 'STABBR' 'ZIP' 'ACCREDAGENCY'\n",
      " 'INSTURL' 'NPCURL' 'HCM2' 'MAIN' 'NUMBRANCH' 'PREDDEG' 'HIGHDEG'\n",
      " 'CONTROL' 'ST_FIPS' 'REGION' 'LOCALE' 'LATITUDE' 'LONGITUDE' 'CCBASIC'\n",
      " 'CCUGPROF' 'CCSIZSET' 'HBCU' 'PBI' 'ANNHI' 'TRIBAL' 'AANAPII' 'HSI'\n",
      " 'NANTI' 'MENONLY' 'WOMENONLY' 'PCIP01' 'PCIP03' 'PCIP04' 'PCIP05'\n",
      " 'PCIP09' 'PCIP10' 'PCIP11' 'PCIP12' 'PCIP13' 'PCIP14' 'PCIP15' 'PCIP16'\n",
      " 'PCIP19' 'PCIP22' 'PCIP23' 'PCIP24' 'PCIP25' 'PCIP26' 'PCIP27' 'PCIP29'\n",
      " 'PCIP30' 'PCIP31' 'PCIP38' 'PCIP39' 'PCIP40' 'PCIP41' 'PCIP42' 'PCIP43'\n",
      " 'PCIP44' 'PCIP45' 'PCIP46' 'PCIP47' 'PCIP48' 'PCIP49' 'PCIP50' 'PCIP51'\n",
      " 'PCIP52' 'PCIP54' 'CIP01CERT1' 'CIP01CERT2' 'CIP01ASSOC' 'CIP01CERT4'\n",
      " 'CIP01BACHL' 'CIP03CERT1' 'CIP03CERT2' 'CIP03ASSOC' 'CIP03CERT4'\n",
      " 'CIP03BACHL' 'CIP04CERT1' 'CIP04CERT2' 'CIP04ASSOC' 'CIP04CERT4'\n",
      " 'CIP04BACHL' 'CIP05CERT1' 'CIP05CERT2' 'CIP05ASSOC' 'CIP05CERT4'\n",
      " 'CIP05BACHL' 'CIP09CERT1' 'CIP09CERT2' 'CIP09ASSOC' 'CIP09CERT4'\n",
      " 'CIP09BACHL' 'CIP10CERT1' 'CIP10CERT2' 'CIP10ASSOC' 'CIP10CERT4'\n",
      " 'CIP10BACHL' 'CIP11CERT1' 'CIP11CERT2' 'CIP11ASSOC' 'CIP11CERT4'\n",
      " 'CIP11BACHL' 'CIP12CERT1' 'CIP12CERT2' 'CIP12ASSOC' 'CIP12CERT4'\n",
      " 'CIP12BACHL' 'CIP13CERT1' 'CIP13CERT2' 'CIP13ASSOC' 'CIP13CERT4'\n",
      " 'CIP13BACHL' 'CIP14CERT1' 'CIP14CERT2' 'CIP14ASSOC' 'CIP14CERT4'\n",
      " 'CIP14BACHL' 'CIP15CERT1' 'CIP15CERT2' 'CIP15ASSOC' 'CIP15CERT4'\n",
      " 'CIP15BACHL' 'CIP16CERT1' 'CIP16CERT2' 'CIP16ASSOC' 'CIP16CERT4'\n",
      " 'CIP16BACHL' 'CIP19CERT1' 'CIP19CERT2' 'CIP19ASSOC' 'CIP19CERT4'\n",
      " 'CIP19BACHL' 'CIP22CERT1' 'CIP22CERT2' 'CIP22ASSOC' 'CIP22CERT4'\n",
      " 'CIP22BACHL' 'CIP23CERT1' 'CIP23CERT2' 'CIP23ASSOC' 'CIP23CERT4'\n",
      " 'CIP23BACHL' 'CIP24CERT1' 'CIP24CERT2' 'CIP24ASSOC' 'CIP24CERT4'\n",
      " 'CIP24BACHL' 'CIP25CERT1' 'CIP25CERT2' 'CIP25ASSOC' 'CIP25CERT4'\n",
      " 'CIP25BACHL' 'CIP26CERT1' 'CIP26CERT2' 'CIP26ASSOC' 'CIP26CERT4'\n",
      " 'CIP26BACHL' 'CIP27CERT1' 'CIP27CERT2' 'CIP27ASSOC' 'CIP27CERT4'\n",
      " 'CIP27BACHL' 'CIP29CERT1' 'CIP29CERT2' 'CIP29ASSOC' 'CIP29CERT4'\n",
      " 'CIP29BACHL' 'CIP30CERT1' 'CIP30CERT2' 'CIP30ASSOC' 'CIP30CERT4'\n",
      " 'CIP30BACHL' 'CIP31CERT1' 'CIP31CERT2' 'CIP31ASSOC' 'CIP31CERT4'\n",
      " 'CIP31BACHL' 'CIP38CERT1' 'CIP38CERT2' 'CIP38ASSOC' 'CIP38CERT4'\n",
      " 'CIP38BACHL' 'CIP39CERT1' 'CIP39CERT2' 'CIP39ASSOC' 'CIP39CERT4'\n",
      " 'CIP39BACHL' 'CIP40CERT1' 'CIP40CERT2' 'CIP40ASSOC' 'CIP40CERT4'\n",
      " 'CIP40BACHL' 'CIP41CERT1' 'CIP41CERT2' 'CIP41ASSOC' 'CIP41CERT4'\n",
      " 'CIP41BACHL' 'CIP42CERT1' 'CIP42CERT2' 'CIP42ASSOC' 'CIP42CERT4'\n",
      " 'CIP42BACHL' 'CIP43CERT1' 'CIP43CERT2' 'CIP43ASSOC' 'CIP43CERT4'\n",
      " 'CIP43BACHL' 'CIP44CERT1' 'CIP44CERT2' 'CIP44ASSOC' 'CIP44CERT4'\n",
      " 'CIP44BACHL' 'CIP45CERT1' 'CIP45CERT2' 'CIP45ASSOC' 'CIP45CERT4'\n",
      " 'CIP45BACHL' 'CIP46CERT1' 'CIP46CERT2' 'CIP46ASSOC' 'CIP46CERT4'\n",
      " 'CIP46BACHL' 'CIP47CERT1' 'CIP47CERT2' 'CIP47ASSOC' 'CIP47CERT4'\n",
      " 'CIP47BACHL' 'CIP48CERT1' 'CIP48CERT2' 'CIP48ASSOC' 'CIP48CERT4'\n",
      " 'CIP48BACHL' 'CIP49CERT1' 'CIP49CERT2' 'CIP49ASSOC' 'CIP49CERT4'\n",
      " 'CIP49BACHL' 'CIP50CERT1' 'CIP50CERT2' 'CIP50ASSOC' 'CIP50CERT4'\n",
      " 'CIP50BACHL' 'CIP51CERT1' 'CIP51CERT2' 'CIP51ASSOC' 'CIP51CERT4'\n",
      " 'CIP51BACHL' 'CIP52CERT1' 'CIP52CERT2' 'CIP52ASSOC' 'CIP52CERT4'\n",
      " 'CIP52BACHL' 'CIP54CERT1' 'CIP54CERT2' 'CIP54ASSOC' 'CIP54CERT4'\n",
      " 'CIP54BACHL' 'DISTANCEONLY' 'UGDS' 'UGDS_WHITE' 'UGDS_BLACK' 'UGDS_HISP'\n",
      " 'UGDS_ASIAN' 'UGDS_AIAN' 'UGDS_NHPI' 'UGDS_2MOR' 'UGDS_NRA' 'UGDS_UNKN'\n",
      " 'PPTUG_EF' 'CURROPER' 'TUITFTE' 'INEXPFTE' 'PCTPELL' 'PCTFLOAN' 'UG25ABV'\n",
      " 'RPY_3YR_RT' 'COMPL_RPY_3YR_RT' 'NONCOM_RPY_3YR_RT' 'LO_INC_RPY_3YR_RT'\n",
      " 'MD_INC_RPY_3YR_RT' 'HI_INC_RPY_3YR_RT' 'DEP_RPY_3YR_RT' 'IND_RPY_3YR_RT'\n",
      " 'PELL_RPY_3YR_RT' 'NOPELL_RPY_3YR_RT' 'FEMALE_RPY_3YR_RT'\n",
      " 'MALE_RPY_3YR_RT' 'FIRSTGEN_RPY_3YR_RT' 'NOTFIRSTGEN_RPY_3YR_RT'\n",
      " 'INC_PCT_LO' 'DEP_STAT_PCT_IND' 'DEP_INC_PCT_LO' 'IND_INC_PCT_LO'\n",
      " 'PAR_ED_PCT_1STGEN' 'INC_PCT_M1' 'INC_PCT_M2' 'INC_PCT_H1' 'INC_PCT_H2'\n",
      " 'DEP_INC_PCT_M1' 'DEP_INC_PCT_M2' 'DEP_INC_PCT_H1' 'DEP_INC_PCT_H2'\n",
      " 'IND_INC_PCT_M1' 'IND_INC_PCT_M2' 'IND_INC_PCT_H1' 'IND_INC_PCT_H2'\n",
      " 'PAR_ED_PCT_MS' 'PAR_ED_PCT_HS' 'PAR_ED_PCT_PS' 'APPL_SCH_PCT_GE2'\n",
      " 'APPL_SCH_PCT_GE3' 'APPL_SCH_PCT_GE4' 'APPL_SCH_PCT_GE5' 'DEP_INC_AVG'\n",
      " 'IND_INC_AVG' 'DEBT_MDN' 'GRAD_DEBT_MDN' 'WDRAW_DEBT_MDN'\n",
      " 'LO_INC_DEBT_MDN' 'MD_INC_DEBT_MDN' 'HI_INC_DEBT_MDN' 'DEP_DEBT_MDN'\n",
      " 'IND_DEBT_MDN' 'PELL_DEBT_MDN' 'NOPELL_DEBT_MDN' 'FEMALE_DEBT_MDN'\n",
      " 'MALE_DEBT_MDN' 'FIRSTGEN_DEBT_MDN' 'NOTFIRSTGEN_DEBT_MDN' 'DEBT_N'\n",
      " 'GRAD_DEBT_N' 'WDRAW_DEBT_N' 'LO_INC_DEBT_N' 'MD_INC_DEBT_N'\n",
      " 'HI_INC_DEBT_N' 'DEP_DEBT_N' 'IND_DEBT_N' 'PELL_DEBT_N' 'NOPELL_DEBT_N'\n",
      " 'FEMALE_DEBT_N' 'MALE_DEBT_N' 'FIRSTGEN_DEBT_N' 'NOTFIRSTGEN_DEBT_N'\n",
      " 'GRAD_DEBT_MDN10YR' 'CUML_DEBT_N' 'CUML_DEBT_P90' 'CUML_DEBT_P75'\n",
      " 'CUML_DEBT_P25' 'CUML_DEBT_P10' 'INC_N' 'DEP_INC_N' 'IND_INC_N'\n",
      " 'DEP_STAT_N' 'PAR_ED_N' 'APPL_SCH_N' 'RPY_3YR_N' 'COMPL_RPY_3YR_N'\n",
      " 'NONCOM_RPY_3YR_N' 'LO_INC_RPY_3YR_N' 'MD_INC_RPY_3YR_N'\n",
      " 'HI_INC_RPY_3YR_N' 'DEP_RPY_3YR_N' 'IND_RPY_3YR_N' 'PELL_RPY_3YR_N'\n",
      " 'NOPELL_RPY_3YR_N' 'FEMALE_RPY_3YR_N' 'MALE_RPY_3YR_N'\n",
      " 'FIRSTGEN_RPY_3YR_N' 'NOTFIRSTGEN_RPY_3YR_N' 'LOAN_EVER' 'PELL_EVER'\n",
      " 'AGE_ENTRY' 'FEMALE' 'MARRIED' 'DEPENDENT' 'VETERAN' 'FIRST_GEN' 'FAMINC'\n",
      " 'MD_FAMINC' 'FAMINC_IND' 'DEBT_MDN_SUPP' 'GRAD_DEBT_MDN_SUPP'\n",
      " 'GRAD_DEBT_MDN10YR_SUPP' 'RPY_3YR_RT_SUPP' 'LO_INC_RPY_3YR_RT_SUPP'\n",
      " 'MD_INC_RPY_3YR_RT_SUPP' 'HI_INC_RPY_3YR_RT_SUPP' 'COMPL_RPY_3YR_RT_SUPP'\n",
      " 'NONCOM_RPY_3YR_RT_SUPP' 'DEP_RPY_3YR_RT_SUPP' 'IND_RPY_3YR_RT_SUPP'\n",
      " 'PELL_RPY_3YR_RT_SUPP' 'NOPELL_RPY_3YR_RT_SUPP' 'FEMALE_RPY_3YR_RT_SUPP'\n",
      " 'MALE_RPY_3YR_RT_SUPP' 'FIRSTGEN_RPY_3YR_RT_SUPP'\n",
      " 'NOTFIRSTGEN_RPY_3YR_RT_SUPP' 'ICLEVEL' 'UGDS_MEN' 'UGDS_WOMEN'\n",
      " 'T4APPROVALDATE' 'D_PCTPELL_PCTFLOAN' 'OPENADMP' 'ACCREDCODE']\n"
     ]
    }
   ],
   "source": [
    "##Columns with fewer missing values\n",
    "df = big.dropna(thresh=len(big) - 1000, axis=1)\n",
    "print(df.columns.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7593, 408)"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}