70. Mental Illness Disparities in Vets.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "81927bb9",
   "metadata": {},
   "source": [
    "# Mental Illness Disparities in Vets\n",
    "\n",
    "## Importing required packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a480da7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from xgboost import XGBRegressor"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab8145e3",
   "metadata": {},
   "source": [
    "## Getting the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ae997c8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Vulnerable_population</th>\n",
       "      <th>Section</th>\n",
       "      <th>Long_title</th>\n",
       "      <th>Short_title</th>\n",
       "      <th>Group</th>\n",
       "      <th>Group1</th>\n",
       "      <th>Subgroup</th>\n",
       "      <th>Mental Illness</th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>Serious Mental Illness</td>\n",
       "      <td>A. Socio-demographic Characteristics</td>\n",
       "      <td>A1. Distribution of SMI among FY13 Veteran VHA...</td>\n",
       "      <td>SMI (Overall)</td>\n",
       "      <td>Overall</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Overall</td>\n",
       "      <td>Mood.Anxiety</td>\n",
       "      <td>1149541.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Serious Mental Illness</td>\n",
       "      <td>A. Socio-demographic Characteristics</td>\n",
       "      <td>A2. Distribution of SMI among FY13 Veteran VHA...</td>\n",
       "      <td>SMI by Sex</td>\n",
       "      <td>Sex</td>\n",
       "      <td>NaN</td>\n",
       "      <td>F</td>\n",
       "      <td>Mood.Anxiety</td>\n",
       "      <td>125993.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Serious Mental Illness</td>\n",
       "      <td>A. Socio-demographic Characteristics</td>\n",
       "      <td>A2. Distribution of SMI among FY13 Veteran VHA...</td>\n",
       "      <td>SMI by Sex</td>\n",
       "      <td>Sex</td>\n",
       "      <td>NaN</td>\n",
       "      <td>M</td>\n",
       "      <td>Mood.Anxiety</td>\n",
       "      <td>1023546.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Serious Mental Illness</td>\n",
       "      <td>A. Socio-demographic Characteristics</td>\n",
       "      <td>A3. Distribution of SMI among FY13 Veteran VHA...</td>\n",
       "      <td>SMI by Age</td>\n",
       "      <td>Age</td>\n",
       "      <td>NaN</td>\n",
       "      <td>18-44</td>\n",
       "      <td>Mood.Anxiety</td>\n",
       "      <td>266131.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Serious Mental Illness</td>\n",
       "      <td>A. Socio-demographic Characteristics</td>\n",
       "      <td>A3. Distribution of SMI among FY13 Veteran VHA...</td>\n",
       "      <td>SMI by Age</td>\n",
       "      <td>Age</td>\n",
       "      <td>NaN</td>\n",
       "      <td>45-64</td>\n",
       "      <td>Mood.Anxiety</td>\n",
       "      <td>529733.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index   Vulnerable_population  ... Mental Illness      Value\n",
       "0      0  Serious Mental Illness  ...   Mood.Anxiety  1149541.0\n",
       "1      1  Serious Mental Illness  ...   Mood.Anxiety   125993.0\n",
       "2      2  Serious Mental Illness  ...   Mood.Anxiety  1023546.0\n",
       "3      3  Serious Mental Illness  ...   Mood.Anxiety   266131.0\n",
       "4      4  Serious Mental Illness  ...   Mood.Anxiety   529733.0\n",
       "\n",
       "[5 rows x 10 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"data/proj70/comma-separated-values-file-1.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "52d2b8e0",
   "metadata": {},
   "source": [
    "## Cleaning dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b2ce9e92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index                      0\n",
       "Vulnerable_population      0\n",
       "Section                    0\n",
       "Long_title                 0\n",
       "Short_title                0\n",
       "Group                      0\n",
       "Group1                   228\n",
       "Subgroup                   0\n",
       "Mental Illness             0\n",
       "Value                     61\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c3a7d324",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "index                      0\n",
       "Vulnerable_population      0\n",
       "Section                    0\n",
       "Long_title                 0\n",
       "Short_title                0\n",
       "Group                      0\n",
       "Group1                   228\n",
       "Subgroup                   0\n",
       "Mental Illness             0\n",
       "Value                      0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[\"Value\"].fillna(np.mean(df[\"Value\"]), inplace=True)\n",
    "df.isna().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55252256",
   "metadata": {},
   "source": [
    "## Pre-Processing data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "92b479b9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1656 entries, 0 to 1655\n",
      "Data columns (total 10 columns):\n",
      " #   Column                 Non-Null Count  Dtype  \n",
      "---  ------                 --------------  -----  \n",
      " 0   index                  1656 non-null   int64  \n",
      " 1   Vulnerable_population  1656 non-null   object \n",
      " 2   Section                1656 non-null   object \n",
      " 3   Long_title             1656 non-null   object \n",
      " 4   Short_title            1656 non-null   object \n",
      " 5   Group                  1656 non-null   object \n",
      " 6   Group1                 1428 non-null   object \n",
      " 7   Subgroup               1656 non-null   object \n",
      " 8   Mental Illness         1656 non-null   object \n",
      " 9   Value                  1656 non-null   float64\n",
      "dtypes: float64(1), int64(1), object(8)\n",
      "memory usage: 129.5+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a3388084",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------\n",
      "Vulnerable_population\n",
      "['Serious Mental Illness']\n",
      "--------------------\n",
      "--------------------\n",
      "Section\n",
      "['A. Socio-demographic Characteristics' 'B. VHA Outpatient Utilization'\n",
      " 'C. Diagnoses']\n",
      "--------------------\n",
      "--------------------\n",
      "Long_title\n",
      "['A1. Distribution of SMI among FY13 Veteran VHA Patients'\n",
      " 'A2. Distribution of SMI among FY13 Veteran VHA Patients by Sex, FY13'\n",
      " 'A3. Distribution of SMI among FY13 Veteran VHA Patients by Age, FY13'\n",
      " 'A4. Distribution of SMI among FY13 Veteran VHA Patients by Race/Ethnicity, FY13'\n",
      " 'A5. Distribution of SMI among FY13 Veteran VHA Patients by Rural/Urban Status, FY13'\n",
      " 'A6. Distribution of SMI among FY13 Veteran VHA Patients by Service-Connected Status, FY13'\n",
      " 'B1. Distribution of SMI among FY13 Veteran VHA Patients by VHA Outpatient Visits, FY13'\n",
      " 'B2. Distribution of SMI among FY13 Veteran VHA Patients by Primary Care Visits, FY13'\n",
      " 'B3. Distribution of SMI among FY13 Veteran VHA Patients by Mental Health/Substance Use Disorder Visits, FY13'\n",
      " 'B4. Distribution of SMI among FY13 Veteran VHA Patients by Emergency Department Visits, FY13'\n",
      " 'B5. Distribution of SMI among FY13 Veteran VHA Patients by Telephone Visits, FY13'\n",
      " 'B6. Distribution of SMI among FY13 Veteran VHA Patients by Fee Outpatient Visits, FY13'\n",
      " 'C1. Distribution of SMI among FY13 Veteran VHA Patients by Domain, FY13'\n",
      " 'C2. Distribution of SMI among FY13 Veteran VHA Patients by Condition, FY13']\n",
      "--------------------\n",
      "--------------------\n",
      "Short_title\n",
      "['SMI (Overall)' 'SMI by Age' 'SMI by Conditions' 'SMI by Domains'\n",
      " 'SMI by Emergency Department Visits' 'SMI by Fee Outpatient Services'\n",
      " 'SMI by Mental Health/Substance Use Disorder Visits'\n",
      " 'SMI by Primary Care Visits' 'SMI by Race/Ethnicity'\n",
      " 'SMI by Rural/Urban Status' 'SMI by Service-Connected Status'\n",
      " 'SMI by Sex' 'SMI by Telephone Visits' 'SMI by VHA Outpatient Visits']\n",
      "--------------------\n",
      "--------------------\n",
      "Group\n",
      "['Age' 'Conditions' 'Domains' 'Encounters' 'Geography' 'Overall'\n",
      " 'Race/Ethnicity' 'Service Connection' 'Sex']\n",
      "--------------------\n",
      "--------------------\n",
      "Group1\n",
      "['BREAST' 'CANCER' 'CARDIOVASCULAR' 'DENTAL' 'DERMATOLOGIC'\n",
      " 'ENDOCRINE/METABOLIC/NUTRITIONAL' 'Emergency Department Visits'\n",
      " 'Fee Outpatient Services' 'GASTROINTESTINAL' 'HEMATOLOGIC/IMMUNOLOGIC'\n",
      " 'INFECTIOUS DISEASE' 'MENTAL HEALTH/SUBSTANCE USE DISORDER'\n",
      " 'MUSCULOSKELETAL' 'Mental Health/Substance use Disorder Visits'\n",
      " 'NEUROLOGIC' 'OTHER' 'Outpatient Visits' 'Primary Care Visits'\n",
      " 'REPRODUCTIVE HEALTH' 'RESPIRATORY' 'SENSE ORGANS' 'Telephone Visits'\n",
      " 'URINARY' nan]\n",
      "--------------------\n",
      "--------------------\n",
      "Subgroup\n",
      "['0 encounters' '1 encounter' '12+ encounters' '18-44' '2 encounters'\n",
      " '3-5 encounters' '45-64' '6-11 encounters' '65+' 'AI/AN' 'Abdominal pain'\n",
      " 'Acute MI' 'Acute stress disorders' 'Adjustment disorders'\n",
      " 'Alcohol use disorders' 'Allergic and other chronic sinusitis/rhinitis'\n",
      " 'Allergies and urticaria' 'Amputation' 'Anemia'\n",
      " 'Anxiety disorders/ other' 'Arrhythmia/Conduction Disorder - Other'\n",
      " 'Asian' 'Asthma' 'Atrial Fibrillation/Flutter'\n",
      " 'Attention Deficit Disorder/Hyperkinetic Disorder' 'BREAST'\n",
      " 'Benign Gynecologic Neoplasms - Other' 'Biliary tract disease'\n",
      " 'Bipolar disorders' 'Black/African American' 'Blindness/Low Vision'\n",
      " 'Breast Conditions/Benign or Unknown'\n",
      " 'Breast conditions abnormal radiologic findings' 'CANCER'\n",
      " 'CARDIOVASCULAR' 'Calculus of urinary tract' 'Cancer - Anal'\n",
      " 'Cancer - Bladder' 'Cancer - Bone/Connective Tissue'\n",
      " 'Cancer - Brain/Nervous System' 'Cancer - Breast'\n",
      " 'Cancer - Bronchopulmonary' 'Cancer - Cervical' 'Cancer - Colorectal'\n",
      " 'Cancer - Esophagus' 'Cancer - Female Reproductive - Other'\n",
      " 'Cancer - Gastric' 'Cancer - Head and Neck' 'Cancer - Hepatobiliary'\n",
      " 'Cancer - Other and Unspecified Primary' 'Cancer - Ovarian'\n",
      " 'Cancer - Pancreatic' 'Cancer - Prostate' 'Cancer - Renal'\n",
      " 'Cancer - Testicular' 'Cancer - Thyroid' 'Cancer - Uterine'\n",
      " 'Carcinoma in Situ - Breast, Ductal or Lobular'\n",
      " 'Carcinoma in Situ - Cervical'\n",
      " 'Carcinoma in Situ - Female Reproductive - Other'\n",
      " 'Carcinoma in Situ - Other' 'Cardiac Conditions - Other'\n",
      " 'Carpal Tunnel Syndrome' 'Cataract' 'Cerebrovascular Accident/TIA'\n",
      " 'Cerebrovascular Disease - Other' 'Cervical Dysplasia/ASCUS'\n",
      " 'Chest Pain/Angina' 'Chronic Obstructive Pulmonary Disease'\n",
      " 'Chronic Pain Syndromes' 'Circulatory System Conditions - Other'\n",
      " 'Coagulation and hemorrhagic disorders' 'Cognitive Disorders - Other'\n",
      " 'Conduct/impulse control disorders' 'Connective Tissue Disease'\n",
      " 'Contraceptive Care Management' 'Coronary Artery Disease' 'DENTAL'\n",
      " 'DERMATOLOGIC' 'Dementia' 'Dental Caries' 'Dental Disorders - Other'\n",
      " 'Depression, possible - Other' 'Dermatologic disorders - Other'\n",
      " 'Diabetes Mellitus'\n",
      " 'Diarrhea/constipation and functional bowel disorders'\n",
      " 'Dissociative disorders' 'Diverticulosis and diverticulitis'\n",
      " 'Dizziness/vertigo' 'Drug use disorders'\n",
      " 'Dyspnea, cough and other respiratory symptoms'\n",
      " 'ENDOCRINE/METABOLIC/NUTRITIONAL' 'Ear disorders - Other'\n",
      " 'Eating disorders' 'Ectopic pregnancy' 'Edema'\n",
      " 'Effects of surgical procedures or medical care'\n",
      " 'Endocrine, Metabolic and Nutritional Disorders - Other' 'Endometriosis'\n",
      " 'Epilepsy- convulsions' 'Esophageal disorders' 'Eye disorders - Other'\n",
      " 'F' 'Female Reproductive Organ Disorders - Other' 'Fibroids'\n",
      " 'Fluid and Electrolyte Disorders' 'Foot deformities' 'Fracture - Other'\n",
      " 'Fracture - hip' 'GASTROINTESTINAL' 'Gastric/Duodenal Disorders - Other'\n",
      " 'Gastroduodenal ulcer' 'Gastrointestinal System Disorders - Other'\n",
      " 'Gastrointestinal hemorrhage' 'Gingivitis/Periodontitis' 'Glaucoma'\n",
      " 'Gout/Crystal Arthropathies' 'HEMATOLOGIC/IMMUNOLOGIC' 'HIV/AIDS'\n",
      " 'Headache' 'Hearing problems' 'Heart Failure'\n",
      " 'Hematologic/Immunologic Conditions - Other' 'Hematuria' 'Hemorrhoids'\n",
      " 'Hepatitis C' 'Hernia' 'Highly Rural' 'Hispanic' 'Housing insufficiency'\n",
      " 'Hypertension' 'INFECTIOUS DISEASE' 'Infections - Other' 'Infertility'\n",
      " 'Inflammatory Bowel Disease' 'Inflammatory Spondyloarthropathies'\n",
      " 'Intracranial Hemorrhage' 'Joint Disorders - Lower Extremity'\n",
      " 'Joint Disorders - Unspecified or Multiple Joints'\n",
      " 'Joint Disorders - Upper Extremity' 'Kidney/Ureter Diseases - Other'\n",
      " 'Leukemias' 'Lipid Disorders' 'Liver disease - Other' 'Loss of Teeth'\n",
      " 'Lymphomas' 'M' 'MENTAL HEALTH/SUD' 'MUSCULOSKELETAL'\n",
      " 'Major Depressive Disorder' 'Malaise and fatigue'\n",
      " 'Male genital disorders' 'Melanoma' 'Menopausal disorders'\n",
      " 'Menstrual disorders' 'Miscarriage' 'Multi-race' 'Multiple myeloma'\n",
      " 'Multiple sclerosis' 'Musculoskeletal conditions - Other'\n",
      " 'Myalgia/Myositis, unspecified' 'Myasthenia Gravis/Myoneuronal Disorders'\n",
      " 'Mycoses' 'NEUROLOGIC' 'NH/OPI' 'Nausea and vomiting'\n",
      " 'Nervous System Symptoms/Disorders - Other' 'No SC' 'OTHER'\n",
      " 'Osteomyelitis/Infectious Arthritis' 'Osteoporosis' 'Other Rural'\n",
      " 'Other injuries and conditions due to external causes'\n",
      " 'Ovarian cyst at least once, FY13' 'Overall' 'Overweight/Obesity' 'PTSD'\n",
      " 'Pancreatic disorders' 'Paralysis - Other' 'Parkinsons disease'\n",
      " 'Pecondition Aortic Aneurysm' 'Peripheral Nerve Disorders'\n",
      " 'Personality disorders' 'Pneumonia' 'Poisoning' 'Polycystic ovaries'\n",
      " 'Polymyalgia Rheumatica' 'Polyp, colorectal'\n",
      " 'Pregnancy Complicated by Other Medical Conditions'\n",
      " 'Pregnancy complicated by DM' 'Pregnancy complicated by HTN'\n",
      " 'Pregnancy or Delivery - Normal'\n",
      " 'Pregnancy with Obstetrical Complications or Prolonged'\n",
      " 'Prolapse of female genital organs' 'Psoriasis'\n",
      " 'Psychiatric disorders, nonspecific' 'Psychosocial factors - Other'\n",
      " 'Psychotic disorders - Other'\n",
      " 'Pulmonary Embolism or Deep Vein Thrombosis' 'REPRODUCTIVE HEALTH'\n",
      " 'RESPIRATORY' 'Refraction disorders' 'Renal failure or Nephropathy'\n",
      " 'Residual Codes' 'Respiratory Conditions - Other'\n",
      " 'Respiratory System Infections - Other'\n",
      " 'Rheumatoid Arthritis and Related Disease' 'SC 0-49' 'SC 100' 'SC 50-99'\n",
      " 'SENSE ORGAN' 'STIs' 'Sarcoidosis' 'Schizophrenia' 'Sexual Dysfunction'\n",
      " 'Skin cancer (non-Melanoma)' 'Skin infection' 'Skin ulcer - chronic'\n",
      " 'Sleep apnea' 'Sleep disturbance - Other' 'Somatiform disorders'\n",
      " 'Spinal cord injury' 'Spine Disorders - Cervical'\n",
      " 'Spine Disorders - Lumbosacral' 'Spine Disorders - Other/Unspecified'\n",
      " 'Symptoms - Other' 'Syncope' 'TBI' 'Thrombocytopenia' 'Thyroid Disorders'\n",
      " 'Tobacco Use Disorder' 'Tobacco Use History' 'Tuberculosis' 'URINARY'\n",
      " 'Unknown' 'Urban' 'Urinary Conditions - Other' 'Urinary Incontinence'\n",
      " 'Urinary Symptoms - Other'\n",
      " 'Urinary Tract Infection (cystitis/ urethritis/ pyelonephritis)'\n",
      " 'Vaginitis/ cervicitis/ and other pelvic inflammatory conditions'\n",
      " 'Valvular Disease' 'Vascular Disease - Other' 'Vasculitis'\n",
      " 'Vitamin D Deficiency' 'White']\n",
      "--------------------\n",
      "--------------------\n",
      "Mental Illness\n",
      "['Mood.Anxiety' 'No.Mental.Health' 'Other.Mental.Health' 'PTSD'\n",
      " 'Serious.Mental.Illness' 'Substance.Abuse']\n",
      "--------------------\n"
     ]
    }
   ],
   "source": [
    "for column in df.keys():\n",
    "    if pd.api.types.is_object_dtype(df[column]):\n",
    "        le = LabelEncoder()\n",
    "        print(\"-\"*20)\n",
    "        print(column)\n",
    "        df[column] = le.fit_transform(df[column])\n",
    "        print(le.classes_)\n",
    "        print(\"-\"*20)\n",
    "    else:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "70d4e99f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>Vulnerable_population</th>\n",
       "      <th>Section</th>\n",
       "      <th>Long_title</th>\n",
       "      <th>Short_title</th>\n",
       "      <th>Group</th>\n",
       "      <th>Group1</th>\n",
       "      <th>Subgroup</th>\n",
       "      <th>Mental Illness</th>\n",
       "      <th>Value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>23</td>\n",
       "      <td>174</td>\n",
       "      <td>0</td>\n",
       "      <td>1149541.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>23</td>\n",
       "      <td>102</td>\n",
       "      <td>0</td>\n",
       "      <td>125993.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>23</td>\n",
       "      <td>146</td>\n",
       "      <td>0</td>\n",
       "      <td>1023546.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>266131.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>23</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>529733.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  Vulnerable_population  Section  ...  Subgroup  Mental Illness      Value\n",
       "0      0                      0        0  ...       174               0  1149541.0\n",
       "1      1                      0        0  ...       102               0   125993.0\n",
       "2      2                      0        0  ...       146               0  1023546.0\n",
       "3      3                      0        0  ...         3               0   266131.0\n",
       "4      4                      0        0  ...         6               0   529733.0\n",
       "\n",
       "[5 rows x 10 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1f2b0486",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1656 entries, 0 to 1655\n",
      "Data columns (total 10 columns):\n",
      " #   Column                 Non-Null Count  Dtype  \n",
      "---  ------                 --------------  -----  \n",
      " 0   index                  1656 non-null   int64  \n",
      " 1   Vulnerable_population  1656 non-null   int64  \n",
      " 2   Section                1656 non-null   int64  \n",
      " 3   Long_title             1656 non-null   int64  \n",
      " 4   Short_title            1656 non-null   int64  \n",
      " 5   Group                  1656 non-null   int64  \n",
      " 6   Group1                 1656 non-null   int64  \n",
      " 7   Subgroup               1656 non-null   int64  \n",
      " 8   Mental Illness         1656 non-null   int64  \n",
      " 9   Value                  1656 non-null   float64\n",
      "dtypes: float64(1), int64(9)\n",
      "memory usage: 129.5 KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "79d181cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.drop(\"index\", inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cde0bdaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop(\"Value\", axis=1)\n",
    "y = df[\"Value\"]\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fff84cc4",
   "metadata": {},
   "source": [
    "## Training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "8de0dfaa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6038819364385503"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rreg = RandomForestRegressor()\n",
    "rreg.fit(X_train, y_train)\n",
    "rreg.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "49a288c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.580323411967071"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xreg = XGBRegressor().fit(X_train.values, y_train.values)\n",
    "xreg.score(X_test.values, y_test.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f85050c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}