Pump_It_Up_Best_Submission.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pump It Up - Best Submission",
      "provenance": [],
      "mount_file_id": "1k2-aZu-Ud-Ru0vb9o0NyQ0WBdACVaHaO",
      "authorship_tag": "ABX9TyNqaGdJeRYzKAJXsHWGl/uH",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Yoshani/Pump-It-Up/blob/master/Pump_It_Up_Best_Submission.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "euKgWFwQJZN8"
      },
      "source": [
        "!pip install catboost\n",
        "!pip install category_encoders"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G_fZp6pPfIOW"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from sklearn import preprocessing\n",
        "import datetime as dt\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import metrics\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "\n",
        "# classifiers\n",
        "from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier, StackingClassifier, VotingClassifier, GradientBoostingClassifier\n",
        "from xgboost.sklearn import XGBClassifier\n",
        "from sklearn.naive_bayes import GaussianNB\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.neural_network import MLPClassifier\n",
        "import lightgbm as lgb\n",
        "from sklearn.linear_model import SGDClassifier\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from catboost import CatBoostClassifier\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W6JFdBsQDWkc"
      },
      "source": [
        "# Pre processing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r0qjcDzpe8Bt"
      },
      "source": [
        "features_df = pd.read_csv(\n",
        "    \"/content/drive/MyDrive/PumpItUp/TrainingSet.csv\", \n",
        "    index_col=\"id\",\n",
        "    low_memory=False\n",
        ")\n",
        "labels_df = pd.read_csv(\n",
        "    \"/content/drive/MyDrive/PumpItUp/TrainingLabels.csv\", \n",
        "    index_col=\"id\"\n",
        ")\n",
        "test_df = pd.read_csv(\n",
        "    \"/content/drive/MyDrive/PumpItUp/TestSet.csv\", \n",
        "    index_col=\"id\",\n",
        "    low_memory=False\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 438
        },
        "id": "GTluLiJVDVCc",
        "outputId": "9a090a88-d309-47bc-a7f7-ca7082289919"
      },
      "source": [
        "features_df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>amount_tsh</th>\n",
              "      <th>date_recorded</th>\n",
              "      <th>funder</th>\n",
              "      <th>gps_height</th>\n",
              "      <th>installer</th>\n",
              "      <th>longitude</th>\n",
              "      <th>latitude</th>\n",
              "      <th>wpt_name</th>\n",
              "      <th>num_private</th>\n",
              "      <th>basin</th>\n",
              "      <th>subvillage</th>\n",
              "      <th>region</th>\n",
              "      <th>region_code</th>\n",
              "      <th>district_code</th>\n",
              "      <th>lga</th>\n",
              "      <th>ward</th>\n",
              "      <th>population</th>\n",
              "      <th>public_meeting</th>\n",
              "      <th>recorded_by</th>\n",
              "      <th>scheme_management</th>\n",
              "      <th>scheme_name</th>\n",
              "      <th>permit</th>\n",
              "      <th>construction_year</th>\n",
              "      <th>extraction_type</th>\n",
              "      <th>extraction_type_group</th>\n",
              "      <th>extraction_type_class</th>\n",
              "      <th>management</th>\n",
              "      <th>management_group</th>\n",
              "      <th>payment</th>\n",
              "      <th>payment_type</th>\n",
              "      <th>water_quality</th>\n",
              "      <th>quality_group</th>\n",
              "      <th>quantity</th>\n",
              "      <th>quantity_group</th>\n",
              "      <th>source</th>\n",
              "      <th>source_type</th>\n",
              "      <th>source_class</th>\n",
              "      <th>waterpoint_type</th>\n",
              "      <th>waterpoint_type_group</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>69572</th>\n",
              "      <td>6000.0</td>\n",
              "      <td>2011-03-14</td>\n",
              "      <td>Roman</td>\n",
              "      <td>1390</td>\n",
              "      <td>Roman</td>\n",
              "      <td>34.938093</td>\n",
              "      <td>-9.856322</td>\n",
              "      <td>none</td>\n",
              "      <td>0</td>\n",
              "      <td>Lake Nyasa</td>\n",
              "      <td>Mnyusi B</td>\n",
              "      <td>Iringa</td>\n",
              "      <td>11</td>\n",
              "      <td>5</td>\n",
              "      <td>Ludewa</td>\n",
              "      <td>Mundindi</td>\n",
              "      <td>109</td>\n",
              "      <td>True</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>VWC</td>\n",
              "      <td>Roman</td>\n",
              "      <td>False</td>\n",
              "      <td>1999</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>pay annually</td>\n",
              "      <td>annually</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>enough</td>\n",
              "      <td>enough</td>\n",
              "      <td>spring</td>\n",
              "      <td>spring</td>\n",
              "      <td>groundwater</td>\n",
              "      <td>communal standpipe</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8776</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2013-03-06</td>\n",
              "      <td>Grumeti</td>\n",
              "      <td>1399</td>\n",
              "      <td>GRUMETI</td>\n",
              "      <td>34.698766</td>\n",
              "      <td>-2.147466</td>\n",
              "      <td>Zahanati</td>\n",
              "      <td>0</td>\n",
              "      <td>Lake Victoria</td>\n",
              "      <td>Nyamara</td>\n",
              "      <td>Mara</td>\n",
              "      <td>20</td>\n",
              "      <td>2</td>\n",
              "      <td>Serengeti</td>\n",
              "      <td>Natta</td>\n",
              "      <td>280</td>\n",
              "      <td>NaN</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>Other</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>2010</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>wug</td>\n",
              "      <td>user-group</td>\n",
              "      <td>never pay</td>\n",
              "      <td>never pay</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>insufficient</td>\n",
              "      <td>insufficient</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34310</th>\n",
              "      <td>25.0</td>\n",
              "      <td>2013-02-25</td>\n",
              "      <td>Lottery Club</td>\n",
              "      <td>686</td>\n",
              "      <td>World vision</td>\n",
              "      <td>37.460664</td>\n",
              "      <td>-3.821329</td>\n",
              "      <td>Kwa Mahundi</td>\n",
              "      <td>0</td>\n",
              "      <td>Pangani</td>\n",
              "      <td>Majengo</td>\n",
              "      <td>Manyara</td>\n",
              "      <td>21</td>\n",
              "      <td>4</td>\n",
              "      <td>Simanjiro</td>\n",
              "      <td>Ngorika</td>\n",
              "      <td>250</td>\n",
              "      <td>True</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>VWC</td>\n",
              "      <td>Nyumba ya mungu pipe scheme</td>\n",
              "      <td>True</td>\n",
              "      <td>2009</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>pay per bucket</td>\n",
              "      <td>per bucket</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>enough</td>\n",
              "      <td>enough</td>\n",
              "      <td>dam</td>\n",
              "      <td>dam</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe multiple</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67743</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2013-01-28</td>\n",
              "      <td>Unicef</td>\n",
              "      <td>263</td>\n",
              "      <td>UNICEF</td>\n",
              "      <td>38.486161</td>\n",
              "      <td>-11.155298</td>\n",
              "      <td>Zahanati Ya Nanyumbu</td>\n",
              "      <td>0</td>\n",
              "      <td>Ruvuma / Southern Coast</td>\n",
              "      <td>Mahakamani</td>\n",
              "      <td>Mtwara</td>\n",
              "      <td>90</td>\n",
              "      <td>63</td>\n",
              "      <td>Nanyumbu</td>\n",
              "      <td>Nanyumbu</td>\n",
              "      <td>58</td>\n",
              "      <td>True</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>VWC</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>1986</td>\n",
              "      <td>submersible</td>\n",
              "      <td>submersible</td>\n",
              "      <td>submersible</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>never pay</td>\n",
              "      <td>never pay</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>dry</td>\n",
              "      <td>dry</td>\n",
              "      <td>machine dbh</td>\n",
              "      <td>borehole</td>\n",
              "      <td>groundwater</td>\n",
              "      <td>communal standpipe multiple</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19728</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2011-07-13</td>\n",
              "      <td>Action In A</td>\n",
              "      <td>0</td>\n",
              "      <td>Artisan</td>\n",
              "      <td>31.130847</td>\n",
              "      <td>-1.825359</td>\n",
              "      <td>Shuleni</td>\n",
              "      <td>0</td>\n",
              "      <td>Lake Victoria</td>\n",
              "      <td>Kyanyamisa</td>\n",
              "      <td>Kagera</td>\n",
              "      <td>18</td>\n",
              "      <td>1</td>\n",
              "      <td>Karagwe</td>\n",
              "      <td>Nyakasimbi</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "      <td>GeoData Consultants Ltd</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>gravity</td>\n",
              "      <td>other</td>\n",
              "      <td>other</td>\n",
              "      <td>never pay</td>\n",
              "      <td>never pay</td>\n",
              "      <td>soft</td>\n",
              "      <td>good</td>\n",
              "      <td>seasonal</td>\n",
              "      <td>seasonal</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       amount_tsh  ... waterpoint_type_group\n",
              "id                 ...                      \n",
              "69572      6000.0  ...    communal standpipe\n",
              "8776          0.0  ...    communal standpipe\n",
              "34310        25.0  ...    communal standpipe\n",
              "67743         0.0  ...    communal standpipe\n",
              "19728         0.0  ...    communal standpipe\n",
              "\n",
              "[5 rows x 39 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 91
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LLqkGqgFqnlF"
      },
      "source": [
        "X = features_df.copy()\n",
        "y = labels_df.copy()\n",
        "X_test = test_df.copy()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mhi_Lz1g3pNB"
      },
      "source": [
        "# Combine train data with test data\n",
        "df=X.append(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MAqGIAFYp7uW"
      },
      "source": [
        "# Drop some columns\n",
        "df.drop(['recorded_by', 'num_private', 'wpt_name', 'extraction_type_group', 'extraction_type', 'payment_type', 'water_quality', 'scheme_management',\n",
        "         'district_code', 'region','region_code', 'subvillage', 'ward', 'waterpoint_type_group', 'quantity_group', 'installer', 'scheme_name'], axis=1, inplace=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        },
        "id": "0tMG9Ve1GYBs",
        "outputId": "ae364d8f-6e60-4537-d878-ecbf448dbe46"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>amount_tsh</th>\n",
              "      <th>date_recorded</th>\n",
              "      <th>funder</th>\n",
              "      <th>gps_height</th>\n",
              "      <th>longitude</th>\n",
              "      <th>latitude</th>\n",
              "      <th>basin</th>\n",
              "      <th>lga</th>\n",
              "      <th>population</th>\n",
              "      <th>public_meeting</th>\n",
              "      <th>permit</th>\n",
              "      <th>construction_year</th>\n",
              "      <th>extraction_type_class</th>\n",
              "      <th>management</th>\n",
              "      <th>management_group</th>\n",
              "      <th>payment</th>\n",
              "      <th>quality_group</th>\n",
              "      <th>quantity</th>\n",
              "      <th>source</th>\n",
              "      <th>source_type</th>\n",
              "      <th>source_class</th>\n",
              "      <th>waterpoint_type</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>id</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>69572</th>\n",
              "      <td>6000.0</td>\n",
              "      <td>2011-03-14</td>\n",
              "      <td>Roman</td>\n",
              "      <td>1390</td>\n",
              "      <td>34.938093</td>\n",
              "      <td>-9.856322</td>\n",
              "      <td>Lake Nyasa</td>\n",
              "      <td>Ludewa</td>\n",
              "      <td>109</td>\n",
              "      <td>True</td>\n",
              "      <td>False</td>\n",
              "      <td>1999</td>\n",
              "      <td>gravity</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>pay annually</td>\n",
              "      <td>good</td>\n",
              "      <td>enough</td>\n",
              "      <td>spring</td>\n",
              "      <td>spring</td>\n",
              "      <td>groundwater</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8776</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2013-03-06</td>\n",
              "      <td>Grumeti</td>\n",
              "      <td>1399</td>\n",
              "      <td>34.698766</td>\n",
              "      <td>-2.147466</td>\n",
              "      <td>Lake Victoria</td>\n",
              "      <td>Serengeti</td>\n",
              "      <td>280</td>\n",
              "      <td>NaN</td>\n",
              "      <td>True</td>\n",
              "      <td>2010</td>\n",
              "      <td>gravity</td>\n",
              "      <td>wug</td>\n",
              "      <td>user-group</td>\n",
              "      <td>never pay</td>\n",
              "      <td>good</td>\n",
              "      <td>insufficient</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34310</th>\n",
              "      <td>25.0</td>\n",
              "      <td>2013-02-25</td>\n",
              "      <td>Lottery Club</td>\n",
              "      <td>686</td>\n",
              "      <td>37.460664</td>\n",
              "      <td>-3.821329</td>\n",
              "      <td>Pangani</td>\n",
              "      <td>Simanjiro</td>\n",
              "      <td>250</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>2009</td>\n",
              "      <td>gravity</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>pay per bucket</td>\n",
              "      <td>good</td>\n",
              "      <td>enough</td>\n",
              "      <td>dam</td>\n",
              "      <td>dam</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe multiple</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>67743</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2013-01-28</td>\n",
              "      <td>Unicef</td>\n",
              "      <td>263</td>\n",
              "      <td>38.486161</td>\n",
              "      <td>-11.155298</td>\n",
              "      <td>Ruvuma / Southern Coast</td>\n",
              "      <td>Nanyumbu</td>\n",
              "      <td>58</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>1986</td>\n",
              "      <td>submersible</td>\n",
              "      <td>vwc</td>\n",
              "      <td>user-group</td>\n",
              "      <td>never pay</td>\n",
              "      <td>good</td>\n",
              "      <td>dry</td>\n",
              "      <td>machine dbh</td>\n",
              "      <td>borehole</td>\n",
              "      <td>groundwater</td>\n",
              "      <td>communal standpipe multiple</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19728</th>\n",
              "      <td>0.0</td>\n",
              "      <td>2011-07-13</td>\n",
              "      <td>Action In A</td>\n",
              "      <td>0</td>\n",
              "      <td>31.130847</td>\n",
              "      <td>-1.825359</td>\n",
              "      <td>Lake Victoria</td>\n",
              "      <td>Karagwe</td>\n",
              "      <td>0</td>\n",
              "      <td>True</td>\n",
              "      <td>True</td>\n",
              "      <td>0</td>\n",
              "      <td>gravity</td>\n",
              "      <td>other</td>\n",
              "      <td>other</td>\n",
              "      <td>never pay</td>\n",
              "      <td>good</td>\n",
              "      <td>seasonal</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>rainwater harvesting</td>\n",
              "      <td>surface</td>\n",
              "      <td>communal standpipe</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       amount_tsh date_recorded  ... source_class              waterpoint_type\n",
              "id                               ...                                          \n",
              "69572      6000.0    2011-03-14  ...  groundwater           communal standpipe\n",
              "8776          0.0    2013-03-06  ...      surface           communal standpipe\n",
              "34310        25.0    2013-02-25  ...      surface  communal standpipe multiple\n",
              "67743         0.0    2013-01-28  ...  groundwater  communal standpipe multiple\n",
              "19728         0.0    2011-07-13  ...      surface           communal standpipe\n",
              "\n",
              "[5 rows x 22 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 96
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v-BW01lWOj_D"
      },
      "source": [
        "# construction_year\n",
        "med = df.loc[df['construction_year'] > 0,'construction_year'].median()\n",
        "df.loc[df['construction_year'] == 0,'construction_year'] = med\n",
        "\n",
        "min_year = 1960\n",
        "df['construction_year'] = df['construction_year'] - min_year\n",
        "\n",
        "# gps_height\n",
        "med = df.loc[df['gps_height'] > 0,'gps_height'].median()\n",
        "df.loc[df['gps_height'] == 0,'gps_height'] = med\n",
        "\n",
        "# funder\n",
        "df['funder'] = df['funder'].fillna(\"Unknown\")\n",
        "\n",
        "# other\n",
        "df = df.fillna(df.median())\n",
        "\n",
        "# label encode\n",
        "for colname in df.select_dtypes([\"object\", \"category\"]):\n",
        "        df[colname], _ = df[colname].factorize()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qHK6T3K14lU3",
        "outputId": "7cdb242e-8909-43bc-9b38-f66a6c4b1a05"
      },
      "source": [
        "# Number of missing values in each column\n",
        "missing_val_count_by_column = (df.isnull().sum())\n",
        "print(missing_val_count_by_column[missing_val_count_by_column > 0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Series([], dtype: int64)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YCVDy-i7whhA"
      },
      "source": [
        "Separate training and test set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "20LkdJiFwj70"
      },
      "source": [
        "X = df[:len(features_df)]\n",
        "X_test = df[len(features_df):]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VIYVn75G9AIz"
      },
      "source": [
        "X_train, X_valid, y_train, y_valid = train_test_split(X, y.status_group, test_size=0.1, random_state=10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "l5bpz_l69Uxg"
      },
      "source": [
        "# Train model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jfTtPwLtNUCH"
      },
      "source": [
        "Define models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7McobnNrAOOA"
      },
      "source": [
        "modelRFC = RandomForestClassifier(n_estimators=1300, min_samples_split=10, criterion='gini')\n",
        "\n",
        "modelETC = ExtraTreesClassifier(n_estimators=1000, min_samples_split=10)\n",
        "\n",
        "modelXGB = XGBClassifier(objective = 'multi:softmax', \n",
        "                      num_class = 3, maximize = False, eval_metric = 'merror', eta = 0.1,\n",
        "                      max_depth = 14, colsample_bytree = 0.4, num_boost_round=1000, nfold=4, early_stopping_rounds=10)\n",
        "\n",
        "modelGB = GradientBoostingClassifier(random_state=8, n_estimators=250)\n",
        "\n",
        "modelLGB = lgb.LGBMClassifier(learning_rate=0.03, n_estimators=478, max_depth=5, min_samples_split=100, min_samples_leaf=100, random_state=27, gamma=0.1,\n",
        "                                                       subsample=0.8, reg_alpha = 0.001, reg_lambda=10, objective = 'binary', boosting_type='goss')\n",
        "\n",
        "modelSVC = SVC(gamma='auto', kernel='rbf')\n",
        "\n",
        "modelCAT = CatBoostClassifier(silent=True, max_ctr_complexity=5, task_type='CPU', iterations=10000,\n",
        "        eval_metric='AUC', od_type='Iter', od_wait=500, loss_function = 'MultiClass', random_seed=42)\n",
        "\n",
        "modelMLP = MLPClassifier(hidden_layer_sizes=(200,100,), learning_rate='invscaling', learning_rate_init=0.01,\n",
        "                          power_t=0.5, max_iter=1000, shuffle=True, tol=1e-4, activation='logistic', \n",
        "                          early_stopping=True, validation_fraction=0.1, n_iter_no_change=12, max_fun=15000,\n",
        "                         random_state=8, solver='adam',batch_size=1024)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bktl17K6rGz1"
      },
      "source": [
        "Single scores"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "odOmT2XO9XhE",
        "outputId": "b11e6226-849c-45cc-ce7e-5cfce5e338ff"
      },
      "source": [
        "modelRFC.fit(X_train, y_train)\n",
        "scoreRFC = modelRFC.score(X_valid,y_valid)\n",
        "scoreRFC"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8223905723905723"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JOaAmZxgIWvH",
        "outputId": "29ff4fcf-3ce8-466a-858d-4cafb763e3f9"
      },
      "source": [
        "modelETC.fit(X_train, y_train)\n",
        "scoreETC = modelETC.score(X_valid,y_valid)\n",
        "scoreETC"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8171717171717172"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fsIB-bapnq2Y",
        "outputId": "fc4bab38-8f09-4011-9cbe-b4e43e5e9c0c"
      },
      "source": [
        "modelXGB.fit(X_train, y_train)\n",
        "scoreXGB = modelXGB.score(X_valid,y_valid)\n",
        "scoreXGB"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8235690235690236"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Ywn1TNqiEPR",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a578100f-41a6-4b1b-9f3e-8f856fc42c2b"
      },
      "source": [
        "modelCAT.fit(X_train, y_train)\n",
        "scoreCAT = modelCAT.score(X_valid,y_valid)\n",
        "scoreCAT"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8028619528619528"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jKg-fJXGrKeK"
      },
      "source": [
        "Ensembling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BU3y-5BcrP7F"
      },
      "source": [
        "estimators_stacked = [\n",
        "              ('xgb', modelXGB),\n",
        "              ('rn', modelRFC)              \n",
        "             ]\n",
        "\n",
        "# Stacking Classifier\n",
        "stacked_estimator =  StackingClassifier(estimators=estimators_stacked,\n",
        "                                    final_estimator=LogisticRegression()  \n",
        "                                  )\n",
        "# Voting Classifier\n",
        "voting_estimator = VotingClassifier(estimators=estimators_stacked, \n",
        "                        voting='hard')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TOjQpaf6rcjJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e0e3c14-b2be-43ef-f001-eccb491ae7a7"
      },
      "source": [
        "stacked_estimator.fit(X_train, y_train)\n",
        "scoreStack = stacked_estimator.score(X_valid,y_valid)\n",
        "scoreStack"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8260942760942761"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dkz5ePnVHtRh",
        "outputId": "e6ca4334-cadc-45ae-e8aa-3ddcf995253f"
      },
      "source": [
        "voting_estimator.fit(X_train, y_train)\n",
        "scoreVoting = voting_estimator.score(X_valid,y_valid)\n",
        "scoreVoting"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.8217171717171717"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EgNdUPvclpeh"
      },
      "source": [
        "# Train on entire dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HmU19tsaltBs"
      },
      "source": [
        "model=stacked_estimator\n",
        "X_train=X\n",
        "y_train = y.status_group\n",
        "\n",
        "model.fit(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CdeLxc1XwxJ_"
      },
      "source": [
        "# Predict"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SEefkErrwyqT"
      },
      "source": [
        "predict=model.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1-RTAKVp-iSS"
      },
      "source": [
        "# Submission"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FN8eZOuQBA0P"
      },
      "source": [
        "submission_df = pd.read_csv(\"/content/drive/MyDrive/PumpItUp/SubmissionFormat.csv\", \n",
        "                            index_col=\"id\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JS2Zm91_BQay"
      },
      "source": [
        "# Make sure we have the rows in the same order\n",
        "np.testing.assert_array_equal(test_df.index.values, \n",
        "                              submission_df.index.values)\n",
        "\n",
        "# Save predictions to submission data frame\n",
        "submission_df[\"status_group\"] = predict"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k7FGyKBz-26A",
        "outputId": "5731281d-e5cf-4a5b-a444-95c0ac8ead03"
      },
      "source": [
        "submission_df['status_group'].unique()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array(['functional', 'non functional', 'functional needs repair'],\n",
              "      dtype=object)"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pC6OLQR1-5-w"
      },
      "source": [
        "submission_df.to_csv('my_submission.csv', index=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 17
        },
        "id": "kDviNFhA6Y1_",
        "outputId": "470a468d-8114-4f67-b082-feb50193e8b0"
      },
      "source": [
        "from google.colab import files\n",
        "files.download('my_submission.csv')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "    async function download(id, filename, size) {\n",
              "      if (!google.colab.kernel.accessAllowed) {\n",
              "        return;\n",
              "      }\n",
              "      const div = document.createElement('div');\n",
              "      const label = document.createElement('label');\n",
              "      label.textContent = `Downloading \"${filename}\": `;\n",
              "      div.appendChild(label);\n",
              "      const progress = document.createElement('progress');\n",
              "      progress.max = size;\n",
              "      div.appendChild(progress);\n",
              "      document.body.appendChild(div);\n",
              "\n",
              "      const buffers = [];\n",
              "      let downloaded = 0;\n",
              "\n",
              "      const channel = await google.colab.kernel.comms.open(id);\n",
              "      // Send a message to notify the kernel that we're ready.\n",
              "      channel.send({})\n",
              "\n",
              "      for await (const message of channel.messages) {\n",
              "        // Send a message to notify the kernel that we're ready.\n",
              "        channel.send({})\n",
              "        if (message.buffers) {\n",
              "          for (const buffer of message.buffers) {\n",
              "            buffers.push(buffer);\n",
              "            downloaded += buffer.byteLength;\n",
              "            progress.value = downloaded;\n",
              "          }\n",
              "        }\n",
              "      }\n",
              "      const blob = new Blob(buffers, {type: 'application/binary'});\n",
              "      const a = document.createElement('a');\n",
              "      a.href = window.URL.createObjectURL(blob);\n",
              "      a.download = filename;\n",
              "      div.appendChild(a);\n",
              "      a.click();\n",
              "      div.remove();\n",
              "    }\n",
              "  "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "download(\"download_b5782585-ed49-4b47-a6e3-dcdf7e2bd9e5\", \"my_submission.csv\", 278339)"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}