ML Pipeline Preparation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ML Pipeline Preparation\n",
    "Follow the instructions below to help you create your ML pipeline.\n",
    "### 1. Import libraries and load data from database.\n",
    "- Import Python libraries\n",
    "- Load dataset from database with [`read_sql_table`](https://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_sql_table.html)\n",
    "- Define feature and target variables X and Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\frank_tubbing\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\frank_tubbing\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\frank_tubbing\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# import libraries\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# Setting up connection to sqlite\n",
    "import sqlite3\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.multioutput import MultiOutputClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "\n",
    "import re\n",
    "\n",
    "nltk.download(['punkt', 'wordnet', 'stopwords'])\n",
    "\n",
    "# Make sure we can see all columns in our dataframes\n",
    "pd.set_option('display.max_columns', 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load data from database\n",
    "conn = sqlite3.connect('data/DisasterResponse.db')\n",
    "df = pd.read_sql('SELECT * FROM messages', conn)\n",
    "# I'm not gonna use the original message in the first place, id column has no added value. Might be that genre is usefull.\n",
    "X = df[['message']]\n",
    "# The data needs to be 1 dimensional for countvectorizer and tfidftransformer\n",
    "X = X.message.tolist()\n",
    "# Get all values that we are trying to predict\n",
    "y = df.drop(columns=['id', 'message', 'original', 'genre'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>related</th>\n",
       "      <th>request</th>\n",
       "      <th>offer</th>\n",
       "      <th>aid_related</th>\n",
       "      <th>medical_help</th>\n",
       "      <th>medical_products</th>\n",
       "      <th>search_and_rescue</th>\n",
       "      <th>security</th>\n",
       "      <th>military</th>\n",
       "      <th>child_alone</th>\n",
       "      <th>water</th>\n",
       "      <th>food</th>\n",
       "      <th>shelter</th>\n",
       "      <th>clothing</th>\n",
       "      <th>money</th>\n",
       "      <th>missing_people</th>\n",
       "      <th>refugees</th>\n",
       "      <th>death</th>\n",
       "      <th>other_aid</th>\n",
       "      <th>infrastructure_related</th>\n",
       "      <th>transport</th>\n",
       "      <th>buildings</th>\n",
       "      <th>electricity</th>\n",
       "      <th>tools</th>\n",
       "      <th>hospitals</th>\n",
       "      <th>shops</th>\n",
       "      <th>aid_centers</th>\n",
       "      <th>other_infrastructure</th>\n",
       "      <th>weather_related</th>\n",
       "      <th>floods</th>\n",
       "      <th>storm</th>\n",
       "      <th>fire</th>\n",
       "      <th>earthquake</th>\n",
       "      <th>cold</th>\n",
       "      <th>other_weather</th>\n",
       "      <th>direct_report</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.0</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.00000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "      <td>26180.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.773415</td>\n",
       "      <td>0.170474</td>\n",
       "      <td>0.004507</td>\n",
       "      <td>0.414095</td>\n",
       "      <td>0.079488</td>\n",
       "      <td>0.050115</td>\n",
       "      <td>0.027617</td>\n",
       "      <td>0.017991</td>\n",
       "      <td>0.032773</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.063751</td>\n",
       "      <td>0.111421</td>\n",
       "      <td>0.088197</td>\n",
       "      <td>0.015432</td>\n",
       "      <td>0.023033</td>\n",
       "      <td>0.011383</td>\n",
       "      <td>0.033384</td>\n",
       "      <td>0.045531</td>\n",
       "      <td>0.13136</td>\n",
       "      <td>0.064973</td>\n",
       "      <td>0.045760</td>\n",
       "      <td>0.050764</td>\n",
       "      <td>0.020321</td>\n",
       "      <td>0.006073</td>\n",
       "      <td>0.010810</td>\n",
       "      <td>0.004584</td>\n",
       "      <td>0.011803</td>\n",
       "      <td>0.043812</td>\n",
       "      <td>0.278189</td>\n",
       "      <td>0.082047</td>\n",
       "      <td>0.093201</td>\n",
       "      <td>0.010772</td>\n",
       "      <td>0.093659</td>\n",
       "      <td>0.020206</td>\n",
       "      <td>0.052445</td>\n",
       "      <td>0.193506</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.435359</td>\n",
       "      <td>0.376056</td>\n",
       "      <td>0.066986</td>\n",
       "      <td>0.492574</td>\n",
       "      <td>0.270504</td>\n",
       "      <td>0.218186</td>\n",
       "      <td>0.163875</td>\n",
       "      <td>0.132920</td>\n",
       "      <td>0.178046</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.244313</td>\n",
       "      <td>0.314659</td>\n",
       "      <td>0.283587</td>\n",
       "      <td>0.123264</td>\n",
       "      <td>0.150011</td>\n",
       "      <td>0.106083</td>\n",
       "      <td>0.179641</td>\n",
       "      <td>0.208470</td>\n",
       "      <td>0.33780</td>\n",
       "      <td>0.246483</td>\n",
       "      <td>0.208968</td>\n",
       "      <td>0.219519</td>\n",
       "      <td>0.141098</td>\n",
       "      <td>0.077696</td>\n",
       "      <td>0.103409</td>\n",
       "      <td>0.067549</td>\n",
       "      <td>0.108000</td>\n",
       "      <td>0.204681</td>\n",
       "      <td>0.448116</td>\n",
       "      <td>0.274442</td>\n",
       "      <td>0.290719</td>\n",
       "      <td>0.103228</td>\n",
       "      <td>0.291360</td>\n",
       "      <td>0.140708</td>\n",
       "      <td>0.222926</td>\n",
       "      <td>0.395054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.00000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            related       request         offer   aid_related  medical_help  \\\n",
       "count  26180.000000  26180.000000  26180.000000  26180.000000  26180.000000   \n",
       "mean       0.773415      0.170474      0.004507      0.414095      0.079488   \n",
       "std        0.435359      0.376056      0.066986      0.492574      0.270504   \n",
       "min        0.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "25%        1.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "50%        1.000000      0.000000      0.000000      0.000000      0.000000   \n",
       "75%        1.000000      0.000000      0.000000      1.000000      0.000000   \n",
       "max        2.000000      1.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "       medical_products  search_and_rescue      security      military  \\\n",
       "count      26180.000000       26180.000000  26180.000000  26180.000000   \n",
       "mean           0.050115           0.027617      0.017991      0.032773   \n",
       "std            0.218186           0.163875      0.132920      0.178046   \n",
       "min            0.000000           0.000000      0.000000      0.000000   \n",
       "25%            0.000000           0.000000      0.000000      0.000000   \n",
       "50%            0.000000           0.000000      0.000000      0.000000   \n",
       "75%            0.000000           0.000000      0.000000      0.000000   \n",
       "max            1.000000           1.000000      1.000000      1.000000   \n",
       "\n",
       "       child_alone         water          food       shelter      clothing  \\\n",
       "count      26180.0  26180.000000  26180.000000  26180.000000  26180.000000   \n",
       "mean           0.0      0.063751      0.111421      0.088197      0.015432   \n",
       "std            0.0      0.244313      0.314659      0.283587      0.123264   \n",
       "min            0.0      0.000000      0.000000      0.000000      0.000000   \n",
       "25%            0.0      0.000000      0.000000      0.000000      0.000000   \n",
       "50%            0.0      0.000000      0.000000      0.000000      0.000000   \n",
       "75%            0.0      0.000000      0.000000      0.000000      0.000000   \n",
       "max            0.0      1.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "              money  missing_people      refugees         death    other_aid  \\\n",
       "count  26180.000000    26180.000000  26180.000000  26180.000000  26180.00000   \n",
       "mean       0.023033        0.011383      0.033384      0.045531      0.13136   \n",
       "std        0.150011        0.106083      0.179641      0.208470      0.33780   \n",
       "min        0.000000        0.000000      0.000000      0.000000      0.00000   \n",
       "25%        0.000000        0.000000      0.000000      0.000000      0.00000   \n",
       "50%        0.000000        0.000000      0.000000      0.000000      0.00000   \n",
       "75%        0.000000        0.000000      0.000000      0.000000      0.00000   \n",
       "max        1.000000        1.000000      1.000000      1.000000      1.00000   \n",
       "\n",
       "       infrastructure_related     transport     buildings   electricity  \\\n",
       "count            26180.000000  26180.000000  26180.000000  26180.000000   \n",
       "mean                 0.064973      0.045760      0.050764      0.020321   \n",
       "std                  0.246483      0.208968      0.219519      0.141098   \n",
       "min                  0.000000      0.000000      0.000000      0.000000   \n",
       "25%                  0.000000      0.000000      0.000000      0.000000   \n",
       "50%                  0.000000      0.000000      0.000000      0.000000   \n",
       "75%                  0.000000      0.000000      0.000000      0.000000   \n",
       "max                  1.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "              tools     hospitals         shops   aid_centers  \\\n",
       "count  26180.000000  26180.000000  26180.000000  26180.000000   \n",
       "mean       0.006073      0.010810      0.004584      0.011803   \n",
       "std        0.077696      0.103409      0.067549      0.108000   \n",
       "min        0.000000      0.000000      0.000000      0.000000   \n",
       "25%        0.000000      0.000000      0.000000      0.000000   \n",
       "50%        0.000000      0.000000      0.000000      0.000000   \n",
       "75%        0.000000      0.000000      0.000000      0.000000   \n",
       "max        1.000000      1.000000      1.000000      1.000000   \n",
       "\n",
       "       other_infrastructure  weather_related        floods         storm  \\\n",
       "count          26180.000000     26180.000000  26180.000000  26180.000000   \n",
       "mean               0.043812         0.278189      0.082047      0.093201   \n",
       "std                0.204681         0.448116      0.274442      0.290719   \n",
       "min                0.000000         0.000000      0.000000      0.000000   \n",
       "25%                0.000000         0.000000      0.000000      0.000000   \n",
       "50%                0.000000         0.000000      0.000000      0.000000   \n",
       "75%                0.000000         1.000000      0.000000      0.000000   \n",
       "max                1.000000         1.000000      1.000000      1.000000   \n",
       "\n",
       "               fire    earthquake          cold  other_weather  direct_report  \n",
       "count  26180.000000  26180.000000  26180.000000   26180.000000   26180.000000  \n",
       "mean       0.010772      0.093659      0.020206       0.052445       0.193506  \n",
       "std        0.103228      0.291360      0.140708       0.222926       0.395054  \n",
       "min        0.000000      0.000000      0.000000       0.000000       0.000000  \n",
       "25%        0.000000      0.000000      0.000000       0.000000       0.000000  \n",
       "50%        0.000000      0.000000      0.000000       0.000000       0.000000  \n",
       "75%        0.000000      0.000000      0.000000       0.000000       0.000000  \n",
       "max        1.000000      1.000000      1.000000       1.000000       1.000000  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Inspect all target columns\n",
    "y.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Turns out that column child_alone has no value. So for this dataset it makes sense to remove this.\n",
    "y.drop(columns=['child_alone'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Write a tokenization function to process your text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    # First remove punctation and lowercase all letters\n",
    "    text = re.sub(r\"[^a-zA-Z0-9]\", \" \", text.lower())\n",
    "    \n",
    "    # tokenize text\n",
    "    tokens = word_tokenize(text)\n",
    "    \n",
    "    stop_words = stopwords.words(\"english\")\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    stemmer = PorterStemmer()\n",
    "    \n",
    "    # lemmatize, stem and remove stop words\n",
    "    tokens = [stemmer.stem(lemmatizer.lemmatize(word)) for word in tokens if word not in stop_words]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Build a machine learning pipeline\n",
    "- You'll find the [MultiOutputClassifier](http://scikit-learn.org/stable/modules/generated/sklearn.multioutput.MultiOutputClassifier.html) helpful for predicting multiple target variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "        ('vect', CountVectorizer(tokenizer=tokenize, max_df=0.75)),\n",
    "        ('tfidf', TfidfTransformer(sublinear_tf=False)),\n",
    "        ('clf', MultiOutputClassifier(RandomForestClassifier(n_estimators=500, min_samples_leaf=1)))\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Train pipeline\n",
    "- Split data into train and test sets\n",
    "- Train pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "\n",
    "# predict on test data\n",
    "y_pred = pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 5. Test your model\n",
    "Report the f1 score, precision and recall for each output category of the dataset. You can do this by iterating through the columns and calling sklearn's `classification_report` on each."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.74      0.39      0.51      1532\n",
      "           1       0.83      0.95      0.89      4963\n",
      "           2       0.40      0.42      0.41        50\n",
      "\n",
      "   micro avg       0.82      0.82      0.82      6545\n",
      "   macro avg       0.65      0.59      0.60      6545\n",
      "weighted avg       0.81      0.82      0.80      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[ 601  922    9]\n",
      " [ 212 4728   23]\n",
      " [   4   25   21]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8174\n",
      "\n",
      "\n",
      "request\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.98      0.94      5436\n",
      "           1       0.85      0.53      0.65      1109\n",
      "\n",
      "   micro avg       0.90      0.90      0.90      6545\n",
      "   macro avg       0.88      0.75      0.80      6545\n",
      "weighted avg       0.90      0.90      0.89      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5331  105]\n",
      " [ 523  586]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9040\n",
      "\n",
      "\n",
      "offer\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6512\n",
      "           1       0.00      0.00      0.00        33\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6512    0]\n",
      " [  33    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9950\n",
      "\n",
      "\n",
      "aid_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.85      0.83      3847\n",
      "           1       0.77      0.72      0.74      2698\n",
      "\n",
      "   micro avg       0.80      0.80      0.80      6545\n",
      "   macro avg       0.79      0.78      0.79      6545\n",
      "weighted avg       0.79      0.80      0.79      6545\n",
      "\n",
      "Confusion Matrix:\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [[3275  572]\n",
      " [ 768 1930]]\n",
      "\n",
      "\n",
      "Accuracy: 0.7953\n",
      "\n",
      "\n",
      "medical_help\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      1.00      0.96      6038\n",
      "           1       0.61      0.05      0.09       507\n",
      "\n",
      "   micro avg       0.92      0.92      0.92      6545\n",
      "   macro avg       0.77      0.52      0.53      6545\n",
      "weighted avg       0.90      0.92      0.89      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6022   16]\n",
      " [ 482   25]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9239\n",
      "\n",
      "\n",
      "medical_products\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6226\n",
      "           1       0.76      0.07      0.13       319\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.86      0.53      0.55      6545\n",
      "weighted avg       0.94      0.95      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6219    7]\n",
      " [ 297   22]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9536\n",
      "\n",
      "\n",
      "search_and_rescue\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.99      6371\n",
      "           1       0.58      0.04      0.08       174\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.78      0.52      0.53      6545\n",
      "weighted avg       0.96      0.97      0.96      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6366    5]\n",
      " [ 167    7]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9737\n",
      "\n",
      "\n",
      "security\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6434\n",
      "           1       0.20      0.01      0.02       111\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.59      0.50      0.50      6545\n",
      "weighted avg       0.97      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6430    4]\n",
      " [ 110    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9826\n",
      "\n",
      "\n",
      "military\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6300\n",
      "           1       0.89      0.03      0.06       245\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.93      0.52      0.52      6545\n",
      "weighted avg       0.96      0.96      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6299    1]\n",
      " [ 237    8]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9636\n",
      "\n",
      "\n",
      "water\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6125\n",
      "           1       0.87      0.37      0.52       420\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.91      0.68      0.75      6545\n",
      "weighted avg       0.95      0.96      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6102   23]\n",
      " [ 266  154]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9558\n",
      "\n",
      "\n",
      "food\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.97      5825\n",
      "           1       0.86      0.68      0.76       720\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.91      0.83      0.87      6545\n",
      "weighted avg       0.95      0.95      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5745   80]\n",
      " [ 233  487]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9522\n",
      "\n",
      "\n",
      "shelter\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      5968\n",
      "           1       0.83      0.41      0.55       577\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.89      0.70      0.76      6545\n",
      "weighted avg       0.94      0.94      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5919   49]\n",
      " [ 340  237]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9406\n",
      "\n",
      "\n",
      "clothing\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6446\n",
      "           1       0.80      0.08      0.15        99\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.89      0.54      0.57      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6444    2]\n",
      " [  91    8]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9858\n",
      "\n",
      "\n",
      "money\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6402\n",
      "           1       1.00      0.03      0.07       143\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.99      0.52      0.53      6545\n",
      "weighted avg       0.98      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6402    0]\n",
      " [ 138    5]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9789\n",
      "\n",
      "\n",
      "missing_people\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6463\n",
      "           1       0.00      0.00      0.00        82\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6463    0]\n",
      " [  82    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9875\n",
      "\n",
      "\n",
      "refugees\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6340\n",
      "           1       0.56      0.02      0.05       205\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.76      0.51      0.52      6545\n",
      "weighted avg       0.96      0.97      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6336    4]\n",
      " [ 200    5]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9688\n",
      "\n",
      "\n",
      "death\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6218\n",
      "           1       0.91      0.10      0.18       327\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.93      0.55      0.58      6545\n",
      "weighted avg       0.95      0.95      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6215    3]\n",
      " [ 295   32]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9545\n",
      "\n",
      "\n",
      "other_aid\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      1.00      0.93      5694\n",
      "           1       0.62      0.04      0.08       851\n",
      "\n",
      "   micro avg       0.87      0.87      0.87      6545\n",
      "   macro avg       0.75      0.52      0.50      6545\n",
      "weighted avg       0.84      0.87      0.82      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5673   21]\n",
      " [ 817   34]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8720\n",
      "\n",
      "\n",
      "infrastructure_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6132\n",
      "           1       0.50      0.00      0.00       413\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.72      0.50      0.49      6545\n",
      "weighted avg       0.91      0.94      0.91      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6131    1]\n",
      " [ 412    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9369\n",
      "\n",
      "\n",
      "transport\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6269\n",
      "           1       0.73      0.06      0.11       276\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.84      0.53      0.54      6545\n",
      "weighted avg       0.95      0.96      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6263    6]\n",
      " [ 260   16]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9594\n",
      "\n",
      "\n",
      "buildings\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6237\n",
      "           1       0.68      0.10      0.17       308\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.82      0.55      0.57      6545\n",
      "weighted avg       0.94      0.96      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6223   14]\n",
      " [ 278   30]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9554\n",
      "\n",
      "\n",
      "electricity\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6398\n",
      "           1       0.67      0.03      0.05       147\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.82      0.51      0.52      6545\n",
      "weighted avg       0.97      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6396    2]\n",
      " [ 143    4]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9778\n",
      "\n",
      "\n",
      "tools\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6514\n",
      "           1       0.00      0.00      0.00        31\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      1.00      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6514    0]\n",
      " [  31    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9953\n",
      "\n",
      "\n",
      "hospitals\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6481\n",
      "           1       0.00      0.00      0.00        64\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6481    0]\n",
      " [  64    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9902\n",
      "\n",
      "\n",
      "shops\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6512\n",
      "           1       0.00      0.00      0.00        33\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6511    1]\n",
      " [  33    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9948\n",
      "\n",
      "\n",
      "aid_centers\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6464\n",
      "           1       0.00      0.00      0.00        81\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      " [[6464    0]\n",
      " [  81    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9876\n",
      "\n",
      "\n",
      "other_infrastructure\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6267\n",
      "           1       0.00      0.00      0.00       278\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.48      0.50      0.49      6545\n",
      "weighted avg       0.92      0.96      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6266    1]\n",
      " [ 278    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9574\n",
      "\n",
      "\n",
      "weather_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.95      0.92      4718\n",
      "           1       0.84      0.72      0.77      1827\n",
      "\n",
      "   micro avg       0.88      0.88      0.88      6545\n",
      "   macro avg       0.87      0.83      0.85      6545\n",
      "weighted avg       0.88      0.88      0.88      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[4474  244]\n",
      " [ 517 1310]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8837\n",
      "\n",
      "\n",
      "floods\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.98      5991\n",
      "           1       0.90      0.51      0.65       554\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.93      0.75      0.81      6545\n",
      "weighted avg       0.95      0.95      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5959   32]\n",
      " [ 271  283]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9537\n",
      "\n",
      "\n",
      "storm\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      5943\n",
      "           1       0.78      0.51      0.62       602\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.86      0.75      0.79      6545\n",
      "weighted avg       0.94      0.94      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5854   89]\n",
      " [ 295  307]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9413\n",
      "\n",
      "\n",
      "fire\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6479\n",
      "           1       0.50      0.02      0.03        66\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.75      0.51      0.51      6545\n",
      "weighted avg       0.99      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6478    1]\n",
      " [  65    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9899\n",
      "\n",
      "\n",
      "earthquake\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.99      5936\n",
      "           1       0.88      0.83      0.85       609\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.93      0.91      0.92      6545\n",
      "weighted avg       0.97      0.97      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5870   66]\n",
      " [ 105  504]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9739\n",
      "\n",
      "\n",
      "cold\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6416\n",
      "           1       0.86      0.05      0.09       129\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.92      0.52      0.54      6545\n",
      "weighted avg       0.98      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6415    1]\n",
      " [ 123    6]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9811\n",
      "\n",
      "\n",
      "other_weather\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6189\n",
      "           1       0.58      0.04      0.07       356\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.77      0.52      0.52      6545\n",
      "weighted avg       0.93      0.95      0.92      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6179   10]\n",
      " [ 342   14]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9462\n",
      "\n",
      "\n",
      "direct_report\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.98      0.92      5309\n",
      "           1       0.80      0.40      0.53      1236\n",
      "\n",
      "   micro avg       0.87      0.87      0.87      6545\n",
      "   macro avg       0.84      0.69      0.73      6545\n",
      "weighted avg       0.86      0.87      0.85      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5182  127]\n",
      " [ 739  497]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8677\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, column in enumerate(y_test.columns):\n",
    "    print(column)\n",
    "    print('_'*60)\n",
    "    print(classification_report(y_test[column], y_pred[:,idx]))\n",
    "    print(\"Confusion Matrix:\\n\", confusion_matrix(y_test[column], y_pred[:,idx]))\n",
    "    print(\"\\n\")\n",
    "    print(\"Accuracy: {0:.4f}\".format((y_test[column] == y_pred[:,idx]).mean()))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 6. Improve your model\n",
    "Use grid search to find better parameters. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "          dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "          lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "          ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "          strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "          tokenizer=<function tokenize at 0x000000000DE77950>,\n",
       "          vocabulary=None)),\n",
       " ('tfidf',\n",
       "  TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)),\n",
       " ('clf',\n",
       "  MultiOutputClassifier(estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "              max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=100, n_jobs=None,\n",
       "              oob_score=False, random_state=None, verbose=0,\n",
       "              warm_start=False),\n",
       "             n_jobs=None))]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv='warn', error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "        strip..._score=False, random_state=None, verbose=0,\n",
       "            warm_start=False),\n",
       "           n_jobs=None))]),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'vect__max_df': [0.5, 0.75, 1], 'tfidf__sublinear_tf': [True, False], 'clf__estimator__n_estimators': [10, 100, 500], 'clf__estimator__min_samples_leaf': [1, 2, 5]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters = {\n",
    "    'vect__max_df': [0.5, 0.75, 1], #0.75\n",
    "    'tfidf__sublinear_tf': [True, False], # False\n",
    "    'clf__estimator__n_estimators' : [10, 100, 500], #500\n",
    "    'clf__estimator__min_samples_leaf': [1, 2, 5] # 1\n",
    "}    \n",
    "# Volgende keer n_jobs=6, cv=5, verbose=2\n",
    "cv = GridSearchCV(pipeline, param_grid=parameters)\n",
    "\n",
    "cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred2 = cv.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 7. Test your model\n",
    "Show the accuracy, precision, and recall of the tuned model.  \n",
    "\n",
    "Since this project focuses on code quality, process, and  pipelines, there is no minimum performance metric needed to pass. However, make sure to fine tune your models for accuracy, precision and recall to make your project stand out - especially for your portfolio!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "          dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "          lowercase=True, max_df=0.75, max_features=None, min_df=1,\n",
       "          ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "          strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "          tokenizer=<function tokenize at 0x000000000DE77950>,\n",
       "          vocabulary=None)),\n",
       " ('tfidf',\n",
       "  TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)),\n",
       " ('clf',\n",
       "  MultiOutputClassifier(estimator=RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "              max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=500, n_jobs=None,\n",
       "              oob_score=False, random_state=None, verbose=0,\n",
       "              warm_start=False),\n",
       "             n_jobs=None))]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.best_estimator_.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.43      0.53      1488\n",
      "           1       0.85      0.95      0.89      5006\n",
      "           2       0.46      0.43      0.44        51\n",
      "\n",
      "   micro avg       0.82      0.82      0.82      6545\n",
      "   macro avg       0.67      0.60      0.62      6545\n",
      "weighted avg       0.81      0.82      0.81      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[ 635  840   13]\n",
      " [ 254 4739   13]\n",
      " [   4   25   22]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8244\n",
      "\n",
      "\n",
      "request\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      0.98      0.94      5425\n",
      "           1       0.83      0.49      0.62      1120\n",
      "\n",
      "   micro avg       0.90      0.90      0.90      6545\n",
      "   macro avg       0.87      0.74      0.78      6545\n",
      "weighted avg       0.89      0.90      0.88      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5312  113]\n",
      " [ 567  553]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8961\n",
      "\n",
      "\n",
      "offer\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6514\n",
      "           1       0.00      0.00      0.00        31\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      1.00      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6514    0]\n",
      " [  31    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9953\n",
      "\n",
      "\n",
      "aid_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.85      0.83      3828\n",
      "           1       0.77      0.72      0.74      2717\n",
      "\n",
      "   micro avg       0.79      0.79      0.79      6545\n",
      "   macro avg       0.79      0.78      0.78      6545\n",
      "weighted avg       0.79      0.79      0.79      6545\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion Matrix:\n",
      " [[3241  587]\n",
      " [ 769 1948]]\n",
      "\n",
      "\n",
      "Accuracy: 0.7928\n",
      "\n",
      "\n",
      "medical_help\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      1.00      0.96      6027\n",
      "           1       0.65      0.08      0.14       518\n",
      "\n",
      "   micro avg       0.92      0.92      0.92      6545\n",
      "   macro avg       0.79      0.54      0.55      6545\n",
      "weighted avg       0.90      0.92      0.90      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6005   22]\n",
      " [ 477   41]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9238\n",
      "\n",
      "\n",
      "medical_products\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6197\n",
      "           1       0.63      0.07      0.12       348\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.79      0.53      0.55      6545\n",
      "weighted avg       0.93      0.95      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6183   14]\n",
      " [ 324   24]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9484\n",
      "\n",
      "\n",
      "search_and_rescue\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.99      6357\n",
      "           1       0.64      0.05      0.09       188\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.81      0.52      0.54      6545\n",
      "weighted avg       0.96      0.97      0.96      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6352    5]\n",
      " [ 179    9]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9719\n",
      "\n",
      "\n",
      "security\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6428\n",
      "           1       0.50      0.02      0.03       117\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.74      0.51      0.51      6545\n",
      "weighted avg       0.97      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6426    2]\n",
      " [ 115    2]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9821\n",
      "\n",
      "\n",
      "military\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6331\n",
      "           1       0.58      0.05      0.09       214\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.77      0.53      0.54      6545\n",
      "weighted avg       0.96      0.97      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6323    8]\n",
      " [ 203   11]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9678\n",
      "\n",
      "\n",
      "water\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6138\n",
      "           1       0.93      0.40      0.55       407\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.94      0.70      0.77      6545\n",
      "weighted avg       0.96      0.96      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6125   13]\n",
      " [ 246  161]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9604\n",
      "\n",
      "\n",
      "food\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.98      0.97      5806\n",
      "           1       0.84      0.65      0.73       739\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.90      0.82      0.85      6545\n",
      "weighted avg       0.94      0.95      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5718   88]\n",
      " [ 262  477]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9465\n",
      "\n",
      "\n",
      "shelter\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      5982\n",
      "           1       0.82      0.40      0.54       563\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.88      0.70      0.76      6545\n",
      "weighted avg       0.94      0.94      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5933   49]\n",
      " [ 335  228]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9413\n",
      "\n",
      "\n",
      "clothing\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6441\n",
      "           1       0.75      0.12      0.20       104\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.87      0.56      0.60      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6437    4]\n",
      " [  92   12]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9853\n",
      "\n",
      "\n",
      "money\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6406\n",
      "           1       0.60      0.02      0.04       139\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.79      0.51      0.52      6545\n",
      "weighted avg       0.97      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6404    2]\n",
      " [ 136    3]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9789\n",
      "\n",
      "\n",
      "missing_people\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6459\n",
      "           1       1.00      0.01      0.02        86\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.99      0.51      0.51      6545\n",
      "weighted avg       0.99      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6459    0]\n",
      " [  85    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9870\n",
      "\n",
      "\n",
      "refugees\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6324\n",
      "           1       0.50      0.00      0.01       221\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.73      0.50      0.50      6545\n",
      "weighted avg       0.95      0.97      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6323    1]\n",
      " [ 220    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9662\n",
      "\n",
      "\n",
      "death\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6240\n",
      "           1       0.91      0.13      0.23       305\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.94      0.57      0.61      6545\n",
      "weighted avg       0.96      0.96      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6236    4]\n",
      " [ 264   41]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9591\n",
      "\n",
      "\n",
      "other_aid\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      1.00      0.93      5700\n",
      "           1       0.56      0.03      0.06       845\n",
      "\n",
      "   micro avg       0.87      0.87      0.87      6545\n",
      "   macro avg       0.71      0.51      0.49      6545\n",
      "weighted avg       0.83      0.87      0.82      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5680   20]\n",
      " [ 820   25]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8717\n",
      "\n",
      "\n",
      "infrastructure_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6126\n",
      "           1       0.50      0.00      0.00       419\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.72      0.50      0.49      6545\n",
      "weighted avg       0.91      0.94      0.91      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6125    1]\n",
      " [ 418    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9360\n",
      "\n",
      "\n",
      "transport\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6245\n",
      "           1       0.85      0.08      0.14       300\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.90      0.54      0.56      6545\n",
      "weighted avg       0.95      0.96      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6241    4]\n",
      " [ 277   23]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9571\n",
      "\n",
      "\n",
      "buildings\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6198\n",
      "           1       0.86      0.12      0.21       347\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.91      0.56      0.59      6545\n",
      "weighted avg       0.95      0.95      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6191    7]\n",
      " [ 305   42]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9523\n",
      "\n",
      "\n",
      "electricity\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6409\n",
      "           1       0.86      0.04      0.08       136\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.92      0.52      0.54      6545\n",
      "weighted avg       0.98      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6408    1]\n",
      " [ 130    6]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9800\n",
      "\n",
      "\n",
      "tools\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6496\n",
      "           1       0.00      0.00      0.00        49\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6496    0]\n",
      " [  49    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9925\n",
      "\n",
      "\n",
      "hospitals\n",
      "____________________________________________________________\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6488\n",
      "           1       0.00      0.00      0.00        57\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6488    0]\n",
      " [  57    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9913\n",
      "\n",
      "\n",
      "shops\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6516\n",
      "           1       0.00      0.00      0.00        29\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      1.00      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6516    0]\n",
      " [  29    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9956\n",
      "\n",
      "\n",
      "aid_centers\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6478\n",
      "           1       0.00      0.00      0.00        67\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6478    0]\n",
      " [  67    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9898\n",
      "\n",
      "\n",
      "other_infrastructure\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6246\n",
      "           1       0.00      0.00      0.00       299\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.48      0.50      0.49      6545\n",
      "weighted avg       0.91      0.95      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6246    0]\n",
      " [ 299    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9543\n",
      "\n",
      "\n",
      "weather_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.95      0.92      4665\n",
      "           1       0.85      0.72      0.78      1880\n",
      "\n",
      "   micro avg       0.88      0.88      0.88      6545\n",
      "   macro avg       0.87      0.83      0.85      6545\n",
      "weighted avg       0.88      0.88      0.88      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[4427  238]\n",
      " [ 527 1353]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8831\n",
      "\n",
      "\n",
      "floods\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6000\n",
      "           1       0.91      0.50      0.64       545\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.93      0.75      0.81      6545\n",
      "weighted avg       0.95      0.95      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5973   27]\n",
      " [ 275  270]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9539\n",
      "\n",
      "\n",
      "storm\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.98      0.97      5911\n",
      "           1       0.77      0.50      0.61       634\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.86      0.74      0.79      6545\n",
      "weighted avg       0.93      0.94      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5816   95]\n",
      " [ 314  320]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9375\n",
      "\n",
      "\n",
      "fire\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6475\n",
      "           1       0.00      0.00      0.00        70\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6475    0]\n",
      " [  70    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9893\n",
      "\n",
      "\n",
      "earthquake\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.98      5893\n",
      "           1       0.88      0.81      0.84       652\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.93      0.90      0.91      6545\n",
      "weighted avg       0.97      0.97      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5820   73]\n",
      " [ 123  529]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9701\n",
      "\n",
      "\n",
      "cold\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6401\n",
      "           1       0.85      0.08      0.14       144\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.91      0.54      0.56      6545\n",
      "weighted avg       0.98      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6399    2]\n",
      " [ 133   11]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9794\n",
      "\n",
      "\n",
      "other_weather\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6188\n",
      "           1       0.57      0.04      0.08       357\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.76      0.52      0.53      6545\n",
      "weighted avg       0.93      0.95      0.92      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6176   12]\n",
      " [ 341   16]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9461\n",
      "\n",
      "\n",
      "direct_report\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.98      0.91      5247\n",
      "           1       0.78      0.35      0.48      1298\n",
      "\n",
      "   micro avg       0.85      0.85      0.85      6545\n",
      "   macro avg       0.82      0.66      0.70      6545\n",
      "weighted avg       0.84      0.85      0.83      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5120  127]\n",
      " [ 844  454]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8516\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, column in enumerate(y_test.columns):\n",
    "    print(column)\n",
    "    print('_'*60)\n",
    "    print(classification_report(y_test[column], y_pred2[:,idx]))\n",
    "    print(\"Confusion Matrix:\\n\", confusion_matrix(y_test[column], y_pred2[:,idx]))\n",
    "    print(\"\\n\")\n",
    "    print(\"Accuracy: {0:.4f}\".format((y_test[column] == y_pred2[:,idx]).mean()))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 8. Try improving your model further. Here are a few ideas:\n",
    "* try other machine learning algorithms\n",
    "* add other features besides the TF-IDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "pipeline = Pipeline([\n",
    "        ('vect', CountVectorizer(tokenizer=tokenize)),\n",
    "        ('tfidf', TfidfTransformer()),\n",
    "        ('clf', MultiOutputClassifier(MultinomialNB()))\n",
    "    ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "pipeline.fit(X_train, y_train)\n",
    "\n",
    "# predict on test data\n",
    "y_pred = pipeline.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.09      0.17      1510\n",
      "           1       0.78      0.99      0.87      4986\n",
      "           2       0.00      0.00      0.00        49\n",
      "\n",
      "   micro avg       0.78      0.78      0.78      6545\n",
      "   macro avg       0.52      0.36      0.35      6545\n",
      "weighted avg       0.77      0.78      0.70      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[ 140 1370    0]\n",
      " [  38 4948    0]\n",
      " [   4   45    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.7774\n",
      "\n",
      "\n",
      "request\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.99      0.92      5429\n",
      "           1       0.84      0.22      0.35      1116\n",
      "\n",
      "   micro avg       0.86      0.86      0.86      6545\n",
      "   macro avg       0.85      0.60      0.63      6545\n",
      "weighted avg       0.86      0.86      0.82      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5382   47]\n",
      " [ 872  244]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8596\n",
      "\n",
      "\n",
      "offer\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6518\n",
      "           1       0.00      0.00      0.00        27\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      1.00      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6518    0]\n",
      " [  27    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9959\n",
      "\n",
      "\n",
      "aid_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.76      0.84      0.80      3834\n",
      "           1       0.74      0.62      0.67      2711\n",
      "\n",
      "   micro avg       0.75      0.75      0.75      6545\n",
      "   macro avg       0.75      0.73      0.74      6545\n",
      "weighted avg       0.75      0.75      0.75      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[3228  606]\n",
      " [1023 1688]]\n",
      "\n",
      "\n",
      "Accuracy: 0.7511\n",
      "\n",
      "\n",
      "medical_help\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      1.00      0.96      6059\n",
      "           1       0.25      0.00      0.00       486\n",
      "\n",
      "   micro avg       0.93      0.93      0.93      6545\n",
      "   macro avg       0.59      0.50      0.48      6545\n",
      "weighted avg       0.88      0.93      0.89      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6056    3]\n",
      " [ 485    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9254\n",
      "\n",
      "\n",
      "medical_products\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6248\n",
      "           1       1.00      0.00      0.01       297\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.98      0.50      0.49      6545\n",
      "weighted avg       0.96      0.95      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6248    0]\n",
      " [ 296    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9548\n",
      "\n",
      "\n",
      "search_and_rescue\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.99      6376\n",
      "           1       0.00      0.00      0.00       169\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.49      0.50      0.49      6545\n",
      "weighted avg       0.95      0.97      0.96      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6376    0]\n",
      " [ 169    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9742\n",
      "\n",
      "\n",
      "security\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6411\n",
      "           1       0.00      0.00      0.00       134\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.49      0.50      0.49      6545\n",
      "weighted avg       0.96      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6411    0]\n",
      " [ 134    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9795\n",
      "\n",
      "\n",
      "military\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6321\n",
      "           1       0.00      0.00      0.00       224\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.48      0.50      0.49      6545\n",
      "weighted avg       0.93      0.97      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6321    0]\n",
      " [ 224    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9658\n",
      "\n",
      "\n",
      "water\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6144\n",
      "           1       0.00      0.00      0.00       401\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.47      0.50      0.48      6545\n",
      "weighted avg       0.88      0.94      0.91      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6144    0]\n",
      " [ 401    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9387\n",
      "\n",
      "\n",
      "food\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      1.00      0.94      5829\n",
      "           1       0.83      0.01      0.03       716\n",
      "\n",
      "   micro avg       0.89      0.89      0.89      6545\n",
      "   macro avg       0.86      0.51      0.49      6545\n",
      "weighted avg       0.89      0.89      0.84      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5827    2]\n",
      " [ 706   10]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8918\n",
      "\n",
      "\n",
      "shelter\n",
      "____________________________________________________________\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      1.00      0.95      5949\n",
      "           1       0.33      0.00      0.00       596\n",
      "\n",
      "   micro avg       0.91      0.91      0.91      6545\n",
      "   macro avg       0.62      0.50      0.48      6545\n",
      "weighted avg       0.86      0.91      0.87      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5947    2]\n",
      " [ 595    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9088\n",
      "\n",
      "\n",
      "clothing\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6459\n",
      "           1       0.00      0.00      0.00        86\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.97      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6459    0]\n",
      " [  86    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9869\n",
      "\n",
      "\n",
      "money\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6406\n",
      "           1       0.00      0.00      0.00       139\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.49      0.50      0.49      6545\n",
      "weighted avg       0.96      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6406    0]\n",
      " [ 139    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9788\n",
      "\n",
      "\n",
      "missing_people\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6478\n",
      "           1       0.00      0.00      0.00        67\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6478    0]\n",
      " [  67    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9898\n",
      "\n",
      "\n",
      "refugees\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6314\n",
      "           1       0.00      0.00      0.00       231\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.48      0.50      0.49      6545\n",
      "weighted avg       0.93      0.96      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6314    0]\n",
      " [ 231    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9647\n",
      "\n",
      "\n",
      "death\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6266\n",
      "           1       0.00      0.00      0.00       279\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.48      0.50      0.49      6545\n",
      "weighted avg       0.92      0.96      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6266    0]\n",
      " [ 279    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9574\n",
      "\n",
      "\n",
      "other_aid\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      1.00      0.93      5656\n",
      "           1       0.00      0.00      0.00       889\n",
      "\n",
      "   micro avg       0.86      0.86      0.86      6545\n",
      "   macro avg       0.43      0.50      0.46      6545\n",
      "weighted avg       0.75      0.86      0.80      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5655    1]\n",
      " [ 889    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8640\n",
      "\n",
      "\n",
      "infrastructure_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      1.00      0.97      6119\n",
      "           1       0.00      0.00      0.00       426\n",
      "\n",
      "   micro avg       0.93      0.93      0.93      6545\n",
      "   macro avg       0.47      0.50      0.48      6545\n",
      "weighted avg       0.87      0.93      0.90      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6118    1]\n",
      " [ 426    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9348\n",
      "\n",
      "\n",
      "transport\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6243\n",
      "           1       0.00      0.00      0.00       302\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.48      0.50      0.49      6545\n",
      "weighted avg       0.91      0.95      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6243    0]\n",
      " [ 302    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9539\n",
      "\n",
      "\n",
      "buildings\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6178\n",
      "           1       0.00      0.00      0.00       367\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.47      0.50      0.49      6545\n",
      "weighted avg       0.89      0.94      0.92      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6178    0]\n",
      " [ 367    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9439\n",
      "\n",
      "\n",
      "electricity\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6391\n",
      "           1       0.00      0.00      0.00       154\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.49      0.50      0.49      6545\n",
      "weighted avg       0.95      0.98      0.96      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6391    0]\n",
      " [ 154    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9765\n",
      "\n",
      "\n",
      "tools\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6499\n",
      "           1       0.00      0.00      0.00        46\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6499    0]\n",
      " [  46    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9930\n",
      "\n",
      "\n",
      "hospitals\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6486\n",
      "           1       0.00      0.00      0.00        59\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6486    0]\n",
      " [  59    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9910\n",
      "\n",
      "\n",
      "shops\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6516\n",
      "           1       0.00      0.00      0.00        29\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      1.00      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6516    0]\n",
      " [  29    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9956\n",
      "\n",
      "\n",
      "aid_centers\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6471\n",
      "           1       0.00      0.00      0.00        74\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6471    0]\n",
      " [  74    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9887\n",
      "\n",
      "\n",
      "other_infrastructure\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6236\n",
      "           1       0.00      0.00      0.00       309\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.48      0.50      0.49      6545\n",
      "weighted avg       0.91      0.95      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6236    0]\n",
      " [ 309    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9528\n",
      "\n",
      "\n",
      "weather_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.97      0.88      4686\n",
      "           1       0.84      0.42      0.56      1859\n",
      "\n",
      "   micro avg       0.81      0.81      0.81      6545\n",
      "   macro avg       0.82      0.70      0.72      6545\n",
      "weighted avg       0.82      0.81      0.79      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[4535  151]\n",
      " [1073  786]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8130\n",
      "\n",
      "\n",
      "floods\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      6008\n",
      "           1       0.40      0.00      0.01       537\n",
      "\n",
      "   micro avg       0.92      0.92      0.92      6545\n",
      "   macro avg       0.66      0.50      0.48      6545\n",
      "weighted avg       0.88      0.92      0.88      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6005    3]\n",
      " [ 535    2]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9178\n",
      "\n",
      "\n",
      "storm\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.90      1.00      0.95      5905\n",
      "           1       0.74      0.03      0.05       640\n",
      "\n",
      "   micro avg       0.90      0.90      0.90      6545\n",
      "   macro avg       0.82      0.51      0.50      6545\n",
      "weighted avg       0.89      0.90      0.86      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5899    6]\n",
      " [ 623   17]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9039\n",
      "\n",
      "\n",
      "fire\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6457\n",
      "           1       0.00      0.00      0.00        88\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.97      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6457    0]\n",
      " [  88    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9866\n",
      "\n",
      "\n",
      "earthquake\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      1.00      0.96      5962\n",
      "           1       0.87      0.12      0.21       583\n",
      "\n",
      "   micro avg       0.92      0.92      0.92      6545\n",
      "   macro avg       0.90      0.56      0.58      6545\n",
      "weighted avg       0.92      0.92      0.89      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5952   10]\n",
      " [ 515   68]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9198\n",
      "\n",
      "\n",
      "cold\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6417\n",
      "           1       0.00      0.00      0.00       128\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.96      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6417    0]\n",
      " [ 128    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9804\n",
      "\n",
      "\n",
      "other_weather\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6173\n",
      "           1       0.00      0.00      0.00       372\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.47      0.50      0.49      6545\n",
      "weighted avg       0.89      0.94      0.92      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6173    0]\n",
      " [ 372    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9432\n",
      "\n",
      "\n",
      "direct_report\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.83      0.99      0.90      5280\n",
      "           1       0.76      0.16      0.27      1265\n",
      "\n",
      "   micro avg       0.83      0.83      0.83      6545\n",
      "   macro avg       0.80      0.57      0.58      6545\n",
      "weighted avg       0.82      0.83      0.78      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5217   63]\n",
      " [1062  203]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8281\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, column in enumerate(y_test.columns):\n",
    "    print(column)\n",
    "    print('_'*60)\n",
    "    print(classification_report(y_test[column], y_pred[:,idx]))\n",
    "    print(\"Confusion Matrix:\\n\", confusion_matrix(y_test[column], y_pred[:,idx]))\n",
    "    print(\"\\n\")\n",
    "    print(\"Accuracy: {0:.4f}\".format((y_test[column] == y_pred[:,idx]).mean()))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 36 candidates, totalling 108 fits\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  38.9s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  1.1min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  41.2s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  35.9s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  40.3s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  38.4s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  40.7s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  40.6s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  37.7s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  30.1s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  32.2s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  36.7s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  39.7s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  40.7s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  41.7s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  42.3s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  37.3s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  40.9s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  37.5s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  33.8s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  35.4s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  39.3s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  36.1s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  41.3s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  40.5s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  38.3s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  38.4s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  40.0s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  31.2s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  35.0s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  36.2s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  37.5s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  30.7s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  40.1s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  45.6s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  43.0s\n",
      "[CV] clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  42.7s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  36.9s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  35.1s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  39.9s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  38.4s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  36.5s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  29.7s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  40.1s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  37.8s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  39.9s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  35.2s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  37.7s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  39.2s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  34.8s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  37.4s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  41.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  39.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  35.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  41.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  39.9s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  42.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  38.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  41.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  33.6s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  39.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  37.4s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  33.7s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  38.1s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  39.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  39.2s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  34.6s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  36.9s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  43.0s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  39.4s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  31.5s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  37.4s\n",
      "[CV] clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=0.5, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  42.4s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  40.0s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  27.4s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  27.1s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  26.5s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  26.7s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  27.2s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  27.0s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  26.8s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=True, vect__max_df=1, total=  28.3s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  27.3s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  27.5s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  26.3s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  26.3s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  26.5s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  26.8s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  28.6s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  28.7s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=True, tfidf__sublinear_tf=False, vect__max_df=1, total=  28.3s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  28.0s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  28.7s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.5, total=  28.2s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  28.7s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  28.6s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=0.75, total=  28.6s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  28.7s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  28.7s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=True, vect__max_df=1, total=  28.7s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  29.0s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  28.2s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.5, total=  28.6s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  28.4s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  28.8s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=0.75, total=  28.6s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  28.4s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  28.5s\n",
      "[CV] clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1 \n",
      "[CV]  clf__estimator__alpha=1, clf__estimator__fit_prior=False, tfidf__sublinear_tf=False, vect__max_df=1, total=  26.7s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done 108 out of 108 | elapsed: 105.3min finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=3, error_score='raise-deprecating',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "        strip...fier(estimator=MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True),\n",
       "           n_jobs=None))]),\n",
       "       fit_params=None, iid='warn', n_jobs=None,\n",
       "       param_grid={'vect__max_df': [0.5, 0.75, 1], 'tfidf__sublinear_tf': [True, False], 'clf__estimator__alpha': [0.1, 0.5, 1], 'clf__estimator__fit_prior': [True, False]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=2)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "parameters = {\n",
    "    'vect__max_df': [0.5, 0.75, 1], #0.75\n",
    "    'tfidf__sublinear_tf': [True, False], # False\n",
    "    'clf__estimator__alpha' : [0.1, 0.5, 1], #500\n",
    "    'clf__estimator__fit_prior': [True, False] # 1\n",
    "}    \n",
    "# Volgende keer n_jobs=6, cv=5, verbose=2\n",
    "cv = GridSearchCV(pipeline, param_grid=parameters, cv=3, verbose=2)\n",
    "\n",
    "cv.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('vect', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "          dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "          lowercase=True, max_df=0.5, max_features=None, min_df=1,\n",
       "          ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "          strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "          tokenizer=<function tokenize at 0x000000000DBFFB70>,\n",
       "          vocabulary=None)),\n",
       " ('tfidf',\n",
       "  TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)),\n",
       " ('clf',\n",
       "  MultiOutputClassifier(estimator=MultinomialNB(alpha=0.1, class_prior=None, fit_prior=True),\n",
       "             n_jobs=None))]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv.best_estimator_.steps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred3 = cv.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.71      0.36      0.48      1510\n",
      "           1       0.83      0.95      0.89      4986\n",
      "           2       0.65      0.35      0.45        49\n",
      "\n",
      "   micro avg       0.81      0.81      0.81      6545\n",
      "   macro avg       0.73      0.55      0.61      6545\n",
      "weighted avg       0.80      0.81      0.79      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[ 549  961    0]\n",
      " [ 219 4758    9]\n",
      " [   2   30   17]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8134\n",
      "\n",
      "\n",
      "request\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.91      0.96      0.94      5429\n",
      "           1       0.74      0.56      0.64      1116\n",
      "\n",
      "   micro avg       0.89      0.89      0.89      6545\n",
      "   macro avg       0.83      0.76      0.79      6545\n",
      "weighted avg       0.88      0.89      0.89      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5204  225]\n",
      " [ 488  628]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8911\n",
      "\n",
      "\n",
      "offer\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6518\n",
      "           1       0.00      0.00      0.00        27\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      1.00      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6518    0]\n",
      " [  27    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9959\n",
      "\n",
      "\n",
      "aid_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.77      0.79      0.78      3834\n",
      "           1       0.69      0.66      0.67      2711\n",
      "\n",
      "   micro avg       0.74      0.74      0.74      6545\n",
      "   macro avg       0.73      0.73      0.73      6545\n",
      "weighted avg       0.74      0.74      0.74      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[3046  788]\n",
      " [ 929 1782]]\n",
      "\n",
      "\n",
      "Accuracy: 0.7377\n",
      "\n",
      "\n",
      "medical_help\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.99      0.96      6059\n",
      "           1       0.53      0.15      0.23       486\n",
      "\n",
      "   micro avg       0.93      0.93      0.93      6545\n",
      "   macro avg       0.73      0.57      0.60      6545\n",
      "weighted avg       0.91      0.93      0.91      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5995   64]\n",
      " [ 414   72]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9270\n",
      "\n",
      "\n",
      "medical_products\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6248\n",
      "           1       0.54      0.11      0.18       297\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.75      0.55      0.58      6545\n",
      "weighted avg       0.94      0.96      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\metrics\\classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[6220   28]\n",
      " [ 264   33]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9554\n",
      "\n",
      "\n",
      "search_and_rescue\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.99      6376\n",
      "           1       0.40      0.01      0.02       169\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.69      0.51      0.50      6545\n",
      "weighted avg       0.96      0.97      0.96      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6373    3]\n",
      " [ 167    2]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9740\n",
      "\n",
      "\n",
      "security\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6411\n",
      "           1       0.00      0.00      0.00       134\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.49      0.50      0.49      6545\n",
      "weighted avg       0.96      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6409    2]\n",
      " [ 134    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9792\n",
      "\n",
      "\n",
      "military\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      1.00      0.98      6321\n",
      "           1       0.63      0.21      0.31       224\n",
      "\n",
      "   micro avg       0.97      0.97      0.97      6545\n",
      "   macro avg       0.80      0.60      0.65      6545\n",
      "weighted avg       0.96      0.97      0.96      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6293   28]\n",
      " [ 177   47]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9687\n",
      "\n",
      "\n",
      "water\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97      6144\n",
      "           1       0.58      0.12      0.20       401\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.76      0.56      0.58      6545\n",
      "weighted avg       0.92      0.94      0.92      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6110   34]\n",
      " [ 354   47]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9407\n",
      "\n",
      "\n",
      "food\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.99      0.95      5829\n",
      "           1       0.72      0.32      0.44       716\n",
      "\n",
      "   micro avg       0.91      0.91      0.91      6545\n",
      "   macro avg       0.82      0.65      0.70      6545\n",
      "weighted avg       0.90      0.91      0.90      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5742   87]\n",
      " [ 490  226]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9118\n",
      "\n",
      "\n",
      "shelter\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.92      0.99      0.96      5949\n",
      "           1       0.69      0.19      0.29       596\n",
      "\n",
      "   micro avg       0.92      0.92      0.92      6545\n",
      "   macro avg       0.81      0.59      0.63      6545\n",
      "weighted avg       0.90      0.92      0.90      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5900   49]\n",
      " [ 485  111]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9184\n",
      "\n",
      "\n",
      "clothing\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6459\n",
      "           1       0.58      0.16      0.25        86\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.79      0.58      0.62      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6449   10]\n",
      " [  72   14]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9875\n",
      "\n",
      "\n",
      "money\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6406\n",
      "           1       0.50      0.03      0.05       139\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.74      0.51      0.52      6545\n",
      "weighted avg       0.97      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6402    4]\n",
      " [ 135    4]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9788\n",
      "\n",
      "\n",
      "missing_people\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6478\n",
      "           1       0.00      0.00      0.00        67\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6477    1]\n",
      " [  67    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9896\n",
      "\n",
      "\n",
      "refugees\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6314\n",
      "           1       0.17      0.00      0.01       231\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.57      0.50      0.50      6545\n",
      "weighted avg       0.94      0.96      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6309    5]\n",
      " [ 230    1]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9641\n",
      "\n",
      "\n",
      "death\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6266\n",
      "           1       0.72      0.12      0.20       279\n",
      "\n",
      "   micro avg       0.96      0.96      0.96      6545\n",
      "   macro avg       0.84      0.56      0.59      6545\n",
      "weighted avg       0.95      0.96      0.95      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6253   13]\n",
      " [ 246   33]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9604\n",
      "\n",
      "\n",
      "other_aid\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.87      0.99      0.93      5656\n",
      "           1       0.36      0.02      0.04       889\n",
      "\n",
      "   micro avg       0.86      0.86      0.86      6545\n",
      "   macro avg       0.61      0.51      0.48      6545\n",
      "weighted avg       0.80      0.86      0.81      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5621   35]\n",
      " [ 869   20]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8619\n",
      "\n",
      "\n",
      "infrastructure_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      1.00      0.97      6119\n",
      "           1       0.25      0.01      0.02       426\n",
      "\n",
      "   micro avg       0.93      0.93      0.93      6545\n",
      "   macro avg       0.59      0.50      0.49      6545\n",
      "weighted avg       0.89      0.93      0.90      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6107   12]\n",
      " [ 422    4]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9337\n",
      "\n",
      "\n",
      "transport\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      1.00      0.98      6243\n",
      "           1       0.59      0.04      0.08       302\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.77      0.52      0.53      6545\n",
      "weighted avg       0.94      0.95      0.94      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6234    9]\n",
      " [ 289   13]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9545\n",
      "\n",
      "\n",
      "buildings\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6178\n",
      "           1       0.61      0.06      0.11       367\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.78      0.53      0.54      6545\n",
      "weighted avg       0.93      0.95      0.92      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6164   14]\n",
      " [ 345   22]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9451\n",
      "\n",
      "\n",
      "electricity\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6391\n",
      "           1       0.67      0.03      0.05       154\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.82      0.51      0.52      6545\n",
      "weighted avg       0.97      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6389    2]\n",
      " [ 150    4]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9768\n",
      "\n",
      "\n",
      "tools\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6499\n",
      "           1       0.00      0.00      0.00        46\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6497    2]\n",
      " [  46    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9927\n",
      "\n",
      "\n",
      "hospitals\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00      6486\n",
      "           1       0.00      0.00      0.00        59\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6485    1]\n",
      " [  59    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9908\n",
      "\n",
      "\n",
      "shops\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00      6516\n",
      "           1       0.00      0.00      0.00        29\n",
      "\n",
      "   micro avg       1.00      1.00      1.00      6545\n",
      "   macro avg       0.50      0.50      0.50      6545\n",
      "weighted avg       0.99      1.00      0.99      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6515    1]\n",
      " [  29    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9954\n",
      "\n",
      "\n",
      "aid_centers\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6471\n",
      "           1       0.00      0.00      0.00        74\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.98      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6469    2]\n",
      " [  74    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9884\n",
      "\n",
      "\n",
      "other_infrastructure\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.98      6236\n",
      "           1       0.22      0.01      0.01       309\n",
      "\n",
      "   micro avg       0.95      0.95      0.95      6545\n",
      "   macro avg       0.59      0.50      0.49      6545\n",
      "weighted avg       0.92      0.95      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6229    7]\n",
      " [ 307    2]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9520\n",
      "\n",
      "\n",
      "weather_related\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.85      0.92      0.89      4686\n",
      "           1       0.76      0.59      0.66      1859\n",
      "\n",
      "   micro avg       0.83      0.83      0.83      6545\n",
      "   macro avg       0.80      0.76      0.77      6545\n",
      "weighted avg       0.82      0.83      0.82      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[4334  352]\n",
      " [ 767 1092]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8290\n",
      "\n",
      "\n",
      "floods\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.99      0.96      6008\n",
      "           1       0.70      0.28      0.40       537\n",
      "\n",
      "   micro avg       0.93      0.93      0.93      6545\n",
      "   macro avg       0.82      0.63      0.68      6545\n",
      "weighted avg       0.92      0.93      0.92      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5944   64]\n",
      " [ 389  148]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9308\n",
      "\n",
      "\n",
      "storm\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.93      0.99      0.96      5905\n",
      "           1       0.72      0.34      0.46       640\n",
      "\n",
      "   micro avg       0.92      0.92      0.92      6545\n",
      "   macro avg       0.83      0.66      0.71      6545\n",
      "weighted avg       0.91      0.92      0.91      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5822   83]\n",
      " [ 423  217]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9227\n",
      "\n",
      "\n",
      "fire\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99      6457\n",
      "           1       0.00      0.00      0.00        88\n",
      "\n",
      "   micro avg       0.99      0.99      0.99      6545\n",
      "   macro avg       0.49      0.50      0.50      6545\n",
      "weighted avg       0.97      0.99      0.98      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6456    1]\n",
      " [  88    0]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9864\n",
      "\n",
      "\n",
      "earthquake\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.99      0.97      5962\n",
      "           1       0.81      0.39      0.53       583\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.88      0.69      0.75      6545\n",
      "weighted avg       0.93      0.94      0.93      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[5910   52]\n",
      " [ 356  227]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9377\n",
      "\n",
      "\n",
      "cold\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99      6417\n",
      "           1       0.38      0.02      0.04       128\n",
      "\n",
      "   micro avg       0.98      0.98      0.98      6545\n",
      "   macro avg       0.68      0.51      0.52      6545\n",
      "weighted avg       0.97      0.98      0.97      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6412    5]\n",
      " [ 125    3]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9801\n",
      "\n",
      "\n",
      "other_weather\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      1.00      0.97      6173\n",
      "           1       0.54      0.04      0.07       372\n",
      "\n",
      "   micro avg       0.94      0.94      0.94      6545\n",
      "   macro avg       0.74      0.52      0.52      6545\n",
      "weighted avg       0.92      0.94      0.92      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[6161   12]\n",
      " [ 358   14]]\n",
      "\n",
      "\n",
      "Accuracy: 0.9435\n",
      "\n",
      "\n",
      "direct_report\n",
      "____________________________________________________________\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.88      0.94      0.91      5280\n",
      "           1       0.66      0.47      0.55      1265\n",
      "\n",
      "   micro avg       0.85      0.85      0.85      6545\n",
      "   macro avg       0.77      0.71      0.73      6545\n",
      "weighted avg       0.84      0.85      0.84      6545\n",
      "\n",
      "Confusion Matrix:\n",
      " [[4969  311]\n",
      " [ 670  595]]\n",
      "\n",
      "\n",
      "Accuracy: 0.8501\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for idx, column in enumerate(y_test.columns):\n",
    "    print(column)\n",
    "    print('_'*60)\n",
    "    print(classification_report(y_test[column], y_pred3[:,idx]))\n",
    "    print(\"Confusion Matrix:\\n\", confusion_matrix(y_test[column], y_pred3[:,idx]))\n",
    "    print(\"\\n\")\n",
    "    print(\"Accuracy: {0:.4f}\".format((y_test[column] == y_pred3[:,idx]).mean()))\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 9. Export your model as a pickle file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pipeline.pkl']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from joblib import dump, load\n",
    "dump(pipeline, 'pipeline.pkl') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['cv.pkl']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dump(cv, 'cv.pkl') "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 10. Use this notebook to complete `train.py`\n",
    "Use the template file attached in the Resources folder to write a script that runs the steps above to create a database and export a model based on a new dataset specified by the user."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}