classify.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('prescriber-info.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>NPI</th>\n",
       "      <th>Gender</th>\n",
       "      <th>State</th>\n",
       "      <th>Credentials</th>\n",
       "      <th>Specialty</th>\n",
       "      <th>ABILIFY</th>\n",
       "      <th>ACETAMINOPHEN.CODEINE</th>\n",
       "      <th>ACYCLOVIR</th>\n",
       "      <th>ADVAIR.DISKUS</th>\n",
       "      <th>AGGRENOX</th>\n",
       "      <th>...</th>\n",
       "      <th>VERAPAMIL.ER</th>\n",
       "      <th>VESICARE</th>\n",
       "      <th>VOLTAREN</th>\n",
       "      <th>VYTORIN</th>\n",
       "      <th>WARFARIN.SODIUM</th>\n",
       "      <th>XARELTO</th>\n",
       "      <th>ZETIA</th>\n",
       "      <th>ZIPRASIDONE.HCL</th>\n",
       "      <th>ZOLPIDEM.TARTRATE</th>\n",
       "      <th>Opioid.Prescriber</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1710982582</td>\n",
       "      <td>M</td>\n",
       "      <td>TX</td>\n",
       "      <td>DDS</td>\n",
       "      <td>Dentist</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1245278100</td>\n",
       "      <td>F</td>\n",
       "      <td>AL</td>\n",
       "      <td>MD</td>\n",
       "      <td>General Surgery</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 256 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          NPI Gender State Credentials        Specialty  ABILIFY  \\\n",
       "0  1710982582      M    TX         DDS          Dentist        0   \n",
       "1  1245278100      F    AL          MD  General Surgery        0   \n",
       "\n",
       "   ACETAMINOPHEN.CODEINE  ACYCLOVIR  ADVAIR.DISKUS  AGGRENOX  \\\n",
       "0                      0          0              0         0   \n",
       "1                      0          0              0         0   \n",
       "\n",
       "         ...          VERAPAMIL.ER  VESICARE  VOLTAREN  VYTORIN  \\\n",
       "0        ...                     0         0         0        0   \n",
       "1        ...                     0         0         0        0   \n",
       "\n",
       "   WARFARIN.SODIUM  XARELTO  ZETIA  ZIPRASIDONE.HCL  ZOLPIDEM.TARTRATE  \\\n",
       "0                0        0      0                0                  0   \n",
       "1                0        0      0                0                 35   \n",
       "\n",
       "   Opioid.Prescriber  \n",
       "0                  1  \n",
       "1                  1  \n",
       "\n",
       "[2 rows x 256 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24237, 256)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn import cross_validation, svm\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos= df[(df[\"Opioid.Prescriber\"]==1)]\n",
    "neg= df[(df[\"Opioid.Prescriber\"]==0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "pos= pos.sample(n=9000)\n",
    "neg= neg.sample(n=9000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot= pd.concat([neg, pos])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot_special=tot.Specialty.unique()\n",
    "tot_creds= tot.Credentials.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def credSpecDict():\n",
    "    cred_dict={}\n",
    "    spec_dict={}\n",
    "    for i in range(len(tot_creds)):\n",
    "        cred_dict[i]= tot_creds[i]\n",
    "    for i in range(len(tot_special)):\n",
    "        spec_dict[i]= tot_special[i]\n",
    "    return cred_dict, spec_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "c, s= credSpecDict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    }
   ],
   "source": [
    "print(list(s.keys())[list(s.values()).index('Dentist')])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "tot.Gender= tot.Gender.replace(['M', 'F'], [1, 0])\n",
    "tot.Credentials=tot.Credentials.replace(tot_creds, [i for i in range(len(tot_creds))])\n",
    "tot.Specialty=tot.Specialty.replace(tot_special, [i for i in range(len(tot_special))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(18000, 256)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tot.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(tot.drop(['State', 'NPI', 'Opioid.Prescriber'], 1))\n",
    "y = np.array(tot['Opioid.Prescriber'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = cross_validation.train_test_split(X, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n",
      "c:\\users\\kimbe\\appdata\\local\\programs\\python\\python36\\lib\\site-packages\\sklearn\\tree\\tree.py:282: DeprecationWarning: The min_impurity_split parameter is deprecated and will be removed in version 0.21. Use the min_impurity_decrease parameter instead.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9238888888888889\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "clf2 = RandomForestClassifier(min_impurity_split=0.05)\n",
    "clf2.fit(X_train, y_train)\n",
    "print(clf2.score(X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of mislabeled points out of a total 3600 points : 292, performance 91.89%\n"
     ]
    }
   ],
   "source": [
    "from sklearn.naive_bayes import GaussianNB\n",
    "gnb = GaussianNB()\n",
    "y_pred = gnb.fit(X_train, y_train).predict(X_test)\n",
    "print(\"Number of mislabeled points out of a total {} points : {}, performance {:05.2f}%\"\n",
    "      .format(\n",
    "          X_test.shape[0],\n",
    "          (y_test != y_pred).sum(),\n",
    "          100*(1-(y_test != y_pred).sum()/X_test.shape[0])\n",
    "))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gnb.predict_proba(X_test)[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extractFeatures():\n",
    "    features= list(tot.columns)\n",
    "    removeFeatures= ['NPI', 'State', 'Opioid.Prescriber']\n",
    "    for i in features:\n",
    "        if i in removeFeatures:\n",
    "            features.remove(i)\n",
    "    return features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def returnPreds(row):\n",
    "    y=gnb.predict(row)\n",
    "    return int(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "new=tot.drop(['State', 'NPI', 'Opioid.Prescriber'],axis= 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "new.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "new=new[extractFeatures()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(new.head(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0, 11,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "       12,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,\n",
       "        0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0,  0],\n",
       "      dtype=int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(new.iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "d= pd.read_csv('demo4.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "254"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.array(d.iloc[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fileEntry():\n",
    "        # open csv file, read in dataframe\n",
    "        # add predictions to df then save it as new file\n",
    "        file =  filedialog.askopenfilename(initialdir = \"/\", \\\n",
    "        title = \"Select file\",filetypes = ((\"csv files\",\"*.csv\"),(\"all files\",\"*.*\")))\n",
    "        valList=[]\n",
    "        if file != None:\n",
    "            data = pd.read_csv(file)\n",
    "            data=data[extractFeatures()]\n",
    "            for i, row in data.iterrows():\n",
    "                val= returnPreds([np.array(data.iloc[i])])\n",
    "                if val==1:\n",
    "                    val= \"Pill Mill\"\n",
    "                else:\n",
    "                    val = \"Not Pill Mill\"\n",
    "                valList.append(val)\n",
    "            v=pd.Series(valList)\n",
    "            data['preds']= v.values\n",
    "#             data.to_csv('predictions.csv')\n",
    "            print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tkinter import filedialog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "fileEntry()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tkinter import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}