data_prep.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np \n",
    "import tensorflow_io as tfio \n",
    "import tensorflow as tf\n",
    "import seaborn as sns \n",
    "import fastai2 \n",
    "import os \n",
    "import category_encoders as ce\n",
    "from   sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Exploring Given Data  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dirctory='/media/tkrsh/ebbc93a5-618a-433c-b301-2406c8ffeca7/siim-isic-melanoma-classification/jpeg/train/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(\"/media/tkrsh/ebbc93a5-618a-433c-b301-2406c8ffeca7/siim-isic-melanoma-classification\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "df= pd.read_csv(\"train.csv\")\n",
    "test=pd.read_csv(\"test.csv\")\n",
    "sub = pd.read_csv(\"sample_submission.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    32542\n",
       "1      584\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.target.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general_challenge</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>benign_malignant</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>male</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>female</td>\n",
       "      <td>50.0</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>nevus</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     image_name  patient_id     sex  age_approx anatom_site_general_challenge  \\\n",
       "0  ISIC_2637011  IP_7279968    male        45.0                     head/neck   \n",
       "1  ISIC_0015719  IP_3075186  female        45.0               upper extremity   \n",
       "2  ISIC_0052212  IP_2842074  female        50.0               lower extremity   \n",
       "3  ISIC_0068279  IP_6890425  female        45.0                     head/neck   \n",
       "4  ISIC_0074268  IP_8723313  female        55.0               upper extremity   \n",
       "\n",
       "  diagnosis benign_malignant  target  \n",
       "0   unknown           benign       0  \n",
       "1   unknown           benign       0  \n",
       "2     nevus           benign       0  \n",
       "3   unknown           benign       0  \n",
       "4   unknown           benign       0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "features=['sex','age_approx','anatom_site_general_challenge']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "targets=['diagnosis','benign_malignant','target']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preproccesing  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1) Filling NaN Values If Any "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sex                               65\n",
       "age_approx                        68\n",
       "anatom_site_general_challenge    527\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[features].isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_na(df):\n",
    "    df[\"anatom_site_general_challenge\"].fillna(value=\"Unkown\",inplace=True)\n",
    "    df[\"age_approx\"].fillna(value=int(df.age_approx.mean()),inplace=True)\n",
    "    df[\"sex\"].fillna(value=\"female\",inplace=True)\n",
    "    return df\n",
    "df=fill_na(df)\n",
    "test=fill_na(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Making sure we have filled all NaN values successfully"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (df[features].isna().sum()).all()==0\n",
    "assert (test[features].isna().sum()).all()==0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2) Encoding Features in Train DataFrame With CatBoostEncoding "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converting Age into 3 Different Categories "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "Means=KMeans(n_clusters=3).fit((df[\"age_approx\"].values).reshape(-1,1))\n",
    "def categorize_age(df,Means):\n",
    "    df[\"Age_Cat\"]=(Means.predict((df[\"age_approx\"].values).reshape(-1,1)))\n",
    "    df=pd.concat([df,pd.get_dummies(df['Age_Cat'],prefix=\"_age\")],axis=1).drop(['Age_Cat'],axis=1)\n",
    "    df=df.drop(\"age_approx\",axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cat_boost_encode(features,df):\n",
    "    Cat_Boost= ce.CatBoostEncoder()\n",
    "    Cat_Boost.fit(df[features],df[\"target\"])\n",
    "    df= df.join(Cat_Boost.transform(df[features]).add_suffix(\"_enc\"))\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def one_hot_encode(df):\n",
    "    df=pd.concat([df,pd.get_dummies(df['sex'])],axis=1).drop(['sex'],axis=1)\n",
    "    df=pd.concat([df,pd.get_dummies(df['anatom_site_general_challenge'])],axis=1).drop(['anatom_site_general_challenge'],axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>age_approx</th>\n",
       "      <th>anatom_site_general_challenge</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>benign_malignant</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>male</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>female</td>\n",
       "      <td>50.0</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>nevus</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>female</td>\n",
       "      <td>45.0</td>\n",
       "      <td>head/neck</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>female</td>\n",
       "      <td>55.0</td>\n",
       "      <td>upper extremity</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     image_name  patient_id     sex  age_approx anatom_site_general_challenge  \\\n",
       "0  ISIC_2637011  IP_7279968    male        45.0                     head/neck   \n",
       "1  ISIC_0015719  IP_3075186  female        45.0               upper extremity   \n",
       "2  ISIC_0052212  IP_2842074  female        50.0               lower extremity   \n",
       "3  ISIC_0068279  IP_6890425  female        45.0                     head/neck   \n",
       "4  ISIC_0074268  IP_8723313  female        55.0               upper extremity   \n",
       "\n",
       "  diagnosis benign_malignant  target  \n",
       "0   unknown           benign       0  \n",
       "1   unknown           benign       0  \n",
       "2     nevus           benign       0  \n",
       "3   unknown           benign       0  \n",
       "4   unknown           benign       0  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Creating Train DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=categorize_age(df,Means)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train=one_hot_encode(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>benign_malignant</th>\n",
       "      <th>target</th>\n",
       "      <th>_age_0</th>\n",
       "      <th>_age_1</th>\n",
       "      <th>_age_2</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "      <th>Unkown</th>\n",
       "      <th>head/neck</th>\n",
       "      <th>lower extremity</th>\n",
       "      <th>oral/genital</th>\n",
       "      <th>palms/soles</th>\n",
       "      <th>torso</th>\n",
       "      <th>upper extremity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>nevus</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     image_name  patient_id diagnosis benign_malignant  target  _age_0  \\\n",
       "0  ISIC_2637011  IP_7279968   unknown           benign       0       0   \n",
       "1  ISIC_0015719  IP_3075186   unknown           benign       0       0   \n",
       "2  ISIC_0052212  IP_2842074     nevus           benign       0       0   \n",
       "3  ISIC_0068279  IP_6890425   unknown           benign       0       0   \n",
       "4  ISIC_0074268  IP_8723313   unknown           benign       0       0   \n",
       "\n",
       "   _age_1  _age_2  female  male  Unkown  head/neck  lower extremity  \\\n",
       "0       0       1       0     1       0          1                0   \n",
       "1       0       1       1     0       0          0                0   \n",
       "2       0       1       1     0       0          0                1   \n",
       "3       0       1       1     0       0          1                0   \n",
       "4       0       1       1     0       0          0                0   \n",
       "\n",
       "   oral/genital  palms/soles  torso  upper extremity  \n",
       "0             0            0      0                0  \n",
       "1             0            0      0                1  \n",
       "2             0            0      0                0  \n",
       "3             0            0      0                0  \n",
       "4             0            0      0                1  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encoding Features In Test DataFrame"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converting age into 3 categories "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "test=categorize_age(test,Means)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>sex</th>\n",
       "      <th>anatom_site_general_challenge</th>\n",
       "      <th>_age_0</th>\n",
       "      <th>_age_1</th>\n",
       "      <th>_age_2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_0052060</td>\n",
       "      <td>IP_3579794</td>\n",
       "      <td>male</td>\n",
       "      <td>Unkown</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0052349</td>\n",
       "      <td>IP_7782715</td>\n",
       "      <td>male</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0058510</td>\n",
       "      <td>IP_7960270</td>\n",
       "      <td>female</td>\n",
       "      <td>torso</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0073313</td>\n",
       "      <td>IP_6375035</td>\n",
       "      <td>female</td>\n",
       "      <td>torso</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0073502</td>\n",
       "      <td>IP_0589375</td>\n",
       "      <td>female</td>\n",
       "      <td>lower extremity</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     image_name  patient_id     sex anatom_site_general_challenge  _age_0  \\\n",
       "0  ISIC_0052060  IP_3579794    male                        Unkown       0   \n",
       "1  ISIC_0052349  IP_7782715    male               lower extremity       1   \n",
       "2  ISIC_0058510  IP_7960270  female                         torso       0   \n",
       "3  ISIC_0073313  IP_6375035  female                         torso       0   \n",
       "4  ISIC_0073502  IP_0589375  female               lower extremity       0   \n",
       "\n",
       "   _age_1  _age_2  \n",
       "0       1       0  \n",
       "1       0       0  \n",
       "2       0       1  \n",
       "3       0       1  \n",
       "4       0       1  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "test=one_hot_encode(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>benign_malignant</th>\n",
       "      <th>target</th>\n",
       "      <th>_age_0</th>\n",
       "      <th>_age_1</th>\n",
       "      <th>_age_2</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "      <th>Unkown</th>\n",
       "      <th>head/neck</th>\n",
       "      <th>lower extremity</th>\n",
       "      <th>oral/genital</th>\n",
       "      <th>palms/soles</th>\n",
       "      <th>torso</th>\n",
       "      <th>upper extremity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>nevus</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     image_name  patient_id diagnosis benign_malignant  target  _age_0  \\\n",
       "0  ISIC_2637011  IP_7279968   unknown           benign       0       0   \n",
       "1  ISIC_0015719  IP_3075186   unknown           benign       0       0   \n",
       "2  ISIC_0052212  IP_2842074     nevus           benign       0       0   \n",
       "3  ISIC_0068279  IP_6890425   unknown           benign       0       0   \n",
       "4  ISIC_0074268  IP_8723313   unknown           benign       0       0   \n",
       "\n",
       "   _age_1  _age_2  female  male  Unkown  head/neck  lower extremity  \\\n",
       "0       0       1       0     1       0          1                0   \n",
       "1       0       1       1     0       0          0                0   \n",
       "2       0       1       1     0       0          0                1   \n",
       "3       0       1       1     0       0          1                0   \n",
       "4       0       1       1     0       0          0                0   \n",
       "\n",
       "   oral/genital  palms/soles  torso  upper extremity  \n",
       "0             0            0      0                0  \n",
       "1             0            0      0                1  \n",
       "2             0            0      0                0  \n",
       "3             0            0      0                0  \n",
       "4             0            0      0                1  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>benign_malignant</th>\n",
       "      <th>target</th>\n",
       "      <th>_age_0</th>\n",
       "      <th>_age_1</th>\n",
       "      <th>_age_2</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "      <th>Unkown</th>\n",
       "      <th>head/neck</th>\n",
       "      <th>lower extremity</th>\n",
       "      <th>oral/genital</th>\n",
       "      <th>palms/soles</th>\n",
       "      <th>torso</th>\n",
       "      <th>upper extremity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>nevus</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     image_name  patient_id diagnosis benign_malignant  target  _age_0  \\\n",
       "0  ISIC_2637011  IP_7279968   unknown           benign       0       0   \n",
       "1  ISIC_0015719  IP_3075186   unknown           benign       0       0   \n",
       "2  ISIC_0052212  IP_2842074     nevus           benign       0       0   \n",
       "3  ISIC_0068279  IP_6890425   unknown           benign       0       0   \n",
       "4  ISIC_0074268  IP_8723313   unknown           benign       0       0   \n",
       "\n",
       "   _age_1  _age_2  female  male  Unkown  head/neck  lower extremity  \\\n",
       "0       0       1       0     1       0          1                0   \n",
       "1       0       1       1     0       0          0                0   \n",
       "2       0       1       1     0       0          0                1   \n",
       "3       0       1       1     0       0          1                0   \n",
       "4       0       1       1     0       0          0                0   \n",
       "\n",
       "   oral/genital  palms/soles  torso  upper extremity  \n",
       "0             0            0      0                0  \n",
       "1             0            0      0                1  \n",
       "2             0            0      0                0  \n",
       "3             0            0      0                0  \n",
       "4             0            0      0                1  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "train['image_name']=train['image_name'] +'.jpg'\n",
    "test['image_name']=test['image_name'] +'.jpg'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image_name</th>\n",
       "      <th>patient_id</th>\n",
       "      <th>diagnosis</th>\n",
       "      <th>benign_malignant</th>\n",
       "      <th>target</th>\n",
       "      <th>_age_0</th>\n",
       "      <th>_age_1</th>\n",
       "      <th>_age_2</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "      <th>Unkown</th>\n",
       "      <th>head/neck</th>\n",
       "      <th>lower extremity</th>\n",
       "      <th>oral/genital</th>\n",
       "      <th>palms/soles</th>\n",
       "      <th>torso</th>\n",
       "      <th>upper extremity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ISIC_2637011.jpg</td>\n",
       "      <td>IP_7279968</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ISIC_0015719.jpg</td>\n",
       "      <td>IP_3075186</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ISIC_0052212.jpg</td>\n",
       "      <td>IP_2842074</td>\n",
       "      <td>nevus</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ISIC_0068279.jpg</td>\n",
       "      <td>IP_6890425</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ISIC_0074268.jpg</td>\n",
       "      <td>IP_8723313</td>\n",
       "      <td>unknown</td>\n",
       "      <td>benign</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         image_name  patient_id diagnosis benign_malignant  target  _age_0  \\\n",
       "0  ISIC_2637011.jpg  IP_7279968   unknown           benign       0       0   \n",
       "1  ISIC_0015719.jpg  IP_3075186   unknown           benign       0       0   \n",
       "2  ISIC_0052212.jpg  IP_2842074     nevus           benign       0       0   \n",
       "3  ISIC_0068279.jpg  IP_6890425   unknown           benign       0       0   \n",
       "4  ISIC_0074268.jpg  IP_8723313   unknown           benign       0       0   \n",
       "\n",
       "   _age_1  _age_2  female  male  Unkown  head/neck  lower extremity  \\\n",
       "0       0       1       0     1       0          1                0   \n",
       "1       0       1       1     0       0          0                0   \n",
       "2       0       1       1     0       0          0                1   \n",
       "3       0       1       1     0       0          1                0   \n",
       "4       0       1       1     0       0          0                0   \n",
       "\n",
       "   oral/genital  palms/soles  torso  upper extremity  \n",
       "0             0            0      0                0  \n",
       "1             0            0      0                1  \n",
       "2             0            0      0                0  \n",
       "3             0            0      0                0  \n",
       "4             0            0      0                1  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.head()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}