reg_train_test_split.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import datasets\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import xgboost as xgb\n",
    "\n",
    "datapath = r'.\\data'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Slice Localization data split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the data: slice_localization\n",
    "df = pd.read_csv(datapath+'\\slice_localization_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>patientId</th>\n",
       "      <th>value0</th>\n",
       "      <th>value1</th>\n",
       "      <th>value2</th>\n",
       "      <th>value3</th>\n",
       "      <th>value4</th>\n",
       "      <th>value5</th>\n",
       "      <th>value6</th>\n",
       "      <th>value7</th>\n",
       "      <th>value8</th>\n",
       "      <th>...</th>\n",
       "      <th>value375</th>\n",
       "      <th>value376</th>\n",
       "      <th>value377</th>\n",
       "      <th>value378</th>\n",
       "      <th>value379</th>\n",
       "      <th>value380</th>\n",
       "      <th>value381</th>\n",
       "      <th>value382</th>\n",
       "      <th>value383</th>\n",
       "      <th>reference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.980381</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>21.803851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.977008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>21.745726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.977008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>21.687600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.977008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>21.629474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>0.976833</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>-0.25</td>\n",
       "      <td>21.571348</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 386 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   patientId  value0  value1  value2  value3  value4  value5  value6  value7  \\\n",
       "0          0     0.0     0.0     0.0     0.0     0.0     0.0   -0.25   -0.25   \n",
       "1          0     0.0     0.0     0.0     0.0     0.0     0.0   -0.25   -0.25   \n",
       "2          0     0.0     0.0     0.0     0.0     0.0     0.0   -0.25   -0.25   \n",
       "3          0     0.0     0.0     0.0     0.0     0.0     0.0   -0.25   -0.25   \n",
       "4          0     0.0     0.0     0.0     0.0     0.0     0.0   -0.25   -0.25   \n",
       "\n",
       "   value8  ...  value375  value376  value377  value378  value379  value380  \\\n",
       "0   -0.25  ...     -0.25  0.980381       0.0       0.0       0.0       0.0   \n",
       "1   -0.25  ...     -0.25  0.977008       0.0       0.0       0.0       0.0   \n",
       "2   -0.25  ...     -0.25  0.977008       0.0       0.0       0.0       0.0   \n",
       "3   -0.25  ...     -0.25  0.977008       0.0       0.0       0.0       0.0   \n",
       "4   -0.25  ...     -0.25  0.976833       0.0       0.0       0.0       0.0   \n",
       "\n",
       "   value381  value382  value383  reference  \n",
       "0       0.0     -0.25     -0.25  21.803851  \n",
       "1       0.0     -0.25     -0.25  21.745726  \n",
       "2       0.0     -0.25     -0.25  21.687600  \n",
       "3       0.0     -0.25     -0.25  21.629474  \n",
       "4       0.0     -0.25     -0.25  21.571348  \n",
       "\n",
       "[5 rows x 386 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 0  1  2  3  4  5  6  7  8  9 10 11 12 13 14 15 16 17 18 19 20 21 22 23\n",
      " 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47\n",
      " 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71\n",
      " 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95\n",
      " 96] 97\n",
      "[ 4  9 14 19 24 29 34 39 44 49 54 59 64 69 74 79 84 89 94]\n",
      "7675 45825\n"
     ]
    }
   ],
   "source": [
    "# Slice data train and test split: Patient id shouldn't get mixed during train and test split\n",
    "u_pid = np.unique(df['patientId'])\n",
    "print(u_pid, len(u_pid))\n",
    "\n",
    "# 80 - 20% split based on patient id: \n",
    "test_idx = np.arange(1, int(len(u_pid)/5) + 1)*5 - 1\n",
    "print(test_idx)\n",
    "test = np.where(np.isin(df['patientId'],test_idx))[0]\n",
    "train = np.where(~np.isin(df['patientId'],test_idx))[0]\n",
    "print(len(test), len(train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(45825, 384) (7675, 384)\n"
     ]
    }
   ],
   "source": [
    "X_train,  y_train = df.iloc[train, 1:385].values, df.iloc[train, 385].values\n",
    "X_test, y_test = df.iloc[test, 1:385].values, df.iloc[test, 385].values\n",
    "print(X_train.shape, X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving train and test into csv file\n",
    "np.savez(datapath + '\\slice_localization_tr.npz', features=X_train, labels=y_train)\n",
    "np.savez(datapath + '\\slice_localization_te.npz', features=X_test, labels=y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Music Year Prediction data split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load the data: YearsPrediction\n",
    "df = pd.read_csv(datapath + '\\YearPredictionMSD.txt', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>81</th>\n",
       "      <th>82</th>\n",
       "      <th>83</th>\n",
       "      <th>84</th>\n",
       "      <th>85</th>\n",
       "      <th>86</th>\n",
       "      <th>87</th>\n",
       "      <th>88</th>\n",
       "      <th>89</th>\n",
       "      <th>90</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>49.94357</td>\n",
       "      <td>21.47114</td>\n",
       "      <td>73.07750</td>\n",
       "      <td>8.74861</td>\n",
       "      <td>-17.40628</td>\n",
       "      <td>-13.09905</td>\n",
       "      <td>-25.01202</td>\n",
       "      <td>-12.23257</td>\n",
       "      <td>7.83089</td>\n",
       "      <td>-2.46783</td>\n",
       "      <td>...</td>\n",
       "      <td>13.01620</td>\n",
       "      <td>-54.40548</td>\n",
       "      <td>58.99367</td>\n",
       "      <td>15.37344</td>\n",
       "      <td>1.11144</td>\n",
       "      <td>-23.08793</td>\n",
       "      <td>68.40795</td>\n",
       "      <td>-1.82223</td>\n",
       "      <td>-27.46348</td>\n",
       "      <td>2.26327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>48.73215</td>\n",
       "      <td>18.42930</td>\n",
       "      <td>70.32679</td>\n",
       "      <td>12.94636</td>\n",
       "      <td>-10.32437</td>\n",
       "      <td>-24.83777</td>\n",
       "      <td>8.76630</td>\n",
       "      <td>-0.92019</td>\n",
       "      <td>18.76548</td>\n",
       "      <td>4.59210</td>\n",
       "      <td>...</td>\n",
       "      <td>5.66812</td>\n",
       "      <td>-19.68073</td>\n",
       "      <td>33.04964</td>\n",
       "      <td>42.87836</td>\n",
       "      <td>-9.90378</td>\n",
       "      <td>-32.22788</td>\n",
       "      <td>70.49388</td>\n",
       "      <td>12.04941</td>\n",
       "      <td>58.43453</td>\n",
       "      <td>26.92061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>50.95714</td>\n",
       "      <td>31.85602</td>\n",
       "      <td>55.81851</td>\n",
       "      <td>13.41693</td>\n",
       "      <td>-6.57898</td>\n",
       "      <td>-18.54940</td>\n",
       "      <td>-3.27872</td>\n",
       "      <td>-2.35035</td>\n",
       "      <td>16.07017</td>\n",
       "      <td>1.39518</td>\n",
       "      <td>...</td>\n",
       "      <td>3.03800</td>\n",
       "      <td>26.05866</td>\n",
       "      <td>-50.92779</td>\n",
       "      <td>10.93792</td>\n",
       "      <td>-0.07568</td>\n",
       "      <td>43.20130</td>\n",
       "      <td>-115.00698</td>\n",
       "      <td>-0.05859</td>\n",
       "      <td>39.67068</td>\n",
       "      <td>-0.66345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>48.24750</td>\n",
       "      <td>-1.89837</td>\n",
       "      <td>36.29772</td>\n",
       "      <td>2.58776</td>\n",
       "      <td>0.97170</td>\n",
       "      <td>-26.21683</td>\n",
       "      <td>5.05097</td>\n",
       "      <td>-10.34124</td>\n",
       "      <td>3.55005</td>\n",
       "      <td>-6.36304</td>\n",
       "      <td>...</td>\n",
       "      <td>34.57337</td>\n",
       "      <td>-171.70734</td>\n",
       "      <td>-16.96705</td>\n",
       "      <td>-46.67617</td>\n",
       "      <td>-12.51516</td>\n",
       "      <td>82.58061</td>\n",
       "      <td>-72.08993</td>\n",
       "      <td>9.90558</td>\n",
       "      <td>199.62971</td>\n",
       "      <td>18.85382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>50.97020</td>\n",
       "      <td>42.20998</td>\n",
       "      <td>67.09964</td>\n",
       "      <td>8.46791</td>\n",
       "      <td>-15.85279</td>\n",
       "      <td>-16.81409</td>\n",
       "      <td>-12.48207</td>\n",
       "      <td>-9.37636</td>\n",
       "      <td>12.63699</td>\n",
       "      <td>0.93609</td>\n",
       "      <td>...</td>\n",
       "      <td>9.92661</td>\n",
       "      <td>-55.95724</td>\n",
       "      <td>64.92712</td>\n",
       "      <td>-17.72522</td>\n",
       "      <td>-1.49237</td>\n",
       "      <td>-7.50035</td>\n",
       "      <td>51.76631</td>\n",
       "      <td>7.88713</td>\n",
       "      <td>55.66926</td>\n",
       "      <td>28.74903</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 90 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         1         2         3         4         5         6         7   \\\n",
       "0  49.94357  21.47114  73.07750   8.74861 -17.40628 -13.09905 -25.01202   \n",
       "1  48.73215  18.42930  70.32679  12.94636 -10.32437 -24.83777   8.76630   \n",
       "2  50.95714  31.85602  55.81851  13.41693  -6.57898 -18.54940  -3.27872   \n",
       "3  48.24750  -1.89837  36.29772   2.58776   0.97170 -26.21683   5.05097   \n",
       "4  50.97020  42.20998  67.09964   8.46791 -15.85279 -16.81409 -12.48207   \n",
       "\n",
       "         8         9        10  ...        81         82        83        84  \\\n",
       "0 -12.23257   7.83089 -2.46783  ...  13.01620  -54.40548  58.99367  15.37344   \n",
       "1  -0.92019  18.76548  4.59210  ...   5.66812  -19.68073  33.04964  42.87836   \n",
       "2  -2.35035  16.07017  1.39518  ...   3.03800   26.05866 -50.92779  10.93792   \n",
       "3 -10.34124   3.55005 -6.36304  ...  34.57337 -171.70734 -16.96705 -46.67617   \n",
       "4  -9.37636  12.63699  0.93609  ...   9.92661  -55.95724  64.92712 -17.72522   \n",
       "\n",
       "         85        86         87        88         89        90  \n",
       "0   1.11144 -23.08793   68.40795  -1.82223  -27.46348   2.26327  \n",
       "1  -9.90378 -32.22788   70.49388  12.04941   58.43453  26.92061  \n",
       "2  -0.07568  43.20130 -115.00698  -0.05859   39.67068  -0.66345  \n",
       "3 -12.51516  82.58061  -72.08993   9.90558  199.62971  18.85382  \n",
       "4  -1.49237  -7.50035   51.76631   7.88713   55.66926  28.74903  \n",
       "\n",
       "[5 rows x 90 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# YearsPrediction data\n",
    "data = df.loc[:, 1:]\n",
    "target = df.loc[:, 0]\n",
    "\n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(463715, 90) (51630, 90)\n"
     ]
    }
   ],
   "source": [
    "# YearsPrediction split\n",
    "cut = 463715 # From the website\n",
    "X_train, y_train = data[:cut].values, target[:cut].values\n",
    "X_test, y_test = data[cut:].values, target[cut:].values\n",
    "\n",
    "print(X_train.shape, X_test.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Saving train and test into npz file\n",
    "np.savez(datapath + '\\YearPredictionMSD_tr.npz', features=X_train, labels=y_train)\n",
    "np.savez(datapath + '\\YearPredictionMSD_te.npz', features=X_test, labels=y_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}