grid_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPRegressor\n",
    "from scipy import sparse\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV\n",
    "import numpy as np\n",
    "from numpy import genfromtxt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.load(\"data/data_prepared.npy\")\n",
    "y = genfromtxt('data/data_prepared_labels.csv', delimiter=',')\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.1, random_state=42)\n",
    "X_train, Y_train = X_train[:50000], y_train[:50000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "regressor = MLPRegressor(max_iter = 50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, ), (100, ), (200, ), (50, 50, ), (100, 100, ), (200, 200, ), (50, 50, 50, ), (100, 100, 100, ), (200, 200, 200, ), (50, 50, 50, 50, ), (100, 100, 100, 100, ), (200, 200, 200, 200, ), "
     ]
    }
   ],
   "source": [
    "for x in range(1, 5):\n",
    "    for y in [50, 100, 200]:\n",
    "        print(\"(\", end = '')\n",
    "        for z in range(x):\n",
    "            print(str(y) + ', ', end = '')\n",
    "        print(\")\", end = ', ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First try to find the best size"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = [\n",
    "    {'batch_size' : [256], 'hidden_layer_sizes' : [(50, ), (100, ), (200, ), (50, 50, ), (100, 100, ), (200, 200, ), (50, 50, 50, ), (100, 100, 100, ), (200, 200, 200, ), (50, 50, 50, 50, ), (100, 100, 100, 100, ), (200, 200, 200, 200, ),]},\n",
    "]\n",
    "grid_search = GridSearchCV(regressor, param_grid, cv = 5, scoring='neg_mean_squared_error', return_train_score=True, verbose=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "343.97044654195463 {'hidden_layer_sizes': (50,)}\n",
      "338.76356011308667 {'hidden_layer_sizes': (100,)}\n",
      "335.19960434802795 {'hidden_layer_sizes': (200,)}\n",
      "333.9572750222429 {'hidden_layer_sizes': (50, 50)}\n",
      "331.02353830870595 {'hidden_layer_sizes': (100, 100)}\n",
      "330.7279477365592 {'hidden_layer_sizes': (200, 200)}\n",
      "329.37718391076174 {'hidden_layer_sizes': (50, 50, 50)}\n",
      "330.6324426599605 {'hidden_layer_sizes': (100, 100, 100)}\n",
      "333.9530040321373 {'hidden_layer_sizes': (200, 200, 200)}\n",
      "330.6829529907919 {'hidden_layer_sizes': (50, 50, 50, 50)}\n",
      "332.8201530916484 {'hidden_layer_sizes': (100, 100, 100, 100)}\n",
      "344.5413814210838 {'hidden_layer_sizes': (200, 200, 200, 200)}\n",
      "341.1986675676969 {'batch_size': 256}\n",
      "331.4010698442091 {'learning_rate_init': 0.01}\n"
     ]
    }
   ],
   "source": [
    "cvres = grid_search.cv_results_\n",
    "for mean_score, params in zip(cvres[\"mean_test_score\"], cvres[\"params\"]):\n",
    "    print(np.sqrt(-mean_score), params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we search for the best batch size and learning rate, also we try different numbers hidden layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = [\n",
    "    {'hidden_layer_sizes' : [(1000, 500, 100), (1000), (500, 250, 100), (100, 50), (200, 100)]},\n",
    "    {'batch_size' : [16, 32, 64, 128]},\n",
    "    {'learning_rate_init' : [0.005, 0.007, 0.01]}\n",
    "]\n",
    "\n",
    "grid_search = GridSearchCV(regressor, param_grid, cv = 5, scoring='neg_mean_squared_error', return_train_score=True, verbose=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_search.fit(X_train, Y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "346.3243302380783 {'hidden_layer_sizes': (1000, 500, 100)}\n",
      "335.9010449163782 {'hidden_layer_sizes': 1000}\n",
      "342.22532887126033 {'hidden_layer_sizes': (500, 250, 100)}\n",
      "338.42153737630036 {'hidden_layer_sizes': (100, 50)}\n",
      "339.49327684992573 {'hidden_layer_sizes': (200, 100)}\n",
      "335.0155828897767 {'batch_size': 16}\n",
      "336.58638198467 {'batch_size': 32}\n",
      "339.4732166896819 {'batch_size': 64}\n",
      "349.2103584093623 {'batch_size': 128}\n",
      "334.7839751822925 {'learning_rate_init': 0.005}\n",
      "334.2307120308362 {'learning_rate_init': 0.007}\n",
      "335.7986563713893 {'learning_rate_init': 0.01}\n"
     ]
    }
   ],
   "source": [
    "cvres = grid_search.cv_results_\n",
    "for mean_score, params in zip(cvres[\"mean_test_score\"], cvres[\"params\"]):\n",
    "    print(np.sqrt(-mean_score), params)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}