6.2-Tuning-Shallow-Model-Keras-Tuner.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install -r https://raw.githubusercontent.com/datamllab/automl-in-action-notebooks/master/requirements.txt\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import keras_tuner as kt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Load data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# Import the dataset loading function from sklearn\n",
    "from sklearn.datasets import load_digits\n",
    "\n",
    "# Load the hand-written digits dataset\n",
    "digits = load_digits()\n",
    "\n",
    "# Get the images and corresponding labels\n",
    "images, labels = digits.images, digits.target\n",
    "images.shape, labels.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## Exploratory data analysis & Data preprocessing &  feature engineering\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# reshape images to vectors\n",
    "n_samples = len(images)\n",
    "X = images.reshape((n_samples, -1))\n",
    "X.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# Split data into train and test subsets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, labels, test_size=0.2, shuffle=False\n",
    ")\n",
    "\n",
    "print(\"Shape of the training data: {}\".format(X_train.shape))\n",
    "print(\"Shape of the testing data: {}\".format(X_test.shape))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "X_train.shape, X_test.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "%%matplotlib inline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# plot first 20 images\n",
    "n = 20\n",
    "_, axes = plt.subplots(2, 10, figsize=(10, 2))\n",
    "plt.tight_layout()\n",
    "for i in range(n):\n",
    "    row, col = i // 10, i % 10\n",
    "    axes[row, col].set_axis_off()\n",
    "    axes[row, col].imshow(\n",
    "        images[\n",
    "            i,\n",
    "        ],\n",
    "        cmap=plt.cm.gray_r,\n",
    "        interpolation=\"nearest\",\n",
    "    )\n",
    "    axes[row, col].set_title(\"Label: %i\" % labels[i])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## KerasTuner tuning: joint hyperparameter tuning and model selection\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from keras_tuner.engine import hyperparameters as hp\n",
    "\n",
    "\n",
    "def build_model(hp):\n",
    "    model_type = hp.Choice(\"model_type\", [\"svm\", \"random_forest\"])\n",
    "    if model_type == \"svm\":\n",
    "        with hp.conditional_scope(\"model_type\", \"svm\"):\n",
    "            model = SVC(\n",
    "                C=hp.Float(\"C\", 1e-3, 10, sampling=\"linear\", default=1),\n",
    "                kernel=hp.Choice(\"kernel_type\", [\"linear\", \"rbf\"], default=\"linear\"),\n",
    "                random_state=42,\n",
    "            )\n",
    "    elif model_type == \"random_forest\":\n",
    "        with hp.conditional_scope(\"model_type\", \"random_forest\"):\n",
    "            model = RandomForestClassifier(\n",
    "                n_estimators=hp.Int(\"n_estimators\", 10, 200, step=10),\n",
    "                max_depth=hp.Int(\"max_depth\", 3, 10),\n",
    "            )\n",
    "    else:\n",
    "        raise ValueError(\"Unrecognized model_type\")\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Random Search Algorithm\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "import sklearn.pipeline\n",
    "\n",
    "random_tuner = kt.tuners.SklearnTuner(\n",
    "    oracle=kt.oracles.RandomSearch(\n",
    "        objective=kt.Objective(\"score\", \"max\"), max_trials=30, seed=42\n",
    "    ),\n",
    "    hypermodel=build_model,\n",
    "    cv=KFold(3, shuffle=True, random_state=42),\n",
    "    overwrite=True,\n",
    "    project_name=\"random_tuner\",\n",
    ")\n",
    "\n",
    "random_tuner.search(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "random_tuner.search_space_summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "random_tuner.results_summary(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# Evaluate the best discovered model\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "best_model = random_tuner.get_best_models(1)[0]\n",
    "best_model.fit(X_train, y_train)\n",
    "y_pred_test = best_model.predict(X_test)\n",
    "test_acc = accuracy_score(y_test, y_pred_test)\n",
    "print(\"The prediction accuracy on test set: {:.2f} %\".format(test_acc * 100))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "## KerasTuner tuning: whole pipeline with shallow models\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from keras_tuner.engine import hyperparameters as hp\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "\n",
    "def build_pipeline(hp):\n",
    "\n",
    "    # select hyperparameters of PCA\n",
    "    n_components = hp.Choice(\"n_components\", [2, 5, 10], default=5)\n",
    "    pca = PCA(n_components=n_components)\n",
    "\n",
    "    # select model type\n",
    "    model_type = hp.Choice(\"model_type\", [\"svm\", \"random_forest\"])\n",
    "    if model_type == \"svm\":\n",
    "        with hp.conditional_scope(\"model_type\", \"svm\"):\n",
    "            model = SVC(\n",
    "                C=hp.Float(\"C\", 1e-3, 10, sampling=\"linear\", default=1),\n",
    "                kernel=hp.Choice(\"kernel_type\", [\"linear\", \"rbf\"], default=\"linear\"),\n",
    "                random_state=42,\n",
    "            )\n",
    "    elif model_type == \"random_forest\":\n",
    "        with hp.conditional_scope(\"model_type\", \"random_forest\"):\n",
    "            model = RandomForestClassifier(\n",
    "                n_estimators=hp.Int(\"n_estimators\", 10, 200, step=10),\n",
    "                max_depth=hp.Int(\"max_depth\", 3, 10),\n",
    "            )\n",
    "    else:\n",
    "        raise ValueError(\"Unrecognized model_type\")\n",
    "\n",
    "    # create pipeline\n",
    "    pipeline = Pipeline([(\"pca\", pca), (\"clf\", model)])\n",
    "\n",
    "    return pipeline\n",
    "\n",
    "\n",
    "tuner = kt.tuners.Sklearn(\n",
    "    oracle=kt.oracles.RandomSearch(\n",
    "        objective=kt.Objective(\"score\", \"max\"), max_trials=3\n",
    "    ),\n",
    "    hypermodel=build_pipeline,\n",
    "    overwrite=True,\n",
    ")\n",
    "tuner.search(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "tuner.results_summary(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# Evaluate the best discovered model\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "best_pipeline = tuner.get_best_models(1)[0]\n",
    "best_pipeline.fit(X_train, y_train)\n",
    "y_pred_test = best_pipeline.predict(X_test)\n",
    "test_acc = accuracy_score(y_test, y_pred_test)\n",
    "print(\"The prediction accuracy on test set: {:.2f} %\".format(test_acc * 100))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "bo_tuner = kt.tuners.SklearnTuner(\n",
    "    oracle=kt.oracles.BayesianOptimization(\n",
    "        objective=kt.Objective(\"score\", \"max\"), max_trials=30, seed=42\n",
    "    ),\n",
    "    hypermodel=build_pipeline,\n",
    "    cv=KFold(3, shuffle=True, random_state=42),\n",
    "    overwrite=True,\n",
    "    project_name=\"bo_tuner\",\n",
    ")\n",
    "\n",
    "bo_tuner.search(X_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "bo_tuner.search_space_summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "bo_tuner.results_summary(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "# Evaluate the best discovered model\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "best_model = bo_tuner.get_best_models(1)[0]\n",
    "best_model.fit(X_train, y_train)\n",
    "y_pred_test = best_model.predict(X_test)\n",
    "test_acc = accuracy_score(y_test, y_pred_test)\n",
    "print(\"The prediction accuracy on test set: {:.2f} %\".format(test_acc * 100))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "# Automated Feature Engineering\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "TRAIN_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/train.csv\"\n",
    "TEST_DATA_URL = \"https://storage.googleapis.com/tf-datasets/titanic/eval.csv\"\n",
    "\n",
    "train_file_path = tf.keras.utils.get_file(\"train.csv\", TRAIN_DATA_URL)\n",
    "test_file_path = tf.keras.utils.get_file(\"eval.csv\", TEST_DATA_URL)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "x_train = pd.read_csv(train_file_path)\n",
    "y_train = x_train.pop(\"survived\")\n",
    "y_train = pd.DataFrame(y_train)\n",
    "\n",
    "x_test = pd.read_csv(test_file_path)\n",
    "y_test = x_test.pop(\"survived\")\n",
    "\n",
    "x_train.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OrdinalEncoder, OneHotEncoder, StandardScaler\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "numerical_columns = [\"age\", \"fare\"]\n",
    "one_hot_columns = [\"n_siblings_spouses\", \"class\"]\n",
    "int_columns = [\"sex\", \"parch\", \"deck\", \"embark_town\", \"alone\"]\n",
    "\n",
    "numerical_transformer = Pipeline(\n",
    "    steps=[\n",
    "        (\"imputer\", SimpleImputer(strategy=\"median\")),\n",
    "        (\"normalizer\", StandardScaler()),\n",
    "    ]\n",
    ")\n",
    "\n",
    "one_hot_transformer = Pipeline(\n",
    "    steps=[\n",
    "        (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"None\")),\n",
    "        (\"one_hot_encoder\", OneHotEncoder(handle_unknown=\"ignore\")),\n",
    "    ]\n",
    ")\n",
    "\n",
    "int_transformer = Pipeline(\n",
    "    steps=[\n",
    "        (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"None\")),\n",
    "        (\n",
    "            \"label_encoder\",\n",
    "            OrdinalEncoder(handle_unknown=\"use_encoded_value\", unknown_value=-1),\n",
    "        ),\n",
    "    ]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "\n",
    "\n",
    "class CategoricalCombination(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, name_a, name_b):\n",
    "        self.name_a = name_a\n",
    "        self.name_b = name_b\n",
    "        self.encoder = OrdinalEncoder(\n",
    "            handle_unknown=\"use_encoded_value\", unknown_value=-1\n",
    "        )\n",
    "\n",
    "    def fit(self, x, y=None, **kwargs):\n",
    "        temp_column = x[self.name_a].astype(str) + x[self.name_b].astype(str)\n",
    "        self.encoder.fit(temp_column.to_frame())\n",
    "        return self\n",
    "\n",
    "    def transform(self, x, **kwargs):\n",
    "        temp_column = x[self.name_a].astype(str) + x[self.name_b].astype(str)\n",
    "        temp_column = self.encoder.transform(temp_column.to_frame())\n",
    "        return temp_column\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "temp_data = pd.DataFrame({\"1\": [\"A\", \"A\", \"A\", \"B\", \"B\", \"B\"], \"2\": [1, 0, 1, 0, 1, 1]})\n",
    "print(temp_data.head(6))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "transformer = CategoricalCombination(\"1\", \"2\")\n",
    "print(transformer.fit_transform(temp_data))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "\n",
    "class MeanEncoder(BaseEstimator, TransformerMixin):\n",
    "    def __init__(self, categorical_name, numerical_name):\n",
    "        self.categorical_name = categorical_name\n",
    "        self.numerical_name = numerical_name\n",
    "        self.means = None\n",
    "\n",
    "    def fit(self, x, y=None, **kwargs):\n",
    "        self.mean = x.groupby(self.categorical_name)[self.numerical_name].mean()\n",
    "        return self\n",
    "\n",
    "    def transform(self, x, **kwargs):\n",
    "        return x[self.categorical_name].map(self.mean).to_frame()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "temp_data = pd.DataFrame(\n",
    "    {\"a\": [\"A\", \"A\", \"B\", \"B\", \"C\", \"C\"], \"b\": [1, 1, 1, 0, 1, -1]}\n",
    ")\n",
    "print(temp_data.head(6))\n",
    "encoder = MeanEncoder(\"a\", \"b\")\n",
    "print(encoder.fit_transform(temp_data).head(6))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "column_transformer = ColumnTransformer(\n",
    "    transformers=[\n",
    "        (\"numerical\", numerical_transformer, numerical_columns),\n",
    "        (\"one_hot\", one_hot_transformer, one_hot_columns),\n",
    "        (\"int\", int_transformer, int_columns),\n",
    "        (\n",
    "            \"categorical_combination\",\n",
    "            CategoricalCombination(\"sex\", \"class\"),\n",
    "            [\"sex\", \"class\"],\n",
    "        ),\n",
    "        (\"mean\", MeanEncoder(\"embark_town\", \"age\"), [\"embark_town\", \"age\"]),\n",
    "    ]\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.feature_selection import mutual_info_classif\n",
    "\n",
    "pipeline = Pipeline(\n",
    "    steps=[\n",
    "        (\"preprocessing\", column_transformer),\n",
    "        (\"feature_selection\", SelectKBest(mutual_info_classif, k=8)),\n",
    "        (\"model\", SVC()),\n",
    "    ]\n",
    ")\n",
    "pipeline.fit(x_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "y_pred = pipeline.predict(x_test)\n",
    "print(\"Accuracy: \", accuracy_score(y_test, y_pred))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "mean_column_pairs = []\n",
    "for int_col in int_columns:\n",
    "    for num_col in numerical_columns:\n",
    "        mean_column_pairs.append((int_col, num_col))\n",
    "\n",
    "cat_column_pairs = []\n",
    "for index1 in range(len(int_columns)):\n",
    "    for index2 in range(index1 + 1, len(int_columns)):\n",
    "        cat_column_pairs.append((int_columns[index1], int_columns[index2]))\n",
    "\n",
    "mean_column_pairs = np.array(mean_column_pairs)\n",
    "cat_column_pairs = np.array(cat_column_pairs)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "transformers = []\n",
    "for index, (col1, col2) in enumerate(cat_column_pairs):\n",
    "    if not hp.Boolean(\"combine_{i}\".format(i=index)):\n",
    "        continue\n",
    "    col1 = str(col1)\n",
    "    col2 = str(col2)\n",
    "    transformers.append((col1 + col2, CategoricalCombination(col1, col2), [col1, col2]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import keras_tuner as kt\n",
    "\n",
    "\n",
    "def build_model(hp):\n",
    "    numerical_transformer = Pipeline(\n",
    "        steps=[\n",
    "            (\"imputer\", SimpleImputer(strategy=\"median\")),\n",
    "            (\"normalizer\", StandardScaler()),\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    one_hot_transformer = Pipeline(\n",
    "        steps=[\n",
    "            (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"None\")),\n",
    "            (\"one_hot_encoder\", OneHotEncoder(handle_unknown=\"ignore\")),\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    int_transformer = Pipeline(\n",
    "        steps=[\n",
    "            (\"imputer\", SimpleImputer(strategy=\"constant\", fill_value=\"None\")),\n",
    "            (\n",
    "                \"label_encoder\",\n",
    "                OrdinalEncoder(handle_unknown=\"use_encoded_value\", unknown_value=-1),\n",
    "            ),\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    transformers = [\n",
    "        (\"numerical\", numerical_transformer, numerical_columns),\n",
    "        (\"one_hot\", one_hot_transformer, one_hot_columns),\n",
    "        (\"int\", int_transformer, int_columns),\n",
    "    ]\n",
    "    for index, (col1, col2) in enumerate(cat_column_pairs):\n",
    "        if not hp.Boolean(\"combine_{i}\".format(i=index)):\n",
    "            continue\n",
    "        col1 = str(col1)\n",
    "        col2 = str(col2)\n",
    "        transformers.append(\n",
    "            (col1 + col2, CategoricalCombination(col1, col2), [col1, col2])\n",
    "        )\n",
    "\n",
    "    for index, (col1, col2) in enumerate(mean_column_pairs):\n",
    "        if not hp.Boolean(\"mean_{i}\".format(i=index)):\n",
    "            continue\n",
    "        col1 = str(col1)\n",
    "        col2 = str(col2)\n",
    "        transformers.append((col1 + col2, MeanEncoder(col1, col2), [col1, col2]))\n",
    "    print(transformers)\n",
    "    pipeline = Pipeline(\n",
    "        steps=[\n",
    "            (\"preprocessing\", ColumnTransformer(transformers=transformers)),\n",
    "            (\"impute\", SimpleImputer(strategy=\"median\")),\n",
    "            (\n",
    "                \"model_selection\",\n",
    "                SelectKBest(\n",
    "                    mutual_info_classif,\n",
    "                    k=hp.Int(\"best_k\", 5, 13 + len(transformers) - 3),\n",
    "                ),\n",
    "            ),\n",
    "            (\"model\", SVC()),\n",
    "        ]\n",
    "    )\n",
    "\n",
    "    return pipeline\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "build_model(kt.HyperParameters()).fit(x_train, y_train)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "import sklearn\n",
    "\n",
    "tuner = kt.SklearnTuner(\n",
    "    kt.oracles.RandomSearchOracle(\n",
    "        objective=kt.Objective(\"score\", \"max\"),\n",
    "        max_trials=10,\n",
    "    ),\n",
    "    build_model,\n",
    "    scoring=metrics.make_scorer(metrics.accuracy_score),\n",
    "    overwrite=True,\n",
    ")\n",
    "tuner.search(x_train, y_train)\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "6.2-Tuning-Shallow-Model-Keras-Tuner",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}