7.4-Evolutionary-Search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "!pip install -r https://raw.githubusercontent.com/datamllab/automl-in-action-notebooks/master/requirements.txt\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Load dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.datasets import fetch_california_housing\n",
    "\n",
    "house_dataset = fetch_california_housing()\n",
    "\n",
    "# Import pandas package to format the data\n",
    "import pandas as pd\n",
    "\n",
    "# Extract features with their names into the a dataframe format\n",
    "data = pd.DataFrame(house_dataset.data, columns=house_dataset.feature_names)\n",
    "\n",
    "# Extract target with their names into a pd.Series object with name MEDV\n",
    "target = pd.Series(house_dataset.target, name=\"MEDV\")\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    data, target, test_size=0.2, random_state=42\n",
    ")\n",
    "X_train, X_val, y_train, y_val = train_test_split(\n",
    "    X_train, y_train, test_size=0.2, shuffle=False\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "X_train.shape, X_test.shape\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Create the LightGBM model building function (search space)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "\n",
    "def build_model(hp):\n",
    "    model = lgb.LGBMRegressor(\n",
    "        boosting_type=\"gbdt\",\n",
    "        num_leaves=hp.Int(\"num_leaves\", 5, 50, step=1),\n",
    "        learning_rate=hp.Float(\"learning_rate\", 1e-3, 1, sampling=\"log\", default=0.01),\n",
    "        n_estimators=hp.Int(\"n_estimators\", 5, 50, step=1),\n",
    "    )\n",
    "\n",
    "    return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Customize tuner\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pickle\n",
    "import tensorflow as tf\n",
    "import keras_tuner as kt\n",
    "import lightgbm as lgb\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "\n",
    "class LightGBMTuner(kt.engine.base_tuner.BaseTuner):\n",
    "    def run_trial(self, trial, X, y, validation_data):\n",
    "        model = self.hypermodel.build(trial.hyperparameters)  # build the model\n",
    "        model.fit(\n",
    "            X_train,\n",
    "            y_train,\n",
    "            eval_set=[validation_data],\n",
    "            eval_metric=\"mse\",\n",
    "            early_stopping_rounds=5,\n",
    "        )  # fit the model\n",
    "        X_val, y_val = validation_data\n",
    "        y_pred = model.predict(\n",
    "            X_val, num_iteration=model.best_iteration_\n",
    "        )  # evaluate the model\n",
    "        eval_mse = mean_squared_error(y_val, y_pred)\n",
    "        self.save_model(trial.trial_id, model)  # save the model to disk\n",
    "        # inform the oracle of the eval result, the result is a dictionary with the metric names as the keys.\n",
    "        return {\"mse\": eval_mse}\n",
    "\n",
    "    def save_model(self, trial_id, model, step=0):\n",
    "        fname = os.path.join(self.get_trial_dir(trial_id), \"model.txt\")\n",
    "        model.booster_.save_model(fname, num_iteration=model.best_iteration_)\n",
    "\n",
    "    def load_model(self, trial):\n",
    "        fname = os.path.join(self.get_trial_dir(trial.trial_id), \"model.txt\")\n",
    "        model = lgb.Booster(model_file=fname)\n",
    "        return model\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Use customized evolutionary search algorithm to tune models\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "import numpy as np\n",
    "from keras_tuner.engine import hyperparameters as hp_module\n",
    "from keras_tuner.engine import oracle as oracle_module\n",
    "from keras_tuner.engine import trial as trial_lib\n",
    "\n",
    "\n",
    "class EvolutionaryOracle(oracle_module.Oracle):\n",
    "    \"\"\"Evolutionary search oracle.\n",
    "\n",
    "        It uses aging evluation algorithm following: https://arxiv.org/pdf/1802.01548.pdf.\n",
    "        # Arguments\n",
    "            objective: String or `kerastuner.Objective`. If a string,\n",
    "              the direction of the optimization (min or max) will be\n",
    "              inferred.\n",
    "            max_trials: Int. Total number of trials\n",
    "                (model configurations) to test at most.\n",
    "                Note that the oracle may interrupt the search\n",
    "                before `max_trial` models have been tested if the search space has been\n",
    "                exhausted.\n",
    "            num_initial_points: (Optional) Int. The number of randomly generated samples\n",
    "                as initial training data for Evolutionary search. If not specified,\n",
    "                a value of 3 times the dimensionality of the hyperparameter space is\n",
    "                used.\n",
    "            population_size: (Optional) Int. The number of trials to form the populations.\n",
    "    candidate_size: (Optional) Int. The number of candidate trials in the tournament\n",
    "    selection.\n",
    "            seed: Int. Random seed.\n",
    "            hyperparameters: HyperParameters class instance.\n",
    "                Can be used to override (or register in advance)\n",
    "                hyperparamters in the search space.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        objective,\n",
    "        max_trials,\n",
    "        num_initial_points=None,\n",
    "        population_size=None,\n",
    "        candidate_size=None,\n",
    "        seed=None,\n",
    "        hyperparameters=None,\n",
    "        *args,\n",
    "        **kwargs\n",
    "    ):\n",
    "        super(EvolutionaryOracle, self).__init__(\n",
    "            objective=objective,\n",
    "            max_trials=max_trials,\n",
    "            hyperparameters=hyperparameters,\n",
    "            seed=seed,\n",
    "            *args,\n",
    "            **kwargs\n",
    "        )\n",
    "        self.population_size = population_size or 20\n",
    "        self.candidate_size = candidate_size or 5\n",
    "        self.num_initial_points = num_initial_points or self.population_size\n",
    "        self.num_initial_points = max(self.num_initial_points, population_size)\n",
    "        self.population_trial_ids = []\n",
    "        self.seed = seed or random.randint(1, 1e4)\n",
    "        self._seed_state = self.seed\n",
    "        self._random_state = np.random.RandomState(self.seed)\n",
    "        self._max_collisions = 100\n",
    "\n",
    "    def _random_populate_space(self):\n",
    "        values = self._random_values()\n",
    "        if values is None:\n",
    "            return {\"status\": trial_lib.TrialStatus.STOPPED, \"values\": None}\n",
    "        return {\"status\": trial_lib.TrialStatus.RUNNING, \"values\": values}\n",
    "\n",
    "    def _num_completed_trials(self):\n",
    "        return len([t for t in self.trials.values() if t.status == \"COMPLETED\"])\n",
    "\n",
    "    def populate_space(self, trial_id):\n",
    "\n",
    "        if self._num_completed_trials() < self.num_initial_points:\n",
    "            return self._random_populate_space()\n",
    "\n",
    "        self.population_trial_ids = self.end_order[-self.population_size :]\n",
    "\n",
    "        # candidate trial selection\n",
    "        candidate_indices = self._random_state.choice(\n",
    "            self.population_size, self.candidate_size, replace=False\n",
    "        )\n",
    "        self.candidate_indices = candidate_indices\n",
    "        candidate_trial_ids = list(\n",
    "            map(self.population_trial_ids.__getitem__, candidate_indices)\n",
    "        )\n",
    "\n",
    "        # get the best candidate based on the performance\n",
    "        candidate_scores = [\n",
    "            self.trials[trial_id].score for trial_id in candidate_trial_ids\n",
    "        ]\n",
    "        best_candidate_trial_id = candidate_trial_ids[np.argmin(candidate_scores)]\n",
    "        best_candidate_trial = self.trials[best_candidate_trial_id]\n",
    "\n",
    "        # mutate the hps of the candidate\n",
    "        values = self._mutate(best_candidate_trial)\n",
    "\n",
    "        if values is None:\n",
    "            return {\"status\": trial_lib.TrialStatus.STOPPED, \"values\": None}\n",
    "\n",
    "        return {\"status\": trial_lib.TrialStatus.RUNNING, \"values\": values}\n",
    "\n",
    "    def _mutate(self, best_trial):\n",
    "\n",
    "        best_hps = best_trial.hyperparameters\n",
    "\n",
    "        # get non-fixed and active hyperparameters in the trial to be mutated\n",
    "        nonfixed_active_hps = [\n",
    "            hp\n",
    "            for hp in self.hyperparameters.space\n",
    "            if not isinstance(hp, hp_module.Fixed) and best_hps.is_active(hp)\n",
    "        ]\n",
    "\n",
    "        # random select a hyperparameter to mutate\n",
    "        hp_to_mutate = self._random_state.choice(nonfixed_active_hps, 1)[0]\n",
    "\n",
    "        collisions = 0\n",
    "        while True:\n",
    "            hps = hp_module.HyperParameters()\n",
    "            # Generate a set of random values.\n",
    "            for hp in self.hyperparameters.space:\n",
    "                hps.merge([hp])\n",
    "                # if not active, do nothing.\n",
    "                # if active, check if selected to be changed.\n",
    "                if hps.is_active(hp):\n",
    "                    # if was active and not selected, do nothing.\n",
    "                    if best_hps.is_active(hp.name) and hp.name != hp_to_mutate.name:\n",
    "                        hps.values[hp.name] = best_hps.values[hp.name]\n",
    "                        continue\n",
    "                    # if was not active or selected, sample.\n",
    "                    hps.values[hp.name] = hp.random_sample(self._seed_state)\n",
    "                    self._seed_state += 1\n",
    "            values = hps.values\n",
    "\n",
    "            # Make sure the new hyperparameters has not been evaluated before\n",
    "            # Keep trying until the set of values is unique,\n",
    "            # or until we exit due to too many collisions.\n",
    "            values_hash = self._compute_values_hash(values)\n",
    "            if values_hash in self._tried_so_far:\n",
    "                collisions += 1\n",
    "                if collisions <= self._max_collisions:\n",
    "                    continue\n",
    "                return None\n",
    "            self._tried_so_far.add(values_hash)\n",
    "            break\n",
    "        return values\n",
    "\n",
    "    def get_state(self):\n",
    "        state = super(EvolutionaryOracle, self).get_state()\n",
    "        state.update(\n",
    "            {\n",
    "                \"num_initial_points\": self.num_initial_points,\n",
    "                \"population_size\": self.population_size,\n",
    "                \"candidate_size\": self.candidate_size,\n",
    "                \"seed\": self.seed,\n",
    "                \"_max_collisions\": self._max_collisions,\n",
    "            }\n",
    "        )\n",
    "        return state\n",
    "\n",
    "    def set_state(self, state):\n",
    "        super(EvolutionaryOracle, self).set_state(state)\n",
    "        self.num_initial_points = state[\"num_initial_points\"]\n",
    "        self.population_size = state[\"population_size\"]\n",
    "        self.candidate_size = state[\"candidate_size\"]\n",
    "        self.population_trial_ids = self.end_order[-self.population_size :]\n",
    "        self.seed = state[\"seed\"]\n",
    "        self._random_state = np.random.RandomState(self.seed)\n",
    "        self._seed_state = self.seed\n",
    "        self._max_collisions = state[\"max_collisions\"]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "evo_tuner_p20c5 = LightGBMTuner(\n",
    "    oracle=EvolutionaryOracle(\n",
    "        objective=kt.Objective(\"mse\", \"min\"),\n",
    "        max_trials=100,\n",
    "        population_size=20,\n",
    "        candidate_size=5,\n",
    "        seed=42,\n",
    "    ),\n",
    "    hypermodel=build_model,\n",
    "    overwrite=True,\n",
    "    project_name=\"evo_tuner_p20c5\",\n",
    ")\n",
    "\n",
    "evo_tuner_p20c5.search(X_train, y_train, validation_data=(X_val, y_val))\n",
    "\n",
    "random_tuner = LightGBMTuner(\n",
    "    oracle=kt.oracles.RandomSearch(\n",
    "        objective=kt.Objective(\"mse\", \"min\"), max_trials=100, seed=42\n",
    "    ),\n",
    "    hypermodel=build_model,\n",
    "    overwrite=True,\n",
    "    project_name=\"random_tuner\",\n",
    ")\n",
    "\n",
    "random_tuner.search(X_train, y_train, validation_data=(X_val, y_val))\n",
    "\n",
    "\n",
    "bo_tuner = LightGBMTuner(\n",
    "    oracle=kt.oracles.BayesianOptimization(\n",
    "        objective=kt.Objective(\"mse\", \"min\"), max_trials=100, seed=42\n",
    "    ),\n",
    "    hypermodel=build_model,\n",
    "    overwrite=True,\n",
    "    project_name=\"bo_tuner\",\n",
    ")\n",
    "\n",
    "bo_tuner.search(X_train, y_train, validation_data=(X_val, y_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "best_model = random_tuner.get_best_models(1)[0]\n",
    "y_pred_test = best_model.predict(X_test)\n",
    "test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "print(\"The prediction MSE on test set: {}\".format(test_mse))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "random_tuner.results_summary(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "best_model = bo_tuner.get_best_models(1)[0]\n",
    "y_pred_test = best_model.predict(X_test)\n",
    "test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "print(\"The prediction MSE on test set: {}\".format(test_mse))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "bo_tuner.results_summary(1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "best_model = evo_tuner_p20c5.get_best_models(1)[0]\n",
    "y_pred_test = best_model.predict(X_test)\n",
    "test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "print(\"The prediction MSE on test set: {}\".format(test_mse))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "evo_tuner_p20c5.results_summary(1)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Plot search curves\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def plot_curve(x, y, xlabel, ylabel, title):\n",
    "    plt.plot(x, y)\n",
    "    plt.xlabel(xlabel)\n",
    "    plt.ylabel(ylabel)\n",
    "    plt.title(title)\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "def plot_curves(\n",
    "    x, ys, xlabel, ylabel, title, ymin, ymax, legend, markers, linestyles, markevery=1\n",
    "):\n",
    "    for i, y in enumerate(ys):\n",
    "        plt.plot(x, y, marker=markers[i], linestyle=linestyles[i], markevery=markevery)\n",
    "    plt.xlabel(xlabel)\n",
    "    plt.ylabel(ylabel)\n",
    "    plt.title(title)\n",
    "    plt.ylim(ymin, ymax)\n",
    "    plt.legend(legend)\n",
    "    plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "mse_evo = [\n",
    "    evo_tuner_p20c5.oracle.get_trial(trial_id).score\n",
    "    for trial_id in evo_tuner_p20c5.oracle.end_order\n",
    "]\n",
    "ids = list(range(len(mse_evo)))\n",
    "plot_curve(\n",
    "    ids, mse_evo, \"Trials in finishing order\", \"Validation MSE\", \"Searched results\"\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "high_value = float(\"inf\")\n",
    "high_mse_evo = []\n",
    "for value in mse_evo:\n",
    "    high_value = min(high_value, value)\n",
    "    high_mse_evo.append(high_value)\n",
    "plot_curve(\n",
    "    ids,\n",
    "    high_mse_evo,\n",
    "    \"Trials in finishing order\",\n",
    "    \"Highest validation MSE so far\",\n",
    "    \"Searched results\",\n",
    ")\n",
    "\n",
    "mse_random = [\n",
    "    random_tuner.oracle.get_trial(trial_id).score\n",
    "    for trial_id in random_tuner.oracle.end_order\n",
    "]\n",
    "mse_bo = [\n",
    "    bo_tuner.oracle.get_trial(trial_id).score for trial_id in bo_tuner.oracle.end_order\n",
    "]\n",
    "mse_evo = [\n",
    "    evo_tuner_p20c5.oracle.get_trial(trial_id).score\n",
    "    for trial_id in evo_tuner_p20c5.oracle.end_order\n",
    "]\n",
    "\n",
    "\n",
    "high_value = float(\"inf\")\n",
    "high_mse_random = []\n",
    "for value in mse_random:\n",
    "    high_value = min(high_value, value)\n",
    "    high_mse_random.append(high_value)\n",
    "\n",
    "high_value = float(\"inf\")\n",
    "high_mse_evo = []\n",
    "for value in mse_evo:\n",
    "    high_value = min(high_value, value)\n",
    "    high_mse_evo.append(high_value)\n",
    "\n",
    "high_value = float(\"inf\")\n",
    "high_mse_bo = []\n",
    "for value in mse_bo:\n",
    "    high_value = min(high_value, value)\n",
    "    high_mse_bo.append(high_value)\n",
    "\n",
    "plot_curves(\n",
    "    ids,\n",
    "    [mse_random, mse_bo, mse_evo],\n",
    "    \"Trials in finishing order\",\n",
    "    \"Validation MSE\",\n",
    "    \"Searched results\",\n",
    "    0,\n",
    "    1.5,\n",
    "    markers=[\"o\", \"+\", \"x\"],\n",
    "    linestyles=[\"-\", \"-.\", \"-\"],\n",
    "    legend=[\"Random search\", \"Bayesian optimization\", \"Aging evolution\"],\n",
    ")\n",
    "plot_curves(\n",
    "    ids,\n",
    "    [high_mse_random, high_mse_bo, high_mse_evo],\n",
    "    \"Trials in finishing order\",\n",
    "    \"Highest validation MSE so far\",\n",
    "    \"Searched results\",\n",
    "    0.2,\n",
    "    0.4,\n",
    "    markers=[\"o\", \"+\", \"x\"],\n",
    "    linestyles=[\"-\", \"-.\", \"-\"],\n",
    "    legend=[\"Random search\", \"Bayesian optimization\", \"Aging evolution\"],\n",
    "    markevery=5,\n",
    ")\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text"
   },
   "source": [
    "### Try to increase the candidate size\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "evo_tuner_p20c10 = LightGBMTuner(\n",
    "    oracle=EvolutionaryOracle(\n",
    "        objective=kt.Objective(\"mse\", \"min\"),\n",
    "        max_trials=100,\n",
    "        population_size=20,\n",
    "        candidate_size=20,\n",
    "        seed=42,\n",
    "    ),\n",
    "    hypermodel=build_model,\n",
    "    overwrite=True,\n",
    "    project_name=\"evo_tuner_p20c10\",\n",
    ")\n",
    "\n",
    "evo_tuner_p20c10.search(X_train, y_train, validation_data=(X_val, y_val))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "best_model = evo_tuner_p20c5.get_best_models(1)[0]\n",
    "y_pred_test = best_model.predict(X_test)\n",
    "test_mse = mean_squared_error(y_test, y_pred_test)\n",
    "print(\"The prediction MSE on test set: {}\".format(test_mse))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab_type": "code"
   },
   "outputs": [],
   "source": [
    "mse_evo_c10 = [\n",
    "    evo_tuner_p20c10.oracle.get_trial(trial_id).score\n",
    "    for trial_id in evo_tuner_p20c10.oracle.end_order\n",
    "]\n",
    "\n",
    "high_value = float(\"inf\")\n",
    "high_mse_evo_c10 = []\n",
    "for value in mse_evo_c10:\n",
    "    high_value = min(high_value, value)\n",
    "    high_mse_evo_c10.append(high_value)\n",
    "\n",
    "plot_curves(\n",
    "    ids,\n",
    "    [mse_random, mse_bo, mse_evo, mse_evo_c10],\n",
    "    \"Trials in finishing order\",\n",
    "    \"Validation MSE\",\n",
    "    \"Searched results\",\n",
    "    0,\n",
    "    1.5,\n",
    "    markers=[\"o\", \"+\", \"x\", \"^\"],\n",
    "    linestyles=[\"-\", \"-.\", \"-\", \":\"],\n",
    "    legend=[\n",
    "        \"Random search\",\n",
    "        \"Bayesian optimization\",\n",
    "        \"Aging evolution\",\n",
    "        \"Aging evolution2\",\n",
    "    ],\n",
    "    markevery=5,\n",
    ")\n",
    "plot_curves(\n",
    "    ids,\n",
    "    [high_mse_random, high_mse_bo, high_mse_evo, high_mse_evo_c10],\n",
    "    \"Trials in finishing order\",\n",
    "    \"Highest validation MSE so far\",\n",
    "    \"Searched results\",\n",
    "    0.2,\n",
    "    0.4,\n",
    "    markers=[\"o\", \"+\", \"x\", \"^\"],\n",
    "    linestyles=[\"-\", \"-.\", \"-\", \":\"],\n",
    "    legend=[\n",
    "        \"Random search\",\n",
    "        \"Bayesian optimization\",\n",
    "        \"Aging evolution (candidate size = 5)\",\n",
    "        \"Aging evolution (candidate size = 20)\",\n",
    "    ],\n",
    "    markevery=5,\n",
    ")\n"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "7.4-Evolutionary-Search",
   "private_outputs": false,
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}