PNN_SavingWeights.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import os\n",
    "import random\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras import backend as K\n",
    "from keras import initializers\n",
    "from keras.layers import Dense, Input, Activation, multiply, Lambda\n",
    "from keras.models import Sequential, Model, load_model\n",
    "from keras.layers.merge import add, concatenate\n",
    "\n",
    "from deap import base, creator, tools, algorithms\n",
    "from multiprocessing import Pool\n",
    "from scoop import futures\n",
    "\n",
    "import copy\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{0: 0, 1: 1, 2: 0.0006535895854646095}\n",
      "{0: 0, 1: 0.00011500694312440574}\n"
     ]
    }
   ],
   "source": [
    "act_dict = {0: 'linear', 1: 'multiply', 2: 'inverse', 3: 'squared', 4: 'sqrt'}\n",
    "np.random.seed(1000)\n",
    "weight_dict = {0: 0, 1: 1, 2: np.random.uniform(0,0.001,1)[0]}\n",
    "bias_dict = {0: 0, 1: np.random.uniform(0,0.001,1)[0]}\n",
    "print(weight_dict)\n",
    "print(bias_dict)\n",
    "nlayers = 4\n",
    "nNodes = [5, 5, 3, 1]\n",
    "nact_terms = sum(nNodes[1:])\n",
    "nweight_terms = sum([nNodes[i-1]*nNodes[i] for i in range(1, nlayers)])\n",
    "nbias_terms = nact_terms\n",
    "p = 0.1\n",
    "save_weights = False\n",
    "save_weights_mutation = False\n",
    "save_weights_crossover = False\n",
    "Load_Old_Population = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "Individual_DB = []\n",
    "MSE_Test_DB = []\n",
    "Weight_Bias_DB = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"PropellantData_v3.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputs = np.array(df[['a','b', 'c', 'd', 'HeatofFormation']])\n",
    "inputs = inputs.astype(np.float)\n",
    "outputs = np.array(df['Isp']).reshape(-1, 1)\n",
    "inputs = np.around(inputs, decimals=6)\n",
    "outputs = np.around(outputs, decimals=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomDense(keras.layers.Layer):\n",
    "    def __init__(self, num_units, input_num, activation, name, trainable_weight, trainable_bias):\n",
    "        super(CustomDense, self).__init__()\n",
    "        self.num_units = num_units\n",
    "        self.activation = Activation(activation)\n",
    "        self.trainable_weight = trainable_weight\n",
    "        self.trainable_bias = trainable_bias\n",
    "        self.name = name\n",
    "        name_w = 'w'+self.name[1:]\n",
    "        name_b = 'b'+self.name[1:]\n",
    "        self.weight = self.add_weight(shape=(input_num, self.num_units), name=name_w, trainable=self.trainable_weight, initializer=\"zeros\")\n",
    "        self.bias = self.add_weight(shape=(self.num_units,), name=name_b, trainable=self.trainable_bias, initializer=\"zeros\")\n",
    "        \n",
    "    def call(self, input):\n",
    "        y = tf.matmul(input, self.weight) + self.bias\n",
    "        y = self.activation(y)\n",
    "        return y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f3(w):\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Individual_O:\n",
    "    # Class to represent an individual\n",
    "    def __init__(self, nLayers, nNodes, max1, max2, max3):\n",
    "        self.nlayers = nLayers\n",
    "        self.nNodes = nNodes\n",
    "        self.nact_terms = sum(self.nNodes[1:])\n",
    "        self.nweight_terms = sum([self.nNodes[i-1]*self.nNodes[i] for i in range(1, self.nlayers)])\n",
    "        self.nbias_terms = self.nact_terms\n",
    "        self.individual = None\n",
    "        self.trainable = None\n",
    "        self.weight_list = None\n",
    "        self.bias_list = None\n",
    "        self.weight_biases = None\n",
    "        self.complexity = None\n",
    "        self.MSE_Test = None\n",
    "        self.Objective = None\n",
    "        self.fitness = None\n",
    "        self.model = None\n",
    "        self.history = None\n",
    "        self.epochs = None\n",
    "        self.max1 = max1\n",
    "        self.max2 = max2\n",
    "        self.max3 = max3\n",
    "    \n",
    "    #Returns a copy of the individual\n",
    "    def object_copy(self):\n",
    "        cp = Individual_O(self.nlayers, self.nNodes, self.max1, self.max2, self.max3)\n",
    "        cp.set_individual(copy.deepcopy(self.individual))\n",
    "        cp.set_weight_bias(copy.deepcopy(self.weight_biases))\n",
    "        return cp\n",
    "        \n",
    "        \n",
    "    # Creates a specific individual\n",
    "    def set_individual(self, individual):\n",
    "        self.individual = individual\n",
    "        self.calc_trainable()\n",
    "        self.calc_complexity()\n",
    "        self.set_random_weight_bias()\n",
    "        \n",
    "    def reset_model(self):\n",
    "        self.Objective = None\n",
    "        self.fitness = None\n",
    "        self.weight_list = None\n",
    "        self.bias_list = None\n",
    "        self.calc_trainable()\n",
    "        self.calc_complexity()\n",
    "        \n",
    "    # Creates a random individual\n",
    "    def randomize_individual(self):\n",
    "        x = []\n",
    "        for l in range(1, self.nlayers):\n",
    "            x.append([])\n",
    "            for n in range(1, self.nNodes[l]+1):\n",
    "                x[l-1].append([])\n",
    "                x[l-1][n-1].append(random.randint(0, self.max1))\n",
    "                for i in range(1, self.nNodes[l-1]+1):\n",
    "                    x[l-1][n-1].append(random.randint(0, self.max2))\n",
    "                x[l-1][n-1].append(random.randint(0, self.max3))\n",
    "        self.individual = x\n",
    "        self.calc_trainable()\n",
    "        self.set_random_weight_bias()\n",
    "        self.calc_complexity()\n",
    "        \n",
    "    # Creates trainable list\n",
    "    def calc_trainable(self):\n",
    "        trainable_list = []\n",
    "        for l in range(1, self.nlayers):\n",
    "            trainable_list.append([])\n",
    "            for n in range(self.nNodes[l]):\n",
    "                trainable_list[l-1].append([])\n",
    "                for i in range(1, nNodes[l-1]+1):\n",
    "                    if (self.individual[l-1][n][i] == 2):\n",
    "                        trainable_list[l-1][n].append(True)\n",
    "                    else:\n",
    "                        trainable_list[l-1][n].append(False)\n",
    "                if (self.individual[l-1][n][i+1] == 1):\n",
    "                    trainable_list[l-1][n].append(True)\n",
    "                else:\n",
    "                    trainable_list[l-1][n].append(False)\n",
    "        self.trainable = trainable_list\n",
    "        \n",
    "    # Calculates complexity of the model\n",
    "    def calc_complexity(self):\n",
    "        actfunc_term = 0\n",
    "        wtbs_term = 0\n",
    "        for l in range(1, self.nlayers):\n",
    "            for n in range(self.nNodes[l]):\n",
    "                actfunc_term += self.individual[l-1][n][0]\n",
    "                wtbs_term += np.sum(self.individual[l-1][n][1:nNodes[l]])\n",
    "                wtbs_term += np.sum(self.individual[l-1][n][nNodes[l]]*2)\n",
    "        self.complexity = actfunc_term + wtbs_term\n",
    "        \n",
    "    # Randomizes all the weights and biases\n",
    "    def set_random_weight_bias(self):\n",
    "        x = []\n",
    "        for l in range(1, self.nlayers):\n",
    "            x.append([])\n",
    "            for n in range(1, self.nNodes[l]+1):\n",
    "                x[l-1].append([])\n",
    "                for i in range(1, self.nNodes[l-1]+1):\n",
    "                    x[l-1][n-1].append(weight_dict[self.individual[l-1][n-1][i]])\n",
    "                x[l-1][n-1].append(bias_dict[self.individual[l-1][n-1][-1]])\n",
    "        self.weight_biases = x\n",
    "        \n",
    "    def format_weight_bias(self, weight, bias):\n",
    "        bias_count = 0\n",
    "        weight_count = 0\n",
    "        x = []\n",
    "        for l in range(1, self.nlayers):\n",
    "            x.append([])\n",
    "            for n in range(1, self.nNodes[l]+1):\n",
    "                x[l-1].append([])\n",
    "                for i in range(1, self.nNodes[l-1]+1):\n",
    "                    x[l-1][n-1].append(float(weight[weight_count]))\n",
    "                    if ((i)%(self.nNodes[l-1])==0):\n",
    "                        x[l-1][n-1].append(float(bias[bias_count]))\n",
    "                    weight_count += 1\n",
    "                    bias_count += 1\n",
    "        self.weight_biases = x\n",
    "        \n",
    "    def set_weight_bias(self, weight_biases):\n",
    "        self.weight_biases = weight_biases\n",
    "        \n",
    "    def update_weight_bias(self):\n",
    "        #Update Weight/Bias List\n",
    "        weight_list = []\n",
    "        bias_list = []\n",
    "        for i in range(0, sum(np.multiply(self.nNodes[:-1], self.nNodes[1:]))):\n",
    "            weights = self.model.layers[i].get_weights()\n",
    "            for weight in weights:\n",
    "                if (weight.shape == (1,1)):\n",
    "                    weight_list.append(weight)\n",
    "                else:\n",
    "                    bias_list.append(weight)\n",
    "        self.weight_list = np.array(weight_list)\n",
    "        self.bias_list = np.array(bias_list)\n",
    "        \n",
    "    # Builds the model\n",
    "    def create_model(self, constraint_strength=10^3):\n",
    "        x = self.individual\n",
    "        self.weight_biases\n",
    "        constraints = []\n",
    "    \n",
    "        inputs = []\n",
    "        for i in range(self.nNodes[0]):\n",
    "            inputs.append(Input(shape=(1,)))\n",
    "    \n",
    "        a = []\n",
    "        for l in range(1, self.nlayers):\n",
    "            a.append([])\n",
    "            for n in range(1, self.nNodes[l]+1):\n",
    "                a[l-1].append([])\n",
    "                if l == 1:\n",
    "                    a[l-1][n-1] = self.create_node(inputs, 'a' + str(l) + str(n), self.trainable[l-1][n-1], x[l-1][n-1], constraints)\n",
    "                else:\n",
    "                    a[l-1][n-1] = self.create_node(a[l-2], 'a' + str(l) + str(n), self.trainable[l-1][n-1], x[l-1][n-1], constraints)\n",
    "    \n",
    "    \n",
    "        # make some adaptations depending on how many root activation functions are used\n",
    "        if len(constraints) > 1:\n",
    "            constraint = add(constraints)\n",
    "        elif len(constraints) == 1:\n",
    "            constraint = constraints[0]\n",
    "        else:\n",
    "            constraint = Lambda(lambda x: x-x)(a[0][0])\n",
    "    \n",
    "        a[-1].append(constraint)\n",
    "        model = Model(inputs=inputs, outputs=a[-1])\n",
    "    \n",
    "        # Learning rate decay/schedule\n",
    "        lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=1e-1,\n",
    "                                                                decay_steps=50000,\n",
    "                                                                decay_rate=0.1,\n",
    "                                                                staircase=False)\n",
    "    \n",
    "        # Optimizers\n",
    "        #optimizer = tf.keras.optimizers.SGD(lr=1e-2)\n",
    "        #optimizer = tf.keras.optimizers.RMSprop(learning_rate=lr_schedule)\n",
    "        optimizer = tf.keras.optimizers.Adam(learning_rate=lr_schedule)\n",
    "        #optimizer = tf.keras.optimizers.Adadelta(learning_rate=lr_schedule)\n",
    "        #optimizer = tf.keras.optimizers.Adagrad(learning_rate=lr_schedule)\n",
    "        #optimizer = tf.keras.optimizers.Adamax(learning_rate=lr_schedule)\n",
    "        #optimizer = tf.keras.optimizers.Nadam(learning_rate=1e-3)\n",
    "        #optimizer = tf.keras.optimizers.Ftrl(lr=1e-3)\n",
    "    \n",
    "        model.compile(loss=['mse','mse'], loss_weights=[1, constraint_strength], optimizer=optimizer)\n",
    "        model.layers.sort(key = lambda x: x.name, reverse=False)\n",
    "    \n",
    "        layer_list = []\n",
    "        for i in range(len(model.layers)):\n",
    "            name = model.layers[i].name\n",
    "            if ( (\"activation\" in name) or (\"input\" in name) or (\"add\" in name) or (\"multiply\" in name) or (\"lambda\" in name)):\n",
    "                continue\n",
    "            else:\n",
    "                layer_list.append(i)\n",
    "    \n",
    "        index = 0\n",
    "        for l in range(1, self.nlayers):\n",
    "            for n in range(1, self.nNodes[l]+1):\n",
    "                for i in range(0, self.nNodes[l-1]):\n",
    "                    if ((i+1)%(self.nNodes[l-1])==0):\n",
    "                        if (model.layers[layer_list[index]].get_weights()[0].shape==(1,1)):\n",
    "                            model.layers[layer_list[index]].set_weights( [ np.array( [[ self.weight_biases[l-1][n-1][i] ]] ),  np.array( [ self.weight_biases[l-1][n-1][i+1] ] ) ] )\n",
    "                        else:\n",
    "                            model.layers[layer_list[index]].set_weights( [ np.array( [ self.weight_biases[l-1][n-1][i+1] ] ),  np.array( [[ self.weight_biases[l-1][n-1][i] ]] ) ] )\n",
    "                    else:\n",
    "                        if (model.layers[layer_list[index]].get_weights()[0].shape==(1,1)):\n",
    "                            model.layers[layer_list[index]].set_weights( [ np.array( [[ self.weight_biases[l-1][n-1][i] ]] ),  np.array( [ 0 ] ) ] )\n",
    "                        else:\n",
    "                            model.layers[layer_list[index]].set_weights( [ np.array( [ 0 ] ),  np.array( [[ self.weight_biases[l-1][n-1][i] ]] ) ] )\n",
    "                    index += 1\n",
    "\n",
    "        self.model = model    \n",
    "    \n",
    "    # Creates nodes\n",
    "    def create_node(self, inputs, name, trainable, x_input, constraints):\n",
    "        base = name\n",
    "        act = act_dict[x_input[0]]\n",
    "  \n",
    "        an = []\n",
    "        n = []\n",
    "        for i in range(len(inputs)):\n",
    "            n = base + str(i + 1)\n",
    "            if i < len(inputs)-1:\n",
    "                an.append(CustomDense(1, 1, activation = 'linear', name=n, trainable_weight=trainable[i], trainable_bias=False) (inputs[i]))\n",
    "            else:\n",
    "                an.append(CustomDense(1, 1, activation = 'linear', name=n, trainable_weight=trainable[i], trainable_bias=trainable[len(trainable)-1]) (inputs[i]))\n",
    "\n",
    "        if (act == \"multiply\"):\n",
    "            an = Activation(lambda x: self.custom_multiply(x, x_inputs=x_input[1:])) (an)\n",
    "        else:\n",
    "            an = add(an)\n",
    "            if (act == \"squared\"):\n",
    "                an = Activation(self.squared_act) (an)\n",
    "            elif (act == \"cubed\"):\n",
    "                an = Activation(self.cubed_act) (an)\n",
    "            elif (act == \"sqrt\"):\n",
    "                an = Activation(self.sqrt_act) (an)\n",
    "            elif (act == \"inverse\"):\n",
    "                an = Activation(self.inverse_act) (an)\n",
    "            else:\n",
    "                an = Activation(act) (an)\n",
    "            \n",
    "        # allow for negative square roots and 4th roots but push them to positive values during back propagation\n",
    "        if act == 'sqrt' or act == '4th_rt':\n",
    "            constraints.append(Activation('relu')(Lambda(lambda x: tf.negative(x))(an)))\n",
    "        return an\n",
    "    \n",
    "    ## Activation functions\n",
    "    \n",
    "    # Multiply Activation\n",
    "    def custom_multiply(self, x, x_inputs):\n",
    "        activation_inputs = []\n",
    "        zero_inputs = []\n",
    "        for t in range(0, len(x)):\n",
    "            if x_inputs[t] != 0:\n",
    "                activation_inputs.append(x[t])\n",
    "            elif (t == len(x)-1) & (x_inputs[len(x)] == 1):\n",
    "                activation_inputs.append(x[t])\n",
    "            else:\n",
    "                zero_inputs.append(x[t])\n",
    "\n",
    "        if activation_inputs == []:\n",
    "            activation_tensor = 0\n",
    "        elif len(activation_inputs) == 1:\n",
    "            activation_tensor = activation_inputs[0]\n",
    "        else:\n",
    "            activation_tensor = multiply(activation_inputs)\n",
    "        \n",
    "        if zero_inputs == []:\n",
    "            zero_tensor = 0\n",
    "        elif len(zero_inputs) == 1:\n",
    "            zero_tensor = zero_inputs[0]\n",
    "        else:\n",
    "            zero_tensor = multiply(zero_inputs)\n",
    "        \n",
    "        if not tf.is_tensor(zero_tensor):\n",
    "            return activation_tensor\n",
    "        elif not tf.is_tensor(activation_tensor):\n",
    "            return zero_tensor\n",
    "        else:\n",
    "            return add([activation_tensor, zero_tensor])\n",
    "        \n",
    "    def inverse_act(self, x):\n",
    "        return (1/x)\n",
    "    \n",
    "    def sqrt_act(self, x):\n",
    "        return tf.sign(x)* tf.sqrt(tf.abs(x))\n",
    "    \n",
    "    def cubed_act(self, x):\n",
    "        return x*x*x\n",
    "    \n",
    "    def squared_act(self, x):\n",
    "        return x*x\n",
    "       \n",
    "    ## Training Functions\n",
    "    def train(self, train_inputs, train_outputs, verbose=False):\n",
    "        mae_es= keras.callbacks.EarlyStopping(monitor='val_loss', patience=1000,\n",
    "                                              min_delta=1e-8, verbose=0, mode='auto', restore_best_weights=True)\n",
    "    \n",
    "        terminate = keras.callbacks.TerminateOnNaN()\n",
    "\n",
    "        EPOCHS = 50000 # Number of EPOCHS\n",
    "        history = self.model.fit([train_inputs[:, i] for i in range(0, nNodes[0])], [train_outputs, np.zeros(train_outputs.shape)], epochs=EPOCHS,\n",
    "                            shuffle=False, batch_size=32, verbose = 0, callbacks=[terminate, mae_es],\n",
    "                            validation_split=0.3)\n",
    "        # Test changing batch size\n",
    "        if verbose:\n",
    "            plt.figure()\n",
    "            plt.xlabel('Epoch')\n",
    "            plt.ylabel('Mean Sq Error')\n",
    "            plt.plot(history.epoch, np.array(history.history['loss']),label='Training loss')\n",
    "            plt.plot(history.epoch, np.array(history.history['val_loss']),label='Val loss')\n",
    "            plt.legend()\n",
    "            plt.show()\n",
    "        self.history = history\n",
    "    \n",
    "    def cv_error(self, inputs, outputs, nSplits):\n",
    "        splits = nSplits\n",
    "        kf = KFold(n_splits=splits, shuffle=True, random_state=42)\n",
    "        kf.get_n_splits(inputs)\n",
    "        self.epochs = 0\n",
    "    \n",
    "        cv_mse_list = []\n",
    "        cv_weight_list = []\n",
    "        cv_bias_list = []\n",
    "\n",
    "        for train_index, test_index in kf.split(inputs):\n",
    "            self.create_model()\n",
    "            if (any(self.trainable) == True):\n",
    "                try:\n",
    "                    trained = True\n",
    "                    self.train(inputs[train_index, :], outputs[train_index], verbose=False)\n",
    "                except:\n",
    "                    trained = False\n",
    "                    self.set_random_weight_bias()\n",
    "            self.update_weight_bias()\n",
    "            #handle nan weights and biases\n",
    "            if (np.isnan(self.weight_list).any()):\n",
    "                cv_mse = 1e50\n",
    "                break\n",
    "            elif (np.isnan(self.bias_list).any()):\n",
    "                cv_mse = 1e50\n",
    "                break\n",
    "            elif not trained:\n",
    "                cv_mse = 1e50\n",
    "                break\n",
    "            else:\n",
    "                cv_mse = self.model.evaluate([inputs[test_index, i] for i in range(0, nNodes[0])], [outputs[test_index], np.zeros(outputs[test_index].shape)], verbose=0)\n",
    "                if (np.isnan(cv_mse).any()):\n",
    "                    cv_mse = 1e50\n",
    "                    break\n",
    "                elif cv_mse[0] == float('inf'):\n",
    "                    cv_mse = 1e50\n",
    "                    break\n",
    "                else:\n",
    "                    cv_mse = cv_mse[1]\n",
    "                    cv_mse = np.around(cv_mse, decimals=6)\n",
    "                    self.epochs += len(self.history.history['loss'])/nSplits\n",
    "                    \n",
    "            if cv_weight_list == []:\n",
    "                cv_weight_list = self.weight_list\n",
    "                cv_bias_list = self.bias_list\n",
    "            else:\n",
    "                cv_weight_list += self.weight_list\n",
    "                cv_bias_list += self.bias_list\n",
    "    \n",
    "            cv_mse_list.append(cv_mse)\n",
    "                \n",
    "        if cv_mse == 1e50:\n",
    "            self.MSE_Test = cv_mse\n",
    "        else:\n",
    "            self.MSE_Test = np.mean(cv_mse_list)\n",
    "            self.format_weight_bias(np.array([weights/splits for weights in cv_weight_list]), np.array([bias/splits for bias in cv_bias_list]))\n",
    "        self.model = None\n",
    "        \n",
    "    def objective_function(self, nSplits=5):\n",
    "        if self.individual not in Individual_DB:\n",
    "            self.cv_error(inputs, outputs, nSplits)\n",
    "            self.history = None\n",
    "            self.Objective = self.MSE_Test + p*self.complexity\n",
    "            self.fitness = self.Objective\n",
    "            \n",
    "            print (\"Individual: \", self.individual, flush=True)\n",
    "            print (\"Objective function: \", self.MSE_Test, self.complexity, self.Objective, self.gen, self.epochs, flush=True)\n",
    "    \n",
    "            Individual_DB.append(copy.deepcopy(self.individual))\n",
    "            MSE_Test_DB.append(copy.deepcopy(self.MSE_Test))\n",
    "            Weight_Bias_DB.append(copy.deepcopy(self.weight_biases))\n",
    "                               \n",
    "        else:\n",
    "            self.MSE_Test = MSE_Test_DB[Individual_DB.index(self.individual)]\n",
    "            self.Objective = self.MSE_Test + p*self.complexity\n",
    "            self.fitness = self.Objective\n",
    "            self.weight_biases = Weight_Bias_DB[Individual_DB.index(self.individual)]\n",
    "                                        \n",
    "        K.clear_session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def init_repeat(nlayers, nNodes, max1=4, max2=2, max3=1):\n",
    "    ind = Individual_O(nlayers, nNodes, max1, max2, max3)\n",
    "    ind.randomize_individual()\n",
    "    return ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_mutation(individual, indpb, max1, max2, max3):\n",
    "    for l in range(1, individual.nlayers):\n",
    "        for n in range(1, individual.nNodes[l]+1):\n",
    "            if random.random() < indpb:\n",
    "                individual.individual[l-1][n-1][0] = random.randint(0, max1)\n",
    "                for i in range(1, individual.nNodes[l-1]+1):\n",
    "                    individual.weight_biases[l-1][n-1][i-1] = weight_dict[individual.individual[l-1][n-1][i]]\n",
    "                individual.weight_biases[l-1][n-1][i] = bias_dict[individual.individual[l-1][n-1][i+1]]\n",
    "            for i in range(1, individual.nNodes[l-1]+1):\n",
    "                if random.random() < indpb:\n",
    "                    individual.individual[l-1][n-1][i] = random.randint(0, max2)\n",
    "                    individual.weight_biases[l-1][n-1][i-1] = weight_dict[individual.individual[l-1][n-1][i]]\n",
    "            if random.random() < indpb:\n",
    "                individual.individual[l-1][n-1][i+1] = random.randint(0, max3)\n",
    "                individual.weight_biases[l-1][n-1][i] = bias_dict[individual.individual[l-1][n-1][i+1]]\n",
    "    if not save_weights:\n",
    "        individual.set_random_weight_bias()\n",
    "    elif not save_weights_mutation:\n",
    "        individual.set_random_weight_bias()\n",
    "        \n",
    "    individual.reset_model()\n",
    "    return individual,"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_crossover(individual1, individual2):\n",
    "    LAYER = random.randint(1, nlayers-1)\n",
    "    NODE = random.randint(0, nNodes[LAYER]-1)\n",
    "    temp = copy.deepcopy(individual1.individual[LAYER-1][NODE])\n",
    "    temp_weights = copy.deepcopy(individual1.weight_biases[LAYER-1][NODE])\n",
    "    individual1.individual[LAYER-1][NODE] = copy.deepcopy(individual2.individual[LAYER-1][NODE])\n",
    "    individual1.weight_biases[LAYER-1][NODE] = copy.deepcopy(individual2.weight_biases[LAYER-1][NODE])\n",
    "    individual2.individual[LAYER-1][NODE] = copy.deepcopy(temp)\n",
    "    individual2.weight_biases[LAYER-1][NODE] = copy.deepcopy(temp_weights)\n",
    "    \n",
    "    if not save_weights:\n",
    "        individual1.set_random_weight_bias()\n",
    "        individual2.set_random_weight_bias()\n",
    "    elif not save_weights_crossover:\n",
    "        weight1 = []\n",
    "        weight2 = []\n",
    "        for i in range(1,individual1.nNodes[LAYER-1]+1):\n",
    "            weight1.append(weight_dict[individual1.individual[LAYER-1][NODE][i]])\n",
    "            weight2.append(weight_dict[individual2.individual[LAYER-1][NODE][i]])\n",
    "        weight1.append(bias_dict[individual1.individual[LAYER-1][NODE][i+1]])\n",
    "        weight2.append(bias_dict[individual2.individual[LAYER-1][NODE][i+1]])\n",
    "        \n",
    "        individual1.weight_biases[LAYER-1][NODE] = weight1\n",
    "        individual2.weight_biases[LAYER-1][NODE] = weight2\n",
    "        \n",
    "    individual1.reset_model()\n",
    "    individual2.reset_model()\n",
    "    return individual1, individual2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective_function(individual):\n",
    "    individual.objective_function()\n",
    "    return [individual.Objective, individual.weight_biases]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_eaSimple(population, toolbox, cxpb, mutpb, ngen, stats=None,\n",
    "             halloffame=None, verbose=__debug__):\n",
    "    \"\"\"This algorithm reproduce the simplest evolutionary algorithm as\n",
    "    presented in chapter 7 of [Back2000]_.\n",
    "\n",
    "    :param population: A list of individuals.\n",
    "    :param toolbox: A :class:`~deap.base.Toolbox` that contains the evolution\n",
    "                    operators.\n",
    "    :param cxpb: The probability of mating two individuals.\n",
    "    :param mutpb: The probability of mutating an individual.\n",
    "    :param ngen: The number of generation.\n",
    "    :param stats: A :class:`~deap.tools.Statistics` object that is updated\n",
    "                  inplace, optional.\n",
    "    :param halloffame: A :class:`~deap.tools.HallOfFame` object that will\n",
    "                       contain the best individuals, optional.\n",
    "    :param verbose: Whether or not to log the statistics.\n",
    "    :returns: The final population\n",
    "    :returns: A class:`~deap.tools.Logbook` with the statistics of the\n",
    "              evolution\n",
    "\n",
    "    The algorithm takes in a population and evolves it in place using the\n",
    "    :meth:`varAnd` method. It returns the optimized population and a\n",
    "    :class:`~deap.tools.Logbook` with the statistics of the evolution. The\n",
    "    logbook will contain the generation number, the number of evaluations for\n",
    "    each generation and the statistics if a :class:`~deap.tools.Statistics` is\n",
    "    given as argument. The *cxpb* and *mutpb* arguments are passed to the\n",
    "    :func:`varAnd` function. The pseudocode goes as follow ::\n",
    "\n",
    "        evaluate(population)\n",
    "        for g in range(ngen):\n",
    "            population = select(population, len(population))\n",
    "            offspring = varAnd(population, toolbox, cxpb, mutpb)\n",
    "            evaluate(offspring)\n",
    "            population = offspring\n",
    "\n",
    "    As stated in the pseudocode above, the algorithm goes as follow. First, it\n",
    "    evaluates the individuals with an invalid fitness. Second, it enters the\n",
    "    generational loop where the selection procedure is applied to entirely\n",
    "    replace the parental population. The 1:1 replacement ratio of this\n",
    "    algorithm **requires** the selection procedure to be stochastic and to\n",
    "    select multiple times the same individual, for example,\n",
    "    :func:`~deap.tools.selTournament` and :func:`~deap.tools.selRoulette`.\n",
    "    Third, it applies the :func:`varAnd` function to produce the next\n",
    "    generation population. Fourth, it evaluates the new individuals and\n",
    "    compute the statistics on this population. Finally, when *ngen*\n",
    "    generations are done, the algorithm returns a tuple with the final\n",
    "    population and a :class:`~deap.tools.Logbook` of the evolution.\n",
    "\n",
    "    .. note::\n",
    "\n",
    "        Using a non-stochastic selection method will result in no selection as\n",
    "        the operator selects *n* individuals from a pool of *n*.\n",
    "\n",
    "    This function expects the :meth:`toolbox.mate`, :meth:`toolbox.mutate`,\n",
    "    :meth:`toolbox.select` and :meth:`toolbox.evaluate` aliases to be\n",
    "    registered in the toolbox.\n",
    "\n",
    "    .. [Back2000] Back, Fogel and Michalewicz, \"Evolutionary Computation 1 :\n",
    "       Basic Algorithms and Operators\", 2000.\n",
    "    \"\"\"\n",
    "    logbook = tools.Logbook()\n",
    "    logbook.header = ['gen', 'nevals'] + (stats.fields if stats else [])\n",
    "\n",
    "    # Evaluate the individuals with an invalid fitness\n",
    "    invalid_ind = [ind for ind in population if ind.fitness == None]\n",
    "    for ind in invalid_ind:\n",
    "        ind.gen = 0\n",
    "    data_stream = toolbox.map(toolbox.evaluate, invalid_ind)\n",
    "    for ind, data in zip(invalid_ind, data_stream):\n",
    "        ind.Objective = data[0]\n",
    "        ind.fitness = data[0]\n",
    "        ind.weight_biases = data[1]\n",
    "    record = stats.compile(population) if stats else {}\n",
    "    logbook.record(gen=0, nevals=len(invalid_ind), **record)\n",
    "    if verbose:\n",
    "        print(logbook.stream)\n",
    "\n",
    "    # Begin the generational process\n",
    "    for gen in range(1, ngen + 1):\n",
    "        # Select the next generation individuals\n",
    "        offspring = toolbox.select(population, len(population))\n",
    "\n",
    "        # Vary the pool of individuals\n",
    "        offspring = custom_varAnd(offspring, toolbox, cxpb, mutpb, gen, ngen)\n",
    "\n",
    "        # Evaluate the individuals with an invalid fitness\n",
    "        invalid_ind = [ind for ind in offspring if ind.Objective == None]\n",
    "        for ind in invalid_ind:\n",
    "            ind.gen = gen\n",
    "        data_stream = toolbox.map(toolbox.evaluate, invalid_ind)\n",
    "        for ind, data in zip(invalid_ind, data_stream):\n",
    "            ind.Objective = data[0]\n",
    "            ind.fitness = data[0]\n",
    "            ind.weight_biases = data[1]\n",
    "\n",
    "        # Replace the current population by the offspring\n",
    "        population[:] = offspring\n",
    "        \n",
    "        # Append the current generation statistics to the logbook\n",
    "        record = stats.compile(population) if stats else {}\n",
    "        logbook.record(gen=gen, nevals=len(invalid_ind), **record)\n",
    "        if verbose:\n",
    "            print(logbook.stream)\n",
    "    return population, logbook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_varAnd(population, toolbox, cxpb, mutpb, gen, ngen):\n",
    "    \"\"\"Part of an evolutionary algorithm applying only the variation part\n",
    "    (crossover **and** mutation). The modified individuals have their\n",
    "    fitness invalidated. The individuals are cloned so returned population is\n",
    "    independent of the input population.\n",
    "\n",
    "    :param population: A list of individuals to vary.\n",
    "    :param toolbox: A :class:`~deap.base.Toolbox` that contains the evolution\n",
    "                    operators.\n",
    "    :param cxpb: The probability of mating two individuals.\n",
    "    :param mutpb: The probability of mutating an individual.\n",
    "    :returns: A list of varied individuals that are independent of their\n",
    "              parents.\n",
    "\n",
    "    The variation goes as follow. First, the parental population\n",
    "    :math:`P_\\mathrm{p}` is duplicated using the :meth:`toolbox.clone` method\n",
    "    and the result is put into the offspring population :math:`P_\\mathrm{o}`.  A\n",
    "    first loop over :math:`P_\\mathrm{o}` is executed to mate pairs of\n",
    "    consecutive individuals. According to the crossover probability *cxpb*, the\n",
    "    individuals :math:`\\mathbf{x}_i` and :math:`\\mathbf{x}_{i+1}` are mated\n",
    "    using the :meth:`toolbox.mate` method. The resulting children\n",
    "    :math:`\\mathbf{y}_i` and :math:`\\mathbf{y}_{i+1}` replace their respective\n",
    "    parents in :math:`P_\\mathrm{o}`. A second loop over the resulting\n",
    "    :math:`P_\\mathrm{o}` is executed to mutate every individual with a\n",
    "    probability *mutpb*. When an individual is mutated it replaces its not\n",
    "    mutated version in :math:`P_\\mathrm{o}`. The resulting :math:`P_\\mathrm{o}`\n",
    "    is returned.\n",
    "\n",
    "    This variation is named *And* because of its propensity to apply both\n",
    "    crossover and mutation on the individuals. Note that both operators are\n",
    "    not applied systematically, the resulting individuals can be generated from\n",
    "    crossover only, mutation only, crossover and mutation, and reproduction\n",
    "    according to the given probabilities. Both probabilities should be in\n",
    "    :math:`[0, 1]`.\n",
    "    \"\"\"\n",
    "    offspring = [ind.object_copy() for ind in population]\n",
    "    #offspring = population\n",
    "    \n",
    "    # Apply crossover and mutation on the offspring\n",
    "    for i in range(1, len(offspring), 2):\n",
    "        if random.random() < cxpb:\n",
    "            offspring[i - 1], offspring[i] = toolbox.mate(offspring[i - 1],\n",
    "                                                          offspring[i])\n",
    "    for i in range(len(offspring)):\n",
    "        if random.random() < mutpb[(gen-1)//(ngen//len(mutpb))]:\n",
    "            offspring[i], = toolbox.mutate(offspring[i], mutpb[(gen-1)//(ngen//len(mutpb))])\n",
    "            \n",
    "    return offspring"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nanohub/psalek/.local/lib/python3.7/site-packages/scoop/fallbacks.py:46: RuntimeWarning: SCOOP was not started properly.\n",
      "Be sure to start your program with the '-m scoop' parameter. You can find further information in the documentation.\n",
      "Your map call has been replaced by the builtin serial Python map().\n",
      "  RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Individual:  [[[4, 1, 0, 2, 0, 2, 0], [0, 1, 2, 1, 0, 0, 1], [0, 1, 0, 2, 0, 0, 1], [4, 2, 1, 2, 2, 0, 0], [0, 1, 0, 1, 2, 0, 0]], [[2, 1, 1, 1, 1, 1, 1], [4, 2, 2, 1, 2, 2, 0], [0, 1, 1, 2, 2, 2, 1]], [[0, 1, 0, 1, 0]]]\n",
      "Objective function:  3.2685180000000003 57.0 8.968518 0 13.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/apps/share64/debian7/anaconda/anaconda-6/lib/python3.7/site-packages/ipykernel_launcher.py:361: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Individual:  [[[1, 0, 1, 2, 2, 2, 0], [4, 1, 2, 2, 0, 2, 0], [4, 0, 1, 1, 1, 0, 1], [3, 1, 2, 2, 0, 2, 1], [4, 0, 2, 1, 0, 2, 0]], [[0, 0, 0, 0, 1, 0, 1], [3, 0, 0, 1, 0, 2, 1], [2, 1, 2, 1, 0, 1, 0]], [[2, 2, 2, 0, 0]]]\n",
      "Objective function:  6.2547305 71.0 13.3547305 0 8.0\n",
      "Batch 1: Invalid loss, terminating training\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[2, 1, 2, 2, 2, 1, 1], [3, 0, 2, 0, 0, 1, 1], [2, 1, 0, 2, 1, 2, 1], [1, 0, 0, 2, 1, 2, 0], [1, 1, 0, 1, 0, 2, 0]], [[2, 1, 2, 2, 2, 2, 0], [1, 1, 1, 1, 2, 2, 1], [3, 0, 2, 1, 2, 0, 1]], [[3, 0, 0, 2, 0]]]\n",
      "Objective function:  1e+50 67.0 1e+50 0 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nanohub/psalek/.local/lib/python3.7/site-packages/keras/callbacks/callbacks.py:846: RuntimeWarning: Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,activation_52_loss,lambda_1_loss\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Individual:  [[[3, 0, 1, 0, 2, 0, 1], [0, 2, 0, 2, 0, 0, 1], [4, 0, 0, 1, 2, 2, 1], [3, 0, 2, 0, 0, 1, 0], [1, 0, 1, 0, 2, 2, 1]], [[2, 0, 0, 0, 1, 2, 0], [0, 2, 1, 2, 2, 1, 0], [1, 1, 1, 1, 2, 0, 0]], [[1, 0, 0, 2, 1]]]\n",
      "Objective function:  433.04602700000004 51.0 438.14602700000006 0 8.0\n",
      "Individual:  [[[0, 2, 0, 2, 1, 2, 0], [1, 0, 2, 2, 0, 0, 1], [2, 2, 2, 0, 0, 1, 1], [4, 1, 2, 0, 1, 2, 0], [1, 0, 2, 1, 0, 2, 1]], [[2, 1, 2, 1, 0, 1, 0], [3, 1, 0, 2, 0, 1, 0], [4, 2, 2, 0, 0, 1, 1]], [[2, 0, 0, 1, 1]]]\n",
      "Objective function:  7.282186 67.0 13.982186 0 6.0\n",
      "gen\tnevals\tavg  \tmin    \tmax  \n",
      "0  \t5     \t2e+49\t8.96852\t1e+50\n",
      "1  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "2  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "3  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "4  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "5  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "6  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "7  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "8  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "9  \t5     \t1e+50\t1e+50  \t1e+50\n",
      "10 \t5     \t1e+50\t1e+50  \t1e+50\n",
      "Individual:  [[[1, 2, 2, 1, 1, 2, 0], [4, 1, 1, 2, 2, 1, 0], [1, 2, 2, 1, 1, 2, 1], [0, 0, 0, 0, 2, 1, 0], [0, 0, 2, 0, 0, 1, 0]], [[2, 1, 2, 0, 0, 1, 1], [2, 1, 0, 1, 0, 1, 1], [3, 0, 2, 0, 1, 2, 1]], [[2, 1, 1, 0, 1]]]\n",
      "Objective function:  199.57841 61.0 205.67840999999999 11 7313.0\n",
      "Individual:  [[[2, 2, 1, 0, 0, 1, 1], [3, 0, 1, 2, 2, 1, 1], [2, 1, 0, 2, 1, 0, 0], [2, 2, 2, 2, 1, 0, 0], [1, 1, 1, 0, 1, 1, 0]], [[2, 0, 2, 1, 0, 2, 0], [1, 0, 2, 1, 0, 1, 0], [0, 0, 2, 1, 2, 1, 0]], [[2, 0, 0, 2, 0]]]\n",
      "Objective function:  6.51036 55.0 12.01036 11 8.0\n",
      "Individual:  [[[2, 2, 1, 0, 0, 1, 1], [3, 0, 2, 0, 1, 1, 1], [4, 2, 2, 2, 1, 0, 0], [4, 0, 0, 0, 1, 2, 0], [4, 1, 2, 0, 0, 0, 0]], [[2, 0, 2, 2, 1, 0, 0], [2, 1, 1, 1, 0, 2, 1], [3, 1, 0, 1, 2, 0, 0]], [[2, 0, 0, 2, 0]]]\n",
      "Objective function:  1.855574 64.0 8.255574000000001 11 6.0\n",
      "11 \t5     \t4e+49\t8.25557\t1e+50\n",
      "Individual:  [[[1, 2, 0, 1, 2, 0, 1], [1, 0, 2, 0, 1, 0, 0], [0, 2, 2, 2, 0, 1, 1], [0, 2, 0, 0, 2, 1, 1], [1, 0, 0, 2, 0, 1, 0]], [[4, 0, 1, 1, 0, 2, 0], [2, 0, 1, 0, 2, 2, 0], [0, 0, 1, 0, 0, 2, 1]], [[4, 0, 0, 2, 1]]]\n",
      "Objective function:  1.0954685 44.0 5.4954685 12 51.5\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[2, 1, 2, 0, 2, 0, 0], [3, 1, 2, 0, 2, 2, 1], [2, 2, 0, 2, 0, 2, 0], [0, 2, 0, 0, 2, 1, 0], [0, 2, 2, 1, 0, 2, 0]], [[4, 1, 0, 0, 2, 2, 0], [4, 0, 2, 2, 2, 2, 1], [3, 2, 2, 2, 1, 0, 1]], [[4, 0, 0, 2, 0]]]\n",
      "Objective function:  1e+50 74.0 1e+50 12 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nanohub/psalek/.local/lib/python3.7/site-packages/keras/callbacks/callbacks.py:846: RuntimeWarning: Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,activation_54_loss,add_10_loss\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Individual:  [[[1, 0, 0, 2, 0, 1, 0], [2, 0, 2, 2, 2, 1, 1], [0, 1, 1, 2, 0, 2, 1], [4, 0, 0, 1, 0, 2, 0], [3, 1, 0, 1, 0, 2, 1]], [[2, 2, 1, 1, 0, 2, 1], [1, 2, 0, 2, 2, 1, 0], [0, 1, 1, 1, 0, 2, 1]], [[0, 2, 2, 2, 1]]]\n",
      "Objective function:  4720162.9296875 63.0 4720169.2296875 12 8.0\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[4, 2, 2, 1, 2, 1, 1], [1, 1, 0, 2, 0, 1, 1], [2, 1, 0, 2, 2, 2, 0], [2, 0, 0, 0, 2, 2, 1], [2, 2, 0, 0, 0, 0, 0]], [[2, 2, 1, 2, 1, 1, 1], [0, 1, 1, 1, 2, 2, 1], [2, 1, 0, 2, 0, 2, 1]], [[0, 1, 0, 1, 1]]]\n",
      "Objective function:  1e+50 64.0 1e+50 12 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nanohub/psalek/.local/lib/python3.7/site-packages/keras/callbacks/callbacks.py:846: RuntimeWarning: Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,activation_53_loss,activation_7_loss\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Individual:  [[[2, 1, 2, 2, 2, 0, 0], [1, 0, 2, 1, 2, 2, 1], [2, 1, 0, 1, 0, 0, 1], [1, 0, 1, 2, 1, 2, 1], [1, 2, 2, 1, 2, 2, 0]], [[2, 1, 0, 2, 0, 0, 1], [1, 1, 0, 1, 2, 2, 1], [4, 0, 0, 0, 2, 0, 0]], [[3, 0, 0, 1, 0]]]\n",
      "Objective function:  12.2066105 62.0 18.4066105 12 7.5\n",
      "12 \t5     \t4e+49\t5.49547\t1e+50\n",
      "Individual:  [[[3, 1, 1, 0, 0, 0, 0], [2, 1, 2, 0, 1, 2, 0], [2, 2, 2, 2, 0, 2, 1], [2, 1, 2, 0, 2, 1, 0], [0, 0, 0, 1, 2, 0, 0]], [[4, 2, 1, 0, 2, 2, 0], [4, 0, 1, 2, 1, 0, 1], [4, 0, 2, 2, 1, 0, 0]], [[1, 2, 2, 0, 0]]]\n",
      "Objective function:  1.5139575 70.0 8.5139575 13 21.5\n",
      "Individual:  [[[4, 1, 0, 0, 2, 0, 0], [2, 1, 2, 0, 0, 0, 1], [0, 1, 0, 2, 2, 2, 1], [2, 0, 0, 0, 2, 2, 0], [4, 1, 2, 2, 0, 2, 1]], [[4, 2, 2, 2, 0, 2, 1], [0, 1, 0, 1, 2, 1, 1], [1, 0, 2, 1, 1, 0, 1]], [[4, 0, 0, 2, 1]]]\n",
      "Objective function:  3.1224225 66.0 9.7224225 13 15.5\n",
      "13 \t5     \t6e+49\t8.51396\t1e+50\n",
      "Individual:  [[[1, 1, 2, 1, 1, 2, 1], [1, 1, 0, 1, 1, 0, 0], [0, 2, 0, 2, 2, 2, 0], [0, 1, 0, 2, 0, 1, 1], [0, 1, 1, 0, 0, 1, 0]], [[2, 0, 0, 2, 1, 2, 1], [0, 0, 2, 0, 2, 2, 0], [3, 1, 0, 2, 0, 1, 1]], [[0, 0, 1, 2, 0]]]\n",
      "Objective function:  1176070485.875 49.0 1176070490.775 14 10.0\n",
      "Individual:  [[[1, 1, 2, 1, 1, 2, 0], [0, 0, 1, 0, 2, 2, 0], [3, 2, 1, 2, 2, 2, 1], [0, 2, 0, 2, 2, 0, 0], [2, 2, 0, 1, 0, 1, 0]], [[2, 0, 1, 2, 2, 2, 1], [1, 2, 0, 2, 1, 2, 0], [0, 0, 0, 1, 2, 1, 1]], [[3, 2, 0, 0, 0]]]\n",
      "Objective function:  58.7836465 67.0 65.4836465 14 9.5\n",
      "Individual:  [[[1, 1, 2, 0, 1, 1, 0], [3, 1, 0, 0, 2, 1, 0], [1, 0, 0, 1, 1, 2, 0], [1, 0, 1, 2, 1, 1, 0], [0, 0, 2, 1, 1, 2, 1]], [[4, 1, 0, 0, 0, 0, 0], [0, 0, 2, 0, 0, 0, 0], [3, 2, 1, 0, 0, 0, 0]], [[2, 2, 0, 1, 0]]]\n",
      "Objective function:  6.4375025 56.0 12.0375025 14 9.0\n",
      "Individual:  [[[0, 1, 0, 1, 2, 2, 1], [3, 2, 2, 2, 2, 2, 1], [2, 1, 0, 2, 2, 2, 1], [2, 2, 0, 1, 1, 1, 1], [4, 2, 1, 2, 2, 0, 1]], [[1, 2, 1, 2, 2, 0, 0], [0, 0, 0, 1, 1, 2, 0], [2, 1, 0, 2, 0, 1, 1]], [[0, 2, 0, 1, 1]]]\n",
      "Objective function:  1.2580580000000001 74.0 8.658058 14 20.0\n",
      "Individual:  [[[4, 0, 1, 0, 2, 0, 1], [1, 1, 0, 1, 0, 0, 1], [1, 1, 2, 0, 0, 1, 1], [2, 1, 2, 2, 1, 0, 1], [3, 2, 0, 1, 0, 1, 0]], [[2, 2, 1, 2, 1, 0, 1], [4, 2, 1, 2, 0, 2, 0], [4, 2, 0, 1, 0, 2, 0]], [[4, 1, 0, 2, 1]]]\n",
      "Objective function:  1.6758 66.0 8.2758 14 6.0\n",
      "14 \t5     \t2.35214e+08\t8.2758 \t1.17607e+09\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[2, 0, 1, 2, 1, 2, 0], [2, 1, 0, 0, 2, 0, 0], [3, 0, 0, 0, 2, 2, 0], [0, 2, 1, 1, 0, 2, 1], [3, 0, 1, 1, 0, 0, 0]], [[2, 0, 2, 0, 2, 2, 0], [1, 0, 0, 2, 2, 2, 1], [1, 1, 2, 2, 2, 1, 1]], [[0, 0, 2, 0, 0]]]\n",
      "Objective function:  1e+50 54.0 1e+50 15 0\n",
      "Individual:  [[[3, 1, 2, 2, 2, 1, 1], [0, 1, 1, 1, 2, 0, 1], [3, 0, 2, 0, 1, 0, 0], [0, 1, 0, 1, 0, 1, 1], [0, 1, 0, 0, 0, 0, 1]], [[0, 0, 0, 0, 0, 2, 0], [0, 0, 2, 0, 2, 2, 1], [3, 0, 0, 2, 0, 1, 1]], [[2, 0, 1, 0, 0]]]\n",
      "Objective function:  10.2756335 39.0 14.1756335 15 11.5\n",
      "15 \t5     \t2e+49      \t14.1756\t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[2, 0, 2, 2, 1, 1, 0], [0, 2, 1, 2, 0, 0, 1], [2, 0, 2, 0, 2, 0, 0], [3, 0, 1, 1, 1, 0, 1], [3, 1, 2, 2, 1, 1, 0]], [[0, 0, 2, 2, 1, 2, 1], [0, 0, 0, 2, 0, 1, 0], [1, 0, 2, 2, 0, 1, 1]], [[0, 0, 0, 0, 0]]]\n",
      "Objective function:  1e+50 54.0 1e+50 16 0\n",
      "Individual:  [[[2, 0, 2, 1, 0, 2, 1], [0, 2, 0, 0, 0, 0, 0], [3, 2, 1, 2, 1, 1, 0], [0, 1, 2, 0, 0, 0, 1], [0, 1, 1, 1, 1, 0, 1]], [[3, 2, 2, 0, 0, 2, 1], [2, 0, 2, 0, 0, 1, 1], [2, 2, 2, 2, 2, 2, 0]], [[1, 0, 0, 1, 0]]]\n",
      "Objective function:  6.337255000000001 51.0 11.437255 16 6.0\n",
      "Individual:  [[[2, 1, 2, 2, 2, 1, 0], [0, 0, 2, 0, 2, 0, 0], [1, 1, 1, 0, 2, 0, 1], [1, 1, 0, 2, 2, 1, 1], [3, 0, 1, 2, 0, 2, 0]], [[2, 0, 2, 1, 2, 2, 0], [3, 2, 0, 2, 0, 1, 0], [3, 0, 1, 2, 2, 1, 0]], [[0, 2, 1, 0, 1]]]\n",
      "Objective function:  112795196.0 65.0 112795202.5 16 10.5\n",
      "Individual:  [[[2, 2, 1, 2, 2, 0, 0], [2, 0, 1, 2, 0, 0, 0], [1, 0, 0, 0, 0, 1, 0], [3, 2, 1, 2, 0, 0, 1], [1, 0, 1, 0, 1, 1, 1]], [[4, 0, 1, 0, 0, 1, 1], [4, 1, 2, 2, 2, 2, 1], [1, 0, 2, 1, 2, 1, 0]], [[0, 2, 0, 1, 1]]]\n",
      "Objective function:  64977772.2175295 55.0 64977777.7175295 16 7.0\n",
      "16 \t5     \t4e+49      \t11.4373\t1e+50      \n",
      "Individual:  [[[2, 1, 2, 2, 2, 2, 0], [3, 1, 2, 2, 1, 2, 0], [3, 2, 0, 0, 2, 2, 0], [1, 2, 1, 2, 0, 1, 1], [0, 1, 1, 1, 0, 0, 0]], [[3, 2, 1, 2, 0, 0, 0], [1, 1, 0, 2, 2, 2, 0], [4, 1, 2, 2, 0, 2, 0]], [[0, 0, 1, 0, 1]]]\n",
      "Objective function:  0.1103075 75.0 7.6103075 17 23.5\n",
      "Individual:  [[[1, 0, 1, 0, 2, 2, 1], [2, 2, 2, 1, 2, 0, 1], [0, 1, 0, 0, 1, 1, 0], [0, 2, 1, 2, 0, 0, 1], [3, 2, 2, 0, 1, 0, 0]], [[2, 0, 2, 1, 1, 2, 1], [1, 0, 1, 2, 2, 2, 0], [3, 1, 0, 2, 0, 2, 1]], [[4, 1, 2, 1, 1]]]\n",
      "Objective function:  203.32718649999998 60.0 209.32718649999998 17 9.5\n",
      "Individual:  [[[2, 1, 2, 1, 1, 0, 0], [0, 2, 1, 2, 0, 0, 1], [0, 1, 1, 0, 2, 2, 1], [0, 2, 1, 0, 0, 0, 0], [4, 0, 0, 0, 2, 1, 1]], [[1, 0, 2, 0, 1, 0, 0], [3, 1, 0, 0, 2, 2, 1], [3, 0, 2, 1, 0, 0, 0]], [[3, 0, 0, 2, 0]]]\n",
      "Objective function:  1192.1179965000001 48.0 1196.9179965 17 1366.5\n",
      "Individual:  [[[2, 2, 1, 2, 1, 1, 1], [0, 2, 1, 2, 2, 2, 1], [3, 0, 2, 0, 1, 1, 0], [3, 1, 1, 1, 2, 2, 0], [3, 2, 2, 0, 0, 1, 0]], [[3, 0, 1, 2, 2, 2, 1], [0, 1, 2, 0, 0, 0, 1], [1, 0, 2, 1, 2, 2, 0]], [[4, 1, 1, 0, 1]]]\n",
      "Objective function:  36349.1206055 72.0 36356.3206055 17 15.5\n",
      "Individual:  [[[2, 0, 0, 2, 1, 1, 0], [4, 1, 1, 1, 2, 1, 1], [2, 0, 2, 0, 1, 2, 1], [3, 1, 1, 1, 2, 0, 1], [1, 1, 0, 0, 1, 2, 1]], [[0, 1, 2, 1, 0, 0, 1], [4, 2, 2, 0, 2, 1, 0], [3, 0, 0, 0, 2, 2, 0]], [[1, 0, 0, 2, 1]]]\n",
      "Objective function:  191155.2622495 59.0 191161.1622495 17 10.5\n",
      "17 \t5     \t45786.3    \t7.61031\t191161     \n",
      "Individual:  [[[2, 1, 1, 2, 1, 0, 0], [1, 1, 1, 1, 2, 0, 1], [0, 0, 2, 0, 1, 2, 0], [3, 1, 1, 1, 0, 1, 1], [0, 1, 1, 2, 1, 2, 0]], [[4, 1, 2, 1, 2, 2, 1], [0, 2, 0, 0, 0, 1, 0], [3, 0, 1, 0, 0, 2, 1]], [[0, 0, 0, 2, 1]]]\n",
      "Objective function:  356976.9404295 52.0 356982.1404295 18 7.0\n",
      "Individual:  [[[4, 0, 1, 1, 2, 2, 0], [1, 1, 1, 2, 2, 0, 1], [3, 0, 0, 0, 1, 0, 1], [3, 1, 0, 2, 0, 1, 1], [1, 1, 1, 0, 0, 0, 1]], [[0, 0, 2, 2, 1, 2, 0], [4, 0, 0, 1, 0, 0, 1], [1, 2, 0, 1, 1, 1, 0]], [[1, 0, 2, 2, 1]]]\n",
      "Objective function:  66392.3130495 52.0 66397.51304949999 18 8.0\n",
      "Individual:  [[[2, 0, 2, 2, 0, 0, 1], [4, 0, 2, 2, 1, 1, 0], [3, 2, 2, 2, 0, 2, 1], [4, 0, 1, 0, 2, 0, 0], [1, 1, 0, 0, 0, 2, 1]], [[3, 2, 2, 1, 0, 0, 1], [4, 1, 2, 0, 2, 2, 0], [3, 0, 2, 0, 2, 1, 1]], [[4, 2, 0, 2, 1]]]\n",
      "Objective function:  1.6420325 72.0 8.8420325 18 908.0\n",
      "Individual:  [[[1, 0, 0, 2, 1, 1, 1], [1, 1, 0, 2, 1, 0, 1], [0, 0, 2, 0, 2, 2, 0], [0, 0, 2, 1, 1, 1, 1], [1, 2, 0, 0, 0, 1, 0]], [[3, 2, 0, 1, 0, 0, 1], [4, 2, 1, 0, 2, 0, 1], [1, 0, 0, 2, 0, 2, 1]], [[3, 0, 0, 1, 1]]]\n",
      "Objective function:  18943.5624335 52.0 18948.7624335 18 10.5\n",
      "18 \t5     \t126700     \t8.84203\t356982     \n",
      "Individual:  [[[4, 1, 1, 1, 2, 2, 0], [2, 2, 1, 1, 0, 0, 1], [3, 0, 0, 2, 0, 1, 1], [0, 0, 0, 1, 1, 1, 0], [0, 0, 2, 1, 1, 1, 0]], [[4, 2, 2, 0, 1, 2, 1], [1, 0, 2, 2, 2, 1, 1], [4, 1, 2, 0, 1, 1, 0]], [[0, 1, 0, 2, 1]]]\n",
      "Objective function:  28.864552 60.0 34.864552 19 22.0\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 1, 0, 2, 2, 0], [2, 2, 0, 0, 2, 0, 0], [4, 0, 2, 0, 1, 2, 1], [3, 2, 1, 1, 0, 0, 0], [3, 1, 1, 2, 1, 1, 0]], [[0, 0, 0, 1, 1, 1, 0], [0, 2, 0, 2, 0, 1, 0], [1, 2, 2, 0, 0, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  1e+50 60.0 1e+50 19 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nanohub/psalek/.local/lib/python3.7/site-packages/keras/callbacks/callbacks.py:846: RuntimeWarning: Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,activation_53_loss,activation_19_loss\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Individual:  [[[4, 1, 2, 2, 2, 1, 0], [0, 0, 2, 0, 0, 0, 1], [3, 0, 1, 0, 2, 2, 1], [3, 1, 0, 1, 0, 1, 0], [1, 0, 1, 2, 1, 2, 0]], [[2, 1, 2, 2, 2, 2, 1], [1, 2, 0, 1, 0, 1, 0], [3, 1, 2, 0, 0, 2, 0]], [[0, 1, 2, 2, 1]]]\n",
      "Objective function:  391.47128150000003 63.0 397.77128150000004 19 6.0\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[1, 2, 0, 2, 1, 0, 1], [3, 1, 0, 2, 0, 2, 0], [2, 2, 1, 0, 1, 2, 1], [1, 2, 1, 1, 2, 1, 0], [0, 2, 2, 1, 2, 2, 0]], [[4, 1, 0, 0, 2, 1, 0], [1, 2, 2, 1, 2, 2, 0], [4, 1, 0, 1, 0, 0, 1]], [[3, 2, 2, 0, 1]]]\n",
      "Objective function:  1e+50 72.0 1e+50 19 4.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nanohub/psalek/.local/lib/python3.7/site-packages/keras/callbacks/callbacks.py:846: RuntimeWarning: Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,activation_108_loss,add_16_loss\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Individual:  [[[3, 1, 2, 2, 2, 2, 1], [4, 2, 2, 1, 2, 0, 1], [3, 2, 1, 0, 2, 0, 0], [3, 2, 2, 1, 0, 2, 1], [0, 1, 2, 2, 2, 2, 1]], [[4, 0, 0, 1, 2, 2, 1], [1, 2, 0, 0, 1, 1, 1], [4, 1, 2, 2, 1, 0, 1]], [[2, 1, 0, 2, 0]]]\n",
      "Objective function:  5.790599 80.0 13.790599 19 7.0\n",
      "19 \t5     \t4e+49      \t13.7906\t1e+50      \n",
      "Individual:  [[[4, 1, 1, 1, 1, 2, 1], [4, 0, 0, 1, 2, 0, 1], [4, 2, 2, 1, 1, 0, 0], [1, 1, 2, 2, 2, 1, 0], [3, 1, 2, 1, 2, 0, 0]], [[4, 1, 1, 0, 2, 1, 0], [1, 2, 0, 1, 0, 1, 1], [3, 0, 1, 0, 0, 1, 1]], [[3, 0, 0, 0, 0]]]\n",
      "Objective function:  6.5003174999999995 66.0 13.1003175 20 6.0\n",
      "Individual:  [[[3, 0, 2, 2, 2, 2, 1], [1, 0, 2, 1, 1, 0, 1], [4, 1, 0, 0, 2, 0, 0], [0, 2, 2, 1, 0, 1, 0], [3, 2, 2, 1, 1, 0, 0]], [[0, 2, 0, 2, 1, 0, 1], [3, 2, 1, 0, 0, 2, 0], [4, 0, 0, 0, 2, 1, 1]], [[0, 2, 2, 0, 0]]]\n",
      "Objective function:  208.4271015 61.0 214.5271015 20 7.0\n",
      "Individual:  [[[1, 0, 1, 1, 1, 1, 1], [3, 0, 0, 2, 0, 2, 0], [4, 0, 1, 0, 0, 1, 0], [4, 0, 2, 1, 2, 0, 0], [0, 2, 2, 0, 2, 2, 1]], [[2, 1, 0, 1, 0, 0, 0], [1, 2, 2, 1, 2, 2, 1], [4, 0, 2, 0, 0, 2, 1]], [[1, 0, 1, 0, 1]]]\n",
      "Objective function:  6.5003174999999995 60.0 12.5003175 20 6.0\n",
      "Individual:  [[[4, 1, 1, 1, 2, 2, 0], [1, 0, 0, 0, 1, 1, 1], [4, 1, 2, 0, 1, 0, 1], [3, 2, 1, 2, 2, 0, 1], [4, 0, 1, 1, 0, 0, 0]], [[3, 1, 0, 0, 1, 2, 0], [1, 0, 1, 1, 0, 0, 0], [2, 1, 2, 2, 0, 0, 1]], [[1, 0, 2, 1, 1]]]\n",
      "Objective function:  10.603276 59.0 16.503276 20 7.5\n",
      "20 \t5     \t2e+49      \t12.5003\t1e+50      \n",
      "Individual:  [[[2, 1, 2, 0, 2, 2, 0], [4, 2, 2, 0, 2, 0, 0], [1, 0, 2, 2, 1, 2, 1], [3, 0, 1, 2, 0, 0, 0], [0, 1, 1, 2, 1, 1, 0]], [[0, 0, 0, 2, 1, 1, 0], [0, 2, 2, 0, 0, 1, 0], [1, 2, 2, 1, 0, 0, 1]], [[2, 0, 1, 2, 1]]]\n",
      "Objective function:  6.5547085 61.0 12.654708500000002 21 9.5\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 2, 1, 2, 2, 0, 0], [2, 1, 0, 1, 1, 1, 0], [4, 2, 2, 0, 1, 2, 1], [1, 1, 2, 1, 0, 0, 0], [3, 1, 1, 1, 1, 1, 0]], [[0, 2, 0, 1, 1, 1, 0], [0, 1, 1, 2, 0, 1, 0], [1, 2, 1, 0, 0, 0, 0]], [[0, 1, 1, 1, 1]]]\n",
      "Objective function:  1e+50 60.0 1e+50 21 0\n",
      "Individual:  [[[1, 1, 1, 0, 2, 1, 0], [2, 0, 2, 0, 0, 0, 1], [3, 0, 1, 0, 1, 2, 0], [1, 0, 1, 1, 0, 0, 0], [2, 2, 1, 2, 1, 2, 0]], [[0, 0, 0, 0, 1, 1, 0], [2, 2, 0, 2, 0, 1, 0], [2, 1, 0, 0, 1, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  283.68832399999997 48.0 288.488324 21 42.5\n",
      "21 \t5     \t6e+49      \t12.6547\t1e+50      \n",
      "Individual:  [[[2, 0, 1, 1, 2, 2, 1], [4, 2, 0, 2, 1, 0, 0], [4, 1, 2, 2, 0, 2, 0], [3, 1, 1, 2, 1, 0, 0], [0, 1, 1, 2, 1, 1, 0]], [[0, 2, 0, 0, 1, 1, 0], [0, 1, 0, 2, 0, 1, 1], [1, 2, 0, 1, 0, 0, 1]], [[3, 1, 1, 2, 1]]]\n",
      "Objective function:  55874946560.0 64.0 55874946566.4 22 6.5\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 2, 1, 2, 2, 2, 0], [2, 0, 0, 0, 1, 0, 0], [4, 0, 2, 0, 1, 2, 1], [3, 1, 1, 1, 0, 2, 0], [3, 1, 2, 2, 2, 1, 0]], [[3, 2, 0, 1, 1, 1, 0], [0, 2, 0, 2, 0, 0, 1], [3, 0, 0, 0, 0, 0, 1]], [[0, 2, 1, 2, 1]]]\n",
      "Objective function:  1e+50 70.0 1e+50 22 0\n",
      "Individual:  [[[3, 0, 1, 0, 2, 2, 0], [3, 2, 2, 0, 1, 0, 0], [4, 0, 0, 1, 1, 2, 1], [1, 0, 2, 1, 1, 2, 0], [3, 1, 1, 0, 0, 1, 0]], [[4, 0, 1, 1, 1, 2, 1], [0, 2, 0, 2, 0, 0, 0], [1, 2, 1, 0, 0, 2, 0]], [[0, 1, 1, 1, 1]]]\n",
      "Objective function:  40365376.296875 63.0 40365382.596875 22 8.5\n",
      "22 \t5     \t6e+49      \t4.03654e+07\t1e+50      \n",
      "Individual:  [[[3, 1, 0, 0, 2, 0, 0], [1, 0, 2, 0, 2, 0, 0], [4, 0, 2, 0, 1, 1, 1], [0, 2, 1, 1, 1, 2, 0], [3, 0, 1, 2, 2, 0, 0]], [[4, 0, 0, 1, 1, 1, 0], [0, 2, 0, 2, 0, 1, 0], [1, 2, 0, 0, 2, 0, 0]], [[4, 0, 2, 2, 1]]]\n",
      "Objective function:  0.269265 56.0 5.869265 23 496.0\n",
      "Individual:  [[[3, 0, 1, 0, 1, 2, 0], [1, 2, 2, 0, 2, 2, 0], [4, 1, 2, 0, 1, 2, 1], [3, 0, 1, 0, 0, 1, 0], [4, 1, 2, 2, 0, 1, 0]], [[0, 0, 2, 1, 1, 1, 1], [0, 2, 0, 0, 0, 1, 0], [1, 2, 0, 0, 1, 2, 1]], [[4, 1, 0, 0, 1]]]\n",
      "Objective function:  7160.8496095 64.0 7167.2496095 23 16.0\n",
      "Individual:  [[[3, 2, 0, 0, 2, 2, 0], [2, 1, 0, 1, 2, 0, 1], [3, 0, 2, 1, 1, 2, 1], [3, 0, 1, 1, 0, 0, 0], [2, 1, 1, 0, 1, 2, 0]], [[0, 2, 1, 0, 1, 0, 1], [0, 2, 0, 0, 0, 2, 1], [3, 2, 2, 0, 0, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  7005.475342 56.0 7011.075342 23 7.0\n",
      "23 \t5     \t4e+49      \t5.86927    \t1e+50      \n",
      "Individual:  [[[4, 1, 0, 0, 1, 1, 0], [0, 0, 0, 0, 2, 0, 0], [2, 0, 2, 0, 1, 0, 1], [0, 2, 2, 1, 0, 2, 0], [3, 1, 1, 2, 1, 1, 0]], [[4, 1, 0, 0, 0, 1, 0], [0, 1, 1, 0, 0, 0, 1], [1, 0, 2, 1, 0, 2, 1]], [[4, 1, 2, 2, 0]]]\n",
      "Objective function:  20.463929999999998 52.0 25.663929999999997 24 9.0\n",
      "Individual:  [[[3, 1, 1, 0, 2, 2, 0], [3, 2, 2, 2, 0, 0, 1], [0, 0, 0, 0, 1, 1, 1], [1, 2, 0, 1, 1, 2, 1], [3, 1, 1, 2, 1, 1, 0]], [[0, 2, 0, 1, 1, 1, 1], [1, 2, 0, 1, 0, 1, 0], [1, 2, 0, 0, 0, 0, 1]], [[0, 2, 2, 2, 1]]]\n",
      "Objective function:  3255959024.0 58.0 3255959029.8 24 8.0\n",
      "24 \t5     \t6e+49      \t25.6639    \t1e+50      \n",
      "Individual:  [[[4, 1, 1, 0, 2, 2, 0], [2, 2, 0, 2, 1, 0, 0], [3, 0, 2, 0, 2, 2, 1], [0, 2, 0, 1, 0, 0, 1], [3, 1, 1, 2, 2, 0, 0]], [[4, 2, 0, 1, 1, 1, 1], [0, 0, 2, 2, 0, 2, 1], [1, 1, 2, 0, 2, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  19.528047 62.0 25.728047 25 7.5\n",
      "Individual:  [[[3, 2, 0, 0, 2, 2, 0], [3, 2, 0, 0, 0, 1, 1], [4, 0, 2, 0, 0, 0, 1], [3, 1, 1, 1, 0, 0, 0], [3, 1, 1, 2, 1, 1, 0]], [[0, 0, 0, 2, 1, 1, 1], [0, 2, 0, 1, 0, 2, 1], [1, 0, 2, 0, 0, 1, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  51262716903424.0 53.0 51262716903429.3 25 8.0\n",
      "Individual:  [[[3, 1, 1, 0, 1, 2, 0], [2, 2, 0, 1, 0, 2, 0], [4, 0, 0, 2, 0, 0, 1], [4, 0, 0, 2, 0, 1, 1], [4, 1, 2, 2, 1, 1, 0]], [[0, 1, 0, 2, 1, 2, 0], [0, 2, 0, 2, 1, 1, 0], [0, 2, 2, 2, 0, 2, 1]], [[0, 1, 1, 0, 1]]]\n",
      "Objective function:  24422.4125975 66.0 24429.012597499997 25 7.0\n",
      "25 \t5     \t4e+49      \t25.728     \t1e+50      \n",
      "Individual:  [[[3, 1, 2, 0, 2, 2, 0], [1, 2, 0, 2, 2, 0, 0], [1, 0, 2, 0, 1, 0, 1], [1, 2, 0, 1, 2, 0, 0], [3, 1, 1, 2, 2, 1, 0]], [[2, 0, 0, 1, 2, 1, 1], [0, 2, 0, 2, 0, 1, 0], [4, 2, 2, 1, 0, 0, 1]], [[0, 1, 1, 0, 1]]]\n",
      "Objective function:  557466.625 62.0 557472.825 26 27.0\n",
      "Individual:  [[[0, 1, 1, 0, 2, 2, 0], [0, 1, 0, 0, 2, 2, 0], [4, 0, 2, 0, 1, 2, 1], [4, 1, 1, 1, 0, 0, 0], [2, 1, 1, 2, 1, 1, 0]], [[0, 0, 0, 2, 1, 1, 1], [0, 2, 0, 0, 1, 1, 1], [1, 2, 1, 0, 1, 0, 0]], [[1, 1, 1, 2, 1]]]\n",
      "Objective function:  88.1640205 55.0 93.6640205 26 9.5\n",
      "26 \t5     \t6e+49      \t93.664     \t1e+50      \n",
      "Individual:  [[[0, 2, 1, 0, 2, 2, 0], [2, 0, 0, 0, 2, 0, 1], [4, 0, 2, 0, 0, 1, 1], [2, 0, 1, 1, 2, 2, 0], [4, 1, 1, 2, 1, 1, 0]], [[0, 0, 0, 1, 1, 1, 0], [4, 2, 1, 1, 0, 1, 0], [1, 0, 1, 0, 2, 1, 1]], [[3, 0, 1, 2, 1]]]\n",
      "Objective function:  28.738982 58.0 34.538982000000004 27 151.0\n",
      "Individual:  [[[4, 1, 1, 1, 2, 2, 0], [2, 2, 0, 2, 2, 0, 0], [4, 1, 2, 0, 1, 2, 1], [3, 2, 1, 1, 0, 2, 0], [3, 0, 2, 2, 1, 1, 0]], [[0, 0, 0, 1, 1, 1, 0], [0, 2, 0, 2, 1, 2, 0], [1, 2, 2, 0, 0, 0, 1]], [[0, 1, 2, 2, 1]]]\n",
      "Objective function:  831863.171875 69.0 831870.071875 27 15.5\n",
      "27 \t5     \t6e+49      \t34.539     \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 0, 2, 0, 2, 0], [2, 2, 0, 0, 1, 0, 0], [4, 0, 2, 0, 1, 0, 1], [1, 1, 0, 0, 2, 2, 0], [3, 1, 2, 2, 1, 2, 1]], [[0, 1, 0, 1, 2, 2, 0], [0, 2, 0, 2, 0, 1, 0], [1, 2, 1, 0, 1, 0, 1]], [[0, 1, 2, 1, 1]]]\n",
      "Objective function:  1e+50 58.0 1e+50 28 0\n",
      "28 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "29 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 1, 0, 1, 2, 0], [2, 2, 0, 0, 0, 0, 0], [0, 0, 1, 0, 1, 2, 1], [4, 2, 1, 1, 2, 1, 0], [3, 1, 1, 2, 0, 1, 0]], [[4, 2, 0, 1, 1, 1, 0], [0, 1, 0, 2, 0, 0, 1], [1, 1, 0, 0, 0, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  1e+50 58.0 1e+50 30 0\n",
      "30 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "31 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[2, 1, 1, 0, 2, 2, 0], [2, 0, 0, 0, 0, 0, 0], [0, 0, 2, 0, 2, 2, 1], [3, 2, 2, 1, 0, 0, 0], [3, 1, 1, 1, 1, 2, 0]], [[0, 0, 0, 1, 1, 0, 0], [0, 2, 0, 2, 0, 1, 1], [3, 2, 2, 0, 0, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  1e+50 56.0 1e+50 32 0\n",
      "32 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "33 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 1, 0, 2, 2, 0], [2, 2, 0, 0, 2, 0, 0], [4, 0, 1, 0, 1, 2, 1], [3, 2, 1, 1, 0, 0, 1], [3, 1, 1, 2, 1, 1, 0]], [[0, 0, 0, 1, 1, 1, 1], [0, 2, 0, 2, 0, 1, 0], [1, 2, 2, 0, 1, 0, 0]], [[0, 1, 1, 2, 1]]]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Objective function:  1e+50 59.0 1e+50 34 0\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 1, 0, 1, 2, 0], [2, 2, 0, 0, 0, 0, 0], [0, 0, 1, 0, 1, 2, 0], [4, 0, 1, 1, 2, 1, 0], [3, 0, 1, 0, 0, 1, 0]], [[4, 2, 0, 1, 1, 0, 0], [0, 1, 0, 2, 0, 0, 1], [4, 1, 0, 0, 0, 0, 1]], [[0, 2, 1, 0, 1]]]\n",
      "Objective function:  1e+50 58.0 1e+50 34 0\n",
      "34 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "35 \t5     \t1e+50      \t1e+50      \t1e+50      \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/nanohub/psalek/.local/lib/python3.7/site-packages/keras/callbacks/callbacks.py:846: RuntimeWarning: Early stopping conditioned on metric `val_loss` which is not available. Available metrics are: loss,activation_55_loss,add_10_loss\n",
      "  (self.monitor, ','.join(list(logs.keys()))), RuntimeWarning\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 1, 2, 1, 2, 0], [2, 2, 0, 0, 0, 0, 0], [0, 0, 1, 1, 1, 2, 1], [1, 1, 1, 1, 1, 1, 0], [3, 1, 1, 1, 0, 1, 0]], [[3, 2, 0, 1, 1, 1, 0], [0, 1, 0, 2, 0, 0, 1], [1, 1, 0, 2, 0, 0, 1]], [[0, 0, 1, 2, 1]]]\n",
      "Objective function:  1e+50 56.0 1e+50 36 0\n",
      "Individual:  [[[2, 0, 1, 0, 2, 2, 0], [4, 0, 0, 0, 1, 0, 0], [0, 0, 2, 0, 2, 2, 1], [3, 2, 2, 1, 2, 0, 0], [3, 1, 1, 1, 0, 2, 0]], [[0, 0, 1, 1, 1, 0, 0], [0, 2, 0, 2, 0, 1, 1], [4, 2, 2, 0, 0, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  86307.048828 61.0 86313.148828 36 7.0\n",
      "36 \t5     \t8e+49      \t86313.1    \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[0, 0, 1, 0, 1, 2, 1], [2, 2, 0, 0, 0, 0, 0], [0, 2, 2, 0, 1, 2, 1], [4, 2, 1, 1, 2, 1, 1], [4, 1, 1, 2, 0, 1, 1]], [[4, 0, 0, 1, 1, 1, 0], [0, 1, 1, 2, 0, 0, 1], [1, 1, 0, 0, 0, 0, 1]], [[0, 0, 1, 2, 1]]]\n",
      "Objective function:  1e+50 55.0 1e+50 37 0\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 2, 0, 0, 1, 0], [2, 2, 0, 0, 0, 0, 0], [3, 0, 1, 2, 1, 2, 1], [4, 2, 1, 1, 2, 1, 0], [3, 1, 1, 2, 2, 1, 0]], [[4, 2, 0, 1, 0, 1, 0], [0, 1, 0, 2, 0, 0, 1], [1, 1, 0, 1, 0, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  1e+50 65.0 1e+50 37 0\n",
      "37 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "Individual:  [[[3, 1, 1, 0, 1, 2, 0], [2, 2, 0, 2, 0, 0, 0], [1, 2, 1, 1, 1, 2, 0], [4, 2, 1, 1, 1, 1, 0], [3, 1, 1, 2, 0, 1, 0]], [[4, 2, 0, 1, 1, 0, 0], [0, 1, 0, 2, 0, 1, 1], [1, 0, 0, 0, 0, 0, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  566017536.0 62.0 566017542.2 38 7.0\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 1, 2, 1, 2, 0], [2, 0, 0, 0, 0, 0, 0], [0, 0, 1, 1, 1, 2, 1], [0, 1, 1, 1, 1, 1, 0], [3, 1, 1, 1, 0, 1, 0]], [[3, 2, 2, 1, 1, 1, 1], [0, 2, 0, 2, 0, 0, 1], [2, 2, 0, 2, 0, 0, 1]], [[0, 0, 1, 2, 0]]]\n",
      "Objective function:  1e+50 58.0 1e+50 38 0\n",
      "38 \t5     \t8e+49      \t5.66018e+08\t1e+50      \n",
      "Individual:  [[[3, 1, 1, 2, 1, 2, 1], [0, 0, 0, 0, 0, 1, 0], [0, 0, 1, 1, 0, 2, 1], [0, 1, 1, 1, 1, 0, 0], [3, 0, 2, 1, 0, 1, 0]], [[3, 0, 2, 1, 1, 2, 0], [0, 2, 0, 2, 0, 1, 1], [2, 2, 2, 2, 0, 1, 1]], [[0, 0, 1, 2, 0]]]\n",
      "Objective function:  4043417.125 55.0 4043422.625 39 16.0\n",
      "Individual:  [[[3, 1, 1, 2, 1, 2, 0], [2, 2, 0, 0, 0, 0, 1], [0, 0, 1, 1, 1, 2, 1], [1, 1, 1, 1, 1, 1, 0], [3, 1, 1, 0, 2, 1, 0]], [[3, 2, 0, 1, 1, 1, 0], [2, 1, 0, 2, 0, 0, 1], [3, 1, 0, 1, 0, 0, 1]], [[0, 0, 1, 2, 1]]]\n",
      "Objective function:  202636110.5 59.0 202636116.4 39 8.0\n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[0, 1, 1, 0, 0, 2, 0], [2, 0, 0, 0, 1, 0, 0], [0, 2, 1, 1, 0, 2, 1], [0, 1, 0, 1, 1, 1, 0], [1, 2, 1, 1, 0, 1, 0]], [[3, 2, 2, 1, 1, 2, 1], [0, 1, 0, 2, 0, 0, 1], [2, 2, 0, 0, 0, 0, 1]], [[0, 0, 1, 0, 0]]]\n",
      "Objective function:  1e+50 47.0 1e+50 39 0\n",
      "Individual:  [[[3, 0, 1, 0, 1, 0, 0], [2, 2, 0, 2, 0, 0, 0], [3, 0, 0, 0, 1, 2, 1], [4, 2, 0, 1, 2, 1, 1], [3, 1, 1, 2, 0, 1, 0]], [[0, 2, 0, 1, 1, 1, 0], [0, 1, 0, 0, 0, 0, 1], [1, 1, 0, 0, 0, 2, 1]], [[0, 1, 1, 2, 1]]]\n",
      "Objective function:  558759920.0 48.0 558759924.8 39 11.0\n",
      "39 \t5     \t4e+49      \t4.04342e+06\t1e+50      \n",
      "40 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "41 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 2, 1, 2, 1, 2, 0], [2, 2, 0, 0, 0, 0, 0], [0, 1, 1, 1, 1, 2, 1], [1, 1, 1, 1, 1, 1, 0], [3, 1, 0, 1, 0, 1, 0]], [[3, 2, 0, 1, 1, 1, 0], [0, 1, 0, 2, 0, 0, 1], [3, 1, 0, 2, 0, 0, 1]], [[0, 0, 1, 2, 1]]]\n",
      "Objective function:  1e+50 59.0 1e+50 42 0\n",
      "Individual:  [[[0, 1, 1, 0, 0, 2, 0], [4, 0, 0, 0, 1, 0, 0], [0, 2, 1, 1, 0, 2, 1], [0, 1, 2, 1, 1, 0, 0], [1, 2, 1, 1, 0, 1, 0]], [[3, 2, 2, 1, 1, 2, 1], [0, 1, 0, 2, 0, 0, 1], [2, 2, 0, 0, 0, 0, 1]], [[3, 0, 1, 0, 0]]]\n",
      "Objective function:  3830.5993350000003 52.0 3835.799335 42 14.5\n",
      "42 \t5     \t8e+49      \t3835.8     \t1e+50      \n",
      "43 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 0, 1, 2, 1, 2, 0], [2, 1, 0, 0, 0, 0, 0], [0, 1, 1, 1, 1, 2, 1], [1, 1, 1, 1, 0, 1, 0], [3, 0, 0, 1, 0, 1, 0]], [[3, 2, 0, 1, 1, 1, 0], [0, 1, 0, 2, 0, 0, 1], [3, 1, 0, 2, 0, 0, 1]], [[0, 0, 1, 2, 1]]]\n",
      "Objective function:  1e+50 54.0 1e+50 44 0\n",
      "44 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "45 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[0, 1, 1, 0, 0, 2, 0], [2, 0, 0, 0, 1, 0, 0], [2, 2, 1, 1, 0, 2, 1], [0, 1, 0, 1, 1, 1, 0], [1, 0, 1, 1, 0, 1, 0]], [[3, 2, 2, 0, 1, 1, 1], [0, 1, 0, 2, 0, 0, 1], [2, 2, 0, 0, 0, 0, 1]], [[0, 1, 1, 0, 0]]]\n",
      "Objective function:  1e+50 47.0 1e+50 46 0\n",
      "46 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "47 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "48 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "49 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "Batch 0: Invalid loss, terminating training\n",
      "Individual:  [[[3, 1, 1, 2, 1, 2, 1], [2, 2, 0, 0, 0, 0, 0], [0, 0, 1, 1, 1, 2, 1], [1, 1, 2, 1, 1, 1, 0], [3, 1, 1, 1, 2, 0, 0]], [[3, 2, 0, 1, 1, 1, 0], [0, 1, 0, 2, 0, 0, 1], [1, 1, 0, 2, 0, 0, 1]], [[0, 0, 1, 2, 1]]]\n",
      "Objective function:  1e+50 57.0 1e+50 50 0\n",
      "Individual:  [[[3, 1, 1, 2, 1, 2, 0], [2, 2, 0, 0, 0, 1, 0], [0, 0, 1, 1, 1, 2, 1], [1, 1, 1, 1, 1, 1, 0], [3, 1, 1, 1, 0, 1, 0]], [[3, 2, 0, 1, 1, 1, 1], [0, 1, 0, 2, 0, 0, 1], [1, 1, 0, 2, 0, 1, 1]], [[0, 0, 1, 2, 1]]]\n",
      "Objective function:  509228932096.0 58.0 509228932101.8 50 7.5\n",
      "50 \t5     \t8e+49      \t5.09229e+11\t1e+50      \n",
      "gen\tnevals\tavg        \tmin        \tmax        \n",
      "0  \t5     \t2e+49      \t8.96852    \t1e+50      \n",
      "1  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "2  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "3  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "4  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "5  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "6  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "7  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "8  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "9  \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "10 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "11 \t5     \t4e+49      \t8.25557    \t1e+50      \n",
      "12 \t5     \t4e+49      \t5.49547    \t1e+50      \n",
      "13 \t5     \t6e+49      \t8.51396    \t1e+50      \n",
      "14 \t5     \t2.35214e+08\t8.2758     \t1.17607e+09\n",
      "15 \t5     \t2e+49      \t14.1756    \t1e+50      \n",
      "16 \t5     \t4e+49      \t11.4373    \t1e+50      \n",
      "17 \t5     \t45786.3    \t7.61031    \t191161     \n",
      "18 \t5     \t126700     \t8.84203    \t356982     \n",
      "19 \t5     \t4e+49      \t13.7906    \t1e+50      \n",
      "20 \t5     \t2e+49      \t12.5003    \t1e+50      \n",
      "21 \t5     \t6e+49      \t12.6547    \t1e+50      \n",
      "22 \t5     \t6e+49      \t4.03654e+07\t1e+50      \n",
      "23 \t5     \t4e+49      \t5.86927    \t1e+50      \n",
      "24 \t5     \t6e+49      \t25.6639    \t1e+50      \n",
      "25 \t5     \t4e+49      \t25.728     \t1e+50      \n",
      "26 \t5     \t6e+49      \t93.664     \t1e+50      \n",
      "27 \t5     \t6e+49      \t34.539     \t1e+50      \n",
      "28 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "29 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "30 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "31 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "32 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "33 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "34 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "35 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "36 \t5     \t8e+49      \t86313.1    \t1e+50      \n",
      "37 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "38 \t5     \t8e+49      \t5.66018e+08\t1e+50      \n",
      "39 \t5     \t4e+49      \t4.04342e+06\t1e+50      \n",
      "40 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "41 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "42 \t5     \t8e+49      \t3835.8     \t1e+50      \n",
      "43 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "44 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "45 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "46 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "47 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "48 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "49 \t5     \t1e+50      \t1e+50      \t1e+50      \n",
      "50 \t5     \t8e+49      \t5.09229e+11\t1e+50      \n"
     ]
    }
   ],
   "source": [
    "################### DEAP #####################\n",
    "#create fitness class and individual class\n",
    "creator.create(\"FitnessMin\", base.Fitness, weights=(-1.0,))\n",
    "creator.create(\"Individual\", Individual_O, fitness=creator.FitnessMin)\n",
    "\n",
    "toolbox = base.Toolbox()\n",
    "#pool = Pool(1)\n",
    "toolbox.register(\"map\", futures.map)\n",
    "#toolbox.register(\"attr_int\", random.randint, 0, 3)\n",
    "\n",
    "#gen = initRepeat(list, random.randint, 3, 7, 4)\n",
    "toolbox.register(\"create_individual\", init_repeat, \n",
    "                 nlayers, nNodes, max1=4, max2=2, max3=1)\n",
    "toolbox.register(\"population\", tools.initRepeat, list, toolbox.create_individual)\n",
    "\n",
    "cxpb = 0\n",
    "mutpb = [0, 0.8, 0.5, 0.2, 0.1]\n",
    "#mutpb = 0.5\n",
    "ngens = 50\n",
    "\n",
    "toolbox.register(\"mate\", custom_crossover)\n",
    "#toolbox.register(\"mutate\", tools.mutUniformInt, low=0, up=3, indpb=mutpb)\n",
    "toolbox.register(\"mutate\", custom_mutation, max1=4, max2=2, max3=1)\n",
    "toolbox.register(\"select\", tools.selTournament, tournsize=80)\n",
    "toolbox.register(\"evaluate\", objective_function)\n",
    "\n",
    "def main():\n",
    "    random.seed(10000)\n",
    "    \n",
    "    if not Load_Old_Population:\n",
    "        population = toolbox.population(n=800)\n",
    "    else:\n",
    "        with open(\"P\" + str(p) + \"_1\", 'rb') as f:\n",
    "            population = pickle.load(f)\n",
    "    stats = tools.Statistics(lambda ind: ind.Objective)\n",
    "    stats.register(\"avg\", np.mean)\n",
    "    stats.register(\"min\", np.min)\n",
    "    stats.register(\"max\", np.max)\n",
    "    pop, logbook = custom_eaSimple(population, toolbox, cxpb, mutpb, ngens, stats=stats, verbose=True)\n",
    "\n",
    "    return pop, logbook\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    pop, logbook = main()\n",
    "    \n",
    "    # Save final population\n",
    "    filename = \"P\" + str(p)\n",
    "    with open(filename, 'wb') as f:\n",
    "        pickle.dump(pop, f)\n",
    "        f.close()\n",
    "        \n",
    "    print (logbook, flush=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}