Experiment3_GA_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FZg_QFpe7W-B"
   },
   "source": [
    "# Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xWATOMlqSBDH"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "import numpy as np\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_OHVp0iz7a9O"
   },
   "outputs": [],
   "source": [
    "class utility:\n",
    "\n",
    "    def read_CSV(self, filename):\n",
    "        df = pd.read_csv(filename)\n",
    "        return df\n",
    "\n",
    "    def get_text_label(self, df):\n",
    "        texts = []  # list of text samples\n",
    "        labels = []  # list of label ids\n",
    "        for index, row in df.iterrows():\n",
    "            if isinstance(row['text_cleaned'], float):\n",
    "                texts.append(str(row['text_cleaned']))\n",
    "            else:\n",
    "                texts.append(row['text_cleaned'])\n",
    "\n",
    "            labels.append(row['target'])\n",
    "\n",
    "        return texts, labels\n",
    "\n",
    "    def tokenize_texts(self, texts):\n",
    "        tokenizer = Tokenizer(num_words=50000)\n",
    "        tokenizer.fit_on_texts(texts)\n",
    "\n",
    "        return tokenizer\n",
    "\n",
    "    def padding_texts(self, texts, maxlen):\n",
    "\n",
    "        texts = pad_sequences(texts, padding='post', maxlen=maxlen)\n",
    "\n",
    "        return texts\n",
    "\n",
    "    def get_metric(self, y_true, y_pred):\n",
    "        accuracyScore = accuracy_score(y_true, y_pred)\n",
    "\n",
    "        precisionScore = precision_score(y_true, y_pred)\n",
    "        recallScore = recall_score(y_true, y_pred)\n",
    "        f1Score = f1_score(y_true, y_pred)\n",
    "\n",
    "        return accuracyScore, precisionScore, recallScore, f1Score\n",
    "\n",
    "    def print_metric(self, accuracyScore, precisionScore, recallScore, f1Score):\n",
    "        print(\"Accuracy: {}\".format(str(accuracyScore)))\n",
    "        print(\"Precision: {}\".format(str(precisionScore)))\n",
    "        print(\"Recall: {}\".format(str(recallScore)))\n",
    "        print(\"F1-Score: {}\".format(str(f1Score)))\n",
    "        print(\"{},{},{},{}\".format(str(accuracyScore), str(precisionScore), str(recallScore), str(f1Score)))\n",
    "        \n",
    "\n",
    "    def get_testing_metric(self, y_test, y_pred):\n",
    "        accuracyScore, precisionScore, recallScore, f1Score = self.get_metric(y_test, y_pred)\n",
    "\n",
    "        return accuracyScore, precisionScore, recallScore, f1Score\n",
    "\n",
    "    def write_df_csv(self, df, out_path):\n",
    "        df.to_csv(out_path, index=False)\n",
    "\n",
    "    def create_embedding_matrix(self, filepath, word_index, embedding_dim):\n",
    "        vocab_size = len(word_index) + 1  # Adding again 1 because of reserved 0 index\n",
    "        embedding_matrix = np.zeros((vocab_size, embedding_dim))\n",
    "\n",
    "        with open(filepath, encoding=\"utf8\") as f:\n",
    "            for line in f:\n",
    "                word, *vector = line.split()\n",
    "                if word in word_index:\n",
    "                    idx = word_index[word]\n",
    "                    embedding_matrix[idx] = np.array(\n",
    "                        vector, dtype=np.float32)[:embedding_dim]\n",
    "\n",
    "        return embedding_matrix\n",
    "\n",
    "    def get_max_length_of_sentences(self, texts):\n",
    "        maxlength = 0\n",
    "        for text in texts:\n",
    "            if (len(text.split()) > maxlength):\n",
    "                maxlength = len(text.split())\n",
    "\n",
    "        return maxlength\n",
    "\n",
    "    def get_training_trial_data(self, textsTraining, labelsTraining, textsTrial, labelsTrial, glovePath):\n",
    "        textsTraining, textsTesting = np.asarray(textsTraining), np.asarray(textsTrial)\n",
    "        y_train, y_val = np.asarray(labelsTraining), np.asarray(labelsTrial)\n",
    "\n",
    "        # Tokenize words\n",
    "        tokenizer = self.tokenize_texts(textsTraining)\n",
    "        X_train = tokenizer.texts_to_sequences(textsTraining)\n",
    "        X_val = tokenizer.texts_to_sequences(textsTesting)\n",
    "\n",
    "        # Adding 1 because of reserved 0 index\n",
    "        vocab_size = len(tokenizer.word_index) + 1\n",
    "\n",
    "        # get maxlen\n",
    "        maxlen = self.get_max_length_of_sentences(textsTraining)\n",
    "\n",
    "        # Pad sequences with zeros\n",
    "        X_train = self.padding_texts(X_train, maxlen)\n",
    "        X_val = self.padding_texts(X_val, maxlen)\n",
    "\n",
    "        embedding_matrix = []\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[0], tokenizer.word_index, 50))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[1], tokenizer.word_index, 100))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[2], tokenizer.word_index, 200))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[3], tokenizer.word_index, 300))\n",
    "\n",
    "        return X_train, X_val, y_train, y_val, vocab_size, maxlen, embedding_matrix\n",
    "\n",
    "    def Average(self, list):\n",
    "        return sum(list) / len(list)\n",
    "\n",
    "    def recall(self, y_true, y_pred):\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(self, y_true, y_pred):\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "\n",
    "    def f1_score(self, y_true, y_pred):\n",
    "        precision = self.precision(y_true, y_pred)\n",
    "        recall = self.recall(y_true, y_pred)\n",
    "        return 2*((precision*recall)/(precision+recall+K.epsilon()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uYAnQP8Ac2Qe"
   },
   "source": [
    "# Finite State Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uKx4XQbmc5HN"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def FSM():\n",
    "    fsm = {}\n",
    "    fsm[0] = {'src': 0, 'dst': 1, 'layer': 'embedding_layer', 'next_path': [1]}\n",
    "    fsm[1] = {'src': 1, 'dst': 2, 'layer': 'convolutional_layer', 'next_path': [2, 4]}\n",
    "    fsm[2] = {'src': 2, 'dst': 3, 'layer': 'maxpooling_layer', 'next_path': [3]}    \n",
    "    fsm[3] = {'src': 3, 'dst': 2, 'layer': 'convolutional_layer', 'next_path': [2, 4]}\n",
    "    fsm[4] = {'src': 2, 'dst': 4, 'layer': 'global_maxpooling_layer', 'next_path': [5]}\n",
    "    fsm[5] = {'src': 4, 'dst': 5, 'layer': 'dense_layer', 'next_path': [6, 7]}\n",
    "    fsm[6] = {'src': 5, 'dst': 5, 'layer': 'dense_layer', 'next_path': [6, 7]}    \n",
    "    fsm[7] = {'src': 5, 'dst': 6, 'layer': 'dropout_layer', 'next_path': [8]}\n",
    "    fsm[8] = {'src': 6, 'dst': 7, 'layer': 'output_layer', 'next_path': []}\n",
    "\n",
    "    return fsm\n",
    "\n",
    "def addConvLayer(idx, toolbox, toolboxes, defaultVal, layerparameters):\n",
    "    toolbox.register('num_filters{}'.format(str(idx)), layerparameters['num_filters'][0],\n",
    "                     layerparameters['num_filters'][1], layerparameters['num_filters'][2])\n",
    "    \n",
    "    toolboxes.append(toolbox.__getattribute__('num_filters{}'.format(str(idx))))\n",
    "    toolbox.register('kernel_size{}'.format(str(idx)), layerparameters['kernel_size'][0],\n",
    "                     layerparameters['kernel_size'][1], layerparameters['kernel_size'][2])\n",
    "    toolboxes.append(toolbox.__getattribute__('kernel_size{}'.format(str(idx))))\n",
    "    toolbox.register('conv_activation_func{}'.format(str(idx)), layerparameters['conv_activation_func'][0],\n",
    "                     layerparameters['conv_activation_func'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('conv_activation_func{}'.format(str(idx))))\n",
    "    toolbox.register('conv_init_mode{}'.format(str(idx)), layerparameters['conv_init_mode'][0],\n",
    "                     layerparameters['conv_init_mode'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('conv_init_mode{}'.format(str(idx))))\n",
    "    toolbox.register('conv_weight_constraint{}'.format(str(idx)), layerparameters['conv_weight_constraint'][0],\n",
    "                     layerparameters['conv_weight_constraint'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('conv_weight_constraint{}'.format(str(idx))))\n",
    "\n",
    "    defaultVal.update({'num_filters{}'.format(str(idx)): 64})\n",
    "    defaultVal.update({'kernel_size{}'.format(str(idx)): 3})\n",
    "    defaultVal.update({'conv_activation_func{}'.format(str(idx)): \"relu\"})\n",
    "    defaultVal.update({'conv_init_mode{}'.format(str(idx)): \"glorot_uniform\"})\n",
    "    defaultVal.update({'conv_weight_constraint{}'.format(str(idx)): 3})\n",
    "\n",
    "\n",
    "def addDenseLayer(idx, toolbox, toolboxes, defaultVal, layerparameters):\n",
    "    toolbox.register('neurons{}'.format(str(idx)), layerparameters['neurons'][0],\n",
    "                     layerparameters['neurons'][1], layerparameters['neurons'][2])\n",
    "    toolboxes.append(toolbox.__getattribute__('neurons{}'.format(str(idx))))\n",
    "    toolbox.register('dense_activation_func{}'.format(str(idx)), layerparameters['dense_activation_func'][0],\n",
    "                     layerparameters['dense_activation_func'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('dense_activation_func{}'.format(str(idx))))\n",
    "    toolbox.register('dense_init_mode{}'.format(str(idx)), layerparameters['dense_init_mode'][0],\n",
    "                     layerparameters['dense_init_mode'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('dense_init_mode{}'.format(str(idx))))\n",
    "    toolbox.register('dense_weight_constraint{}'.format(str(idx)), layerparameters['dense_weight_constraint'][0],\n",
    "                     layerparameters['dense_weight_constraint'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('dense_weight_constraint{}'.format(str(idx))))\n",
    "\n",
    "    defaultVal.update({'neurons{}'.format(str(idx)): 1})\n",
    "    defaultVal.update({'dense_activation_func{}'.format(str(idx)): \"relu\"})\n",
    "    defaultVal.update({'dense_init_mode{}'.format(str(idx)): \"glorot_uniform\"})\n",
    "    defaultVal.update({'dense_weight_constraint{}'.format(str(idx)): 3})\n",
    "\n",
    "\n",
    "def addMaxPoolingLayer(idx, toolbox, toolboxes, defaultVal, layerparameters):\n",
    "    toolbox.register('pool_size{}'.format(str(idx)), layerparameters['pool_size'][0],\n",
    "                     layerparameters['pool_size'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('pool_size{}'.format(str(idx))))\n",
    "\n",
    "    defaultVal.update({'pool_size{}'.format(str(idx)): 5})\n",
    "\n",
    "\n",
    "def addDropoutLayer(idx, toolbox, toolboxes, defaultVal, layerparameters):\n",
    "    toolbox.register('dropout_rate{}'.format(str(idx)), layerparameters['dropout_rate'][0],\n",
    "                     layerparameters['dropout_rate'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('dropout_rate{}'.format(str(idx))))\n",
    "\n",
    "    defaultVal.update({'dropout_rate{}'.format(str(idx)): 0.2})\n",
    "\n",
    "def getLayerSize(layer, conv_idx, dense_idx, dropout_idx, maxpooling_idx):\n",
    "    if layer == 'convolutional_layer':\n",
    "        conv_idx += 1\n",
    "    elif layer == 'dense_layer':\n",
    "        dense_idx += 1\n",
    "    elif layer == 'dropout_layer':\n",
    "        dropout_idx += 1\n",
    "    elif layer == 'maxpooling_layer':\n",
    "        maxpooling_idx += 1\n",
    "    return conv_idx, dense_idx, dropout_idx, maxpooling_idx\n",
    "\n",
    "\n",
    "def getMaxLayerSize(conv_idx, dense_idx, dropout_idx, maxpooling_idx, max_conv_idx, max_dense_idx, max_dropout_idx,\n",
    "                    max_maxpooling_idx):\n",
    "    if conv_idx > max_conv_idx:\n",
    "        max_conv_idx = conv_idx\n",
    "    if dense_idx > max_dense_idx:\n",
    "        max_dense_idx = dense_idx\n",
    "    if dropout_idx > max_dropout_idx:\n",
    "        max_dropout_idx = dropout_idx\n",
    "    if maxpooling_idx > max_maxpooling_idx:\n",
    "        max_maxpooling_idx = maxpooling_idx\n",
    "\n",
    "    return max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx\n",
    "\n",
    "\n",
    "def addLayerToolboxes(max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx, toolbox, toolboxes, defaultVal,\n",
    "                      layerparameters):\n",
    "    idx = 0\n",
    "    while idx < max_conv_idx:\n",
    "        idx += 1\n",
    "        addConvLayer(idx, toolbox, toolboxes, defaultVal, layerparameters)\n",
    "\n",
    "    idx = 0\n",
    "    while idx < max_dense_idx:\n",
    "        idx += 1\n",
    "        addDenseLayer(idx, toolbox, toolboxes, defaultVal, layerparameters)\n",
    "\n",
    "    idx = 0\n",
    "    while idx < max_maxpooling_idx:\n",
    "        idx += 1\n",
    "        addMaxPoolingLayer(idx, toolbox, toolboxes, defaultVal, layerparameters)\n",
    "\n",
    "    idx = 0\n",
    "    while idx < max_dropout_idx:\n",
    "        idx += 1\n",
    "        addDropoutLayer(idx, toolbox, toolboxes, defaultVal, layerparameters)\n",
    "\n",
    "\n",
    "def generateFSM(n_pop, layerparameters, toolbox, toolboxes, defaultVal):\n",
    "    fsm = FSM()\n",
    "\n",
    "    path_ind = {}\n",
    "    max_conv_idx = 0\n",
    "    max_dense_idx = 0\n",
    "    max_dropout_idx = 0\n",
    "    max_maxpooling_idx = 0\n",
    "\n",
    "    for ind in range(0, n_pop):\n",
    "        idx = conv_idx = dense_idx = dropout_idx = maxpooling_idx = 0\n",
    "        path = [fsm[idx]['layer']]\n",
    "        while len(fsm[idx]['next_path']) != 0:\n",
    "            idx = random.choice(fsm[idx]['next_path'])\n",
    "            layer = fsm[idx]['layer']\n",
    "            path.append(layer)\n",
    "            conv_idx, dense_idx, dropout_idx, maxpooling_idx = getLayerSize(layer, conv_idx, dense_idx, dropout_idx,\n",
    "                                                                            maxpooling_idx)\n",
    "\n",
    "        max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx = getMaxLayerSize(conv_idx, dense_idx,\n",
    "                                                                                           dropout_idx, maxpooling_idx,\n",
    "                                                                                           max_conv_idx, max_dense_idx,\n",
    "                                                                                           max_dropout_idx,\n",
    "                                                                                           max_maxpooling_idx)\n",
    "\n",
    "        path_ind[ind] = path\n",
    "\n",
    "    addLayerToolboxes(max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx, toolbox, toolboxes, defaultVal,\n",
    "                      layerparameters)\n",
    "\n",
    "    return path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx\n",
    "\n",
    "\n",
    "def openFSM(df, layerparameters, toolbox, toolboxes, defaultVal):\n",
    "    path_ind = {}\n",
    "    fitnesses = []\n",
    "\n",
    "    hyperparams = [s for s in list(df.columns) if not 'Unnamed' in s]\n",
    "\n",
    "    max_conv_idx = sum('num_filters' in s for s in hyperparams)\n",
    "    max_dense_idx = sum('neurons' in s for s in hyperparams)\n",
    "    max_dropout_idx = sum('dropout_rate' in s for s in hyperparams)\n",
    "    max_maxpooling_idx = sum('pool_size' in s for s in hyperparams)\n",
    "\n",
    "    for index, row in df.iterrows():\n",
    "        path = [s for s in row if 'layer' in str(s)]\n",
    "        fitness = [s for s in row if str(s).replace('.', '', 1).isdigit()]\n",
    "        fitnesses.append(tuple([float(fitness[0])]))\n",
    "        path_ind[index] = path\n",
    "\n",
    "    addLayerToolboxes(max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx, toolbox, toolboxes, defaultVal,\n",
    "                      layerparameters)\n",
    "\n",
    "    return path_ind, fitnesses, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-QjUHS9ZWs7U"
   },
   "source": [
    "# CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7DObu8fBWr48"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "class CNN:\n",
    "\n",
    "    def cnn_model(self, vocab_size, maxlen, embedding_matrix, indiv, path):\n",
    "        model = tf.keras.models.Sequential()\n",
    "        conv_idx = dense_idx = dropout_idx = maxpooling_idx = 0\n",
    "        for layer in path:\n",
    "            if layer == 'embedding_layer':\n",
    "                model.add(\n",
    "                    tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=indiv['output_dim'],\n",
    "                                     weights=[embedding_matrix], input_length=maxlen, trainable=True))\n",
    "            elif layer == 'convolutional_layer':\n",
    "                conv_idx += 1\n",
    "                model.add(tf.keras.layers.Conv1D(indiv['num_filters{}'.format(str(conv_idx))], \n",
    "                                                 indiv['kernel_size{}'.format(str(conv_idx))],\n",
    "                                        kernel_initializer=indiv['conv_init_mode{}'.format(str(conv_idx))],\n",
    "                                        activation=indiv['conv_activation_func{}'.format(str(conv_idx))],\n",
    "                                        kernel_constraint=tf.keras.constraints.max_norm(indiv['conv_weight_constraint{}'.format(str(conv_idx))]),\n",
    "                                        data_format='channels_first'))\n",
    "            elif layer == 'dense_layer':\n",
    "                dense_idx += 1\n",
    "                model.add(tf.keras.layers.Dense(indiv['neurons{}'.format(str(dense_idx))],\n",
    "                                       kernel_initializer=indiv['dense_init_mode{}'.format(str(dense_idx))],\n",
    "                                       activation=indiv['dense_activation_func{}'.format(str(dense_idx))],\n",
    "                                       kernel_constraint=tf.keras.constraints.max_norm(indiv['dense_weight_constraint{}'.format(str(dense_idx))])))\n",
    "            elif layer == 'dropout_layer':\n",
    "                dropout_idx += 1\n",
    "                model.add(tf.keras.layers.Dropout(indiv['dropout_rate{}'.format(str(dropout_idx))]))\n",
    "            elif layer == 'maxpooling_layer':\n",
    "                maxpooling_idx += 1\n",
    "                model.add(tf.keras.layers.MaxPooling1D(indiv['pool_size{}'.format(str(maxpooling_idx))]))\n",
    "            elif layer == 'global_maxpooling_layer':\n",
    "                model.add(tf.keras.layers.GlobalMaxPooling1D())\n",
    "            elif layer == 'output_layer':\n",
    "                model.add(tf.keras.layers.Dense(1, kernel_initializer=indiv['output_init_mode'], activation='sigmoid'))\n",
    "\n",
    "        if indiv['optimizer'] == 'sgd':\n",
    "            opt = tf.keras.optimizers.SGD(lr=indiv['learning_rate'], momentum=indiv['momentum'], decay=0.0,\n",
    "                                 nesterov=False)\n",
    "        elif indiv['optimizer'] == 'rmsprop':\n",
    "            opt = tf.keras.optimizers.RMSprop(lr=indiv['learning_rate'], rho=0.9, epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adagrad':\n",
    "            opt = tf.keras.optimizers.Adagrad(lr=indiv['learning_rate'], epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adadelta':\n",
    "            opt = tf.keras.optimizers.Adadelta(lr=indiv['learning_rate'], rho=0.95, epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adam':\n",
    "            opt = tf.keras.optimizers.Adam(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                  decay=0.0, amsgrad=False)\n",
    "        elif indiv['optimizer'] == 'adamax':\n",
    "            opt = tf.keras.optimizers.Adamax(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                    decay=0.0)\n",
    "        elif indiv['optimizer'] == 'nadam':\n",
    "            opt = tf.keras.optimizers.Nadam(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                   schedule_decay=0.004)\n",
    "        \n",
    "        util = utility()\n",
    "        model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[util.f1_score])\n",
    "\n",
    "        return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jsfyjDRoWVMU"
   },
   "source": [
    "# Fitness Calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hn9qqGyhWS8x"
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import os\n",
    "from time import sleep\n",
    "import gc\n",
    "\n",
    "util = utility()\n",
    "cnn = CNN()\n",
    "\n",
    "def FitnessCalculation(individual, cfold, defaultVal, resultsPath, testing_name):\n",
    "    indiv = collections.OrderedDict()\n",
    "    i = 0\n",
    "    for key in defaultVal.keys():\n",
    "        indiv[key] = individual[i]\n",
    "        i += 1\n",
    "\n",
    "    path = individual[len(defaultVal):len(individual)]\n",
    "    \n",
    "    return crossfold(indiv, path, cfold, resultsPath, testing_name)\n",
    "\n",
    "\n",
    "def crossfold(indiv, path, fold, resultsPath, testing_name):\n",
    "    if indiv['output_dim'] == 50:\n",
    "        embedding_mtx = fold['embedding_matrix'][0]\n",
    "    elif indiv['output_dim'] == 100:\n",
    "        embedding_mtx = fold['embedding_matrix'][1]\n",
    "    elif indiv['output_dim'] == 200:\n",
    "        embedding_mtx = fold['embedding_matrix'][2]\n",
    "    elif indiv['output_dim'] == 300:\n",
    "        embedding_mtx = fold['embedding_matrix'][3]\n",
    "\n",
    "    model = cnn.cnn_model(fold['vocab_size'], fold['maxlen'], embedding_mtx,\n",
    "                          indiv, path)\n",
    "    \n",
    "    #early stopping\n",
    "    #save the best model\n",
    "    callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_f1_score', mode='max', verbose=False, patience=10), \n",
    "                 tf.keras.callbacks.ModelCheckpoint(\"{}{}.h5\".format(resultsPath, testing_name), monitor='val_f1_score', mode='max', verbose=False, \n",
    "                                  save_best_only=True)]\n",
    "\n",
    "    model.fit(fold['X_train'], fold['y_train'], epochs=indiv['epochs'], verbose=False, \n",
    "              validation_data=(fold['X_val'], fold['y_val']), use_multiprocessing=False,\n",
    "              batch_size=indiv['batch_size'], callbacks=callbacks)\n",
    "    \n",
    "    dependencies = {\n",
    "    'f1_score': util.f1_score\n",
    "    }\n",
    "\n",
    "    # load the saved model\n",
    "    for x in range(0, 10):  # try 4 times\n",
    "        try:\n",
    "            # msg.send()\n",
    "            saved_model = tf.keras.models.load_model(\"{}{}.h5\".format(resultsPath, testing_name), custom_objects=dependencies)\n",
    "            str_error = None\n",
    "        except Exception as e:\n",
    "            print('An error occurs when loading saved model.')\n",
    "            str_error = e\n",
    "            pass\n",
    "\n",
    "        if str_error:\n",
    "            sleep(5)  # wait for 2 seconds before trying to fetch the data again\n",
    "        else:\n",
    "            break\n",
    "    \n",
    "\n",
    "    y_pred = saved_model.predict_classes(fold['X_val'])\n",
    "\n",
    "    os.remove(\"{}{}.h5\".format(resultsPath, testing_name))\n",
    "    del embedding_mtx\n",
    "    gc.collect()\n",
    "\n",
    "    # CNN metrics\n",
    "    accuracyScore, precisionScore, recallScore, f1Score = util.get_testing_metric(fold['y_val'], y_pred)\n",
    "    return f1Score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EJ38f_Y7WJSv"
   },
   "source": [
    "# Genetic Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2mE6C_3bWHcU"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "from operator import attrgetter\n",
    "from deap import base\n",
    "from deap import creator\n",
    "from deap import tools\n",
    "import time\n",
    "import datetime\n",
    "import math\n",
    "from scipy.spatial import distance\n",
    "import itertools\n",
    "\n",
    "\n",
    "class GeneticAlgorithm:\n",
    "    __slots__ = (\n",
    "        \"toolbox\", \"toolboxes\", \"cross_rate\", \"mut_rate\", \"n_pop\", \"n_gen\", \"resultsPath\", \"testing_name\", \"cfold\",\n",
    "        \"globalparameters\", \"layerparameters\", \"defaultVal\", \"path_ind\", \"max_conv_idx\", \"max_maxpooling_idx\",\n",
    "        \"max_dense_idx\", \"max_dropout_idx\")\n",
    "\n",
    "    def __init__(self, toolbox, toolboxes, cross_rate, mut_rate, n_pop, n_gen, resultsPath, testing_name,\n",
    "                 cfold, globalparameters, layerparameters, defaultVal, path_ind, max_conv_idx, max_maxpooling_idx,\n",
    "                 max_dense_idx, max_dropout_idx):\n",
    "        self.toolbox = toolbox\n",
    "        self.toolboxes = toolboxes\n",
    "        self.cross_rate = cross_rate\n",
    "        self.mut_rate = mut_rate\n",
    "        self.n_pop = n_pop\n",
    "        self.n_gen = n_gen\n",
    "        self.resultsPath = resultsPath\n",
    "        self.testing_name = testing_name\n",
    "        self.cfold = cfold\n",
    "        self.globalparameters = globalparameters\n",
    "        self.layerparameters = layerparameters\n",
    "        self.defaultVal = defaultVal\n",
    "        self.path_ind = path_ind\n",
    "        self.max_conv_idx = max_conv_idx\n",
    "        self.max_maxpooling_idx = max_maxpooling_idx\n",
    "        self.max_dense_idx = max_dense_idx\n",
    "        self.max_dropout_idx = max_dropout_idx\n",
    "\n",
    "    def fitnessCalc(self, individual):\n",
    "        i = 0\n",
    "        if len(individual.fitness.values) == 0:\n",
    "            if (0 in individual or '' in individual or 'False' in individual or None in individual):\n",
    "                for param in self.defaultVal:\n",
    "                    if individual[i] == 0 or individual[i] == '' or individual[i] == 'False' or individual[i] == None:\n",
    "                        individual[i] = self.defaultVal[param]\n",
    "                    i += 1\n",
    "\n",
    "            fc = FitnessCalculation(individual, self.cfold, self.defaultVal, self.resultsPath, self.testing_name)\n",
    "        else:\n",
    "            fc = individual.fitness.values[0]\n",
    "        print('{} {}'.format(datetime.datetime.now(), fc))\n",
    "        return fc,\n",
    "\n",
    "    def write_result(self):\n",
    "        # Create Testing Results\n",
    "        f = open(\"{}{}.csv\".format(self.resultsPath, self.testing_name), \"a+\")\n",
    "        text = \"i,min,max,mean,std,avgdistance,time,CR,MR\"\n",
    "        for param in self.defaultVal:\n",
    "            text = \"{},{}\".format(text, param)\n",
    "        text = \"{}\\n\".format(text)\n",
    "        f.write(text)\n",
    "        f.close()\n",
    "\n",
    "        # Create Last Population file\n",
    "        f = open(\"{}{}lastpop.csv\".format(self.resultsPath, self.testing_name), 'a+')\n",
    "        text = \"i,f1score\"\n",
    "        for param in self.defaultVal:\n",
    "            text = \"{},{}\".format(text, param)\n",
    "        text = \"{}\\n\".format(text)\n",
    "        f.write(text)\n",
    "        f.close()\n",
    "\n",
    "    def std_calc(self, fits, length):\n",
    "        mean = sum(fits) / length\n",
    "        sum2 = sum(x * x for x in fits)\n",
    "        std = abs(sum2 / length - mean ** 2) ** 0.5\n",
    "\n",
    "        return mean, std\n",
    "    \n",
    "    def distance_calc(self, pop):\n",
    "        distances = []\n",
    "        for subset in itertools.combinations(pop, 2):\n",
    "            distances.append(distance.hamming(subset[0][0:subset[0].index('embedding_layer')],\n",
    "                                              subset[1][0:subset[1].index('embedding_layer')]))\n",
    "\n",
    "        avgDistance = sum(distances) / len(distances)\n",
    "        \n",
    "        return avgDistance\n",
    "\n",
    "    def invalid_fitness_calc(self, pop):\n",
    "        # Evaluate the individuals with an invalid fitness\n",
    "        invalid_ind = [ind for ind in pop if not ind.fitness.valid]\n",
    "        fitnesses = map(self.toolbox.evaluate, invalid_ind)\n",
    "        for ind, fit in zip(invalid_ind, fitnesses):\n",
    "            ind.fitness.values = fit\n",
    "\n",
    "    def mutHyperparam(self, individual, indpb):\n",
    "        toolboxesSize = len(self.toolboxes)\n",
    "        fsm = FSM()\n",
    "\n",
    "        # Mutation for the Hyperparameter Chromosomes\n",
    "        for i in range(toolboxesSize):\n",
    "            if random.random() < indpb:\n",
    "                if len(self.toolboxes[i].args) == 1:\n",
    "                    individual[i] = self.toolboxes[i].func(self.toolboxes[i].args[0])\n",
    "                else:\n",
    "                    individual[i] = self.toolboxes[i].func(self.toolboxes[i].args[0], self.toolboxes[i].args[1])\n",
    "\n",
    "        # Mutation for the Architecture Chromosomes\n",
    "        archChrom = individual[individual.index('convolutional_layer'):individual.index('output_layer')]\n",
    "        size = len(archChrom)\n",
    "\n",
    "        for i in range(1, size):\n",
    "            if random.random() < indpb:\n",
    "                if (i>=size):\n",
    "                    break\n",
    "                                \n",
    "                if (archChrom[i] == 'global_maxpooling_layer' or archChrom[i] == 'maxpooling_layer' or archChrom[i] == 'dropout_layer'):\n",
    "                    continue\n",
    "\n",
    "                selectMutType = random.randint(0, 1)\n",
    "                # Remove the layer\n",
    "                if selectMutType == 0:\n",
    "#                     print('individual before remove', individual)\n",
    "                        if (archChrom[i] == 'convolutional_layer') and (archChrom[i+1] == 'maxpooling_layer'):\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            size -= 2\n",
    "                        elif (archChrom[i] == 'dense_layer'):\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            size -= 1\n",
    "\n",
    "                # Add a layer\n",
    "                elif selectMutType == 1:\n",
    "#                     print('individual before add', individual)\n",
    "                    if (archChrom[i] == 'convolutional_layer' and \n",
    "                        individual.count('convolutional_layer') < self.max_conv_idx):                    \n",
    "                        archChrom.insert(i, 'convolutional_layer')\n",
    "                        archChrom.insert(i+1, 'maxpooling_layer')\n",
    "                    elif (archChrom[i] == 'dense_layer' and \n",
    "                        individual.count('dense_layer') < self.max_dense_idx):\n",
    "                        archChrom.insert(i, 'dense_layer')\n",
    "\n",
    "                individual[individual.index('convolutional_layer'):individual.index('output_layer')] = archChrom\n",
    "#                 print('individual after', individual)\n",
    "        return individual,\n",
    "\n",
    "    def cxTwoPoint(self, ind1, ind2, pop, offspring):\n",
    "        # Crossover for hyperparameter chromosomes\n",
    "        size = ind1.index('embedding_layer')\n",
    "        selectCxType = random.randint(0, 2)\n",
    "        # One point crossover\n",
    "        if selectCxType == 0:\n",
    "#             print('ind1 before one-point crossover:', ind1)\n",
    "#             print('ind2 before one-point crossover:', ind2)\n",
    "            cxpoint = random.randint(1, size - 1)\n",
    "            ind1[cxpoint:], ind2[cxpoint:] = ind2[cxpoint:], ind1[cxpoint:]\n",
    "#             print('ind1 after one-point crossover:', ind1)\n",
    "#             print('ind2 after one-point crossover:', ind2)\n",
    "        # Two-point crossover\n",
    "        elif selectCxType == 1:\n",
    "#             print('ind1 before two-point crossover:', ind1)\n",
    "#             print('ind2 before two-point crossover:', ind2)\n",
    "            cxpoint1 = random.randint(1, size - 1)\n",
    "            cxpoint2 = random.randint(1, size - 1)\n",
    "            if cxpoint2 >= cxpoint1:\n",
    "                cxpoint2 += 1\n",
    "            else:  # Swap the two cx points\n",
    "                cxpoint1, cxpoint2 = cxpoint2, cxpoint1\n",
    "\n",
    "            ind1[cxpoint1:cxpoint2], ind2[cxpoint1:cxpoint2] \\\n",
    "                = ind2[cxpoint1:cxpoint2], ind1[cxpoint1:cxpoint2]\n",
    "#             print('ind1 after two-point crossover:', ind1)\n",
    "#             print('ind2 after two-point crossover:', ind2)\n",
    "        # Uniform crossover\n",
    "        elif selectCxType == 2:\n",
    "#             print('ind1 before uniform crossover:', ind1)\n",
    "#             print('ind2 before uniform crossover:', ind2)\n",
    "            for i in range(size):\n",
    "                if random.random() < self.cross_rate:\n",
    "                    ind1[i], ind2[i] = ind2[i], ind1[i]\n",
    "#             print('ind1 after uniform crossover:', ind1)\n",
    "#             print('ind2 after uniform crossover:', ind2)\n",
    "\n",
    "        # Crossover for architecture chromosomes\n",
    "        # One-cut point crossover from the Global MaxPooling layer\n",
    "        cxpoint1 = ind1.index('global_maxpooling_layer')\n",
    "        cxpoint2 = ind2.index('global_maxpooling_layer')\n",
    "        ind1[cxpoint1:], ind2[cxpoint2:] = ind2[cxpoint2:], ind1[cxpoint1:]\n",
    "\n",
    "        max_drop_layer = max(ind1.count('dropout_layer'), ind2.count('dropout_layer'))\n",
    "        if max_drop_layer > self.max_dropout_idx:\n",
    "            idx = self.max_dropout_idx\n",
    "            self.max_dropout_idx = max_drop_layer\n",
    "\n",
    "            while idx < self.max_dropout_idx:\n",
    "                idx += 1\n",
    "                addDropoutLayer(idx, self.toolbox, self.toolboxes, self.defaultVal, self.layerparameters)\n",
    "\n",
    "                for ind in pop + offspring:\n",
    "                    ind.insert(size, random.uniform(0, 1))\n",
    "                size += 1\n",
    "\n",
    "                self.write_result()\n",
    "\n",
    "        return ind1, ind2\n",
    "\n",
    "    def runGA(self, lastPop=[], lastFitnesses=[]):\n",
    "        creator.create(\"FitnessMax\", base.Fitness, weights=(1.0,))\n",
    "        creator.create(\"Individual\", list, fitness=creator.FitnessMax)\n",
    "\n",
    "        self.toolbox.register(\"individual\", tools.initCycle, creator.Individual,\n",
    "                              self.toolboxes, n=1)\n",
    "        self.toolbox.register(\"population\", tools.initRepeat, list, self.toolbox.individual)\n",
    "        self.toolbox.register(\"evaluate\", self.fitnessCalc)\n",
    "        self.toolbox.register(\"mate\", self.cxTwoPoint)\n",
    "        self.toolbox.register(\"mutate\", self.mutHyperparam, indpb=self.mut_rate)\n",
    "        self.toolbox.register(\"select\", tools.selBest)\n",
    "\n",
    "        pop = self.toolbox.population(n=self.n_pop)\n",
    "\n",
    "        idx = 0\n",
    "        for ind in pop:\n",
    "            if lastPop:\n",
    "                ind[:] = lastPop[idx]\n",
    "            ind.extend(self.path_ind[idx])\n",
    "            idx += 1\n",
    "        \n",
    "        if lastFitnesses:\n",
    "            # Fitnesses from previous population\n",
    "            fitnesses = lastFitnesses\n",
    "        else:\n",
    "            # Evaluate the entire population\n",
    "            fitnesses = list(map(self.toolbox.evaluate, pop))\n",
    "\n",
    "        for ind, fit in zip(pop, fitnesses):\n",
    "            ind.fitness.values = fit\n",
    "\n",
    "        self.write_result()\n",
    "        \n",
    "        \n",
    "        g = 0\n",
    "        while g < self.n_gen:\n",
    "            then = time.time()\n",
    "            g = g + 1\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"-- Generation %i --\" % g))          \n",
    "            \n",
    "            # Select the next generation individuals\n",
    "            offspring = self.toolbox.select(pop, len(pop))\n",
    "            # Clone the selected individuals\n",
    "            offspring = list(map(self.toolbox.clone, offspring))\n",
    "\n",
    "            # Apply crossover and mutation on the offspring\n",
    "            for child1, child2 in zip(offspring[::2], offspring[1::2]):\n",
    "                if random.random() < self.cross_rate:\n",
    "                    self.toolbox.mate(child1, child2, pop, offspring)\n",
    "                    del child1.fitness.values\n",
    "                    del child2.fitness.values\n",
    "\n",
    "            for mutant in offspring:\n",
    "                if random.random() < self.mut_rate:\n",
    "                    self.toolbox.mutate(mutant)\n",
    "                    del mutant.fitness.values\n",
    "\n",
    "            # Evaluate the individuals with an invalid fitness\n",
    "            self.invalid_fitness_calc(offspring)\n",
    "\n",
    "            pop[:] = self.toolbox.select(pop + offspring, self.n_pop)\n",
    "\n",
    "            # Gather all the fitnesses in one list and print the stats\n",
    "            fits = [ind.fitness.values[0] for ind in pop]\n",
    "\n",
    "            length = len(pop)\n",
    "            mean, std = self.std_calc(fits, length)\n",
    "            avgDistance = self.distance_calc(pop)\n",
    "            best = max(pop, key=attrgetter(\"fitness\"))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Min %s\" % min(fits)))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Max %s\" % max(fits)))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Avg %s\" % mean))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Std %s\" % std))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  AvgDistance %s\" % avgDistance))\n",
    "            print('{} {}'.format(datetime.datetime.now(), best))\n",
    "\n",
    "            now = time.time()\n",
    "            diff = now - then\n",
    "\n",
    "            # save testing data\n",
    "            f = open(\"{}{}.csv\".format(self.resultsPath, self.testing_name), 'a')\n",
    "            text = \"{0},{1},{2},{3},{4},{5},{6},{7},{8}\".format(g,min(fits), max(fits), mean, std, avgDistance, diff, self.cross_rate, self.mut_rate)\n",
    "            for param in best:\n",
    "                text = \"{},{}\".format(text, param)\n",
    "            text = \"{}\\n\".format(text)\n",
    "            f.write(text)\n",
    "            f.close()\n",
    "\n",
    "            # save last population data\n",
    "            f = open(\"{}{}lastpop.csv\".format(self.resultsPath, self.testing_name), 'a')\n",
    "            for ind in pop:\n",
    "                text = \"{0},{1}\".format(g,ind.fitness.values[0])\n",
    "                for param in ind:\n",
    "                    text = \"{},{}\".format(text, param)\n",
    "                text = \"{}\\n\".format(text)             \n",
    "                f.write(text)\n",
    "\n",
    "            f.close()            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kUA1C18jfyE6"
   },
   "source": [
    "# Project path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "9OZ4ryogfYra",
    "outputId": "84ac2846-ae86-4784-81ab-7cd413c89af5"
   },
   "outputs": [],
   "source": [
    "import os \n",
    "# path\n",
    "\n",
    "training_path = 'trainPreprocessed.csv'\n",
    "population_path = 'NewPop.csv'\n",
    "root_path = '/lab/dbms/fatyanosa'\n",
    "datasetPath = '{}/Dataset/Disaster Tweets/'.format(root_path)\n",
    "resultsPath = '{}/Server1/Disaster Tweets/Results/'.format(root_path)\n",
    "testing_name = \"Experiment3_GA_CNN\"\n",
    "glovePath = ['{}/Glove/glove.6B.50d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.100d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.200d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.300d.txt'.format(root_path)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8O1VRBE73ivB"
   },
   "source": [
    "# Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6XqVtrSu3kq4"
   },
   "outputs": [],
   "source": [
    "# crossover rate is the probability with which two individuals\n",
    "cross_rate = 0.8\n",
    "\n",
    "# mutation rate is the probability for mutating an individual\n",
    "mut_rate = 0.2\n",
    "\n",
    "# number of population\n",
    "n_pop = 30\n",
    "\n",
    "# number of generation\n",
    "n_gen = 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X6kUHesFXh17"
   },
   "source": [
    "# Main Program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "U4oAqAYAUmVT",
    "outputId": "b9c2e51a-6a4d-4eca-8b05-8397a9fc9216",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-01 06:44:13.033777 -- Generation 31 --\n",
      "2020-04-01 06:44:45.158575 0.7753981328940143\n",
      "2020-04-01 06:45:03.392898 0.7660455486542443\n",
      "2020-04-01 06:45:24.362473 0.7756410256410257\n",
      "2020-04-01 06:45:42.285389 0.776590330788804\n",
      "2020-04-01 06:45:58.130119 0.7735042735042735\n",
      "2020-04-01 06:46:16.322869 0.7720020586721563\n",
      "2020-04-01 06:46:34.865235 0.7743338771071234\n",
      "2020-04-01 06:47:04.237413 0.7761194029850746\n",
      "2020-04-01 06:47:31.105567 0.7820796460176991\n",
      "2020-04-01 06:47:51.498990 0.7739938080495357\n",
      "2020-04-01 06:48:08.187626 0.7839917906618779\n",
      "2020-04-01 06:48:23.812791 0.777217741935484\n",
      "2020-04-01 06:48:40.949665 0.7832618025751074\n",
      "2020-04-01 06:48:58.147227 0.7675557974959173\n",
      "2020-04-01 06:49:15.489541 0.7774769897130481\n",
      "2020-04-01 06:49:32.413828 0.771004942339374\n",
      "2020-04-01 06:49:47.935379 0.7737384698860553\n",
      "2020-04-01 06:50:05.403211 0.7721320062860136\n",
      "2020-04-01 06:50:22.616593 0.7680084745762711\n",
      "2020-04-01 06:50:39.452629 0.7765567765567766\n",
      "2020-04-01 06:50:55.609117 0.7717750826901874\n",
      "2020-04-01 06:51:11.314074 0.77938808373591\n",
      "2020-04-01 06:51:28.684354 0.7777193904361533\n",
      "2020-04-01 06:51:47.358532 0.7741228070175439\n",
      "2020-04-01 06:52:02.899807 0.7673160173160173\n",
      "2020-04-01 06:52:20.016156 0.7748583204533747\n",
      "2020-04-01 06:52:33.980847 0.0\n",
      "2020-04-01 06:52:34.010247   Min 0.7871569135163109\n",
      "2020-04-01 06:52:34.010360   Max 0.791710945802337\n",
      "2020-04-01 06:52:34.010392   Avg 0.7875522418068104\n",
      "2020-04-01 06:52:34.010418   Std 0.0008687919516613053\n",
      "2020-04-01 06:52:34.010797   AvgDistance 0.11092866265280066\n",
      "2020-04-01 06:52:34.010849 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 06:52:34.014627 -- Generation 32 --\n",
      "2020-04-01 06:52:50.560077 0.7764317180616741\n",
      "2020-04-01 06:53:10.042138 0.7805907172995781\n",
      "2020-04-01 06:53:33.619062 0.7832093517534537\n",
      "2020-04-01 06:54:01.732307 0.7822318526543879\n",
      "2020-04-01 06:54:24.501321 0.7833859095688748\n",
      "2020-04-01 06:54:43.547638 0.77989417989418\n",
      "2020-04-01 06:55:02.269406 0.779874213836478\n",
      "2020-04-01 06:55:26.245774 0.780952380952381\n",
      "2020-04-01 06:55:45.950455 0.7747281201450026\n",
      "2020-04-01 06:56:03.247122 0.7768773635872501\n",
      "2020-04-01 06:56:20.015161 0.7780061664953751\n",
      "2020-04-01 06:56:37.744237 0.7766470273165507\n",
      "2020-04-01 06:56:54.374154 0.7730812013348165\n",
      "2020-04-01 06:57:11.663664 0.77868406742795\n",
      "2020-04-01 06:57:24.857584 0.7707006369426751\n",
      "2020-04-01 06:57:41.461954 0.7821989528795812\n",
      "2020-04-01 06:58:02.084177 0.7636761487964988\n",
      "2020-04-01 06:58:20.127152 0.7763941526800217\n",
      "2020-04-01 06:58:35.896017 0.773346794548208\n",
      "2020-04-01 06:58:50.892506 0.7688022284122563\n",
      "2020-04-01 06:58:50.912210   Min 0.7871569135163109\n",
      "2020-04-01 06:58:50.912838   Max 0.791710945802337\n",
      "2020-04-01 06:58:50.912934   Avg 0.7876078389073652\n",
      "2020-04-01 06:58:50.912967   Std 0.000856738279469032\n",
      "2020-04-01 06:58:50.912998   AvgDistance 0.1250501733260356\n",
      "2020-04-01 06:58:50.913050 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 06:58:50.917647 -- Generation 33 --\n",
      "2020-04-01 06:59:11.443493 0.7773109243697478\n",
      "2020-04-01 06:59:30.491065 0.775144813059505\n",
      "2020-04-01 07:00:01.081752 0.7797333333333334\n",
      "2020-04-01 07:00:33.687777 0.7862513426423201\n",
      "2020-04-01 07:00:45.589335 0.0\n",
      "2020-04-01 07:01:04.020342 0.775\n",
      "2020-04-01 07:01:22.182721 0.7790262172284644\n",
      "2020-04-01 07:01:41.882156 0.7855183763027976\n",
      "2020-04-01 07:01:58.372053 0.7759958613554061\n",
      "2020-04-01 07:02:20.002182 0.7705286839145107\n",
      "2020-04-01 07:02:37.981261 0.783625730994152\n",
      "2020-04-01 07:03:09.877711 0.783739837398374\n",
      "2020-04-01 07:03:38.117750 0.7801418439716311\n",
      "2020-04-01 07:04:05.389125 0.7811158798283263\n",
      "2020-04-01 07:04:23.601387 0.771399798590131\n",
      "2020-04-01 07:04:41.415319 0.7745901639344263\n",
      "2020-04-01 07:05:01.851071 0.7712625462229266\n",
      "2020-04-01 07:05:18.876216 0.7725490196078432\n",
      "2020-04-01 07:05:38.984510 0.7652264445601249\n",
      "2020-04-01 07:05:59.298093 0.785289345592212\n",
      "2020-04-01 07:06:15.587054 0.7706725468577729\n",
      "2020-04-01 07:06:36.181473 0.7648283038501561\n",
      "2020-04-01 07:06:54.444755 0.7760084925690021\n",
      "2020-04-01 07:07:09.982586 0.7777777777777778\n",
      "2020-04-01 07:07:27.549125 0.7681007345225603\n",
      "2020-04-01 07:07:44.230144 0.7779527559055118\n",
      "2020-04-01 07:08:01.493491 0.7711468604047743\n",
      "2020-04-01 07:08:18.809725 0.7723404255319148\n",
      "2020-04-01 07:08:18.830616   Min 0.7871569135163109\n",
      "2020-04-01 07:08:18.830672   Max 0.791710945802337\n",
      "2020-04-01 07:08:18.830688   Avg 0.7876078389073652\n",
      "2020-04-01 07:08:18.830700   Std 0.000856738279469032\n",
      "2020-04-01 07:08:18.830716   AvgDistance 0.1250501733260356\n",
      "2020-04-01 07:08:18.830741 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 07:08:18.834268 -- Generation 34 --\n",
      "2020-04-01 07:08:45.373336 0.7691453940066592\n",
      "2020-04-01 07:09:15.416546 0.7767088607594936\n",
      "2020-04-01 07:09:34.433281 0.7784903798717315\n",
      "2020-04-01 07:09:53.335797 0.7771911298838438\n",
      "2020-04-01 07:10:11.733745 0.771290840921264\n",
      "2020-04-01 07:10:23.936045 0.0\n",
      "2020-04-01 07:10:43.725826 0.7873254564983888\n",
      "2020-04-01 07:11:01.933139 0.7853736089030205\n",
      "2020-04-01 07:11:17.742809 0.7730263157894738\n",
      "2020-04-01 07:11:46.718026 0.7783902976846747\n",
      "2020-04-01 07:12:17.676349 0.7804878048780488\n",
      "2020-04-01 07:12:35.184803 0.7745249101181305\n",
      "2020-04-01 07:12:51.800506 0.7628524046434495\n",
      "2020-04-01 07:13:14.274633 0.7808896210873146\n",
      "2020-04-01 07:13:38.862643 0.7691453940066592\n",
      "2020-04-01 07:13:56.844998 0.7833247555326815\n",
      "2020-04-01 07:14:17.830207 0.78860103626943\n",
      "2020-04-01 07:14:37.159543 0.7797716150081567\n",
      "2020-04-01 07:14:54.811059 0.7711111111111112\n",
      "2020-04-01 07:15:14.155215 0.7693965517241379\n",
      "2020-04-01 07:15:33.231124 0.766112266112266\n",
      "2020-04-01 07:15:51.584056 0.7644829453167299\n",
      "2020-04-01 07:16:10.708267 0.7756202804746495\n",
      "2020-04-01 07:16:30.105238 0.7598065556152607\n",
      "2020-04-01 07:16:49.374837 0.7775305895439377\n",
      "2020-04-01 07:16:49.407259   Min 0.7871569135163109\n",
      "2020-04-01 07:16:49.407384   Max 0.791710945802337\n",
      "2020-04-01 07:16:49.407406   Avg 0.7878911000157746\n",
      "2020-04-01 07:16:49.407788   Std 0.0011331252798253725\n",
      "2020-04-01 07:16:49.407913   AvgDistance 0.13231162196679472\n",
      "2020-04-01 07:16:49.407940 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 07:16:49.411133 -- Generation 35 --\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-01 07:17:14.279525 0.7729323308270676\n",
      "2020-04-01 07:17:39.552341 0.7723404255319148\n",
      "2020-04-01 07:18:10.035159 0.7855973813420621\n",
      "2020-04-01 07:18:34.483653 0.7709727225939268\n",
      "2020-04-01 07:18:57.581353 0.7817522032141005\n",
      "2020-04-01 07:19:17.149115 0.7815845824411135\n",
      "2020-04-01 07:19:32.254090 0.7728008634646519\n",
      "2020-04-01 07:19:54.347120 0.7817312798725438\n",
      "2020-04-01 07:20:17.352196 0.7717797969000534\n",
      "2020-04-01 07:20:36.365151 0.7817047817047817\n",
      "2020-04-01 07:21:07.232084 0.7813658020116464\n",
      "2020-04-01 07:21:40.479337 0.7782515991471215\n",
      "2020-04-01 07:22:10.396669 0.7744680851063829\n",
      "2020-04-01 07:22:36.443217 0.7778925619834711\n",
      "2020-04-01 07:23:00.485069 0.7753584705257567\n",
      "2020-04-01 07:23:21.187560 0.7795193312434693\n",
      "2020-04-01 07:23:40.144954 0.7825600894354388\n",
      "2020-04-01 07:24:00.949785 0.7709963484611373\n",
      "2020-04-01 07:24:20.146251 0.7714443829551744\n",
      "2020-04-01 07:24:41.305419 0.7785522788203754\n",
      "2020-04-01 07:24:59.408462 0.771852666300165\n",
      "2020-04-01 07:25:15.053421 0.7747933884297522\n",
      "2020-04-01 07:25:33.772810 0.7749318801089919\n",
      "2020-04-01 07:25:51.416172 0.7649315068493151\n",
      "2020-04-01 07:26:05.571799 0.0\n",
      "2020-04-01 07:26:05.602496   Min 0.7871569135163109\n",
      "2020-04-01 07:26:05.602587   Max 0.791710945802337\n",
      "2020-04-01 07:26:05.602605   Avg 0.7880509695295038\n",
      "2020-04-01 07:26:05.602633   Std 0.0011295361710167385\n",
      "2020-04-01 07:26:05.602654   AvgDistance 0.12837073526728734\n",
      "2020-04-01 07:26:05.602672 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 07:26:05.605758 -- Generation 36 --\n",
      "2020-04-01 07:26:31.739676 0.7717277486910996\n",
      "2020-04-01 07:27:03.134098 0.7698744769874478\n",
      "2020-04-01 07:27:22.760730 0.7791378392762107\n",
      "2020-04-01 07:27:41.327326 0.7744680851063829\n",
      "2020-04-01 07:28:01.025981 0.7715355805243446\n",
      "2020-04-01 07:28:19.945875 0.78082929456112\n",
      "2020-04-01 07:28:38.309179 0.7736450584484591\n",
      "2020-04-01 07:29:08.105559 0.7792482795129699\n",
      "2020-04-01 07:29:39.928891 0.7706329113924051\n",
      "2020-04-01 07:30:00.660069 0.7810335641981887\n",
      "2020-04-01 07:30:19.907632 0.7819013726487035\n",
      "2020-04-01 07:30:39.523576 0.7695494562402899\n",
      "2020-04-01 07:30:59.962824 0.7818666666666667\n",
      "2020-04-01 07:31:21.821626 0.7789585547290117\n",
      "2020-04-01 07:31:36.139314 0.0\n",
      "2020-04-01 07:31:54.642641 0.7781350482315113\n",
      "2020-04-01 07:32:17.861585 0.771358024691358\n",
      "2020-04-01 07:32:34.815374 0.7821727019498607\n",
      "2020-04-01 07:33:06.640944 0.7841880341880343\n",
      "2020-04-01 07:33:40.970533 0.7802022352315061\n",
      "2020-04-01 07:34:12.217427 0.7780172413793103\n",
      "2020-04-01 07:34:33.405058 0.7731182795698924\n",
      "2020-04-01 07:34:54.120223 0.7869209809264305\n",
      "2020-04-01 07:35:03.039716 0.72\n",
      "2020-04-01 07:35:19.902928 0.7606557377049181\n",
      "2020-04-01 07:35:31.286292 0.0\n",
      "2020-04-01 07:35:31.306883   Min 0.7871569135163109\n",
      "2020-04-01 07:35:31.307120   Max 0.791710945802337\n",
      "2020-04-01 07:35:31.307142   Avg 0.7880715386062407\n",
      "2020-04-01 07:35:31.307166   Std 0.0011164653597992096\n",
      "2020-04-01 07:35:31.307183   AvgDistance 0.12840722495894932\n",
      "2020-04-01 07:35:31.307198 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 07:35:31.310840 -- Generation 37 --\n",
      "2020-04-01 07:35:59.580777 0.779014308426073\n",
      "2020-04-01 07:36:26.962055 0.7756410256410257\n",
      "2020-04-01 07:36:45.695076 0.7838528359734287\n",
      "2020-04-01 07:37:04.366662 0.7759269210102095\n",
      "2020-04-01 07:37:20.916702 0.0\n",
      "2020-04-01 07:37:43.952440 0.7790821771611526\n",
      "2020-04-01 07:38:06.770505 0.7785234899328858\n",
      "2020-04-01 07:38:29.652267 0.7834728033472802\n",
      "2020-04-01 07:38:50.867869 0.7804072647220693\n",
      "2020-04-01 07:39:20.220205 0.7864229765013055\n",
      "2020-04-01 07:39:35.252521 0.0\n",
      "2020-04-01 07:39:56.684164 0.7753623188405797\n",
      "2020-04-01 07:40:19.577860 0.7822364901016586\n",
      "2020-04-01 07:40:40.818838 0.7691472026072788\n",
      "2020-04-01 07:41:03.792410 0.7835051546391752\n",
      "2020-04-01 07:41:23.957812 0.78575225943647\n",
      "2020-04-01 07:41:58.991719 0.7827027027027028\n",
      "2020-04-01 07:42:42.702604 0.7774798927613941\n",
      "2020-04-01 07:43:18.054865 0.7870115575123832\n",
      "2020-04-01 07:43:48.577635 0.785635359116022\n",
      "2020-04-01 07:44:07.852126 0.7754259163655137\n",
      "2020-04-01 07:45:09.257541 0.7183673469387756\n",
      "2020-04-01 07:45:32.484104 0.7724583109198494\n",
      "2020-04-01 07:45:49.605686 0.7680945347119646\n",
      "2020-04-01 07:46:10.058069 0.768581971534001\n",
      "2020-04-01 07:46:31.565358 0.7796086508753862\n",
      "2020-04-01 07:46:31.585957   Min 0.78731547293603\n",
      "2020-04-01 07:46:31.586005   Max 0.791710945802337\n",
      "2020-04-01 07:46:31.586021   Avg 0.7881065666300587\n",
      "2020-04-01 07:46:31.586047   Std 0.0010952613181871708\n",
      "2020-04-01 07:46:31.586064   AvgDistance 0.11921182266009861\n",
      "2020-04-01 07:46:31.586079 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 07:46:31.589890 -- Generation 38 --\n",
      "2020-04-01 07:47:02.550814 0.78848167539267\n",
      "2020-04-01 07:47:30.303932 0.7784877529286475\n",
      "2020-04-01 07:47:52.728656 0.7660455486542443\n",
      "2020-04-01 07:48:11.108476 0.7703398558187436\n",
      "2020-04-01 07:48:29.413187 0.7692307692307692\n",
      "2020-04-01 07:48:50.766034 0.777415852334419\n",
      "2020-04-01 07:49:20.784523 0.7796785899429756\n",
      "2020-04-01 07:49:44.307535 0.7788461538461537\n",
      "2020-04-01 07:49:59.355350 0.0\n",
      "2020-04-01 07:50:20.494893 0.7796070100902814\n",
      "2020-04-01 07:50:41.179664 0.7765451664025357\n",
      "2020-04-01 07:51:04.363514 0.7794037940379404\n",
      "2020-04-01 07:51:25.423450 0.7700592353257943\n",
      "2020-04-01 07:51:46.469782 0.7774122807017544\n",
      "2020-04-01 07:52:07.385927 0.7705442902881536\n",
      "2020-04-01 07:52:45.413273 0.7824245632609846\n",
      "2020-04-01 07:53:23.997131 0.7800650054171181\n",
      "2020-04-01 07:53:59.372816 0.7868155236576289\n",
      "2020-04-01 07:54:38.602817 0.78529568460309\n",
      "2020-04-01 07:55:01.238238 0.7790633608815427\n",
      "2020-04-01 07:55:24.036894 0.7738534528202425\n",
      "2020-04-01 07:55:24.065868   Min 0.7876154263986959\n",
      "2020-04-01 07:55:24.065953   Max 0.791710945802337\n",
      "2020-04-01 07:55:24.065970   Avg 0.7882860657667974\n",
      "2020-04-01 07:55:24.066007   Std 0.0010653798236560275\n",
      "2020-04-01 07:55:24.066027   AvgDistance 0.11081919357781446\n",
      "2020-04-01 07:55:24.066551 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 07:55:24.070382 -- Generation 39 --\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-01 07:55:52.721653 0.7829880043620502\n",
      "2020-04-01 07:56:28.299289 0.7771368641845832\n",
      "2020-04-01 07:56:48.905114 0.7715317173800929\n",
      "2020-04-01 07:57:11.552548 0.7767313019390581\n",
      "2020-04-01 07:57:34.882124 0.7700205338809035\n",
      "2020-04-01 07:57:56.025084 0.7710084033613446\n",
      "2020-04-01 07:58:15.774830 0.7777188328912467\n",
      "2020-04-01 07:58:44.279653 0.7743933918430562\n",
      "2020-04-01 07:59:08.010099 0.7752688172043011\n",
      "2020-04-01 07:59:51.367657 0.7609903991915109\n",
      "2020-04-01 08:00:12.507211 0.774585635359116\n",
      "2020-04-01 08:00:38.055784 0.7743293003682272\n",
      "2020-04-01 08:01:00.995114 0.7799268165185572\n",
      "2020-04-01 08:01:23.903749 0.7751046025104602\n",
      "2020-04-01 08:01:46.135184 0.778125\n",
      "2020-04-01 08:02:02.061088 0.7654584221748401\n",
      "2020-04-01 08:02:24.805351 0.7776584317937703\n",
      "2020-04-01 08:02:47.468625 0.7735341581495427\n",
      "2020-04-01 08:03:11.178586 0.7824742268041237\n",
      "2020-04-01 08:03:38.491144 0.7701030927835052\n",
      "2020-04-01 08:04:49.280507 0.7393310265282583\n",
      "2020-04-01 08:04:49.299561   Min 0.787682333873581\n",
      "2020-04-01 08:04:49.299601   Max 0.791710945802337\n",
      "2020-04-01 08:04:49.299615   Avg 0.7884021704620487\n",
      "2020-04-01 08:04:49.299634   Std 0.0010518020378948018\n",
      "2020-04-01 08:04:49.299659   AvgDistance 0.10490786352855369\n",
      "2020-04-01 08:04:49.299674 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 08:04:49.302439 -- Generation 40 --\n",
      "2020-04-01 08:05:19.942884 0.7804878048780488\n",
      "2020-04-01 08:05:51.739194 0.7788710512687725\n",
      "2020-04-01 08:06:12.734367 0.7751351351351352\n",
      "2020-04-01 08:06:34.879519 0.7744360902255638\n",
      "2020-04-01 08:06:55.276474 0.766050054406964\n",
      "2020-04-01 08:07:17.383217 0.7749077490774907\n",
      "2020-04-01 08:07:39.993467 0.7712707182320442\n",
      "2020-04-01 08:08:01.181637 0.7702632937532267\n",
      "2020-04-01 08:08:22.725288 0.7693989071038251\n",
      "2020-04-01 08:08:43.835503 0.7745604963805583\n",
      "2020-04-01 08:09:09.736106 0.7703782631859352\n",
      "2020-04-01 08:09:42.530897 0.7863340563991322\n",
      "2020-04-01 08:10:13.884744 0.7840349917987972\n",
      "2020-04-01 08:10:51.057481 0.7762162162162163\n",
      "2020-04-01 08:11:19.712076 0.7746403835908364\n",
      "2020-04-01 08:11:51.871023 0.7666486193827831\n",
      "2020-04-01 08:12:14.482576 0.7728459530026109\n",
      "2020-04-01 08:12:28.807604 0.7700592353257943\n",
      "2020-04-01 08:12:50.441489 0.7763713080168776\n",
      "2020-04-01 08:13:13.609257 0.7864229765013055\n",
      "2020-04-01 08:13:36.411326 0.7777182645956079\n",
      "2020-04-01 08:13:57.966513 0.7733910221741481\n",
      "2020-04-01 08:14:20.745045 0.7703473302229134\n",
      "2020-04-01 08:14:45.803853 0.7763941526800217\n",
      "2020-04-01 08:15:10.411326 0.7708108108108108\n",
      "2020-04-01 08:15:34.269949 0.7801127626858022\n",
      "2020-04-01 08:15:57.661275 0.7804610318331503\n",
      "2020-04-01 08:16:21.586205 0.7773741567202905\n",
      "2020-04-01 08:16:45.609986 0.7764044943820225\n",
      "2020-04-01 08:17:08.239920 0.7735753908219869\n",
      "2020-04-01 08:17:08.258229   Min 0.787682333873581\n",
      "2020-04-01 08:17:08.258270   Max 0.791710945802337\n",
      "2020-04-01 08:17:08.258285   Avg 0.7884021704620487\n",
      "2020-04-01 08:17:08.258310   Std 0.0010518020378948018\n",
      "2020-04-01 08:17:08.258328   AvgDistance 0.10490786352855369\n",
      "2020-04-01 08:17:08.258343 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 08:17:08.262036 -- Generation 41 --\n",
      "2020-04-01 08:17:39.444324 0.7855648535564853\n",
      "2020-04-01 08:18:10.128625 0.775\n",
      "2020-04-01 08:18:30.870877 0.7688706846108836\n",
      "2020-04-01 08:18:56.471215 0.0\n",
      "2020-04-01 08:19:19.991347 0.7752622860298178\n",
      "2020-04-01 08:19:40.888172 0.768814293448836\n",
      "2020-04-01 08:20:03.320156 0.7702773417059131\n",
      "2020-04-01 08:20:27.080855 0.7728260869565217\n",
      "2020-04-01 08:20:50.134704 0.7694840834248079\n",
      "2020-04-01 08:21:13.681120 0.7794656888423259\n",
      "2020-04-01 08:21:45.177457 0.7759446514103248\n",
      "2020-04-01 08:22:22.609552 0.7813163481953289\n",
      "2020-04-01 08:22:53.342758 0.7685774946921444\n",
      "2020-04-01 08:23:26.608334 0.7771428571428571\n",
      "2020-04-01 08:23:50.421318 0.7767094017094016\n",
      "2020-04-01 08:24:14.533000 0.7691460055096419\n",
      "2020-04-01 08:24:39.695426 0.7737373737373736\n",
      "2020-04-01 08:25:04.865922 0.7771117166212534\n",
      "2020-04-01 08:25:29.906408 0.7794759825327512\n",
      "2020-04-01 08:25:56.112437 0.770345596432553\n",
      "2020-04-01 08:26:20.384343 0.7748691099476441\n",
      "2020-04-01 08:26:45.371704 0.778856526429342\n",
      "2020-04-01 08:27:00.243195 0.0\n",
      "2020-04-01 08:27:00.263389   Min 0.787682333873581\n",
      "2020-04-01 08:27:00.263429   Max 0.791710945802337\n",
      "2020-04-01 08:27:00.263444   Avg 0.7884594385925466\n",
      "2020-04-01 08:27:00.263790   Std 0.0010344044318788253\n",
      "2020-04-01 08:27:00.263950   AvgDistance 0.11180441525269166\n",
      "2020-04-01 08:27:00.263968 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 08:27:00.268257 -- Generation 42 --\n",
      "2020-04-01 08:27:23.408877 0.7785016286644952\n",
      "2020-04-01 08:27:45.842029 0.7697262479871175\n",
      "2020-04-01 08:28:08.729538 0.7754874651810585\n",
      "2020-04-01 08:28:32.222236 0.7682119205298014\n",
      "2020-04-01 08:28:54.148677 0.7730192719486081\n",
      "2020-04-01 08:29:18.450807 0.7703141928494043\n",
      "2020-04-01 08:29:53.250970 0.7749318801089919\n",
      "2020-04-01 08:30:31.657260 0.7798810167658194\n",
      "2020-04-01 08:31:06.993331 0.7827004219409283\n",
      "2020-04-01 08:31:41.399865 0.7767616998386229\n",
      "2020-04-01 08:32:13.937561 0.781545302946081\n",
      "2020-04-01 08:32:41.725128 0.7683673469387755\n",
      "2020-04-01 08:33:08.001005 0.7676869041645761\n",
      "2020-04-01 08:33:24.697668 0.0\n",
      "2020-04-01 08:33:51.614728 0.7787021630615641\n",
      "2020-04-01 08:34:18.128751 0.7727272727272728\n",
      "2020-04-01 08:34:56.481829 0.77989417989418\n",
      "2020-04-01 08:35:20.872057 0.7777158774373258\n",
      "2020-04-01 08:35:46.588060 0.776824034334764\n",
      "2020-04-01 08:36:10.233455 0.7833859095688748\n",
      "2020-04-01 08:36:34.338853 0.7804878048780488\n",
      "2020-04-01 08:36:34.357219   Min 0.787682333873581\n",
      "2020-04-01 08:36:34.357263   Max 0.791710945802337\n",
      "2020-04-01 08:36:34.357279   Avg 0.7888877581298669\n",
      "2020-04-01 08:36:34.357303   Std 0.0012175159125471814\n",
      "2020-04-01 08:36:34.357749   AvgDistance 0.12461229702609078\n",
      "2020-04-01 08:36:34.357867 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 08:36:34.361218 -- Generation 43 --\n",
      "2020-04-01 08:37:03.497150 0.7824675324675324\n",
      "2020-04-01 08:37:42.928281 0.7747551686615886\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-01 08:38:16.484783 0.7795396419437339\n",
      "2020-04-01 08:38:39.994467 0.776330076004343\n",
      "2020-04-01 08:39:05.048466 0.7693116132422491\n",
      "2020-04-01 08:39:29.378496 0.7810972297664314\n",
      "2020-04-01 08:39:54.794598 0.7648351648351648\n",
      "2020-04-01 08:40:19.028758 0.7688864017907107\n",
      "2020-04-01 08:40:42.807224 0.7677208287895312\n",
      "2020-04-01 08:41:08.837435 0.7654190827622562\n",
      "2020-04-01 08:41:34.107780 0.7777188328912467\n",
      "2020-04-01 08:41:59.533816 0.781798245614035\n",
      "2020-04-01 08:42:21.424056 0.776824034334764\n",
      "2020-04-01 08:42:44.029012 0.7727514635444385\n",
      "2020-04-01 08:43:08.366338 0.7745358090185677\n",
      "2020-04-01 08:43:31.642070 0.7507274490785646\n",
      "2020-04-01 08:44:10.666452 0.7765333333333334\n",
      "2020-04-01 08:44:47.110263 0.7777777777777778\n",
      "2020-04-01 08:45:21.328125 0.7809224318658282\n",
      "2020-04-01 08:45:50.398009 0.7727272727272728\n",
      "2020-04-01 08:46:09.438979 0.0\n",
      "2020-04-01 08:46:46.325623 0.779164405860011\n",
      "2020-04-01 08:47:12.640372 0.7800914169629254\n",
      "2020-04-01 08:47:45.057357 0.7795698924731183\n",
      "2020-04-01 08:48:14.847474 0.7736331119059785\n",
      "2020-04-01 08:48:39.902776 0.7821835958718087\n",
      "2020-04-01 08:49:05.838151 0.777656078860898\n",
      "2020-04-01 08:49:29.105631 0.774064837905237\n",
      "2020-04-01 08:49:29.123843   Min 0.787682333873581\n",
      "2020-04-01 08:49:29.123885   Max 0.791710945802337\n",
      "2020-04-01 08:49:29.123901   Avg 0.7890669519272053\n",
      "2020-04-01 08:49:29.123925   Std 0.0012726800029203439\n",
      "2020-04-01 08:49:29.123952   AvgDistance 0.1251961320926844\n",
      "2020-04-01 08:49:29.123968 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 08:49:29.128160 -- Generation 44 --\n",
      "2020-04-01 08:50:10.190897 0.7826535880227156\n",
      "2020-04-01 08:50:39.159591 0.7783197831978319\n",
      "2020-04-01 08:51:09.651440 0.7824726134585289\n",
      "2020-04-01 08:51:36.529674 0.778751933986591\n",
      "2020-04-01 08:52:03.160082 0.7858672376873662\n",
      "2020-04-01 08:52:26.700868 0.7699018538713195\n",
      "2020-04-01 08:52:51.686435 0.7700803212851406\n",
      "2020-04-01 08:53:17.883099 0.786206896551724\n",
      "2020-04-01 08:53:42.804420 0.7684887459807075\n",
      "2020-04-01 08:54:05.774838 0.7688053097345133\n",
      "2020-04-01 08:54:29.365157 0.7713052858683926\n",
      "2020-04-01 08:54:54.057079 0.7803837953091685\n",
      "2020-04-01 08:55:18.680783 0.769309462915601\n",
      "2020-04-01 08:55:44.649328 0.7771855010660981\n",
      "2020-04-01 08:56:06.800941 0.0\n",
      "2020-04-01 08:56:32.774146 0.7751677852348994\n",
      "2020-04-01 08:56:51.856033 0.7766990291262136\n",
      "2020-04-01 08:57:13.938820 0.0\n",
      "2020-04-01 08:57:37.419738 0.7749866381614109\n",
      "2020-04-01 08:58:11.345737 0.7862714508580343\n",
      "2020-04-01 08:58:42.944786 0.7679586563307494\n",
      "2020-04-01 08:59:17.504561 0.7769347496206372\n",
      "2020-04-01 08:59:50.967613 0.780040733197556\n",
      "2020-04-01 09:00:27.189582 0.7783094098883572\n",
      "2020-04-01 09:00:55.117499 0.7763226820324778\n",
      "2020-04-01 09:00:55.136120   Min 0.787942887361184\n",
      "2020-04-01 09:00:55.136163   Max 0.791710945802337\n",
      "2020-04-01 09:00:55.136178   Avg 0.7891292288927588\n",
      "2020-04-01 09:00:55.136214   Std 0.0012281696391775888\n",
      "2020-04-01 09:00:55.136232   AvgDistance 0.12439335887611798\n",
      "2020-04-01 09:00:55.136246 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 09:00:55.140651 -- Generation 45 --\n",
      "2020-04-01 09:01:34.028646 0.7821835958718087\n",
      "2020-04-01 09:01:59.021440 0.7715680954964732\n",
      "2020-04-01 09:02:24.052420 0.7766884531590413\n",
      "2020-04-01 09:02:49.938359 0.7675040085515769\n",
      "2020-04-01 09:03:18.415788 0.7797333333333334\n",
      "2020-04-01 09:03:41.901240 0.7802597402597404\n",
      "2020-04-01 09:04:08.584094 0.778361344537815\n",
      "2020-04-01 09:04:33.737622 0.7703862660944206\n",
      "2020-04-01 09:04:58.840817 0.772256728778468\n",
      "2020-04-01 09:05:22.865562 0.7834051724137931\n",
      "2020-04-01 09:05:48.547722 0.7870762711864407\n",
      "2020-04-01 09:06:15.082966 0.7869884575026233\n",
      "2020-04-01 09:06:37.699540 0.7763975155279503\n",
      "2020-04-01 09:07:03.911481 0.7782994261867503\n",
      "2020-04-01 09:07:30.003817 0.775796866558617\n",
      "2020-04-01 09:07:55.014918 0.7746863066012001\n",
      "2020-04-01 09:08:23.577255 0.778816199376947\n",
      "2020-04-01 09:08:59.815354 0.7801494130202774\n",
      "2020-04-01 09:09:34.603544 0.7809110629067244\n",
      "2020-04-01 09:10:10.893514 0.7794759825327512\n",
      "2020-04-01 09:10:47.200863 0.7775965198477433\n",
      "2020-04-01 09:11:24.165993 0.784704033525406\n",
      "2020-04-01 09:11:59.895241 0.7783669141039236\n",
      "2020-04-01 09:12:38.262504 0.7861333333333332\n",
      "2020-04-01 09:13:21.615158 0.7787144362486829\n",
      "2020-04-01 09:13:21.635074   Min 0.787942887361184\n",
      "2020-04-01 09:13:21.635286   Max 0.791710945802337\n",
      "2020-04-01 09:13:21.635308   Avg 0.7895784779704605\n",
      "2020-04-01 09:13:21.635333   Std 0.0013187861933771855\n",
      "2020-04-01 09:13:21.635646   AvgDistance 0.11895639481846415\n",
      "2020-04-01 09:13:21.635667 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 09:13:21.640081 -- Generation 46 --\n",
      "2020-04-01 09:14:00.778942 0.7756174461376774\n",
      "2020-04-01 09:14:37.516958 0.7853736089030205\n",
      "2020-04-01 09:15:17.726829 0.7807486631016043\n",
      "2020-04-01 09:15:56.996416 0.7775330396475771\n",
      "2020-04-01 09:16:29.796833 0.7806970509383377\n",
      "2020-04-01 09:17:04.591021 0.7806691449814127\n",
      "2020-04-01 09:17:45.182433 0.7754874651810585\n",
      "2020-04-01 09:18:20.680587 0.7756962025316456\n",
      "2020-04-01 09:18:48.376505 0.7755324959038776\n",
      "2020-04-01 09:19:13.984068 0.7755324959038776\n",
      "2020-04-01 09:19:30.594379 0.7511363636363635\n",
      "2020-04-01 09:19:57.216199 0.7793478260869565\n",
      "2020-04-01 09:20:22.557312 0.7684563758389262\n",
      "2020-04-01 09:20:50.043976 0.7764334560757497\n",
      "2020-04-01 09:21:14.560714 0.7707997852925389\n",
      "2020-04-01 09:21:42.084446 0.780278670953912\n",
      "2020-04-01 09:22:07.023671 0.779947229551451\n",
      "2020-04-01 09:22:35.332520 0.767268862911796\n",
      "2020-04-01 09:23:02.954763 0.7753584705257567\n",
      "2020-04-01 09:23:28.882471 0.7743672590199246\n",
      "2020-04-01 09:23:57.722807 0.7693977211068909\n",
      "2020-04-01 09:24:31.561822 0.7746192893401015\n",
      "2020-04-01 09:25:03.997843 0.780565936999466\n",
      "2020-04-01 09:25:43.339486 0.7857142857142858\n",
      "2020-04-01 09:26:21.317047 0.7753743760399335\n",
      "2020-04-01 09:26:21.337871   Min 0.78860103626943\n",
      "2020-04-01 09:26:21.338110   Max 0.791710945802337\n",
      "2020-04-01 09:26:21.338133   Avg 0.7896445952759792\n",
      "2020-04-01 09:26:21.338149   Std 0.0012577230084696404\n",
      "2020-04-01 09:26:21.338166   AvgDistance 0.09633278598795847\n",
      "2020-04-01 09:26:21.338182 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 09:26:21.340722 -- Generation 47 --\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-01 09:26:43.348034 0.0\n",
      "2020-04-01 09:27:17.679300 0.7826552462526767\n",
      "2020-04-01 09:27:46.435990 0.7720782654680063\n",
      "2020-04-01 09:28:24.247528 0.7833247555326815\n",
      "2020-04-01 09:28:50.546823 0.7767416346681295\n",
      "2020-04-01 09:29:18.966166 0.7745415318230853\n",
      "2020-04-01 09:29:48.394017 0.7701525054466231\n",
      "2020-04-01 09:30:17.776244 0.7760869565217391\n",
      "2020-04-01 09:30:45.387237 0.7834051724137931\n",
      "2020-04-01 09:31:13.666825 0.7845484221980413\n",
      "2020-04-01 09:31:41.667626 0.773763201778766\n",
      "2020-04-01 09:32:08.621443 0.7719298245614036\n",
      "2020-04-01 09:32:36.770416 0.7746863066012001\n",
      "2020-04-01 09:33:03.806296 0.7768052516411378\n",
      "2020-04-01 09:33:31.105828 0.7759446514103248\n",
      "2020-04-01 09:33:58.190352 0.7768773635872501\n",
      "2020-04-01 09:34:23.546875 0.7672369103162261\n",
      "2020-04-01 09:34:51.617891 0.7733755942947702\n",
      "2020-04-01 09:35:16.827727 0.7697127937336815\n",
      "2020-04-01 09:35:42.644040 0.7721454173067076\n",
      "2020-04-01 09:36:10.310699 0.7745535714285715\n",
      "2020-04-01 09:36:36.730406 0.7759231158320689\n",
      "2020-04-01 09:37:05.709729 0.774617067833698\n",
      "2020-04-01 09:37:33.820862 0.773827923750644\n",
      "2020-04-01 09:37:33.839960   Min 0.78860103626943\n",
      "2020-04-01 09:37:33.840194   Max 0.791710945802337\n",
      "2020-04-01 09:37:33.840215   Avg 0.7900878165200697\n",
      "2020-04-01 09:37:33.840233   Std 0.0013274585318046636\n",
      "2020-04-01 09:37:33.840247   AvgDistance 0.11822660098522199\n",
      "2020-04-01 09:37:33.840264 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 09:37:33.843935 -- Generation 48 --\n",
      "2020-04-01 09:38:12.873959 0.7833152762730228\n",
      "2020-04-01 09:38:59.484358 0.7796432318992655\n",
      "2020-04-01 09:39:41.607563 0.7793427230046948\n",
      "2020-04-01 09:40:15.464309 0.7793030623020063\n",
      "2020-04-01 09:40:54.822627 0.7712707182320442\n",
      "2020-04-01 09:41:32.696879 0.7795148247978436\n",
      "2020-04-01 09:42:09.215737 0.7792343995804929\n",
      "2020-04-01 09:42:47.617498 0.7735155018392013\n",
      "2020-04-01 09:43:21.806348 0.7864935064935065\n",
      "2020-04-01 09:43:54.333911 0.7762605042016808\n",
      "2020-04-01 09:44:31.564788 0.7757638529259451\n",
      "2020-04-01 09:45:07.018012 0.7757510729613734\n",
      "2020-04-01 09:45:33.762907 0.7638668779714739\n",
      "2020-04-01 09:46:03.427264 0.7794117647058824\n",
      "2020-04-01 09:46:33.027620 0.7749737118822293\n",
      "2020-04-01 09:47:03.423658 0.7584722969338352\n",
      "2020-04-01 09:47:31.747478 0.7853231106243154\n",
      "2020-04-01 09:47:58.983751 0.7702991452991452\n",
      "2020-04-01 09:48:29.401907 0.7770374167093796\n",
      "2020-04-01 09:48:58.288184 0.773445732349842\n",
      "2020-04-01 09:49:27.362120 0.7626262626262625\n",
      "2020-04-01 09:50:01.077297 0.7560728744939271\n",
      "2020-04-01 09:50:24.406784 0.7800103039670274\n",
      "2020-04-01 09:50:53.714206 0.7704109589041095\n",
      "2020-04-01 09:51:21.946123 0.7768685215493726\n",
      "2020-04-01 09:51:49.462584 0.7772461456671984\n",
      "2020-04-01 09:52:18.144042 0.7669256381798002\n",
      "2020-04-01 09:52:46.991387 0.7767616998386229\n",
      "2020-04-01 09:52:47.009946   Min 0.789029535864978\n",
      "2020-04-01 09:52:47.009986   Max 0.791710945802337\n",
      "2020-04-01 09:52:47.010001   Avg 0.7901020998399212\n",
      "2020-04-01 09:52:47.010022   Std 0.0013136172280054847\n",
      "2020-04-01 09:52:47.010041   AvgDistance 0.11822660098522199\n",
      "2020-04-01 09:52:47.010067 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 09:52:47.013682 -- Generation 49 --\n",
      "2020-04-01 09:53:26.820209 0.7804347826086957\n",
      "2020-04-01 09:54:00.356647 0.7787798408488064\n",
      "2020-04-01 09:54:37.932525 0.7804878048780488\n",
      "2020-04-01 09:55:14.272860 0.7777777777777778\n",
      "2020-04-01 09:55:48.341836 0.7862433862433864\n",
      "2020-04-01 09:56:18.268459 0.7763088313061872\n",
      "2020-04-01 09:56:57.632915 0.7790262172284644\n",
      "2020-04-01 09:57:27.995952 0.7792768483540204\n",
      "2020-04-01 09:58:02.647096 0.7799016930638996\n",
      "2020-04-01 09:58:43.172520 0.7841913676547062\n",
      "2020-04-01 09:59:12.625824 0.7821409359870899\n",
      "2020-04-01 09:59:42.512206 0.778575372722253\n",
      "2020-04-01 10:00:10.465317 0.7845117845117846\n",
      "2020-04-01 10:00:40.745906 0.7681396617566829\n",
      "2020-04-01 10:01:12.022847 0.7619559376679205\n",
      "2020-04-01 10:01:40.883553 0.7771368641845832\n",
      "2020-04-01 10:02:11.305269 0.7775530839231547\n",
      "2020-04-01 10:02:41.217463 0.7789815817984832\n",
      "2020-04-01 10:03:08.170280 0.7779596290234588\n",
      "2020-04-01 10:03:39.888681 0.7684265551489807\n",
      "2020-04-01 10:04:11.538941 0.7731456415809421\n",
      "2020-04-01 10:04:42.950998 0.7882599580712787\n",
      "2020-04-01 10:05:09.423251 0.7712871287128713\n",
      "2020-04-01 10:05:40.315990 0.777835859905907\n",
      "2020-04-01 10:06:10.683782 0.7720588235294118\n",
      "2020-04-01 10:06:42.221695 0.7508702138239682\n",
      "2020-04-01 10:07:13.128175 0.7724137931034482\n",
      "2020-04-01 10:07:46.362413 0.7742963356346256\n",
      "2020-04-01 10:07:46.384290   Min 0.789029535864978\n",
      "2020-04-01 10:07:46.384335   Max 0.791710945802337\n",
      "2020-04-01 10:07:46.384351   Avg 0.7902808605024118\n",
      "2020-04-01 10:07:46.384364   Std 0.0013377223069520484\n",
      "2020-04-01 10:07:46.384391   AvgDistance 0.1226053639846747\n",
      "2020-04-01 10:07:46.384420 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-01 10:07:46.389108 -- Generation 50 --\n",
      "2020-04-01 10:08:27.717007 0.7703943814154511\n",
      "2020-04-01 10:09:06.687171 0.7846320346320346\n",
      "2020-04-01 10:09:47.131693 0.7790821771611526\n",
      "2020-04-01 10:10:13.427498 0.7678767876787678\n",
      "2020-04-01 10:10:49.807738 0.7788259958071279\n",
      "2020-04-01 10:11:31.637565 0.7769110764430576\n",
      "2020-04-01 10:12:08.511979 0.7677314564158094\n",
      "2020-04-01 10:12:48.331152 0.7798607391537226\n",
      "2020-04-01 10:13:28.828423 0.7815845824411135\n",
      "2020-04-01 10:14:06.952200 0.7784810126582278\n",
      "2020-04-01 10:14:50.760752 0.7806731813246471\n",
      "2020-04-01 10:15:26.161236 0.7805405405405405\n",
      "2020-04-01 10:15:56.282227 0.7776000000000001\n",
      "2020-04-01 10:16:29.350827 0.7723190702588484\n",
      "2020-04-01 10:16:59.508930 0.767268862911796\n",
      "2020-04-01 10:17:32.527842 0.7617486338797814\n",
      "2020-04-01 10:18:06.094287 0.7746701846965699\n",
      "2020-04-01 10:18:43.724574 0.7665544332210998\n",
      "2020-04-01 10:19:15.235849 0.7715680954964732\n",
      "2020-04-01 10:19:45.934674 0.7810972297664314\n",
      "2020-04-01 10:20:16.576597 0.7778369738945125\n",
      "2020-04-01 10:20:45.820776 0.0\n",
      "2020-04-01 10:21:17.960970 0.7548179871520343\n",
      "2020-04-01 10:21:48.942406 0.7629994526546251\n",
      "2020-04-01 10:21:48.961091   Min 0.789029535864978\n",
      "2020-04-01 10:21:48.961322   Max 0.791710945802337\n",
      "2020-04-01 10:21:48.961344   Avg 0.7904596211649023\n",
      "2020-04-01 10:21:48.961359   Std 0.0013377223069105516\n",
      "2020-04-01 10:21:48.961698   AvgDistance 0.1226053639846747\n",
      "2020-04-01 10:21:48.961733 [67, 83, 'adamax', 0.004253707676122, 0.9, 'glorot_uniform', 100, 426, 5, 'sigmoid', 'glorot_uniform', 3, 269, 3, 'softsign', 'glorot_uniform', 3, 131, 2, 'linear', 'glorot_uniform', 3, 340, 1, 'softmax', 'glorot_uniform', 3, 29, 'relu', 'glorot_uniform', 3, 24, 'relu', 'glorot_uniform', 3, 15, 'relu', 'glorot_uniform', 3, 6, 'relu', 'glorot_uniform', 3, 3, 'relu', 'glorot_uniform', 3, 4, 'relu', 'glorot_uniform', 3, 23, 'relu', 'glorot_uniform', 3, 18, 'relu', 'glorot_uniform', 3, 5, 5, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from deap import base\n",
    "import warnings; warnings.simplefilter('ignore')\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    globalparameters = []\n",
    "    globalparameters.append((\"epochs\", random.randint, 1, 100))\n",
    "    globalparameters.append((\"batch_size\", random.randint, 32, 256))\n",
    "    globalparameters.append((\"optimizer\", random.choice, ['sgd', 'rmsprop', \n",
    "                                                          'adagrad', 'adadelta',\n",
    "                                                          'adam', 'adamax', \n",
    "                                                          'nadam']))\n",
    "    globalparameters.append((\"learning_rate\", random.uniform, 1e-4, 1e-2))\n",
    "    globalparameters.append((\"momentum\", random.choice, [0.9]))\n",
    "    globalparameters.append((\"output_init_mode\", random.choice, ['glorot_uniform']))\n",
    "    globalparameters.append((\"output_dim\", random.choice, [100]))\n",
    "\n",
    "    layerparameters = {}\n",
    "    layerparameters[\"num_filters\"] = [random.randint, 32, 512]\n",
    "    layerparameters[\"kernel_size\"] = [random.randint, 1, 5]\n",
    "    layerparameters[\"conv_activation_func\"] = [random.choice,\n",
    "                                               ['relu', 'softmax', 'elu', 'selu',\n",
    "                                                'softplus', 'softsign', 'tanh',\n",
    "                                                'sigmoid', 'hard_sigmoid', 'linear']]\n",
    "    layerparameters[\"conv_init_mode\"] = [random.choice,\n",
    "                                         ['glorot_uniform']]\n",
    "    layerparameters[\"conv_weight_constraint\"] = [random.choice, [3]]\n",
    "    layerparameters[\"neurons\"] = [random.randint, 1, 30]\n",
    "    layerparameters[\"dense_activation_func\"] = [random.choice,\n",
    "                                                ['relu']]\n",
    "    layerparameters[\"dense_init_mode\"] = [random.choice,\n",
    "                                          ['glorot_uniform']]\n",
    "    layerparameters[\"dense_weight_constraint\"] = [random.choice, [3]]\n",
    "    layerparameters[\"pool_size\"] = [random.choice, [5]]\n",
    "    layerparameters[\"dropout_rate\"] = [random.choice, [0.2]]\n",
    "\n",
    "    defaultVal = collections.OrderedDict([\n",
    "        (\"epochs\", 10),\n",
    "        (\"batch_size\", 32),\n",
    "        (\"optimizer\", \"adam\"),\n",
    "        (\"learning_rate\", 1e-4),\n",
    "        (\"momentum\", 0.9),\n",
    "        (\"output_init_mode\", \"glorot_uniform\"),\n",
    "        (\"output_dim\", 100)]\n",
    "    )\n",
    "    \n",
    "    # object class\n",
    "    util = utility()\n",
    "    toolbox = base.Toolbox()\n",
    "    toolboxes = []\n",
    "\n",
    "    # Attribute generator\n",
    "    for hyper in globalparameters:\n",
    "        if len(hyper) == 3:\n",
    "            toolbox.register(hyper[0], hyper[1], hyper[2])\n",
    "        else:\n",
    "            toolbox.register(hyper[0], hyper[1], hyper[2], hyper[3])\n",
    "\n",
    "    toolboxes.append(toolbox.epochs)\n",
    "    toolboxes.append(toolbox.batch_size)\n",
    "    toolboxes.append(toolbox.optimizer)\n",
    "    toolboxes.append(toolbox.learning_rate)\n",
    "    toolboxes.append(toolbox.momentum)\n",
    "    toolboxes.append(toolbox.output_init_mode)\n",
    "    toolboxes.append(toolbox.output_dim)\n",
    "\n",
    "    path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx = generateFSM(n_pop, layerparameters,\n",
    "                                                                                             toolbox, toolboxes,\n",
    "                                                                                             defaultVal)\n",
    "\n",
    "#     # Read population data\n",
    "#     dfPopulation = util.read_CSV(\"{}{}\".format(resultsPath, population_path))\n",
    "\n",
    "#     path_ind, fitnesses, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx = openFSM(dfPopulation, layerparameters,\n",
    "#                                                                                          toolbox, toolboxes, defaultVal)\n",
    "#     dfPopulation = dfPopulation.drop(columns=[col for col in dfPopulation if col not in defaultVal])\n",
    "\n",
    "#     population = dfPopulation.loc[:, ~dfPopulation.columns.str.match('Unnamed')].values.tolist()\n",
    "\n",
    "    # Read data\n",
    "    df = util.read_CSV(\"{}{}\".format(datasetPath, training_path))\n",
    "\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    dfTraining, dfTrial = train_test_split(df, test_size = 0.3, random_state=42)\n",
    "\n",
    "    textsTraining, labelsTraining = util.get_text_label(dfTraining)\n",
    "    textsTrial, labelsTrial = util.get_text_label(dfTrial)\n",
    "    cfold = {}\n",
    "\n",
    "    X_train, X_val, y_train, y_val, vocab_size, maxlen, embedding_matrix = util.get_training_trial_data(\n",
    "        textsTraining, labelsTraining, textsTrial, labelsTrial, glovePath)\n",
    "    cfold= {'X_train': X_train, 'X_val': X_val, 'y_train': y_train, 'y_val': y_val, 'vocab_size': vocab_size,\n",
    "                  'maxlen': maxlen, 'embedding_matrix': embedding_matrix}\n",
    "                  \n",
    "    ga = GeneticAlgorithm(toolbox, toolboxes, cross_rate, mut_rate, n_pop, n_gen, resultsPath, testing_name,\n",
    "                          cfold, globalparameters, layerparameters, defaultVal, path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx)\n",
    "    ga.runGA()\n",
    "#     ga.runGA(population, fitnesses)\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "GA-CNN.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}