Experiment4_GA_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FZg_QFpe7W-B"
   },
   "source": [
    "# Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xWATOMlqSBDH"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "import numpy as np\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_OHVp0iz7a9O"
   },
   "outputs": [],
   "source": [
    "class utility:\n",
    "\n",
    "    def read_CSV(self, filename):\n",
    "        df = pd.read_csv(filename)\n",
    "        return df\n",
    "\n",
    "    def get_text_label(self, df):\n",
    "        texts = []  # list of text samples\n",
    "        labels = []  # list of label ids\n",
    "        for index, row in df.iterrows():\n",
    "            if isinstance(row['text_cleaned'], float):\n",
    "                texts.append(str(row['text_cleaned']))\n",
    "            else:\n",
    "                texts.append(row['text_cleaned'])\n",
    "\n",
    "            labels.append(row['target'])\n",
    "\n",
    "        return texts, labels\n",
    "\n",
    "    def tokenize_texts(self, texts):\n",
    "        tokenizer = Tokenizer(num_words=50000)\n",
    "        tokenizer.fit_on_texts(texts)\n",
    "\n",
    "        return tokenizer\n",
    "\n",
    "    def padding_texts(self, texts, maxlen):\n",
    "\n",
    "        texts = pad_sequences(texts, padding='post', maxlen=maxlen)\n",
    "\n",
    "        return texts\n",
    "\n",
    "    def get_metric(self, y_true, y_pred):\n",
    "        accuracyScore = accuracy_score(y_true, y_pred)\n",
    "\n",
    "        precisionScore = precision_score(y_true, y_pred)\n",
    "        recallScore = recall_score(y_true, y_pred)\n",
    "        f1Score = f1_score(y_true, y_pred)\n",
    "\n",
    "        return accuracyScore, precisionScore, recallScore, f1Score\n",
    "\n",
    "    def print_metric(self, accuracyScore, precisionScore, recallScore, f1Score):\n",
    "        print(\"Accuracy: {}\".format(str(accuracyScore)))\n",
    "        print(\"Precision: {}\".format(str(precisionScore)))\n",
    "        print(\"Recall: {}\".format(str(recallScore)))\n",
    "        print(\"F1-Score: {}\".format(str(f1Score)))\n",
    "        print(\"{},{},{},{}\".format(str(accuracyScore), str(precisionScore), str(recallScore), str(f1Score)))\n",
    "        \n",
    "\n",
    "    def get_testing_metric(self, y_test, y_pred):\n",
    "        accuracyScore, precisionScore, recallScore, f1Score = self.get_metric(y_test, y_pred)\n",
    "\n",
    "        return accuracyScore, precisionScore, recallScore, f1Score\n",
    "\n",
    "    def write_df_csv(self, df, out_path):\n",
    "        df.to_csv(out_path, index=False)\n",
    "\n",
    "    def create_embedding_matrix(self, filepath, word_index, embedding_dim):\n",
    "        vocab_size = len(word_index) + 1  # Adding again 1 because of reserved 0 index\n",
    "        embedding_matrix = np.zeros((vocab_size, embedding_dim))\n",
    "\n",
    "        with open(filepath, encoding=\"utf8\") as f:\n",
    "            for line in f:\n",
    "                word, *vector = line.split()\n",
    "                if word in word_index:\n",
    "                    idx = word_index[word]\n",
    "                    embedding_matrix[idx] = np.array(\n",
    "                        vector, dtype=np.float32)[:embedding_dim]\n",
    "\n",
    "        return embedding_matrix\n",
    "\n",
    "    def get_max_length_of_sentences(self, texts):\n",
    "        maxlength = 0\n",
    "        for text in texts:\n",
    "            if (len(text.split()) > maxlength):\n",
    "                maxlength = len(text.split())\n",
    "\n",
    "        return maxlength\n",
    "\n",
    "    def get_training_trial_data(self, textsTraining, labelsTraining, textsTrial, labelsTrial, glovePath):\n",
    "        textsTraining, textsTesting = np.asarray(textsTraining), np.asarray(textsTrial)\n",
    "        y_train, y_val = np.asarray(labelsTraining), np.asarray(labelsTrial)\n",
    "\n",
    "        # Tokenize words\n",
    "        tokenizer = self.tokenize_texts(textsTraining)\n",
    "        X_train = tokenizer.texts_to_sequences(textsTraining)\n",
    "        X_val = tokenizer.texts_to_sequences(textsTesting)\n",
    "\n",
    "        # Adding 1 because of reserved 0 index\n",
    "        vocab_size = len(tokenizer.word_index) + 1\n",
    "\n",
    "        # get maxlen\n",
    "        maxlen = self.get_max_length_of_sentences(textsTraining)\n",
    "\n",
    "        # Pad sequences with zeros\n",
    "        X_train = self.padding_texts(X_train, maxlen)\n",
    "        X_val = self.padding_texts(X_val, maxlen)\n",
    "\n",
    "        embedding_matrix = []\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[0], tokenizer.word_index, 50))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[1], tokenizer.word_index, 100))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[2], tokenizer.word_index, 200))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[3], tokenizer.word_index, 300))\n",
    "\n",
    "        return X_train, X_val, y_train, y_val, vocab_size, maxlen, embedding_matrix\n",
    "\n",
    "    def Average(self, list):\n",
    "        return sum(list) / len(list)\n",
    "\n",
    "    def recall(self, y_true, y_pred):\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(self, y_true, y_pred):\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "\n",
    "    def f1_score(self, y_true, y_pred):\n",
    "        precision = self.precision(y_true, y_pred)\n",
    "        recall = self.recall(y_true, y_pred)\n",
    "        return 2*((precision*recall)/(precision+recall+K.epsilon()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uYAnQP8Ac2Qe"
   },
   "source": [
    "# Finite State Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uKx4XQbmc5HN"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def FSM():\n",
    "    fsm = {}\n",
    "    fsm[0] = {'src': 0, 'dst': 1, 'layer': 'embedding_layer', 'next_path': [1]}\n",
    "    fsm[1] = {'src': 1, 'dst': 2, 'layer': 'convolutional_layer', 'next_path': [2, 4]}\n",
    "    fsm[2] = {'src': 2, 'dst': 3, 'layer': 'maxpooling_layer', 'next_path': [3]}    \n",
    "    fsm[3] = {'src': 3, 'dst': 2, 'layer': 'convolutional_layer', 'next_path': [2, 4]}\n",
    "    fsm[4] = {'src': 2, 'dst': 4, 'layer': 'global_maxpooling_layer', 'next_path': [5]}\n",
    "    fsm[5] = {'src': 4, 'dst': 5, 'layer': 'dense_layer', 'next_path': [6, 7]}\n",
    "    fsm[6] = {'src': 5, 'dst': 5, 'layer': 'dense_layer', 'next_path': [6, 7]}    \n",
    "    fsm[7] = {'src': 5, 'dst': 6, 'layer': 'dropout_layer', 'next_path': [8]}\n",
    "    fsm[8] = {'src': 6, 'dst': 7, 'layer': 'output_layer', 'next_path': []}\n",
    "\n",
    "    return fsm\n",
    "\n",
    "def getLayerSize(layer, conv_idx, dense_idx, dropout_idx, maxpooling_idx):\n",
    "    if layer == 'convolutional_layer':\n",
    "        conv_idx += 1\n",
    "    elif layer == 'dense_layer':\n",
    "        dense_idx += 1\n",
    "    elif layer == 'dropout_layer':\n",
    "        dropout_idx += 1\n",
    "    elif layer == 'maxpooling_layer':\n",
    "        maxpooling_idx += 1\n",
    "    return conv_idx, dense_idx, dropout_idx, maxpooling_idx\n",
    "\n",
    "\n",
    "def getMaxLayerSize(conv_idx, dense_idx, dropout_idx, maxpooling_idx, max_conv_idx, max_dense_idx, max_dropout_idx,\n",
    "                    max_maxpooling_idx):\n",
    "    if conv_idx > max_conv_idx:\n",
    "        max_conv_idx = conv_idx\n",
    "    if dense_idx > max_dense_idx:\n",
    "        max_dense_idx = dense_idx\n",
    "    if dropout_idx > max_dropout_idx:\n",
    "        max_dropout_idx = dropout_idx\n",
    "    if maxpooling_idx > max_maxpooling_idx:\n",
    "        max_maxpooling_idx = maxpooling_idx\n",
    "\n",
    "    return max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx\n",
    "\n",
    "def generateFSM(n_pop):\n",
    "    fsm = FSM()\n",
    "\n",
    "    path_ind = {}\n",
    "    max_conv_idx = 0\n",
    "    max_dense_idx = 0\n",
    "    max_dropout_idx = 0\n",
    "    max_maxpooling_idx = 0\n",
    "\n",
    "    for ind in range(0, n_pop):\n",
    "        idx = conv_idx = dense_idx = dropout_idx = maxpooling_idx = 0\n",
    "        path = [fsm[idx]['layer']]\n",
    "        while len(fsm[idx]['next_path']) != 0:\n",
    "            idx = random.choice(fsm[idx]['next_path'])\n",
    "            layer = fsm[idx]['layer']\n",
    "            path.append(layer)\n",
    "            conv_idx, dense_idx, dropout_idx, maxpooling_idx = getLayerSize(layer, conv_idx, dense_idx, dropout_idx,\n",
    "                                                                            maxpooling_idx)\n",
    "\n",
    "        max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx = getMaxLayerSize(conv_idx, dense_idx,\n",
    "                                                                                           dropout_idx, maxpooling_idx,\n",
    "                                                                                           max_conv_idx, max_dense_idx,\n",
    "                                                                                           max_dropout_idx,\n",
    "                                                                                           max_maxpooling_idx)\n",
    "\n",
    "        path_ind[ind] = path\n",
    "\n",
    "    return path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx\n",
    "\n",
    "\n",
    "def openFSM(df):\n",
    "    path_ind = {}\n",
    "    fitnesses = []\n",
    "\n",
    "    hyperparams = [s for s in list(df.columns) if not 'Unnamed' in s]\n",
    "\n",
    "    max_conv_idx = sum('num_filters' in s for s in hyperparams)\n",
    "    max_dense_idx = sum('neurons' in s for s in hyperparams)\n",
    "    max_dropout_idx = sum('dropout_rate' in s for s in hyperparams)\n",
    "    max_maxpooling_idx = sum('pool_size' in s for s in hyperparams)\n",
    "\n",
    "    for index, row in df.iterrows():\n",
    "        path = [s for s in row if 'layer' in str(s)]\n",
    "        fitness = [s for s in row if str(s).replace('.', '', 1).isdigit()]\n",
    "        fitnesses.append(tuple([float(fitness[0])]))\n",
    "        path_ind[index] = path\n",
    "\n",
    "    return path_ind, fitnesses, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-QjUHS9ZWs7U"
   },
   "source": [
    "# CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7DObu8fBWr48"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "class CNN:\n",
    "\n",
    "    def cnn_model(self, vocab_size, maxlen, embedding_matrix, indiv, path):\n",
    "        model = tf.keras.models.Sequential()\n",
    "        conv_idx = dense_idx = dropout_idx = maxpooling_idx = 0\n",
    "        for layer in path:\n",
    "            if layer == 'embedding_layer':\n",
    "                model.add(\n",
    "                    tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=indiv['output_dim'],\n",
    "                                     weights=[embedding_matrix], input_length=maxlen, trainable=True))\n",
    "            elif layer == 'convolutional_layer':\n",
    "                conv_idx += 1\n",
    "                model.add(tf.keras.layers.Conv1D(indiv['num_filters'], \n",
    "                                                 indiv['kernel_size'],\n",
    "                                        kernel_initializer=indiv['conv_init_mode'],\n",
    "                                        activation=indiv['conv_activation_func'],\n",
    "                                        kernel_constraint=tf.keras.constraints.max_norm(indiv['conv_weight_constraint']),\n",
    "                                        data_format='channels_first'))\n",
    "            elif layer == 'dense_layer':\n",
    "                dense_idx += 1\n",
    "                model.add(tf.keras.layers.Dense(indiv['neurons'],\n",
    "                                       kernel_initializer=indiv['dense_init_mode'],\n",
    "                                       activation=indiv['dense_activation_func'],\n",
    "                                       kernel_constraint=tf.keras.constraints.max_norm(indiv['dense_weight_constraint'])))\n",
    "            elif layer == 'dropout_layer':\n",
    "                dropout_idx += 1\n",
    "                model.add(tf.keras.layers.Dropout(indiv['dropout_rate']))\n",
    "            elif layer == 'maxpooling_layer':\n",
    "                maxpooling_idx += 1\n",
    "                model.add(tf.keras.layers.MaxPooling1D(indiv['pool_size']))\n",
    "            elif layer == 'global_maxpooling_layer':\n",
    "                model.add(tf.keras.layers.GlobalMaxPooling1D())\n",
    "            elif layer == 'output_layer':\n",
    "                model.add(tf.keras.layers.Dense(1, kernel_initializer=indiv['output_init_mode'], activation='sigmoid'))\n",
    "\n",
    "        if indiv['optimizer'] == 'sgd':\n",
    "            opt = tf.keras.optimizers.SGD(lr=indiv['learning_rate'], momentum=indiv['momentum'], decay=0.0,\n",
    "                                 nesterov=False)\n",
    "        elif indiv['optimizer'] == 'rmsprop':\n",
    "            opt = tf.keras.optimizers.RMSprop(lr=indiv['learning_rate'], rho=0.9, epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adagrad':\n",
    "            opt = tf.keras.optimizers.Adagrad(lr=indiv['learning_rate'], epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adadelta':\n",
    "            opt = tf.keras.optimizers.Adadelta(lr=indiv['learning_rate'], rho=0.95, epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adam':\n",
    "            opt = tf.keras.optimizers.Adam(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                  decay=0.0, amsgrad=False)\n",
    "        elif indiv['optimizer'] == 'adamax':\n",
    "            opt = tf.keras.optimizers.Adamax(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                    decay=0.0)\n",
    "        elif indiv['optimizer'] == 'nadam':\n",
    "            opt = tf.keras.optimizers.Nadam(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                   schedule_decay=0.004)\n",
    "        \n",
    "        util = utility()\n",
    "        model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[util.f1_score])\n",
    "\n",
    "        return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jsfyjDRoWVMU"
   },
   "source": [
    "# Fitness Calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hn9qqGyhWS8x"
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import os\n",
    "from time import sleep\n",
    "import gc\n",
    "\n",
    "util = utility()\n",
    "cnn = CNN()\n",
    "\n",
    "def FitnessCalculation(individual, cfold, defaultVal, resultsPath, testing_name):\n",
    "    indiv = collections.OrderedDict()\n",
    "    i = 0\n",
    "    for key in defaultVal.keys():\n",
    "        indiv[key] = individual[i]\n",
    "        i += 1\n",
    "\n",
    "    path = individual[len(defaultVal):len(individual)]\n",
    "    \n",
    "    return crossfold(indiv, path, cfold, resultsPath, testing_name)\n",
    "\n",
    "\n",
    "def crossfold(indiv, path, fold, resultsPath, testing_name):\n",
    "    if indiv['output_dim'] == 50:\n",
    "        embedding_mtx = fold['embedding_matrix'][0]\n",
    "    elif indiv['output_dim'] == 100:\n",
    "        embedding_mtx = fold['embedding_matrix'][1]\n",
    "    elif indiv['output_dim'] == 200:\n",
    "        embedding_mtx = fold['embedding_matrix'][2]\n",
    "    elif indiv['output_dim'] == 300:\n",
    "        embedding_mtx = fold['embedding_matrix'][3]\n",
    "\n",
    "    model = cnn.cnn_model(fold['vocab_size'], fold['maxlen'], embedding_mtx,\n",
    "                          indiv, path)\n",
    "    \n",
    "    #early stopping\n",
    "    #save the best model\n",
    "    callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_f1_score', mode='max', verbose=False, patience=10), \n",
    "                 tf.keras.callbacks.ModelCheckpoint(\"{}{}.h5\".format(resultsPath, testing_name), monitor='val_f1_score', mode='max', verbose=False, \n",
    "                                  save_best_only=True)]\n",
    "\n",
    "    model.fit(fold['X_train'], fold['y_train'], epochs=indiv['epochs'], verbose=False, \n",
    "              validation_data=(fold['X_val'], fold['y_val']), use_multiprocessing=False,\n",
    "              batch_size=indiv['batch_size'], callbacks=callbacks)\n",
    "    \n",
    "    dependencies = {\n",
    "    'f1_score': util.f1_score\n",
    "    }\n",
    "\n",
    "    # load the saved model\n",
    "    for x in range(0, 4):  # try 4 times\n",
    "        try:\n",
    "            # msg.send()\n",
    "            saved_model = tf.keras.models.load_model(\"{}{}.h5\".format(resultsPath, testing_name), custom_objects=dependencies)\n",
    "            str_error = None\n",
    "        except Exception as e:\n",
    "            print('An error occurs when loading saved model.')\n",
    "            str_error = e\n",
    "            pass\n",
    "\n",
    "        if str_error:\n",
    "            sleep(5)  # wait for 2 seconds before trying to fetch the data again\n",
    "        else:\n",
    "            break\n",
    "    \n",
    "\n",
    "    y_pred = saved_model.predict_classes(fold['X_val'])\n",
    "\n",
    "    os.remove(\"{}{}.h5\".format(resultsPath, testing_name))\n",
    "    del embedding_mtx\n",
    "    gc.collect()\n",
    "\n",
    "    # CNN metrics\n",
    "    accuracyScore, precisionScore, recallScore, f1Score = util.get_testing_metric(fold['y_val'], y_pred)\n",
    "    return f1Score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EJ38f_Y7WJSv"
   },
   "source": [
    "# Genetic Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2mE6C_3bWHcU"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "from operator import attrgetter\n",
    "from deap import base\n",
    "from deap import creator\n",
    "from deap import tools\n",
    "import time\n",
    "import datetime\n",
    "import math\n",
    "from scipy.spatial import distance\n",
    "import itertools\n",
    "\n",
    "\n",
    "class GeneticAlgorithm:\n",
    "    __slots__ = (\n",
    "        \"toolbox\", \"toolboxes\", \"cross_rate\", \"mut_rate\", \"n_pop\", \"n_gen\", \"resultsPath\", \"testing_name\", \"cfold\",\n",
    "        \"globalparameters\", \"defaultVal\", \"path_ind\", \"max_conv_idx\", \"max_maxpooling_idx\",\n",
    "        \"max_dense_idx\", \"max_dropout_idx\")\n",
    "\n",
    "    def __init__(self, toolbox, toolboxes, cross_rate, mut_rate, n_pop, n_gen, resultsPath, testing_name,\n",
    "                 cfold, globalparameters, defaultVal, path_ind, max_conv_idx, max_maxpooling_idx,\n",
    "                 max_dense_idx, max_dropout_idx):\n",
    "        self.toolbox = toolbox\n",
    "        self.toolboxes = toolboxes\n",
    "        self.cross_rate = cross_rate\n",
    "        self.mut_rate = mut_rate\n",
    "        self.n_pop = n_pop\n",
    "        self.n_gen = n_gen\n",
    "        self.resultsPath = resultsPath\n",
    "        self.testing_name = testing_name\n",
    "        self.cfold = cfold\n",
    "        self.globalparameters = globalparameters\n",
    "        self.defaultVal = defaultVal\n",
    "        self.path_ind = path_ind\n",
    "        self.max_conv_idx = max_conv_idx\n",
    "        self.max_maxpooling_idx = max_maxpooling_idx\n",
    "        self.max_dense_idx = max_dense_idx\n",
    "        self.max_dropout_idx = max_dropout_idx\n",
    "\n",
    "    def fitnessCalc(self, individual):\n",
    "        i = 0\n",
    "        if len(individual.fitness.values) == 0:\n",
    "            if (0 in individual or '' in individual or 'False' in individual or None in individual):\n",
    "                for param in self.defaultVal:\n",
    "                    if individual[i] == 0 or individual[i] == '' or individual[i] == 'False' or individual[i] == None:\n",
    "                        individual[i] = self.defaultVal[param]\n",
    "                    i += 1\n",
    "\n",
    "            fc = FitnessCalculation(individual, self.cfold, self.defaultVal, self.resultsPath, self.testing_name)\n",
    "        else:\n",
    "            fc = individual.fitness.values[0]\n",
    "        print('{} {}'.format(datetime.datetime.now(), fc))\n",
    "        return fc,\n",
    "\n",
    "    def write_result(self):\n",
    "        # Create Testing Results\n",
    "        f = open(\"{}{}.csv\".format(self.resultsPath, self.testing_name), \"a+\")\n",
    "        text = \"i,min,max,mean,std,avgdistance,time,CR,MR\"\n",
    "        for param in self.defaultVal:\n",
    "            text = \"{},{}\".format(text, param)\n",
    "        text = \"{}\\n\".format(text)\n",
    "        f.write(text)\n",
    "        f.close()\n",
    "\n",
    "        # Create Last Population file\n",
    "        f = open(\"{}{}lastpop.csv\".format(self.resultsPath, self.testing_name), 'a+')\n",
    "        text = \"i,f1score\"\n",
    "        for param in self.defaultVal:\n",
    "            text = \"{},{}\".format(text, param)\n",
    "        text = \"{}\\n\".format(text)\n",
    "        f.write(text)\n",
    "        f.close()\n",
    "\n",
    "    def std_calc(self, fits, length):\n",
    "        mean = sum(fits) / length\n",
    "        sum2 = sum(x * x for x in fits)\n",
    "        std = abs(sum2 / length - mean ** 2) ** 0.5\n",
    "\n",
    "        return mean, std\n",
    "    \n",
    "    def distance_calc(self, pop):\n",
    "        distances = []\n",
    "        for subset in itertools.combinations(pop, 2):\n",
    "            distances.append(distance.hamming(subset[0][0:subset[0].index('embedding_layer')],\n",
    "                                              subset[1][0:subset[1].index('embedding_layer')]))\n",
    "\n",
    "        avgDistance = sum(distances) / len(distances)\n",
    "        \n",
    "        return avgDistance\n",
    "\n",
    "    def invalid_fitness_calc(self, pop):\n",
    "        # Evaluate the individuals with an invalid fitness\n",
    "        invalid_ind = [ind for ind in pop if not ind.fitness.valid]\n",
    "        fitnesses = map(self.toolbox.evaluate, invalid_ind)\n",
    "        for ind, fit in zip(invalid_ind, fitnesses):\n",
    "            ind.fitness.values = fit\n",
    "\n",
    "    def mutHyperparam(self, individual, indpb):\n",
    "        toolboxesSize = len(self.toolboxes)\n",
    "        fsm = FSM()\n",
    "#         mutatefsm = mutateFSM()\n",
    "#         addfsm = addFSM()\n",
    "\n",
    "        # Mutation for the Hyperparameter Chromosomes\n",
    "        for i in range(toolboxesSize):\n",
    "            if random.random() < indpb:\n",
    "                if len(self.toolboxes[i].args) == 1:\n",
    "                    individual[i] = self.toolboxes[i].func(self.toolboxes[i].args[0])\n",
    "                else:\n",
    "                    individual[i] = self.toolboxes[i].func(self.toolboxes[i].args[0], self.toolboxes[i].args[1])\n",
    "\n",
    "        # Mutation for the Architecture Chromosomes\n",
    "        archChrom = individual[individual.index('convolutional_layer'):individual.index('output_layer')]\n",
    "        size = len(archChrom)\n",
    "\n",
    "        for i in range(1, size):\n",
    "            if random.random() < indpb:\n",
    "                if (i>=size):\n",
    "                    break\n",
    "                                \n",
    "                if (archChrom[i] == 'global_maxpooling_layer' or archChrom[i] == 'maxpooling_layer' or archChrom[i] == 'dropout_layer'):\n",
    "                    continue\n",
    "\n",
    "                selectMutType = random.randint(0, 1)\n",
    "                # Remove the layer\n",
    "                if selectMutType == 0:\n",
    "#                     print('individual before remove', individual)\n",
    "                        if (archChrom[i] == 'convolutional_layer') and (archChrom[i+1] == 'maxpooling_layer'):\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            size -= 2\n",
    "                        elif (archChrom[i] == 'dense_layer'):\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            size -= 1\n",
    "\n",
    "                # Add a layer\n",
    "                elif selectMutType == 1:\n",
    "#                     print('individual before add', individual)\n",
    "                    if (archChrom[i] == 'convolutional_layer'):                    \n",
    "                        archChrom.insert(i, 'convolutional_layer')\n",
    "                        archChrom.insert(i+1, 'maxpooling_layer')\n",
    "                    elif (archChrom[i] == 'dense_layer'):\n",
    "                        archChrom.insert(i, 'dense_layer')\n",
    "\n",
    "                individual[individual.index('convolutional_layer'):individual.index('output_layer')] = archChrom\n",
    "#                 print('individual after', individual)\n",
    "        return individual,\n",
    "\n",
    "    def cxTwoPoint(self, ind1, ind2, pop, offspring):\n",
    "        # Crossover for hyperparameter chromosomes\n",
    "        size = ind1.index('embedding_layer')\n",
    "        selectCxType = random.randint(0, 2)\n",
    "        # One point crossover\n",
    "        if selectCxType == 0:\n",
    "#             print('ind1 before one-point crossover:', ind1)\n",
    "#             print('ind2 before one-point crossover:', ind2)\n",
    "            cxpoint = random.randint(1, size - 1)\n",
    "            ind1[cxpoint:], ind2[cxpoint:] = ind2[cxpoint:], ind1[cxpoint:]\n",
    "#             print('ind1 after one-point crossover:', ind1)\n",
    "#             print('ind2 after one-point crossover:', ind2)\n",
    "        # Two-point crossover\n",
    "        elif selectCxType == 1:\n",
    "#             print('ind1 before two-point crossover:', ind1)\n",
    "#             print('ind2 before two-point crossover:', ind2)\n",
    "            cxpoint1 = random.randint(1, size - 1)\n",
    "            cxpoint2 = random.randint(1, size - 1)\n",
    "            if cxpoint2 >= cxpoint1:\n",
    "                cxpoint2 += 1\n",
    "            else:  # Swap the two cx points\n",
    "                cxpoint1, cxpoint2 = cxpoint2, cxpoint1\n",
    "\n",
    "            ind1[cxpoint1:cxpoint2], ind2[cxpoint1:cxpoint2] \\\n",
    "                = ind2[cxpoint1:cxpoint2], ind1[cxpoint1:cxpoint2]\n",
    "#             print('ind1 after two-point crossover:', ind1)\n",
    "#             print('ind2 after two-point crossover:', ind2)\n",
    "        # Uniform crossover\n",
    "        elif selectCxType == 2:\n",
    "#             print('ind1 before uniform crossover:', ind1)\n",
    "#             print('ind2 before uniform crossover:', ind2)\n",
    "            for i in range(size):\n",
    "                if random.random() < self.cross_rate:\n",
    "                    ind1[i], ind2[i] = ind2[i], ind1[i]\n",
    "#             print('ind1 after uniform crossover:', ind1)\n",
    "#             print('ind2 after uniform crossover:', ind2)\n",
    "\n",
    "        # Crossover for architecture chromosomes\n",
    "        # One-cut point crossover from the Global MaxPooling layer\n",
    "        cxpoint1 = ind1.index('global_maxpooling_layer')\n",
    "        cxpoint2 = ind2.index('global_maxpooling_layer')\n",
    "        ind1[cxpoint1:], ind2[cxpoint2:] = ind2[cxpoint2:], ind1[cxpoint1:]\n",
    "\n",
    "        return ind1, ind2\n",
    "\n",
    "    def runGA(self, lastPop=[], lastFitnesses=[]):\n",
    "        creator.create(\"FitnessMax\", base.Fitness, weights=(1.0,))\n",
    "        creator.create(\"Individual\", list, fitness=creator.FitnessMax)\n",
    "\n",
    "        self.toolbox.register(\"individual\", tools.initCycle, creator.Individual,\n",
    "                              self.toolboxes, n=1)\n",
    "        self.toolbox.register(\"population\", tools.initRepeat, list, self.toolbox.individual)\n",
    "        self.toolbox.register(\"evaluate\", self.fitnessCalc)\n",
    "        self.toolbox.register(\"mate\", self.cxTwoPoint)\n",
    "        self.toolbox.register(\"mutate\", self.mutHyperparam, indpb=self.mut_rate)\n",
    "        self.toolbox.register(\"select\", tools.selBest)\n",
    "\n",
    "        pop = self.toolbox.population(n=self.n_pop)\n",
    "\n",
    "        idx = 0\n",
    "        for ind in pop:\n",
    "            if lastPop:\n",
    "                ind[:] = lastPop[idx]\n",
    "            ind.extend(self.path_ind[idx])\n",
    "            idx += 1\n",
    "        \n",
    "        if lastFitnesses:\n",
    "            # Fitnesses from previous population\n",
    "            fitnesses = lastFitnesses\n",
    "        else:\n",
    "            # Evaluate the entire population\n",
    "            fitnesses = list(map(self.toolbox.evaluate, pop))\n",
    "\n",
    "        for ind, fit in zip(pop, fitnesses):\n",
    "            ind.fitness.values = fit\n",
    "\n",
    "        self.write_result()\n",
    "        \n",
    "        g = 0\n",
    "        while g < self.n_gen:\n",
    "            then = time.time()\n",
    "            g = g + 1\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"-- Generation %i --\" % g))          \n",
    "            \n",
    "            # Select the next generation individuals\n",
    "            offspring = self.toolbox.select(pop, len(pop))\n",
    "            # Clone the selected individuals\n",
    "            offspring = list(map(self.toolbox.clone, offspring))\n",
    "\n",
    "            # Apply crossover and mutation on the offspring\n",
    "            for child1, child2 in zip(offspring[::2], offspring[1::2]):\n",
    "                if random.random() < self.cross_rate:\n",
    "                    self.toolbox.mate(child1, child2, pop, offspring)\n",
    "                    del child1.fitness.values\n",
    "                    del child2.fitness.values\n",
    "\n",
    "            for mutant in offspring:\n",
    "                if random.random() < self.mut_rate:\n",
    "                    self.toolbox.mutate(mutant)\n",
    "                    del mutant.fitness.values\n",
    "\n",
    "            # Evaluate the individuals with an invalid fitness\n",
    "            self.invalid_fitness_calc(offspring)\n",
    "\n",
    "            pop[:] = self.toolbox.select(pop + offspring, self.n_pop)\n",
    "\n",
    "            # Gather all the fitnesses in one list and print the stats\n",
    "            fits = [ind.fitness.values[0] for ind in pop]\n",
    "\n",
    "            length = len(pop)\n",
    "            mean, std = self.std_calc(fits, length)\n",
    "            avgDistance = self.distance_calc(pop)\n",
    "            best = max(pop, key=attrgetter(\"fitness\"))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Min %s\" % min(fits)))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Max %s\" % max(fits)))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Avg %s\" % mean))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Std %s\" % std))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  AvgDistance %s\" % avgDistance))\n",
    "            print('{} {}'.format(datetime.datetime.now(), best))\n",
    "\n",
    "            now = time.time()\n",
    "            diff = now - then\n",
    "\n",
    "            # save testing data\n",
    "            f = open(\"{}{}.csv\".format(self.resultsPath, self.testing_name), 'a')\n",
    "            text = \"{0},{1},{2},{3},{4},{5},{6},{7},{8}\".format(g,min(fits), max(fits), mean, std, avgDistance, diff, self.cross_rate, self.mut_rate)\n",
    "            for param in best:\n",
    "                text = \"{},{}\".format(text, param)\n",
    "            text = \"{}\\n\".format(text)\n",
    "            f.write(text)\n",
    "            f.close()\n",
    "\n",
    "            # save last population data\n",
    "            f = open(\"{}{}lastpop.csv\".format(self.resultsPath, self.testing_name), 'a')\n",
    "            for ind in pop:\n",
    "                text = \"{0},{1}\".format(g,ind.fitness.values[0])\n",
    "                for param in ind:\n",
    "                    text = \"{},{}\".format(text, param)\n",
    "                text = \"{}\\n\".format(text)             \n",
    "                f.write(text)\n",
    "\n",
    "            f.close()            "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kUA1C18jfyE6"
   },
   "source": [
    "# Project path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "9OZ4ryogfYra",
    "outputId": "84ac2846-ae86-4784-81ab-7cd413c89af5"
   },
   "outputs": [],
   "source": [
    "training_path = 'trainPreprocessed.csv'\n",
    "population_path = 'NewPop.csv'\n",
    "root_path = '/lab/dbms/fatyanosa'\n",
    "datasetPath = '{}/Dataset/Disaster Tweets/'.format(root_path)\n",
    "resultsPath = '{}/Server2/Disaster Tweets/Results/'.format(root_path)\n",
    "testing_name = \"Experiment4_GA_CNN\"\n",
    "glovePath = ['{}/Glove/glove.6B.50d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.100d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.200d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.300d.txt'.format(root_path)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8O1VRBE73ivB"
   },
   "source": [
    "# Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6XqVtrSu3kq4"
   },
   "outputs": [],
   "source": [
    "# crossover rate is the probability with which two individuals\n",
    "cross_rate = 0.8\n",
    "\n",
    "# mutation rate is the probability for mutating an individual\n",
    "mut_rate = 0.2\n",
    "\n",
    "# number of population\n",
    "n_pop = 30\n",
    "\n",
    "# number of generation\n",
    "n_gen = 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X6kUHesFXh17"
   },
   "source": [
    "# Main Program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "U4oAqAYAUmVT",
    "outputId": "b9c2e51a-6a4d-4eca-8b05-8397a9fc9216",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-02 18:42:55.444801 -- Generation 14 --\n",
      "2020-04-02 18:43:21.317979 0.7682191780821918\n",
      "2020-04-02 18:43:34.069998 0.7782561894510225\n",
      "2020-04-02 18:43:49.810592 0.7810140237324704\n",
      "2020-04-02 18:44:06.839462 0.7736147757255936\n",
      "2020-04-02 18:44:25.402440 0.7823275862068966\n",
      "2020-04-02 18:44:47.436378 0.7801268498942917\n",
      "2020-04-02 18:45:01.378564 0.776470588235294\n",
      "2020-04-02 18:45:14.016359 0.7806970509383377\n",
      "2020-04-02 18:45:34.767616 0.7821409359870899\n",
      "2020-04-02 18:45:45.540955 0.7726775956284153\n",
      "2020-04-02 18:46:07.510240 0.7828389830508475\n",
      "2020-04-02 18:46:22.340169 0.783695652173913\n",
      "2020-04-02 18:46:25.615127 0.0082389289392379\n",
      "2020-04-02 18:46:47.316837 0.7881981032665965\n",
      "2020-04-02 18:47:09.232486 0.7807999999999999\n",
      "2020-04-02 18:47:16.314408 0.0\n",
      "2020-04-02 18:47:22.803430 0.46724546172059983\n",
      "2020-04-02 18:47:39.476215 0.7845244492208491\n",
      "2020-04-02 18:47:48.574479 0.7744401966138722\n",
      "2020-04-02 18:47:58.918151 0.7737909516380655\n",
      "2020-04-02 18:47:58.940100   Min 0.785368478\n",
      "2020-04-02 18:47:58.940217   Max 0.789359392\n",
      "2020-04-02 18:47:58.940243   Avg 0.7863121447755532\n",
      "2020-04-02 18:47:58.940284   Std 0.0009808363771676243\n",
      "2020-04-02 18:47:58.940310   AvgDistance 0.057854406130268377\n",
      "2020-04-02 18:47:58.940332 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'tanh', 'uniform', 3, 24, 'softmax', 'he_uniform', 2, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 18:47:58.942815 -- Generation 15 --\n",
      "2020-04-02 18:48:18.416197 0.7900701565029682\n",
      "2020-04-02 18:48:33.781475 0.7712\n",
      "2020-04-02 18:48:41.137628 0.0\n",
      "2020-04-02 18:49:02.372925 0.7819314641744549\n",
      "2020-04-02 18:49:21.694356 0.7824267782426778\n",
      "2020-04-02 18:49:42.671980 0.7857900318133616\n",
      "2020-04-02 18:49:57.994574 0.7837415320479416\n",
      "2020-04-02 18:50:11.191017 0.784934497816594\n",
      "2020-04-02 18:50:37.238549 0.7828843106180664\n",
      "2020-04-02 18:50:58.764814 0.7781975175391258\n",
      "2020-04-02 18:51:18.761014 0.7865404837013671\n",
      "2020-04-02 18:51:37.831511 0.7760388559093362\n",
      "2020-04-02 18:51:59.828455 0.7793575566087415\n",
      "2020-04-02 18:52:18.769091 0.7789358200767964\n",
      "2020-04-02 18:52:34.213068 0.7769940314704287\n",
      "2020-04-02 18:52:47.257204 0.7730812013348165\n",
      "2020-04-02 18:53:01.967050 0.7863436123348017\n",
      "2020-04-02 18:53:19.811810 0.7551462621885158\n",
      "2020-04-02 18:53:34.105319 0.7787798408488064\n",
      "2020-04-02 18:53:34.118136   Min 0.785942492\n",
      "2020-04-02 18:53:34.118178   Max 0.7900701565029682\n",
      "2020-04-02 18:53:34.118205   Avg 0.786751374226858\n",
      "2020-04-02 18:53:34.118219   Std 0.0010632913463222305\n",
      "2020-04-02 18:53:34.118235   AvgDistance 0.061685823754789454\n",
      "2020-04-02 18:53:34.118249 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 4, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 18:53:34.120804 -- Generation 16 --\n",
      "2020-04-02 18:53:45.870863 0.7662874870734229\n",
      "2020-04-02 18:54:08.628410 0.7806176783812566\n",
      "2020-04-02 18:54:29.864472 0.7790762381747358\n",
      "2020-04-02 18:54:51.822917 0.7815126050420168\n",
      "2020-04-02 18:55:09.332212 0.7743271221532091\n",
      "2020-04-02 18:55:26.973970 0.7794037940379404\n",
      "2020-04-02 18:55:50.183423 0.7774261603375526\n",
      "2020-04-02 18:56:05.156091 0.7774227902023428\n",
      "2020-04-02 18:56:26.329897 0.7760388559093362\n",
      "2020-04-02 18:56:47.020888 0.7840785169029445\n",
      "2020-04-02 18:57:06.553377 0.7851772287862514\n",
      "2020-04-02 18:57:24.350219 0.7811816192560175\n",
      "2020-04-02 18:57:42.460400 0.7845903418339664\n",
      "2020-04-02 18:58:00.555977 0.7767517653449213\n",
      "2020-04-02 18:58:17.155008 0.7803837953091685\n",
      "2020-04-02 18:58:40.230746 0.7825171142706687\n",
      "2020-04-02 18:58:46.601028 0.7448121144139092\n",
      "2020-04-02 18:59:02.058339 0.780982905982906\n",
      "2020-04-02 18:59:16.766380 0.7817204301075268\n",
      "2020-04-02 18:59:32.884877 0.7791077257889009\n",
      "2020-04-02 18:59:52.949726 0.7771428571428571\n",
      "2020-04-02 19:00:12.398725 0.7754442649434571\n",
      "2020-04-02 19:00:31.229750 0.7653230259525123\n",
      "2020-04-02 19:00:47.813091 0.7698154180238871\n",
      "2020-04-02 19:01:02.056576 0.7773109243697478\n",
      "2020-04-02 19:01:16.268191 0.7745152354570637\n",
      "2020-04-02 19:01:16.283453   Min 0.785942492\n",
      "2020-04-02 19:01:16.283495   Max 0.7900701565029682\n",
      "2020-04-02 19:01:16.283528   Avg 0.786817559826858\n",
      "2020-04-02 19:01:16.283545   Std 0.0010366758797765102\n",
      "2020-04-02 19:01:16.283561   AvgDistance 0.05146871008939983\n",
      "2020-04-02 19:01:16.283576 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 4, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 19:01:16.286186 -- Generation 17 --\n",
      "2020-04-02 19:01:32.723127 0.7782470960929252\n",
      "2020-04-02 19:01:56.108605 0.7822364901016586\n",
      "2020-04-02 19:02:15.927816 0.7803814713896458\n",
      "2020-04-02 19:02:36.239236 0.7713498622589531\n",
      "2020-04-02 19:02:51.148088 0.7708219923788787\n",
      "2020-04-02 19:03:07.705476 0.7778969957081545\n",
      "2020-04-02 19:03:30.009520 0.7766470273165507\n",
      "2020-04-02 19:03:48.282542 0.7805929919137466\n",
      "2020-04-02 19:04:12.536903 0.7827980402830703\n",
      "2020-04-02 19:04:34.762876 0.7792068595927117\n",
      "2020-04-02 19:04:56.857790 0.7738801942795467\n",
      "2020-04-02 19:05:15.304808 0.7831074035453598\n",
      "2020-04-02 19:05:34.186706 0.7814854682454252\n",
      "2020-04-02 19:05:55.342420 0.7776572668112799\n",
      "2020-04-02 19:06:17.709384 0.7805676855895196\n",
      "2020-04-02 19:06:31.001438 0.7618025751072961\n",
      "2020-04-02 19:06:48.940228 0.7816216216216215\n",
      "2020-04-02 19:07:12.980564 0.7813658020116464\n",
      "2020-04-02 19:07:35.346346 0.7850568489442338\n",
      "2020-04-02 19:07:57.972475 0.7733478973238668\n",
      "2020-04-02 19:08:14.149506 0.7731000546746856\n",
      "2020-04-02 19:08:37.777173 0.788421052631579\n",
      "2020-04-02 19:08:54.835787 0.7804359383306751\n",
      "2020-04-02 19:09:10.116536 0.781183932346723\n",
      "2020-04-02 19:09:26.552931 0.7744774477447744\n",
      "2020-04-02 19:09:42.651037 0.7787234042553192\n",
      "2020-04-02 19:10:01.109080 0.7743362831858408\n",
      "2020-04-02 19:10:15.705920 0.7757313109425786\n",
      "2020-04-02 19:10:15.721858   Min 0.785979819\n",
      "2020-04-02 19:10:15.721903   Max 0.7900701565029682\n",
      "2020-04-02 19:10:15.721929   Avg 0.7869307828479104\n",
      "2020-04-02 19:10:15.721955   Std 0.0010469886054741263\n",
      "2020-04-02 19:10:15.721971   AvgDistance 0.045083014048531315\n",
      "2020-04-02 19:10:15.721986 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 4, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 19:10:15.725154 -- Generation 18 --\n",
      "2020-04-02 19:10:40.350159 0.7791509940891994\n",
      "2020-04-02 19:11:02.626056 0.7854426941879413\n",
      "2020-04-02 19:11:21.916121 0.7800108636610539\n",
      "2020-04-02 19:11:38.455300 0.7769028871391077\n",
      "2020-04-02 19:12:02.392625 0.7784496976360638\n",
      "2020-04-02 19:12:23.320592 0.7830139823925428\n",
      "2020-04-02 19:12:47.669737 0.7799680340969631\n",
      "2020-04-02 19:13:11.564299 0.7774261603375526\n",
      "2020-04-02 19:13:35.162978 0.7815384615384615\n",
      "2020-04-02 19:13:55.772868 0.7796426637791012\n",
      "2020-04-02 19:14:13.942492 0.7742980561555075\n",
      "2020-04-02 19:14:26.778075 0.7835164835164835\n",
      "2020-04-02 19:14:46.350011 0.7818181818181817\n",
      "2020-04-02 19:15:05.805585 0.7801652892561984\n",
      "2020-04-02 19:15:28.855711 0.7829333333333334\n",
      "2020-04-02 19:15:37.873495 0.0\n",
      "2020-04-02 19:15:57.425817 0.7873154729360305\n",
      "2020-04-02 19:16:21.803022 0.7832538420773715\n",
      "2020-04-02 19:16:44.368084 0.7836990595611285\n",
      "2020-04-02 19:17:07.272237 0.7820580474934036\n",
      "2020-04-02 19:17:19.021066 0.7778353184878302\n",
      "2020-04-02 19:17:37.760121 0.7861271676300577\n",
      "2020-04-02 19:17:55.752876 0.7848804891606449\n",
      "2020-04-02 19:17:55.773408   Min 0.786026201\n",
      "2020-04-02 19:17:55.773518   Max 0.7900701565029682\n",
      "2020-04-02 19:17:55.773560   Avg 0.787390651750212\n",
      "2020-04-02 19:17:55.773583   Std 0.0011128433914459341\n",
      "2020-04-02 19:17:55.773611   AvgDistance 0.04802043422733093\n",
      "2020-04-02 19:17:55.773634 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 4, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 19:17:55.776356 -- Generation 19 --\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-02 19:18:14.152323 0.7776628748707342\n",
      "2020-04-02 19:18:30.359974 0.7835703001579779\n",
      "2020-04-02 19:18:47.916617 0.7792768483540204\n",
      "2020-04-02 19:19:09.228322 0.7872913301023154\n",
      "2020-04-02 19:19:30.910957 0.7837415320479416\n",
      "2020-04-02 19:19:48.539988 0.7857528332433891\n",
      "2020-04-02 19:20:05.539212 0.7792768483540204\n",
      "2020-04-02 19:20:25.137631 0.7776584317937703\n",
      "2020-04-02 19:20:49.485356 0.7776572668112799\n",
      "2020-04-02 19:21:06.792987 0.7864184008762322\n",
      "2020-04-02 19:21:31.432003 0.7798213347346296\n",
      "2020-04-02 19:21:56.161988 0.7774261603375526\n",
      "2020-04-02 19:22:17.746921 0.7797522886375875\n",
      "2020-04-02 19:22:41.802626 0.7789363920750781\n",
      "2020-04-02 19:23:07.317911 0.783625730994152\n",
      "2020-04-02 19:23:29.814680 0.7836822329575953\n",
      "2020-04-02 19:23:52.691594 0.7861333333333332\n",
      "2020-04-02 19:24:14.680967 0.780539397144368\n",
      "2020-04-02 19:24:33.902023 0.7789585547290117\n",
      "2020-04-02 19:24:57.913857 0.7866738312735088\n",
      "2020-04-02 19:25:15.604254 0.7771245323356495\n",
      "2020-04-02 19:25:33.021901 0.7803108808290156\n",
      "2020-04-02 19:25:53.173481 0.7809626825310978\n",
      "2020-04-02 19:26:12.060938 0.7729994556341862\n",
      "2020-04-02 19:26:12.072957   Min 0.7865404837013671\n",
      "2020-04-02 19:26:12.073007   Max 0.7900701565029682\n",
      "2020-04-02 19:26:12.073024   Avg 0.7878446000184948\n",
      "2020-04-02 19:26:12.073036   Std 0.0011496197111072134\n",
      "2020-04-02 19:26:12.073061   AvgDistance 0.04712643678160936\n",
      "2020-04-02 19:26:12.073076 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 4, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 19:26:12.076159 -- Generation 20 --\n",
      "2020-04-02 19:26:34.571445 0.7708442579968537\n",
      "2020-04-02 19:26:56.591392 0.7815587266739845\n",
      "2020-04-02 19:27:19.105567 0.7812840043525572\n",
      "2020-04-02 19:27:36.851344 0.7773741567202905\n",
      "2020-04-02 19:27:52.743607 0.7790383576445165\n",
      "2020-04-02 19:28:10.380578 0.7790262172284644\n",
      "2020-04-02 19:28:31.464611 0.7772972972972972\n",
      "2020-04-02 19:28:57.049495 0.7813378302417088\n",
      "2020-04-02 19:29:21.279055 0.7708108108108108\n",
      "2020-04-02 19:29:36.245382 0.7799016930638996\n",
      "2020-04-02 19:29:54.107420 0.7751249305941144\n",
      "2020-04-02 19:30:14.002209 0.7751091703056768\n",
      "2020-04-02 19:30:33.795326 0.7776000000000001\n",
      "2020-04-02 19:30:53.179208 0.7786666666666666\n",
      "2020-04-02 19:31:15.643421 0.782703886152162\n",
      "2020-04-02 19:31:39.338368 0.7758241758241758\n",
      "2020-04-02 19:32:03.833292 0.7860262008733626\n",
      "2020-04-02 19:32:27.024885 0.7824657534246575\n",
      "2020-04-02 19:32:49.908316 0.788717402873869\n",
      "2020-04-02 19:33:10.875813 0.782051282051282\n",
      "2020-04-02 19:33:25.038835 0.7734333154793787\n",
      "2020-04-02 19:33:48.437178 0.7799564270152506\n",
      "2020-04-02 19:33:48.448410   Min 0.786851567\n",
      "2020-04-02 19:33:48.448453   Max 0.7900701565029682\n",
      "2020-04-02 19:33:48.448476   Avg 0.7880237119252929\n",
      "2020-04-02 19:33:48.448492   Std 0.0010673927495259646\n",
      "2020-04-02 19:33:48.448508   AvgDistance 0.0541507024265647\n",
      "2020-04-02 19:33:48.448522 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 4, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 19:33:48.450982 -- Generation 21 --\n",
      "2020-04-02 19:34:14.070725 0.778575372722253\n",
      "2020-04-02 19:34:38.839770 0.782608695652174\n",
      "2020-04-02 19:34:57.561794 0.7866242038216561\n",
      "2020-04-02 19:35:22.352604 0.7809319764327799\n",
      "2020-04-02 19:35:39.445434 0.7784679089026915\n",
      "2020-04-02 19:35:55.704785 0.7807545106615638\n",
      "2020-04-02 19:36:18.887746 0.7836691410392365\n",
      "2020-04-02 19:36:39.718670 0.7746863066012001\n",
      "2020-04-02 19:37:04.870737 0.7837693539775761\n",
      "2020-04-02 19:37:22.536384 0.7787993510005408\n",
      "2020-04-02 19:37:47.143137 0.7784877529286475\n",
      "2020-04-02 19:38:10.511657 0.7855603448275862\n",
      "2020-04-02 19:38:29.749519 0.7816826411075614\n",
      "2020-04-02 19:38:55.382050 0.7866884888161484\n",
      "2020-04-02 19:39:13.493608 0.7807840971838763\n",
      "2020-04-02 19:39:31.533696 0.782656421514819\n",
      "2020-04-02 19:39:49.194826 0.7769784172661871\n",
      "2020-04-02 19:40:11.042861 0.7832538420773715\n",
      "2020-04-02 19:40:35.308283 0.7796257796257796\n",
      "2020-04-02 19:40:59.906003 0.7885032537960954\n",
      "2020-04-02 19:41:21.781512 0.7797716150081567\n",
      "2020-04-02 19:41:45.609789 0.7820372398685651\n",
      "2020-04-02 19:42:09.304943 0.7809939923539049\n",
      "2020-04-02 19:42:35.103476 0.7887931034482758\n",
      "2020-04-02 19:43:00.145344 0.7778981581798483\n",
      "2020-04-02 19:43:26.289409 0.7753351206434317\n",
      "2020-04-02 19:43:49.651609 0.7780149413020278\n",
      "2020-04-02 19:44:10.214698 0.7809319764327799\n",
      "2020-04-02 19:44:10.235030   Min 0.786851567\n",
      "2020-04-02 19:44:10.235129   Max 0.7900701565029682\n",
      "2020-04-02 19:44:10.235171   Avg 0.7881845661313062\n",
      "2020-04-02 19:44:10.235193   Std 0.000989854519839853\n",
      "2020-04-02 19:44:10.235219   AvgDistance 0.05581098339719051\n",
      "2020-04-02 19:44:10.235241 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 4, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 19:44:10.244961 -- Generation 22 --\n",
      "2020-04-02 19:44:36.053371 0.782608695652174\n",
      "2020-04-02 19:45:01.932079 0.779498131340096\n",
      "2020-04-02 19:45:24.568871 0.7881720430107527\n",
      "2020-04-02 19:45:48.164776 0.7794817556848228\n",
      "2020-04-02 19:46:05.259487 0.7830744509908945\n",
      "2020-04-02 19:46:31.450203 0.7763941526800217\n",
      "2020-04-02 19:46:56.438193 0.7790262172284644\n",
      "2020-04-02 19:47:23.321055 0.7769028871391077\n",
      "2020-04-02 19:47:48.991363 0.7837985769020251\n",
      "2020-04-02 19:48:16.318698 0.7783145464304885\n",
      "2020-04-02 19:48:40.499316 0.7795918367346939\n",
      "2020-04-02 19:49:06.542404 0.7807775377969762\n",
      "2020-04-02 19:49:31.074118 0.7806731813246471\n",
      "2020-04-02 19:49:55.674284 0.784356328082564\n",
      "2020-04-02 19:50:09.208296 0.7714129841789416\n",
      "2020-04-02 19:50:32.128419 0.7845828933474128\n",
      "2020-04-02 19:50:54.839706 0.7747844827586208\n",
      "2020-04-02 19:51:13.061573 0.7847682119205299\n",
      "2020-04-02 19:51:32.391710 0.773417059131345\n",
      "2020-04-02 19:51:53.381548 0.781183932346723\n",
      "2020-04-02 19:52:39.538693 0.7501397428731135\n",
      "2020-04-02 19:52:58.187042 0.7769784172661871\n",
      "2020-04-02 19:53:25.126343 0.781769436997319\n",
      "2020-04-02 19:53:40.131922 0.7711632987438557\n",
      "2020-04-02 19:54:07.579511 0.7814293166405843\n",
      "2020-04-02 19:54:33.368108 0.779498131340096\n",
      "2020-04-02 19:54:59.929054 0.781888459414688\n",
      "2020-04-02 19:55:23.394780 0.7842720510095642\n",
      "2020-04-02 19:55:48.723878 0.7828539199097575\n",
      "2020-04-02 19:56:14.550345 0.7828911748781807\n",
      "2020-04-02 19:56:14.563097   Min 0.786851567\n",
      "2020-04-02 19:56:14.563138   Max 0.7900701565029682\n",
      "2020-04-02 19:56:14.563164   Avg 0.7882285819983312\n",
      "2020-04-02 19:56:14.563178   Std 0.000958462438136442\n",
      "2020-04-02 19:56:14.563705   AvgDistance 0.057982120051085786\n",
      "2020-04-02 19:56:14.563723 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 4, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 19:56:14.566450 -- Generation 23 --\n",
      "2020-04-02 19:56:39.001449 0.7826552462526767\n",
      "2020-04-02 19:57:05.661445 0.7782608695652173\n",
      "2020-04-02 19:57:31.223892 0.7837259100642399\n",
      "2020-04-02 19:57:56.664089 0.7793030623020063\n",
      "2020-04-02 19:58:13.490087 0.7809939923539049\n",
      "2020-04-02 19:58:35.499640 0.7740540540540541\n",
      "2020-04-02 19:59:00.041779 0.7876106194690266\n",
      "2020-04-02 19:59:23.746131 0.7823750671681892\n",
      "2020-04-02 19:59:51.119547 0.7804878048780488\n",
      "2020-04-02 20:00:14.781726 0.7902439024390244\n",
      "2020-04-02 20:00:35.749773 0.783023872679045\n",
      "2020-04-02 20:01:02.814512 0.788806758183738\n",
      "2020-04-02 20:01:25.734201 0.7846994535519126\n",
      "2020-04-02 20:01:52.197138 0.7857900318133616\n",
      "2020-04-02 20:02:15.478580 0.779014308426073\n",
      "2020-04-02 20:02:37.814849 0.7758530183727035\n",
      "2020-04-02 20:03:01.055714 0.7782470960929252\n",
      "2020-04-02 20:03:18.277197 0.7791249341064839\n",
      "2020-04-02 20:03:38.559605 0.7848804891606449\n",
      "2020-04-02 20:03:57.807230 0.7808510638297872\n",
      "2020-04-02 20:04:29.479893 0.7799016930638996\n",
      "2020-04-02 20:04:57.960944 0.7720234917245061\n",
      "2020-04-02 20:05:25.901561 0.7861842105263158\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-02 20:05:52.836937 0.7837837837837837\n",
      "2020-04-02 20:06:16.976586 0.7836822329575953\n",
      "2020-04-02 20:06:44.534540 0.7797913234486545\n",
      "2020-04-02 20:07:05.221648 0.7793281653746771\n",
      "2020-04-02 20:07:05.234424   Min 0.7873154729360305\n",
      "2020-04-02 20:07:05.234477   Max 0.7902439024390244\n",
      "2020-04-02 20:07:05.234496   Avg 0.7885244190095724\n",
      "2020-04-02 20:07:05.234510   Std 0.000883605651637079\n",
      "2020-04-02 20:07:05.234526   AvgDistance 0.05708812260536427\n",
      "2020-04-02 20:07:05.234540 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 3, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 20:07:05.237091 -- Generation 24 --\n",
      "2020-04-02 20:07:31.484109 0.7791780821917808\n",
      "2020-04-02 20:08:00.441065 0.7821782178217822\n",
      "2020-04-02 20:08:21.653264 0.7732772225144662\n",
      "2020-04-02 20:08:44.175782 0.7793791574279378\n",
      "2020-04-02 20:09:09.442753 0.7787610619469026\n",
      "2020-04-02 20:09:33.758556 0.7792768483540204\n",
      "2020-04-02 20:09:59.543395 0.7813319349764027\n",
      "2020-04-02 20:10:25.948015 0.7842301545018647\n",
      "2020-04-02 20:10:52.926097 0.7827956989247311\n",
      "2020-04-02 20:11:20.320737 0.78143550998381\n",
      "2020-04-02 20:11:49.234716 0.7796610169491526\n",
      "2020-04-02 20:12:11.586486 0.7709611451942741\n",
      "2020-04-02 20:12:40.076488 0.7786177105831533\n",
      "2020-04-02 20:13:08.632140 0.785289345592212\n",
      "2020-04-02 20:13:38.706401 0.7800963081861958\n",
      "2020-04-02 20:14:04.778196 0.7837123215230037\n",
      "2020-04-02 20:14:29.176020 0.7803278688524589\n",
      "2020-04-02 20:14:49.435040 0.7809110629067244\n",
      "2020-04-02 20:15:10.262347 0.7747368421052631\n",
      "2020-04-02 20:15:31.072827 0.7767416346681295\n",
      "2020-04-02 20:15:53.246524 0.7867450561197221\n",
      "2020-04-02 20:16:09.325827 0.7763578274760383\n",
      "2020-04-02 20:16:31.484316 0.7762803234501348\n",
      "2020-04-02 20:17:00.083558 0.783023872679045\n",
      "2020-04-02 20:17:00.105199   Min 0.7876200640000001\n",
      "2020-04-02 20:17:00.105297   Max 0.7902439024390244\n",
      "2020-04-02 20:17:00.105340   Avg 0.788801551966928\n",
      "2020-04-02 20:17:00.105362   Std 0.0008814611910824724\n",
      "2020-04-02 20:17:00.105386   AvgDistance 0.06309067688378062\n",
      "2020-04-02 20:17:00.105409 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'hard_sigmoid', 'uniform', 3, 24, 'softmax', 'he_uniform', 1, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 20:17:00.107923 -- Generation 25 --\n",
      "2020-04-02 20:17:26.856473 0.7721032399780341\n",
      "2020-04-02 20:17:53.962884 0.77289972899729\n",
      "2020-04-02 20:18:21.956032 0.7867768595041322\n",
      "2020-04-02 20:18:44.617714 0.7822798487304161\n",
      "2020-04-02 20:19:15.577952 0.7654986522911051\n",
      "2020-04-02 20:19:42.642813 0.7760869565217391\n",
      "2020-04-02 20:20:04.311862 0.7775401069518716\n",
      "2020-04-02 20:20:27.288547 0.7810026385224275\n",
      "2020-04-02 20:20:48.303931 0.7947165657677491\n",
      "2020-04-02 20:21:10.993715 0.7771051183269125\n",
      "2020-04-02 20:21:39.825016 0.7810026385224275\n",
      "2020-04-02 20:22:06.515724 0.7785016286644952\n",
      "2020-04-02 20:22:32.298854 0.7762947143619862\n",
      "2020-04-02 20:23:00.992449 0.7793814432989691\n",
      "2020-04-02 20:23:27.673328 0.7765681700362882\n",
      "2020-04-02 20:23:56.266554 0.7866379310344827\n",
      "2020-04-02 20:24:23.672589 0.7817418677859391\n",
      "2020-04-02 20:24:54.120852 0.7933440687063876\n",
      "2020-04-02 20:25:17.975681 0.7742980561555075\n",
      "2020-04-02 20:25:42.427142 0.7813698630136987\n",
      "2020-04-02 20:26:05.948890 0.7774827403080191\n",
      "2020-04-02 20:26:29.069362 0.7736842105263159\n",
      "2020-04-02 20:26:54.979012 0.7773584905660377\n",
      "2020-04-02 20:26:54.992153   Min 0.7881981032665965\n",
      "2020-04-02 20:26:54.992207   Max 0.7947165657677491\n",
      "2020-04-02 20:26:54.992225   Avg 0.789444837638498\n",
      "2020-04-02 20:26:54.992239   Std 0.0014056136717117858\n",
      "2020-04-02 20:26:54.992261   AvgDistance 0.05874840357598992\n",
      "2020-04-02 20:26:54.992277 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'tanh', 'uniform', 3, 24, 'softmax', 'he_uniform', 2, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 20:26:54.994986 -- Generation 26 --\n",
      "2020-04-02 20:27:20.080673 0.7805405405405405\n",
      "2020-04-02 20:27:40.214084 0.7734503565551288\n",
      "2020-04-02 20:28:06.647603 0.7842931937172775\n",
      "2020-04-02 20:28:34.704620 0.7838427947598253\n",
      "2020-04-02 20:28:59.954857 0.7819706498951783\n",
      "2020-04-02 20:29:28.287431 0.7871529667936854\n",
      "2020-04-02 20:29:55.186232 0.7792349726775957\n",
      "2020-04-02 20:30:22.593157 0.7818574514038877\n",
      "2020-04-02 20:30:51.795833 0.7833698030634573\n",
      "2020-04-02 20:31:17.368943 0.7854507116499737\n",
      "2020-04-02 20:31:39.950066 0.7739544732662784\n",
      "2020-04-02 20:32:04.596216 0.7808930425752856\n",
      "2020-04-02 20:32:25.932441 0.7782515991471215\n",
      "2020-04-02 20:32:53.448773 0.7836822329575953\n",
      "2020-04-02 20:33:21.963743 0.7824675324675324\n",
      "2020-04-02 20:33:50.456310 0.7854122621564482\n",
      "2020-04-02 20:34:14.676461 0.7794817556848228\n",
      "2020-04-02 20:34:44.602123 0.7866018368449487\n",
      "2020-04-02 20:35:12.264554 0.7887771307570143\n",
      "2020-04-02 20:35:42.677636 0.7815126050420168\n",
      "2020-04-02 20:36:12.420798 0.7756613756613755\n",
      "2020-04-02 20:36:38.280873 0.7779544250132485\n",
      "2020-04-02 20:37:08.529857 0.7796070100902814\n",
      "2020-04-02 20:37:30.725420 0.7801268498942917\n",
      "2020-04-02 20:37:58.798116 0.7818574514038877\n",
      "2020-04-02 20:37:58.819556   Min 0.788421052631579\n",
      "2020-04-02 20:37:58.819663   Max 0.7947165657677491\n",
      "2020-04-02 20:37:58.819687   Avg 0.7894644580337926\n",
      "2020-04-02 20:37:58.820071   Std 0.001392035656974245\n",
      "2020-04-02 20:37:58.820100   AvgDistance 0.05708812260536408\n",
      "2020-04-02 20:37:58.820120 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'tanh', 'uniform', 3, 24, 'softmax', 'he_uniform', 2, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 20:37:58.823282 -- Generation 27 --\n",
      "2020-04-02 20:38:28.986644 0.7724645296899634\n",
      "2020-04-02 20:38:52.075259 0.7859879584017516\n",
      "2020-04-02 20:39:16.038227 0.7805907172995781\n",
      "2020-04-02 20:39:46.874140 0.7774227902023428\n",
      "2020-04-02 20:40:16.573548 0.786206896551724\n",
      "2020-04-02 20:40:45.670096 0.7840000000000001\n",
      "2020-04-02 20:41:15.847607 0.783695652173913\n",
      "2020-04-02 20:41:51.498984 0.772234273318872\n",
      "2020-04-02 20:42:14.453862 0.7721382289416846\n",
      "2020-04-02 20:42:43.537689 0.7862796833773087\n",
      "2020-04-02 20:43:10.952027 0.7831715210355986\n",
      "2020-04-02 20:43:41.900949 0.7829255596043726\n",
      "2020-04-02 20:44:10.724169 0.775082690187431\n",
      "2020-04-02 20:44:41.930187 0.7790821771611526\n",
      "2020-04-02 20:45:06.725918 0.7867803837953092\n",
      "2020-04-02 20:45:35.920481 0.7830290010741138\n",
      "2020-04-02 20:46:01.552267 0.781029263370333\n",
      "2020-04-02 20:46:27.388090 0.7754214246873301\n",
      "2020-04-02 20:46:56.787941 0.7798810167658194\n",
      "2020-04-02 20:47:26.724189 0.7806205770277626\n",
      "2020-04-02 20:47:54.505992 0.780952380952381\n",
      "2020-04-02 20:48:21.593708 0.7866154690071312\n",
      "2020-04-02 20:48:50.648576 0.7763088313061872\n",
      "2020-04-02 20:49:20.878821 0.7809319764327799\n",
      "2020-04-02 20:49:52.347425 0.7821835958718087\n",
      "2020-04-02 20:50:16.940230 0.7835269271383315\n",
      "2020-04-02 20:50:16.953258   Min 0.788430637\n",
      "2020-04-02 20:50:16.953302   Max 0.7947165657677491\n",
      "2020-04-02 20:50:16.953327   Avg 0.789665798973186\n",
      "2020-04-02 20:50:16.953341   Std 0.0013410705312094022\n",
      "2020-04-02 20:50:16.953356   AvgDistance 0.06526181353767582\n",
      "2020-04-02 20:50:16.953371 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'tanh', 'uniform', 3, 24, 'softmax', 'he_uniform', 2, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 20:50:16.955895 -- Generation 28 --\n",
      "2020-04-02 20:50:45.872509 0.78330658105939\n",
      "2020-04-02 20:51:17.713009 0.7823750671681892\n",
      "2020-04-02 20:51:47.179164 0.781789638932496\n",
      "2020-04-02 20:52:12.156118 0.7728459530026109\n",
      "2020-04-02 20:52:40.786011 0.7812666311868015\n",
      "2020-04-02 20:53:12.315557 0.7796610169491526\n",
      "2020-04-02 20:53:36.788644 0.783739837398374\n",
      "2020-04-02 20:54:04.317301 0.7854077253218883\n",
      "2020-04-02 20:54:28.307755 0.7798607391537226\n",
      "2020-04-02 20:54:53.859938 0.7875593041644702\n",
      "2020-04-02 20:55:25.096907 0.7891832229580574\n",
      "2020-04-02 20:55:48.293785 0.7831134564643799\n",
      "2020-04-02 20:56:08.529463 0.7774227902023428\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-04-02 20:56:37.731618 0.7752021563342318\n",
      "2020-04-02 20:56:37.744468   Min 0.788806758183738\n",
      "2020-04-02 20:56:37.744520   Max 0.7947165657677491\n",
      "2020-04-02 20:56:37.744815   Avg 0.7902627524704595\n",
      "2020-04-02 20:56:37.744833   Std 0.001580260034273206\n",
      "2020-04-02 20:56:37.744849   AvgDistance 0.07854406130268235\n",
      "2020-04-02 20:56:37.744864 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'tanh', 'uniform', 3, 24, 'softmax', 'he_uniform', 2, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 20:56:37.747114 -- Generation 29 --\n",
      "2020-04-02 20:57:00.843924 0.7788259958071279\n",
      "2020-04-02 20:57:25.865476 0.7822878228782287\n",
      "2020-04-02 20:57:47.971183 0.7811340752517223\n",
      "2020-04-02 20:58:18.855384 0.7765843179377016\n",
      "2020-04-02 20:58:35.495810 0.0\n",
      "2020-04-02 20:59:04.999320 0.7787056367432151\n",
      "2020-04-02 20:59:35.922626 0.7826541274817137\n",
      "2020-04-02 21:00:06.603178 0.788368336025848\n",
      "2020-04-02 21:00:36.722652 0.7812666311868015\n",
      "2020-04-02 21:01:01.745582 0.7783251231527094\n",
      "2020-04-02 21:01:31.181744 0.7736757624398074\n",
      "2020-04-02 21:02:01.974507 0.7806487080813633\n",
      "2020-04-02 21:02:28.856267 0.7800429184549357\n",
      "2020-04-02 21:02:58.840285 0.7813319349764027\n",
      "2020-04-02 21:03:30.195457 0.7805417357656164\n",
      "2020-04-02 21:04:02.662256 0.7785598267460747\n",
      "2020-04-02 21:04:34.230904 0.7825148407987048\n",
      "2020-04-02 21:05:06.221193 0.7911494873178628\n",
      "2020-04-02 21:05:29.929652 0.7698924731182795\n",
      "2020-04-02 21:05:52.952680 0.7845010615711253\n",
      "2020-04-02 21:06:14.319792 0.77868406742795\n",
      "2020-04-02 21:06:47.328063 0.7809722948248824\n",
      "2020-04-02 21:07:13.735823 0.7774193548387097\n",
      "2020-04-02 21:07:41.858031 0.7760555852485301\n",
      "2020-04-02 21:08:05.819632 0.787335092348285\n",
      "2020-04-02 21:08:32.238718 0.7819314641744549\n",
      "2020-04-02 21:08:58.510368 0.7841218053289831\n",
      "2020-04-02 21:08:58.523047   Min 0.788806758183738\n",
      "2020-04-02 21:08:58.523276   Max 0.7947165657677491\n",
      "2020-04-02 21:08:58.523297   Avg 0.7903829567189047\n",
      "2020-04-02 21:08:58.523312   Std 0.0015391202531219567\n",
      "2020-04-02 21:08:58.523618   AvgDistance 0.0770114942528739\n",
      "2020-04-02 21:08:58.523635 [41, 219, 'adamax', 0.0063873, 0.9, 'glorot_uniform', 100, 490, 2, 'tanh', 'uniform', 3, 24, 'softmax', 'he_uniform', 2, 5, 0.2, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-04-02 21:08:58.525739 -- Generation 30 --\n",
      "2020-04-02 21:09:24.606944 0.7815217391304349\n",
      "2020-04-02 21:09:49.213375 0.7699890470974807\n",
      "2020-04-02 21:10:19.389925 0.7790456431535269\n",
      "2020-04-02 21:10:51.325481 0.7807203389830508\n",
      "2020-04-02 21:11:21.629714 0.7782608695652173\n",
      "2020-04-02 21:11:53.669600 0.7834602829162133\n",
      "2020-04-02 21:12:25.730921 0.7813163481953289\n",
      "2020-04-02 21:12:53.546290 0.7855973813420621\n",
      "2020-04-02 21:13:25.548556 0.7825613079019075\n",
      "2020-04-02 21:13:56.511411 0.7833511205976522\n",
      "2020-04-02 21:14:25.914890 0.7815126050420168\n",
      "2020-04-02 21:14:58.874717 0.7829928216454998\n",
      "2020-04-02 21:15:30.350173 0.7831932773109244\n",
      "2020-04-02 21:16:00.941741 0.7867965367965368\n",
      "2020-04-02 21:16:27.051224 0.7809110629067244\n",
      "2020-04-02 21:16:58.043647 0.7791780821917808\n",
      "2020-04-02 21:17:19.597609 0.7682403433476395\n",
      "2020-04-02 21:17:44.884857 0.7849056603773584\n",
      "2020-04-02 21:18:10.471893 0.7792068595927117\n",
      "2020-04-02 21:18:26.810640 0.0\n",
      "2020-04-02 21:18:59.669017 0.78330658105939\n",
      "2020-04-02 21:19:30.025015 0.7778342653787493\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from deap import base\n",
    "import warnings; warnings.simplefilter('ignore')\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    globalparameters = []\n",
    "    globalparameters.append((\"epochs\", random.randint, 1, 100))\n",
    "    globalparameters.append((\"batch_size\", random.randint, 32, 256))\n",
    "    globalparameters.append((\"optimizer\", random.choice, ['sgd', 'rmsprop', \n",
    "                                                          'adagrad', 'adadelta',\n",
    "                                                          'adam', 'adamax', \n",
    "                                                          'nadam']))\n",
    "    globalparameters.append((\"learning_rate\", random.uniform, 1e-4, 1e-2))\n",
    "    globalparameters.append((\"momentum\", random.choice, [0.9]))\n",
    "    globalparameters.append((\"output_init_mode\", random.choice, ['glorot_uniform']))\n",
    "    globalparameters.append((\"output_dim\", random.choice, [100]))\n",
    "    globalparameters.append((\"num_filters\", random.randint, 32, 512))\n",
    "    globalparameters.append((\"kernel_size\", random.randint, 1, 5))\n",
    "    globalparameters.append((\"conv_activation_func\", random.choice,\n",
    "                                               ['relu', 'softmax', 'elu', 'selu',\n",
    "                                                'softplus', 'softsign', 'tanh',\n",
    "                                                'sigmoid', 'hard_sigmoid', 'linear']))\n",
    "    globalparameters.append((\"conv_init_mode\", random.choice,\n",
    "                                         ['zeros',\n",
    "                                          'ones',\n",
    "                                          'uniform',\n",
    "                                          'normal',\n",
    "                                          'glorot_normal',\n",
    "                                          'glorot_uniform',\n",
    "                                          'he_normal',\n",
    "                                          'he_uniform',\n",
    "                                          'lecun_normal',\n",
    "                                          'lecun_uniform']))\n",
    "    globalparameters.append((\"conv_weight_constraint\", random.randint, 1, 5))\n",
    "    globalparameters.append((\"neurons\", random.randint, 1, 30))\n",
    "    globalparameters.append((\"dense_activation_func\", random.choice,\n",
    "                                                ['relu', 'softmax', 'elu', 'selu',\n",
    "                                                 'softplus', 'softsign', 'tanh',\n",
    "                                                 'sigmoid', 'hard_sigmoid', 'linear']))\n",
    "    globalparameters.append((\"dense_init_mode\", random.choice,\n",
    "                                          ['zeros',\n",
    "                                           'ones',\n",
    "                                           'uniform',\n",
    "                                           'normal',\n",
    "                                           'glorot_normal',\n",
    "                                           'glorot_uniform',\n",
    "                                           'he_normal',\n",
    "                                           'he_uniform',\n",
    "                                           'lecun_normal',\n",
    "                                           'lecun_uniform']))\n",
    "    globalparameters.append((\"dense_weight_constraint\", random.randint, 1, 5))\n",
    "    globalparameters.append((\"pool_size\", random.choice, [5]))\n",
    "    globalparameters.append((\"dropout_rate\", random.choice, [0.2]))\n",
    "\n",
    "    defaultVal = collections.OrderedDict([\n",
    "        (\"epochs\", 10),\n",
    "        (\"batch_size\", 32),\n",
    "        (\"optimizer\", \"adam\"),\n",
    "        (\"learning_rate\", 1e-4),\n",
    "        (\"momentum\", 0.9),\n",
    "        (\"output_init_mode\", \"glorot_uniform\"),\n",
    "        (\"output_dim\", 100),\n",
    "        ('num_filters', 64),\n",
    "        ('kernel_size', 3),\n",
    "        ('conv_activation_func', \"relu\"),\n",
    "        ('conv_init_mode', \"glorot_uniform\"),\n",
    "        ('conv_weight_constraint', 3),\n",
    "        ('neurons', 1),\n",
    "        ('dense_activation_func', \"relu\"),\n",
    "        ('dense_init_mode', \"glorot_uniform\"),\n",
    "        ('dense_weight_constraint', 3),\n",
    "        ('pool_size', 5),\n",
    "        ('dropout_rate', 0.2)]\n",
    "    )\n",
    "    \n",
    "    # object class\n",
    "    util = utility()\n",
    "    toolbox = base.Toolbox()\n",
    "    toolboxes = []\n",
    "\n",
    "    # Attribute generator\n",
    "    for hyper in globalparameters:\n",
    "        if len(hyper) == 3:\n",
    "            toolbox.register(hyper[0], hyper[1], hyper[2])\n",
    "        else:\n",
    "            toolbox.register(hyper[0], hyper[1], hyper[2], hyper[3])\n",
    "\n",
    "    toolboxes.append(toolbox.epochs)\n",
    "    toolboxes.append(toolbox.batch_size)\n",
    "    toolboxes.append(toolbox.optimizer)\n",
    "    toolboxes.append(toolbox.learning_rate)\n",
    "    toolboxes.append(toolbox.momentum)\n",
    "    toolboxes.append(toolbox.output_init_mode)\n",
    "    toolboxes.append(toolbox.output_dim)\n",
    "    toolboxes.append(toolbox.num_filters)\n",
    "    toolboxes.append(toolbox.kernel_size)\n",
    "    toolboxes.append(toolbox.conv_activation_func)\n",
    "    toolboxes.append(toolbox.conv_init_mode)\n",
    "    toolboxes.append(toolbox.conv_weight_constraint)\n",
    "    toolboxes.append(toolbox.neurons)\n",
    "    toolboxes.append(toolbox.dense_activation_func)\n",
    "    toolboxes.append(toolbox.dense_init_mode)\n",
    "    toolboxes.append(toolbox.dense_weight_constraint)\n",
    "    toolboxes.append(toolbox.pool_size)\n",
    "    toolboxes.append(toolbox.dropout_rate)\n",
    "    \n",
    "\n",
    "    path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx = generateFSM(n_pop)\n",
    "\n",
    "#     # Read population data\n",
    "#     dfPopulation = util.read_CSV(\"{}{}\".format(resultsPath, population_path))\n",
    "\n",
    "#     path_ind, fitnesses, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx = openFSM(dfPopulation)\n",
    "#     dfPopulation = dfPopulation.drop(columns=[col for col in dfPopulation if col not in defaultVal])\n",
    "\n",
    "#     population = dfPopulation.loc[:, ~dfPopulation.columns.str.match('Unnamed')].values.tolist()\n",
    "\n",
    "    # Read data\n",
    "    df = util.read_CSV(\"{}{}\".format(datasetPath, training_path))\n",
    "\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    dfTraining, dfTrial = train_test_split(df, test_size = 0.3, random_state=42)\n",
    "\n",
    "    textsTraining, labelsTraining = util.get_text_label(dfTraining)\n",
    "    textsTrial, labelsTrial = util.get_text_label(dfTrial)\n",
    "    cfold = {}\n",
    "\n",
    "    X_train, X_val, y_train, y_val, vocab_size, maxlen, embedding_matrix = util.get_training_trial_data(\n",
    "        textsTraining, labelsTraining, textsTrial, labelsTrial, glovePath)\n",
    "    cfold= {'X_train': X_train, 'X_val': X_val, 'y_train': y_train, 'y_val': y_val, 'vocab_size': vocab_size,\n",
    "                  'maxlen': maxlen, 'embedding_matrix': embedding_matrix}\n",
    "                  \n",
    "    ga = GeneticAlgorithm(toolbox, toolboxes, cross_rate, mut_rate, n_pop, n_gen, resultsPath, testing_name,\n",
    "                          cfold, globalparameters, defaultVal, path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx)\n",
    "    ga.runGA()\n",
    "#     ga.runGA(population, fitnesses)\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "GA-CNN.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}