Experiment7_GA_CNN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FZg_QFpe7W-B"
   },
   "source": [
    "# Utility"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xWATOMlqSBDH"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The history saving thread hit an unexpected error (DatabaseError('database disk image is malformed')).History will not be written to the database.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n",
    "import numpy as np\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_OHVp0iz7a9O"
   },
   "outputs": [],
   "source": [
    "class utility:\n",
    "\n",
    "    def read_CSV(self, filename):\n",
    "        df = pd.read_csv(filename)\n",
    "        return df\n",
    "\n",
    "    def get_text_label(self, df):\n",
    "        texts = []  # list of text samples\n",
    "        labels = []  # list of label ids\n",
    "        for index, row in df.iterrows():\n",
    "            if isinstance(row['text_cleaned'], float):\n",
    "                texts.append(str(row['text_cleaned']))\n",
    "            else:\n",
    "                texts.append(row['text_cleaned'])\n",
    "\n",
    "            labels.append(row['target'])\n",
    "\n",
    "        return texts, labels\n",
    "\n",
    "    def tokenize_texts(self, texts):\n",
    "        tokenizer = Tokenizer(num_words=50000)\n",
    "        tokenizer.fit_on_texts(texts)\n",
    "\n",
    "        return tokenizer\n",
    "\n",
    "    def padding_texts(self, texts, maxlen):\n",
    "\n",
    "        texts = pad_sequences(texts, padding='post', maxlen=maxlen)\n",
    "\n",
    "        return texts\n",
    "\n",
    "    def get_metric(self, y_true, y_pred):\n",
    "        accuracyScore = accuracy_score(y_true, y_pred)\n",
    "\n",
    "        precisionScore = precision_score(y_true, y_pred)\n",
    "        recallScore = recall_score(y_true, y_pred)\n",
    "        f1Score = f1_score(y_true, y_pred)\n",
    "\n",
    "        return accuracyScore, precisionScore, recallScore, f1Score\n",
    "\n",
    "    def print_metric(self, accuracyScore, precisionScore, recallScore, f1Score):\n",
    "        print(\"Accuracy: {}\".format(str(accuracyScore)))\n",
    "        print(\"Precision: {}\".format(str(precisionScore)))\n",
    "        print(\"Recall: {}\".format(str(recallScore)))\n",
    "        print(\"F1-Score: {}\".format(str(f1Score)))\n",
    "        print(\"{},{},{},{}\".format(str(accuracyScore), str(precisionScore), str(recallScore), str(f1Score)))\n",
    "        \n",
    "\n",
    "    def get_testing_metric(self, y_test, y_pred):\n",
    "        accuracyScore, precisionScore, recallScore, f1Score = self.get_metric(y_test, y_pred)\n",
    "\n",
    "        return accuracyScore, precisionScore, recallScore, f1Score\n",
    "\n",
    "    def write_df_csv(self, df, out_path):\n",
    "        df.to_csv(out_path, index=False)\n",
    "\n",
    "    def create_embedding_matrix(self, filepath, word_index, embedding_dim):\n",
    "        vocab_size = len(word_index) + 1  # Adding again 1 because of reserved 0 index\n",
    "        embedding_matrix = np.zeros((vocab_size, embedding_dim))\n",
    "\n",
    "        with open(filepath, encoding=\"utf8\") as f:\n",
    "            for line in f:\n",
    "                word, *vector = line.split()\n",
    "                if word in word_index:\n",
    "                    idx = word_index[word]\n",
    "                    embedding_matrix[idx] = np.array(\n",
    "                        vector, dtype=np.float32)[:embedding_dim]\n",
    "\n",
    "        return embedding_matrix\n",
    "\n",
    "    def get_max_length_of_sentences(self, texts):\n",
    "        maxlength = 0\n",
    "        for text in texts:\n",
    "            if (len(text.split()) > maxlength):\n",
    "                maxlength = len(text.split())\n",
    "\n",
    "        return maxlength\n",
    "\n",
    "    def get_training_trial_data(self, textsTraining, labelsTraining, textsTrial, labelsTrial, glovePath):\n",
    "        textsTraining, textsTesting = np.asarray(textsTraining), np.asarray(textsTrial)\n",
    "        y_train, y_val = np.asarray(labelsTraining), np.asarray(labelsTrial)\n",
    "\n",
    "        # Tokenize words\n",
    "        tokenizer = self.tokenize_texts(textsTraining)\n",
    "        X_train = tokenizer.texts_to_sequences(textsTraining)\n",
    "        X_val = tokenizer.texts_to_sequences(textsTesting)\n",
    "\n",
    "        # Adding 1 because of reserved 0 index\n",
    "        vocab_size = len(tokenizer.word_index) + 1\n",
    "\n",
    "        # get maxlen\n",
    "        maxlen = self.get_max_length_of_sentences(textsTraining)\n",
    "\n",
    "        # Pad sequences with zeros\n",
    "        X_train = self.padding_texts(X_train, maxlen)\n",
    "        X_val = self.padding_texts(X_val, maxlen)\n",
    "\n",
    "        embedding_matrix = []\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[0], tokenizer.word_index, 50))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[1], tokenizer.word_index, 100))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[2], tokenizer.word_index, 200))\n",
    "        embedding_matrix.append(self.create_embedding_matrix(glovePath[3], tokenizer.word_index, 300))\n",
    "\n",
    "        return X_train, X_val, y_train, y_val, vocab_size, maxlen, embedding_matrix\n",
    "\n",
    "    def Average(self, list):\n",
    "        return sum(list) / len(list)\n",
    "\n",
    "    def recall(self, y_true, y_pred):\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "        recall = true_positives / (possible_positives + K.epsilon())\n",
    "        return recall\n",
    "\n",
    "    def precision(self, y_true, y_pred):\n",
    "        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "        precision = true_positives / (predicted_positives + K.epsilon())\n",
    "        return precision\n",
    "\n",
    "    def f1_score(self, y_true, y_pred):\n",
    "        precision = self.precision(y_true, y_pred)\n",
    "        recall = self.recall(y_true, y_pred)\n",
    "        return 2*((precision*recall)/(precision+recall+K.epsilon()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uYAnQP8Ac2Qe"
   },
   "source": [
    "# Finite State Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uKx4XQbmc5HN"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "def FSM():\n",
    "    fsm = {}\n",
    "    fsm[0] = {'src': 0, 'dst': 1, 'layer': 'embedding_layer', 'next_path': [1]}\n",
    "    fsm[1] = {'src': 1, 'dst': 2, 'layer': 'convolutional_layer', 'next_path': [2, 4]}\n",
    "    fsm[2] = {'src': 2, 'dst': 3, 'layer': 'maxpooling_layer', 'next_path': [3]}    \n",
    "    fsm[3] = {'src': 3, 'dst': 2, 'layer': 'convolutional_layer', 'next_path': [2, 4]}\n",
    "    fsm[4] = {'src': 2, 'dst': 4, 'layer': 'global_maxpooling_layer', 'next_path': [5]}\n",
    "    fsm[5] = {'src': 4, 'dst': 5, 'layer': 'dense_layer', 'next_path': [6, 7]}\n",
    "    fsm[6] = {'src': 5, 'dst': 5, 'layer': 'dense_layer', 'next_path': [6, 7]}    \n",
    "    fsm[7] = {'src': 5, 'dst': 6, 'layer': 'dropout_layer', 'next_path': [8]}\n",
    "    fsm[8] = {'src': 6, 'dst': 7, 'layer': 'output_layer', 'next_path': []}\n",
    "\n",
    "    return fsm\n",
    "\n",
    "def addFSM():\n",
    "    fsm = {}\n",
    "    fsm['convolutional_layer'] = {'before': ['maxpooling_layer'],\n",
    "                                  'add': ['convolutional_layer', 'maxpooling_layer']}\n",
    "    fsm['dense_layer'] = {'before': ['global_maxpooling_layer', 'dense_layer'],\n",
    "                          'add': ['dense_layer']}\n",
    "\n",
    "    return fsm\n",
    "\n",
    "def addConvLayer(idx, toolbox, toolboxes, defaultVal, layerparameters):\n",
    "    toolbox.register('num_filters{}'.format(str(idx)), layerparameters['num_filters'][0],\n",
    "                     layerparameters['num_filters'][1], layerparameters['num_filters'][2])\n",
    "    \n",
    "    toolboxes.append(toolbox.__getattribute__('num_filters{}'.format(str(idx))))\n",
    "    toolbox.register('kernel_size{}'.format(str(idx)), layerparameters['kernel_size'][0],\n",
    "                     layerparameters['kernel_size'][1], layerparameters['kernel_size'][2])\n",
    "    toolboxes.append(toolbox.__getattribute__('kernel_size{}'.format(str(idx))))\n",
    "    toolbox.register('conv_activation_func{}'.format(str(idx)), layerparameters['conv_activation_func'][0],\n",
    "                     layerparameters['conv_activation_func'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('conv_activation_func{}'.format(str(idx))))\n",
    "    toolbox.register('conv_init_mode{}'.format(str(idx)), layerparameters['conv_init_mode'][0],\n",
    "                     layerparameters['conv_init_mode'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('conv_init_mode{}'.format(str(idx))))\n",
    "    toolbox.register('conv_weight_constraint{}'.format(str(idx)), layerparameters['conv_weight_constraint'][0],\n",
    "                     layerparameters['conv_weight_constraint'][1], layerparameters['conv_weight_constraint'][2])\n",
    "    toolboxes.append(toolbox.__getattribute__('conv_weight_constraint{}'.format(str(idx))))\n",
    "\n",
    "    defaultVal.update({'num_filters{}'.format(str(idx)): 64})\n",
    "    defaultVal.update({'kernel_size{}'.format(str(idx)): 3})\n",
    "    defaultVal.update({'conv_activation_func{}'.format(str(idx)): \"relu\"})\n",
    "    defaultVal.update({'conv_init_mode{}'.format(str(idx)): \"glorot_uniform\"})\n",
    "    defaultVal.update({'conv_weight_constraint{}'.format(str(idx)): 3})\n",
    "\n",
    "\n",
    "def addDenseLayer(idx, toolbox, toolboxes, defaultVal, layerparameters):\n",
    "    toolbox.register('neurons{}'.format(str(idx)), layerparameters['neurons'][0],\n",
    "                     layerparameters['neurons'][1], layerparameters['neurons'][2])\n",
    "    toolboxes.append(toolbox.__getattribute__('neurons{}'.format(str(idx))))\n",
    "    toolbox.register('dense_activation_func{}'.format(str(idx)), layerparameters['dense_activation_func'][0],\n",
    "                     layerparameters['dense_activation_func'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('dense_activation_func{}'.format(str(idx))))\n",
    "    toolbox.register('dense_init_mode{}'.format(str(idx)), layerparameters['dense_init_mode'][0],\n",
    "                     layerparameters['dense_init_mode'][1])\n",
    "    toolboxes.append(toolbox.__getattribute__('dense_init_mode{}'.format(str(idx))))\n",
    "    toolbox.register('dense_weight_constraint{}'.format(str(idx)), layerparameters['dense_weight_constraint'][0],\n",
    "                     layerparameters['dense_weight_constraint'][1], layerparameters['dense_weight_constraint'][2])\n",
    "    toolboxes.append(toolbox.__getattribute__('dense_weight_constraint{}'.format(str(idx))))\n",
    "\n",
    "    defaultVal.update({'neurons{}'.format(str(idx)): 1})\n",
    "    defaultVal.update({'dense_activation_func{}'.format(str(idx)): \"relu\"})\n",
    "    defaultVal.update({'dense_init_mode{}'.format(str(idx)): \"glorot_uniform\"})\n",
    "    defaultVal.update({'dense_weight_constraint{}'.format(str(idx)): 3})\n",
    "\n",
    "\n",
    "def addMaxPoolingLayer(idx, toolbox, toolboxes, defaultVal, layerparameters):\n",
    "    toolbox.register('pool_size{}'.format(str(idx)), layerparameters['pool_size'][0],\n",
    "                     layerparameters['pool_size'][1], layerparameters['pool_size'][2])\n",
    "    toolboxes.append(toolbox.__getattribute__('pool_size{}'.format(str(idx))))\n",
    "\n",
    "    defaultVal.update({'pool_size{}'.format(str(idx)): 5})\n",
    "\n",
    "\n",
    "def addDropoutLayer(idx, toolbox, toolboxes, defaultVal, layerparameters):\n",
    "    toolbox.register('dropout_rate{}'.format(str(idx)), layerparameters['dropout_rate'][0],\n",
    "                     layerparameters['dropout_rate'][1], layerparameters['dropout_rate'][2])\n",
    "    toolboxes.append(toolbox.__getattribute__('dropout_rate{}'.format(str(idx))))\n",
    "\n",
    "    defaultVal.update({'dropout_rate{}'.format(str(idx)): 0.2})\n",
    "\n",
    "def getLayerSize(layer, conv_idx, dense_idx, dropout_idx, maxpooling_idx):\n",
    "    if layer == 'convolutional_layer':\n",
    "        conv_idx += 1\n",
    "    elif layer == 'dense_layer':\n",
    "        dense_idx += 1\n",
    "    elif layer == 'dropout_layer':\n",
    "        dropout_idx += 1\n",
    "    elif layer == 'maxpooling_layer':\n",
    "        maxpooling_idx += 1\n",
    "    return conv_idx, dense_idx, dropout_idx, maxpooling_idx\n",
    "\n",
    "\n",
    "def getMaxLayerSize(conv_idx, dense_idx, dropout_idx, maxpooling_idx, max_conv_idx, max_dense_idx, max_dropout_idx,\n",
    "                    max_maxpooling_idx):\n",
    "    if conv_idx > max_conv_idx:\n",
    "        max_conv_idx = conv_idx\n",
    "    if dense_idx > max_dense_idx:\n",
    "        max_dense_idx = dense_idx\n",
    "    if dropout_idx > max_dropout_idx:\n",
    "        max_dropout_idx = dropout_idx\n",
    "    if maxpooling_idx > max_maxpooling_idx:\n",
    "        max_maxpooling_idx = maxpooling_idx\n",
    "\n",
    "    return max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx\n",
    "\n",
    "\n",
    "def addLayerToolboxes(max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx, toolbox, toolboxes, defaultVal,\n",
    "                      layerparameters):\n",
    "    idx = 0\n",
    "    while idx < max_conv_idx:\n",
    "        idx += 1\n",
    "        addConvLayer(idx, toolbox, toolboxes, defaultVal, layerparameters)\n",
    "\n",
    "    idx = 0\n",
    "    while idx < max_dense_idx:\n",
    "        idx += 1\n",
    "        addDenseLayer(idx, toolbox, toolboxes, defaultVal, layerparameters)\n",
    "\n",
    "    idx = 0\n",
    "    while idx < max_maxpooling_idx:\n",
    "        idx += 1\n",
    "        addMaxPoolingLayer(idx, toolbox, toolboxes, defaultVal, layerparameters)\n",
    "\n",
    "    idx = 0\n",
    "    while idx < max_dropout_idx:\n",
    "        idx += 1\n",
    "        addDropoutLayer(idx, toolbox, toolboxes, defaultVal, layerparameters)\n",
    "\n",
    "\n",
    "def generateFSM(n_pop, layerparameters, toolbox, toolboxes, defaultVal):\n",
    "    fsm = FSM()\n",
    "\n",
    "    path_ind = {}\n",
    "    max_conv_idx = 0\n",
    "    max_dense_idx = 0\n",
    "    max_dropout_idx = 0\n",
    "    max_maxpooling_idx = 0\n",
    "\n",
    "    for ind in range(0, n_pop):\n",
    "        idx = conv_idx = dense_idx = dropout_idx = maxpooling_idx = 0\n",
    "        path = [fsm[idx]['layer']]\n",
    "        while len(fsm[idx]['next_path']) != 0:\n",
    "            idx = random.choice(fsm[idx]['next_path'])\n",
    "            layer = fsm[idx]['layer']\n",
    "            path.append(layer)\n",
    "            conv_idx, dense_idx, dropout_idx, maxpooling_idx = getLayerSize(layer, conv_idx, dense_idx, dropout_idx,\n",
    "                                                                            maxpooling_idx)\n",
    "\n",
    "        max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx = getMaxLayerSize(conv_idx, dense_idx,\n",
    "                                                                                           dropout_idx, maxpooling_idx,\n",
    "                                                                                           max_conv_idx, max_dense_idx,\n",
    "                                                                                           max_dropout_idx,\n",
    "                                                                                           max_maxpooling_idx)\n",
    "\n",
    "        path_ind[ind] = path\n",
    "\n",
    "    addLayerToolboxes(max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx, toolbox, toolboxes, defaultVal,\n",
    "                      layerparameters)\n",
    "\n",
    "    return path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx\n",
    "\n",
    "\n",
    "def openFSM(df, layerparameters, toolbox, toolboxes, defaultVal):\n",
    "    path_ind = {}\n",
    "    fitnesses = []\n",
    "\n",
    "    hyperparams = [s for s in list(df.columns) if not 'Unnamed' in s]\n",
    "\n",
    "    max_conv_idx = sum('num_filters' in s for s in hyperparams)\n",
    "    max_dense_idx = sum('neurons' in s for s in hyperparams)\n",
    "    max_dropout_idx = sum('dropout_rate' in s for s in hyperparams)\n",
    "    max_maxpooling_idx = sum('pool_size' in s for s in hyperparams)\n",
    "\n",
    "    for index, row in df.iterrows():\n",
    "        path = [s for s in row if 'layer' in str(s)]\n",
    "        fitness = [s for s in row if str(s).replace('.', '', 1).isdigit()]\n",
    "        fitnesses.append(tuple([float(fitness[0])]))\n",
    "        path_ind[index] = path\n",
    "\n",
    "    addLayerToolboxes(max_conv_idx, max_dense_idx, max_dropout_idx, max_maxpooling_idx, toolbox, toolboxes, defaultVal,\n",
    "                      layerparameters)\n",
    "\n",
    "    return path_ind, fitnesses, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-QjUHS9ZWs7U"
   },
   "source": [
    "# CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7DObu8fBWr48"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "class CNN:\n",
    "\n",
    "    def cnn_model(self, vocab_size, maxlen, embedding_matrix, indiv, path):\n",
    "        model = tf.keras.models.Sequential()\n",
    "        conv_idx = dense_idx = dropout_idx = maxpooling_idx = 0\n",
    "        for layer in path:\n",
    "            if layer == 'embedding_layer':\n",
    "                model.add(\n",
    "                    tf.keras.layers.Embedding(input_dim=vocab_size, output_dim=indiv['output_dim'],\n",
    "                                     weights=[embedding_matrix], input_length=maxlen, trainable=True))\n",
    "            elif layer == 'convolutional_layer':\n",
    "                conv_idx += 1\n",
    "                model.add(tf.keras.layers.Conv1D(indiv['num_filters{}'.format(str(conv_idx))], \n",
    "                                                 indiv['kernel_size{}'.format(str(conv_idx))],\n",
    "                                        kernel_initializer=indiv['conv_init_mode{}'.format(str(conv_idx))],\n",
    "                                        activation=indiv['conv_activation_func{}'.format(str(conv_idx))],\n",
    "                                        kernel_constraint=tf.keras.constraints.max_norm(indiv['conv_weight_constraint{}'.format(str(conv_idx))]),\n",
    "                                        data_format='channels_first'))\n",
    "            elif layer == 'dense_layer':\n",
    "                dense_idx += 1\n",
    "                model.add(tf.keras.layers.Dense(indiv['neurons{}'.format(str(dense_idx))],\n",
    "                                       kernel_initializer=indiv['dense_init_mode{}'.format(str(dense_idx))],\n",
    "                                       activation=indiv['dense_activation_func{}'.format(str(dense_idx))],\n",
    "                                       kernel_constraint=tf.keras.constraints.max_norm(indiv['dense_weight_constraint{}'.format(str(dense_idx))])))\n",
    "            elif layer == 'dropout_layer':\n",
    "                dropout_idx += 1\n",
    "                model.add(tf.keras.layers.Dropout(indiv['dropout_rate{}'.format(str(dropout_idx))]))\n",
    "            elif layer == 'maxpooling_layer':\n",
    "                maxpooling_idx += 1\n",
    "                model.add(tf.keras.layers.MaxPooling1D(indiv['pool_size{}'.format(str(maxpooling_idx))]))\n",
    "            elif layer == 'global_maxpooling_layer':\n",
    "                model.add(tf.keras.layers.GlobalMaxPooling1D())\n",
    "            elif layer == 'output_layer':\n",
    "                model.add(tf.keras.layers.Dense(1, kernel_initializer=indiv['output_init_mode'], activation='sigmoid'))\n",
    "\n",
    "        if indiv['optimizer'] == 'sgd':\n",
    "            opt = tf.keras.optimizers.SGD(lr=indiv['learning_rate'], momentum=indiv['momentum'], decay=0.0,\n",
    "                                 nesterov=False)\n",
    "        elif indiv['optimizer'] == 'rmsprop':\n",
    "            opt = tf.keras.optimizers.RMSprop(lr=indiv['learning_rate'], rho=0.9, epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adagrad':\n",
    "            opt = tf.keras.optimizers.Adagrad(lr=indiv['learning_rate'], epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adadelta':\n",
    "            opt = tf.keras.optimizers.Adadelta(lr=indiv['learning_rate'], rho=0.95, epsilon=None, decay=0.0)\n",
    "        elif indiv['optimizer'] == 'adam':\n",
    "            opt = tf.keras.optimizers.Adam(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                  decay=0.0, amsgrad=False)\n",
    "        elif indiv['optimizer'] == 'adamax':\n",
    "            opt = tf.keras.optimizers.Adamax(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                    decay=0.0)\n",
    "        elif indiv['optimizer'] == 'nadam':\n",
    "            opt = tf.keras.optimizers.Nadam(lr=indiv['learning_rate'], beta_1=0.9, beta_2=0.999, epsilon=None,\n",
    "                                   schedule_decay=0.004)\n",
    "        \n",
    "        util = utility()\n",
    "        model.compile(optimizer=opt, loss='binary_crossentropy', metrics=[util.f1_score])\n",
    "\n",
    "        return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jsfyjDRoWVMU"
   },
   "source": [
    "# Fitness Calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hn9qqGyhWS8x"
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import os\n",
    "from time import sleep\n",
    "import gc\n",
    "\n",
    "util = utility()\n",
    "cnn = CNN()\n",
    "\n",
    "def FitnessCalculation(individual, cfold, defaultVal, resultsPath, testing_name):\n",
    "    indiv = collections.OrderedDict()\n",
    "    i = 0\n",
    "    for key in defaultVal.keys():\n",
    "        indiv[key] = individual[i]\n",
    "        i += 1\n",
    "\n",
    "    path = individual[len(defaultVal):len(individual)]\n",
    "    \n",
    "    return crossfold(indiv, path, cfold, resultsPath, testing_name)\n",
    "\n",
    "\n",
    "def crossfold(indiv, path, fold, resultsPath, testing_name):\n",
    "    if indiv['output_dim'] == 50:\n",
    "        embedding_mtx = fold['embedding_matrix'][0]\n",
    "    elif indiv['output_dim'] == 100:\n",
    "        embedding_mtx = fold['embedding_matrix'][1]\n",
    "    elif indiv['output_dim'] == 200:\n",
    "        embedding_mtx = fold['embedding_matrix'][2]\n",
    "    elif indiv['output_dim'] == 300:\n",
    "        embedding_mtx = fold['embedding_matrix'][3]\n",
    "\n",
    "    model = cnn.cnn_model(fold['vocab_size'], fold['maxlen'], embedding_mtx,\n",
    "                          indiv, path)\n",
    "    \n",
    "    #early stopping\n",
    "    #save the best model\n",
    "    callbacks = [tf.keras.callbacks.EarlyStopping(monitor='val_f1_score', mode='max', verbose=False, patience=10), \n",
    "                 tf.keras.callbacks.ModelCheckpoint(\"{}{}.h5\".format(resultsPath, testing_name), monitor='val_f1_score', mode='max', verbose=False, \n",
    "                                  save_best_only=True)]\n",
    "\n",
    "    model.fit(fold['X_train'], fold['y_train'], epochs=indiv['epochs'], verbose=False, \n",
    "              validation_data=(fold['X_val'], fold['y_val']), use_multiprocessing=False,\n",
    "              batch_size=indiv['batch_size'], callbacks=callbacks)\n",
    "    \n",
    "    dependencies = {\n",
    "    'f1_score': util.f1_score\n",
    "    }\n",
    "\n",
    "    # load the saved model\n",
    "    for x in range(0, 10):  # try 4 times\n",
    "        try:\n",
    "            # msg.send()\n",
    "            saved_model = tf.keras.models.load_model(\"{}{}.h5\".format(resultsPath, testing_name), custom_objects=dependencies)\n",
    "            str_error = None\n",
    "        except Exception as e:\n",
    "            print('An error occurs when loading saved model.')\n",
    "            str_error = e\n",
    "            pass\n",
    "\n",
    "        if str_error:\n",
    "            sleep(5)  # wait for 2 seconds before trying to fetch the data again\n",
    "        else:\n",
    "            break\n",
    "    \n",
    "\n",
    "    y_pred = saved_model.predict_classes(fold['X_val'])\n",
    "\n",
    "    os.remove(\"{}{}.h5\".format(resultsPath, testing_name))\n",
    "\n",
    "    # CNN metrics\n",
    "    accuracyScore, precisionScore, recallScore, f1Score = util.get_testing_metric(fold['y_val'], y_pred)\n",
    "    \n",
    "    del embedding_mtx, indiv, path, fold, resultsPath, testing_name, model, callbacks, saved_model, accuracyScore, precisionScore, recallScore\n",
    "    gc.collect()\n",
    "    \n",
    "    return f1Score\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EJ38f_Y7WJSv"
   },
   "source": [
    "# Genetic Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2mE6C_3bWHcU"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "from operator import attrgetter\n",
    "from deap import base\n",
    "from deap import creator\n",
    "from deap import tools\n",
    "import time\n",
    "import datetime\n",
    "import math\n",
    "from scipy.spatial import distance\n",
    "import itertools\n",
    "\n",
    "\n",
    "class GeneticAlgorithm:\n",
    "    __slots__ = (\n",
    "        \"toolbox\", \"toolboxes\", \"cross_rate\", \"mut_rate\", \"n_pop\", \"n_gen\", \"resultsPath\", \"testing_name\", \"cfold\",\n",
    "        \"globalparameters\", \"layerparameters\", \"defaultVal\", \"path_ind\", \"max_conv_idx\", \"max_maxpooling_idx\",\n",
    "        \"max_dense_idx\", \"max_dropout_idx\")\n",
    "\n",
    "    def __init__(self, toolbox, toolboxes, cross_rate, mut_rate, n_pop, n_gen, resultsPath, testing_name,\n",
    "                 cfold, globalparameters, layerparameters, defaultVal, path_ind, max_conv_idx, max_maxpooling_idx,\n",
    "                 max_dense_idx, max_dropout_idx):\n",
    "        self.toolbox = toolbox\n",
    "        self.toolboxes = toolboxes\n",
    "        self.cross_rate = cross_rate\n",
    "        self.mut_rate = mut_rate\n",
    "        self.n_pop = n_pop\n",
    "        self.n_gen = n_gen\n",
    "        self.resultsPath = resultsPath\n",
    "        self.testing_name = testing_name\n",
    "        self.cfold = cfold\n",
    "        self.globalparameters = globalparameters\n",
    "        self.layerparameters = layerparameters\n",
    "        self.defaultVal = defaultVal\n",
    "        self.path_ind = path_ind\n",
    "        self.max_conv_idx = max_conv_idx\n",
    "        self.max_maxpooling_idx = max_maxpooling_idx\n",
    "        self.max_dense_idx = max_dense_idx\n",
    "        self.max_dropout_idx = max_dropout_idx\n",
    "\n",
    "    def fitnessCalc(self, individual):\n",
    "        i = 0\n",
    "        if len(individual.fitness.values) == 0:\n",
    "            if (0 in individual or '' in individual or 'False' in individual or None in individual):\n",
    "                for param in self.defaultVal:\n",
    "                    if individual[i] == 0 or individual[i] == '' or individual[i] == 'False' or individual[i] == None:\n",
    "                        individual[i] = self.defaultVal[param]\n",
    "                    i += 1\n",
    "\n",
    "            fc = FitnessCalculation(individual, self.cfold, self.defaultVal, self.resultsPath, self.testing_name)\n",
    "        else:\n",
    "            fc = individual.fitness.values[0]\n",
    "        print('{} {}'.format(datetime.datetime.now(), fc))\n",
    "        return fc,\n",
    "\n",
    "    def write_result(self):\n",
    "        # Create Testing Results\n",
    "        f = open(\"{}{}.csv\".format(self.resultsPath, self.testing_name), \"a+\")\n",
    "        text = \"i,min,max,mean,std,avgdistance,time,CR,MR\"\n",
    "        for param in self.defaultVal:\n",
    "            text = \"{},{}\".format(text, param)\n",
    "        text = \"{}\\n\".format(text)\n",
    "        f.write(text)\n",
    "        f.close()\n",
    "\n",
    "        # Create Last Population file\n",
    "        f = open(\"{}{}lastpop.csv\".format(self.resultsPath, self.testing_name), 'a+')\n",
    "        text = \"i,f1score\"\n",
    "        for param in self.defaultVal:\n",
    "            text = \"{},{}\".format(text, param)\n",
    "        text = \"{}\\n\".format(text)\n",
    "        f.write(text)\n",
    "        f.close()\n",
    "\n",
    "    def std_calc(self, fits, length):\n",
    "        mean = sum(fits) / length\n",
    "        sum2 = sum(x * x for x in fits)\n",
    "        std = abs(sum2 / length - mean ** 2) ** 0.5\n",
    "\n",
    "        return mean, std\n",
    "    \n",
    "    def distance_calc(self, pop):\n",
    "        distances = []\n",
    "        for subset in itertools.combinations(pop, 2):\n",
    "            distances.append(distance.hamming(subset[0][0:subset[0].index('embedding_layer')],\n",
    "                                              subset[1][0:subset[1].index('embedding_layer')]))\n",
    "\n",
    "        avgDistance = sum(distances) / len(distances)\n",
    "        \n",
    "        return avgDistance\n",
    "\n",
    "    def invalid_fitness_calc(self, pop):\n",
    "        # Evaluate the individuals with an invalid fitness\n",
    "        invalid_ind = [ind for ind in pop if not ind.fitness.valid]\n",
    "        fitnesses = map(self.toolbox.evaluate, invalid_ind)\n",
    "        for ind, fit in zip(invalid_ind, fitnesses):\n",
    "            ind.fitness.values = fit\n",
    "\n",
    "    def mutHyperparam(self, individual, indpb):\n",
    "        toolboxesSize = len(self.toolboxes)\n",
    "        fsm = FSM()\n",
    "        \n",
    "        # Mutation for the Hyperparameter Chromosomes\n",
    "        for i in range(toolboxesSize):\n",
    "            if random.random() < indpb:\n",
    "                if len(self.toolboxes[i].args) == 1:\n",
    "                    individual[i] = self.toolboxes[i].func(self.toolboxes[i].args[0])\n",
    "                else:\n",
    "                    individual[i] = self.toolboxes[i].func(self.toolboxes[i].args[0], self.toolboxes[i].args[1])\n",
    "\n",
    "        # Mutation for the Architecture Chromosomes\n",
    "        archChrom = individual[individual.index('convolutional_layer'):individual.index('output_layer')]\n",
    "        size = len(archChrom)\n",
    "\n",
    "        for i in range(1, size):\n",
    "            if random.random() < indpb:\n",
    "                if (i>=size):\n",
    "                    break\n",
    "                                \n",
    "                if (archChrom[i] == 'global_maxpooling_layer' or archChrom[i] == 'maxpooling_layer' or archChrom[i] == 'dropout_layer'):\n",
    "                    continue\n",
    "\n",
    "                selectMutType = random.randint(0, 1)\n",
    "                # Remove the layer\n",
    "                if selectMutType == 0:\n",
    "#                     print('individual before remove', individual)\n",
    "                        if (archChrom[i] == 'convolutional_layer') and (archChrom[i+1] == 'maxpooling_layer'):\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            size -= 2\n",
    "                        elif (archChrom[i] == 'dense_layer'):\n",
    "                            archChrom.remove(archChrom[i])\n",
    "                            size -= 1\n",
    "\n",
    "                # Add a layer\n",
    "                elif selectMutType == 1:\n",
    "#                     print('individual before add', individual)\n",
    "                    if (archChrom[i] == 'convolutional_layer' and \n",
    "                        individual.count('convolutional_layer') < self.max_conv_idx):                    \n",
    "                        archChrom.insert(i, 'convolutional_layer')\n",
    "                        archChrom.insert(i+1, 'maxpooling_layer')\n",
    "                    elif (archChrom[i] == 'dense_layer' and \n",
    "                        individual.count('dense_layer') < self.max_dense_idx):\n",
    "                        archChrom.insert(i, 'dense_layer')\n",
    "\n",
    "                individual[individual.index('convolutional_layer'):individual.index('output_layer')] = archChrom\n",
    "#                 print('individual after', individual)\n",
    "        return individual,\n",
    "\n",
    "    def cxTwoPoint(self, ind1, ind2, pop, offspring):\n",
    "        # Crossover for hyperparameter chromosomes\n",
    "        size = ind1.index('embedding_layer')\n",
    "        selectCxType = random.randint(0, 2)\n",
    "        # One point crossover\n",
    "        if selectCxType == 0:\n",
    "#             print('ind1 before one-point crossover:', ind1)\n",
    "#             print('ind2 before one-point crossover:', ind2)\n",
    "            cxpoint = random.randint(1, size - 1)\n",
    "            ind1[cxpoint:], ind2[cxpoint:] = ind2[cxpoint:], ind1[cxpoint:]\n",
    "#             print('ind1 after one-point crossover:', ind1)\n",
    "#             print('ind2 after one-point crossover:', ind2)\n",
    "        # Two-point crossover\n",
    "        elif selectCxType == 1:\n",
    "#             print('ind1 before two-point crossover:', ind1)\n",
    "#             print('ind2 before two-point crossover:', ind2)\n",
    "            cxpoint1 = random.randint(1, size - 1)\n",
    "            cxpoint2 = random.randint(1, size - 1)\n",
    "            if cxpoint2 >= cxpoint1:\n",
    "                cxpoint2 += 1\n",
    "            else:  # Swap the two cx points\n",
    "                cxpoint1, cxpoint2 = cxpoint2, cxpoint1\n",
    "\n",
    "            ind1[cxpoint1:cxpoint2], ind2[cxpoint1:cxpoint2] \\\n",
    "                = ind2[cxpoint1:cxpoint2], ind1[cxpoint1:cxpoint2]\n",
    "#             print('ind1 after two-point crossover:', ind1)\n",
    "#             print('ind2 after two-point crossover:', ind2)\n",
    "        # Uniform crossover\n",
    "        elif selectCxType == 2:\n",
    "#             print('ind1 before uniform crossover:', ind1)\n",
    "#             print('ind2 before uniform crossover:', ind2)\n",
    "            for i in range(size):\n",
    "                if random.random() < self.cross_rate:\n",
    "                    ind1[i], ind2[i] = ind2[i], ind1[i]\n",
    "#             print('ind1 after uniform crossover:', ind1)\n",
    "#             print('ind2 after uniform crossover:', ind2)\n",
    "\n",
    "        # Crossover for architecture chromosomes\n",
    "        # One-cut point crossover from the Global MaxPooling layer\n",
    "        cxpoint1 = ind1.index('global_maxpooling_layer')\n",
    "        cxpoint2 = ind2.index('global_maxpooling_layer')\n",
    "        ind1[cxpoint1:], ind2[cxpoint2:] = ind2[cxpoint2:], ind1[cxpoint1:]\n",
    "\n",
    "        max_drop_layer = max(ind1.count('dropout_layer'), ind2.count('dropout_layer'))\n",
    "        if max_drop_layer > self.max_dropout_idx:\n",
    "            idx = self.max_dropout_idx\n",
    "            self.max_dropout_idx = max_drop_layer\n",
    "\n",
    "            while idx < self.max_dropout_idx:\n",
    "                idx += 1\n",
    "                addDropoutLayer(idx, self.toolbox, self.toolboxes, self.defaultVal, self.layerparameters)\n",
    "\n",
    "                for ind in pop + offspring:\n",
    "                    ind.insert(size, random.uniform(0, 1))\n",
    "                size += 1\n",
    "\n",
    "                self.write_result()\n",
    "\n",
    "        return ind1, ind2\n",
    "\n",
    "    def runGA(self, lastPop=[], lastFitnesses=[]):\n",
    "        creator.create(\"FitnessMax\", base.Fitness, weights=(1.0,))\n",
    "        creator.create(\"Individual\", list, fitness=creator.FitnessMax)\n",
    "\n",
    "        self.toolbox.register(\"individual\", tools.initCycle, creator.Individual,\n",
    "                              self.toolboxes, n=1)\n",
    "        self.toolbox.register(\"population\", tools.initRepeat, list, self.toolbox.individual)\n",
    "        self.toolbox.register(\"evaluate\", self.fitnessCalc)\n",
    "        self.toolbox.register(\"mate\", self.cxTwoPoint)\n",
    "        self.toolbox.register(\"mutate\", self.mutHyperparam, indpb=self.mut_rate)\n",
    "        self.toolbox.register(\"select\", tools.selBest)\n",
    "\n",
    "        pop = self.toolbox.population(n=self.n_pop)\n",
    "\n",
    "        idx = 0\n",
    "        for ind in pop:\n",
    "            if lastPop:\n",
    "                ind[:] = lastPop[idx]\n",
    "            ind.extend(self.path_ind[idx])\n",
    "            idx += 1\n",
    "        \n",
    "        if lastFitnesses:\n",
    "            # Fitnesses from previous population\n",
    "            fitnesses = lastFitnesses\n",
    "        else:\n",
    "            # Evaluate the entire population\n",
    "            fitnesses = list(map(self.toolbox.evaluate, pop))\n",
    "\n",
    "        for ind, fit in zip(pop, fitnesses):\n",
    "            ind.fitness.values = fit\n",
    "\n",
    "        self.write_result()\n",
    "        \n",
    "        \n",
    "        g = 0\n",
    "        while g < self.n_gen:\n",
    "            then = time.time()\n",
    "            g = g + 1\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"-- Generation %i --\" % g))          \n",
    "            \n",
    "            # Select the next generation individuals\n",
    "            offspring = self.toolbox.select(pop, len(pop))\n",
    "            # Clone the selected individuals\n",
    "            offspring = list(map(self.toolbox.clone, offspring))\n",
    "\n",
    "            # Apply crossover and mutation on the offspring\n",
    "            for child1, child2 in zip(offspring[::2], offspring[1::2]):\n",
    "                if random.random() < self.cross_rate:\n",
    "                    self.toolbox.mate(child1, child2, pop, offspring)\n",
    "                    del child1.fitness.values\n",
    "                    del child2.fitness.values\n",
    "\n",
    "            for mutant in offspring:\n",
    "                if random.random() < self.mut_rate:\n",
    "                    self.toolbox.mutate(mutant)\n",
    "                    del mutant.fitness.values\n",
    "\n",
    "            # Evaluate the individuals with an invalid fitness\n",
    "            self.invalid_fitness_calc(offspring)\n",
    "\n",
    "            pop[:] = self.toolbox.select(pop + offspring, self.n_pop)\n",
    "\n",
    "            # Gather all the fitnesses in one list and print the stats\n",
    "            fits = [ind.fitness.values[0] for ind in pop]\n",
    "\n",
    "            length = len(pop)\n",
    "            mean, std = self.std_calc(fits, length)\n",
    "            avgDistance = self.distance_calc(pop)\n",
    "            best = max(pop, key=attrgetter(\"fitness\"))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Min %s\" % min(fits)))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Max %s\" % max(fits)))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Avg %s\" % mean))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  Std %s\" % std))\n",
    "            print('{} {}'.format(datetime.datetime.now(), \"  AvgDistance %s\" % avgDistance))\n",
    "            print('{} {}'.format(datetime.datetime.now(), best))\n",
    "\n",
    "            now = time.time()\n",
    "            diff = now - then\n",
    "\n",
    "            # save testing data\n",
    "            f = open(\"{}{}.csv\".format(self.resultsPath, self.testing_name), 'a')\n",
    "            text = \"{0},{1},{2},{3},{4},{5},{6},{7},{8}\".format(g,min(fits), max(fits), mean, std, avgDistance, diff, self.cross_rate, self.mut_rate)\n",
    "            for param in best:\n",
    "                text = \"{},{}\".format(text, param)\n",
    "            text = \"{}\\n\".format(text)\n",
    "            f.write(text)\n",
    "            f.close()\n",
    "\n",
    "            # save last population data\n",
    "            f = open(\"{}{}lastpop.csv\".format(self.resultsPath, self.testing_name), 'a')\n",
    "            for ind in pop:\n",
    "                text = \"{0},{1}\".format(g,ind.fitness.values[0])\n",
    "                for param in ind:\n",
    "                    text = \"{},{}\".format(text, param)\n",
    "                text = \"{}\\n\".format(text)             \n",
    "                f.write(text)\n",
    "\n",
    "            f.close()\n",
    "            \n",
    "            # Create NewPop\n",
    "            f = open(\"{}{}.csv\".format(self.resultsPath, 'NewPop'), 'w+')\n",
    "            text = \"f1score,{}\\n\".format(','.join(map(str, self.defaultVal)))\n",
    "\n",
    "            for ind in pop:            \n",
    "                text = \"{}{},{}\\n\".format(text,ind.fitness.values[0],','.join(map(str,  ind))) \n",
    "            \n",
    "            f.write(text)\n",
    "            f.close()\n",
    "            \n",
    "            del offspring, length, mean, std, best, avgDistance, then, now, diff, text, fits\n",
    "            gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kUA1C18jfyE6"
   },
   "source": [
    "# Project path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "9OZ4ryogfYra",
    "outputId": "84ac2846-ae86-4784-81ab-7cd413c89af5"
   },
   "outputs": [],
   "source": [
    "training_path = 'trainPreprocessed.csv'\n",
    "population_path = 'NewPop.csv'\n",
    "root_path = '/lab/dbms/fatyanosa'\n",
    "datasetPath = '{}/Dataset/Disaster Tweets/'.format(root_path)\n",
    "resultsPath = '{}/Server1/Disaster Tweets/Paper GA-CNN/Results/'.format(root_path)\n",
    "testing_name = \"Experiment7_GA_CNN\"\n",
    "glovePath = ['{}/Glove/glove.6B.50d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.100d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.200d.txt'.format(root_path),\n",
    "             '{}/Glove/glove.6B.300d.txt'.format(root_path)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "8O1VRBE73ivB"
   },
   "source": [
    "# Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "6XqVtrSu3kq4"
   },
   "outputs": [],
   "source": [
    "# crossover rate is the probability with which two individuals\n",
    "cross_rate = 0.8\n",
    "\n",
    "# mutation rate is the probability for mutating an individual\n",
    "mut_rate = 0.2\n",
    "\n",
    "# number of population\n",
    "n_pop = 30\n",
    "\n",
    "# number of generation\n",
    "n_gen = 30"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "X6kUHesFXh17"
   },
   "source": [
    "# Main Program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "U4oAqAYAUmVT",
    "outputId": "b9c2e51a-6a4d-4eca-8b05-8397a9fc9216",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-07-23 12:58:39.789395 -- Generation 23 --\n",
      "WARNING:tensorflow:Large dropout rate: 0.630617 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.630617 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.630617 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "WARNING:tensorflow:Large dropout rate: 0.630617 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "2020-07-23 12:59:11.246738 0.7876200640341515\n",
      "WARNING:tensorflow:Large dropout rate: 0.630617 (>0.5). In TensorFlow 2.x, dropout() uses dropout rate instead of keep_prob. Please ensure that this is intended.\n",
      "2020-07-23 12:59:56.885330 0.7929936305732483\n",
      "2020-07-23 13:00:26.764633 0.7932535364526659\n",
      "2020-07-23 13:00:54.884524 0.7948854555141184\n",
      "2020-07-23 13:01:32.923430 0.7765843179377016\n",
      "2020-07-23 13:01:59.914274 0.7870900572618428\n",
      "2020-07-23 13:02:29.131359 0.7807430664573521\n",
      "2020-07-23 13:02:54.912818 0.7922998986828773\n",
      "2020-07-23 13:03:19.351179 0.7756376887038\n",
      "2020-07-23 13:03:50.108263 0.7774358974358975\n",
      "2020-07-23 13:04:32.765361 0.7793176972281449\n",
      "2020-07-23 13:05:02.170670 0.7747551686615886\n",
      "2020-07-23 13:05:40.151838 0.7943405760485094\n",
      "2020-07-23 13:06:20.605680 0.7939546599496222\n",
      "2020-07-23 13:07:04.972105 0.7866805411030178\n",
      "2020-07-23 13:07:52.846493 0.7807712625462229\n",
      "2020-07-23 13:08:16.122523 0.7791942886282509\n",
      "2020-07-23 13:08:38.954607 0.7760388559093362\n",
      "2020-07-23 13:09:03.900198 0.7807203389830508\n",
      "2020-07-23 13:09:26.369260 0.7879428873611846\n",
      "2020-07-23 13:09:47.640771 0.7834051724137931\n",
      "2020-07-23 13:10:09.167850 0.7778375470683162\n",
      "2020-07-23 13:10:30.735191 0.7804878048780488\n",
      "2020-07-23 13:12:09.437705 0.7663355985363304\n",
      "2020-07-23 13:12:53.189205 0.7875700458481916\n",
      "2020-07-23 13:13:34.110922 0.7868686868686868\n",
      "2020-07-23 13:13:52.728357 0.7843551797040169\n",
      "2020-07-23 13:13:52.751509   Min 0.7942583732057416\n",
      "2020-07-23 13:13:52.751600   Max 0.8002159827213823\n",
      "2020-07-23 13:13:52.751620   Avg 0.7950589615732278\n",
      "2020-07-23 13:13:52.751650   Std 0.0011687194115070588\n",
      "2020-07-23 13:13:52.751667   AvgDistance 0.3406735228876801\n",
      "2020-07-23 13:13:52.751682 [56, 42, 'adamax', 0.003722738523755628, 0.9360068577872628, 'glorot_normal', 200, 449, 1, 'linear', 'zeros', 5, 277, 2, 'sigmoid', 'uniform', 3, 195, 4, 'hard_sigmoid', 'lecun_uniform', 1, 326, 3, 'softplus', 'uniform', 5, 224, 5, 'relu', 'glorot_uniform', 5, 12, 'tanh', 'glorot_normal', 5, 20, 'sigmoid', 'glorot_uniform', 1, 10, 'hard_sigmoid', 'lecun_uniform', 1, 24, 'elu', 'lecun_normal', 5, 22, 'selu', 'lecun_uniform', 5, 3, 2, 6, 2, 0.6306174692005132, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-07-23 13:13:53.103090 -- Generation 24 --\n",
      "2020-07-23 13:14:36.036293 0.7841726618705036\n",
      "2020-07-23 13:15:24.409905 0.775880469583778\n",
      "2020-07-23 13:15:49.960157 0.7838676318510858\n",
      "2020-07-23 13:16:14.459601 0.7904451682953312\n",
      "2020-07-23 13:16:41.959070 0.7878128400435257\n",
      "2020-07-23 13:17:11.701050 0.7946949602122017\n",
      "2020-07-23 13:17:42.097539 0.786480686695279\n",
      "2020-07-23 13:18:37.426412 0.7808370044052864\n",
      "2020-07-23 13:19:25.625386 0.7817721518987342\n",
      "2020-07-23 13:20:13.995286 0.782422293676313\n",
      "2020-07-23 13:21:04.291579 0.7857142857142857\n",
      "2020-07-23 13:21:52.868013 0.7699629825489159\n",
      "2020-07-23 13:22:30.785118 0.7876823338735819\n",
      "2020-07-23 13:22:55.101894 0.7778332501248127\n",
      "2020-07-23 13:24:19.644912 0.7777777777777778\n",
      "2020-07-23 13:24:35.252382 0.026395939086294413\n",
      "2020-07-23 13:24:49.784508 0.0\n",
      "2020-07-23 13:25:10.424662 0.7807807807807807\n",
      "2020-07-23 13:25:33.379626 0.786505007907222\n",
      "2020-07-23 13:25:55.569532 0.7889125799573561\n",
      "2020-07-23 13:26:19.781606 0.7873154729360305\n",
      "2020-07-23 13:26:43.624780 0.7851605758582502\n",
      "2020-07-23 13:27:05.832142 0.7903225806451613\n",
      "2020-07-23 13:27:49.510708 0.7854922279792745\n",
      "2020-07-23 13:28:43.519977 0.7809722948248824\n",
      "2020-07-23 13:29:25.304415 0.7735229759299781\n",
      "2020-07-23 13:29:25.337460   Min 0.7943037974683544\n",
      "2020-07-23 13:29:25.337589   Max 0.8002159827213823\n",
      "2020-07-23 13:29:25.337617   Avg 0.7952001283586916\n",
      "2020-07-23 13:29:25.337655   Std 0.0011190233552166359\n",
      "2020-07-23 13:29:25.337681   AvgDistance 0.31812865497076076\n",
      "2020-07-23 13:29:25.337704 [56, 42, 'adamax', 0.003722738523755628, 0.9360068577872628, 'glorot_normal', 200, 449, 1, 'linear', 'zeros', 5, 277, 2, 'sigmoid', 'uniform', 3, 195, 4, 'hard_sigmoid', 'lecun_uniform', 1, 326, 3, 'softplus', 'uniform', 5, 224, 5, 'relu', 'glorot_uniform', 5, 12, 'tanh', 'glorot_normal', 5, 20, 'sigmoid', 'glorot_uniform', 1, 10, 'hard_sigmoid', 'lecun_uniform', 1, 24, 'elu', 'lecun_normal', 5, 22, 'selu', 'lecun_uniform', 5, 3, 2, 6, 2, 0.6306174692005132, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-07-23 13:29:26.027682 -- Generation 25 --\n",
      "2020-07-23 13:30:09.261907 0.7889807162534435\n",
      "2020-07-23 13:30:46.007627 0.7823936696340257\n",
      "2020-07-23 13:31:09.741232 0.7786666666666666\n",
      "2020-07-23 13:31:32.296418 0.781657113079729\n",
      "2020-07-23 13:32:02.038805 0.7839912280701754\n",
      "2020-07-23 13:32:32.551237 0.7905785970302098\n",
      "2020-07-23 13:32:59.549363 0.7795527156549521\n",
      "2020-07-23 13:33:37.020239 0.7759866592551418\n",
      "2020-07-23 13:34:07.363374 0.7819148936170213\n",
      "2020-07-23 13:34:38.568938 0.7780195865070728\n",
      "2020-07-23 13:35:07.854026 0.7866596082583378\n",
      "2020-07-23 13:35:51.683700 0.7885117493472584\n",
      "2020-07-23 13:36:43.040903 0.7843766720171214\n",
      "2020-07-23 13:37:24.400232 0.780565936999466\n",
      "2020-07-23 13:38:05.686996 0.7805676855895196\n",
      "2020-07-23 13:38:47.113720 0.7810140237324704\n",
      "2020-07-23 13:39:44.107849 0.7894179894179895\n",
      "2020-07-23 13:40:10.761215 0.7824692677712454\n",
      "2020-07-23 13:40:52.141081 0.7618059159314997\n",
      "2020-07-23 13:41:19.308603 0.7777188328912467\n",
      "2020-07-23 13:41:42.748739 0.7787610619469026\n",
      "2020-07-23 13:42:05.036258 0.7808845958312151\n",
      "2020-07-23 13:42:26.162805 0.7886710239651417\n",
      "2020-07-23 13:42:49.698647 0.7808370044052864\n",
      "2020-07-23 13:43:12.007390 0.7824709609292502\n",
      "2020-07-23 13:43:37.049932 0.7790697674418605\n",
      "2020-07-23 13:43:37.070048   Min 0.7943037974683544\n",
      "2020-07-23 13:43:37.070126   Max 0.8002159827213823\n",
      "2020-07-23 13:43:37.070140   Avg 0.7953237139597655\n",
      "2020-07-23 13:43:37.070165   Std 0.0010629878606038027\n",
      "2020-07-23 13:43:37.070356   AvgDistance 0.2611816898568258\n",
      "2020-07-23 13:43:37.070373 [56, 42, 'adamax', 0.003722738523755628, 0.9360068577872628, 'glorot_normal', 200, 449, 1, 'linear', 'zeros', 5, 277, 2, 'sigmoid', 'uniform', 3, 195, 4, 'hard_sigmoid', 'lecun_uniform', 1, 326, 3, 'softplus', 'uniform', 5, 224, 5, 'relu', 'glorot_uniform', 5, 12, 'tanh', 'glorot_normal', 5, 20, 'sigmoid', 'glorot_uniform', 1, 10, 'hard_sigmoid', 'lecun_uniform', 1, 24, 'elu', 'lecun_normal', 5, 22, 'selu', 'lecun_uniform', 5, 3, 2, 6, 2, 0.6306174692005132, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-07-23 13:43:38.057949 -- Generation 26 --\n",
      "2020-07-23 13:44:23.975767 0.7909238249594813\n",
      "2020-07-23 13:44:46.912531 0.787222523010287\n",
      "2020-07-23 13:45:12.066938 0.7882599580712787\n",
      "2020-07-23 13:45:37.493398 0.7874865156418555\n",
      "2020-07-23 13:46:06.468223 0.7863811092806149\n",
      "2020-07-23 13:46:35.484174 0.7804621848739497\n",
      "2020-07-23 13:47:07.030116 0.7920043219881145\n",
      "2020-07-23 13:47:36.453454 0.7846575342465753\n",
      "2020-07-23 13:47:48.579831 0.5944615384615385\n",
      "2020-07-23 13:48:06.930703 0.0\n",
      "2020-07-23 13:48:37.550623 0.783695652173913\n",
      "2020-07-23 13:49:08.335309 0.7834602829162133\n",
      "2020-07-23 13:49:34.930973 0.7870900572618428\n",
      "2020-07-23 13:50:07.966763 0.7773664727657323\n",
      "2020-07-23 13:50:37.600654 0.7901878914405011\n",
      "2020-07-23 13:51:05.383289 0.7856041131105399\n",
      "2020-07-23 13:51:35.238370 0.79033105622701\n",
      "2020-07-23 13:52:02.685454 0.7816216216216215\n",
      "2020-07-23 13:52:32.614265 0.7878787878787877\n",
      "2020-07-23 13:53:19.332881 0.7885888945491594\n",
      "2020-07-23 13:53:59.247109 0.7858672376873662\n",
      "2020-07-23 13:54:52.477804 0.7801566579634466\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-07-23 13:55:23.100639 0.7914209115281501\n",
      "2020-07-23 13:56:08.811016 0.7899786780383794\n",
      "2020-07-23 13:56:32.134266 0.7895277207392197\n",
      "2020-07-23 13:56:53.808534 0.7820927723840345\n",
      "2020-07-23 13:56:53.827207   Min 0.7946949602122017\n",
      "2020-07-23 13:56:53.827263   Max 0.8002159827213823\n",
      "2020-07-23 13:56:53.827279   Avg 0.7953871123736271\n",
      "2020-07-23 13:56:53.827293   Std 0.0010205748234780542\n",
      "2020-07-23 13:56:53.827320   AvgDistance 0.23359548296027413\n",
      "2020-07-23 13:56:53.827336 [56, 42, 'adamax', 0.003722738523755628, 0.9360068577872628, 'glorot_normal', 200, 449, 1, 'linear', 'zeros', 5, 277, 2, 'sigmoid', 'uniform', 3, 195, 4, 'hard_sigmoid', 'lecun_uniform', 1, 326, 3, 'softplus', 'uniform', 5, 224, 5, 'relu', 'glorot_uniform', 5, 12, 'tanh', 'glorot_normal', 5, 20, 'sigmoid', 'glorot_uniform', 1, 10, 'hard_sigmoid', 'lecun_uniform', 1, 24, 'elu', 'lecun_normal', 5, 22, 'selu', 'lecun_uniform', 5, 3, 2, 6, 2, 0.6306174692005132, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-07-23 13:56:55.397682 -- Generation 27 --\n",
      "2020-07-23 13:57:26.278577 0.7717934480843976\n",
      "2020-07-23 13:58:20.204093 0.7837837837837838\n",
      "2020-07-23 13:58:50.100068 0.786021505376344\n",
      "2020-07-23 13:59:17.303741 0.7846735024284943\n",
      "2020-07-23 13:59:46.551287 0.779947229551451\n",
      "2020-07-23 14:00:21.739953 0.7866400797607179\n",
      "2020-07-23 14:00:52.961012 0.7885416666666667\n",
      "2020-07-23 14:01:15.240323 0.7551560021152829\n",
      "2020-07-23 14:01:43.877488 0.7879763821792807\n",
      "2020-07-23 14:02:11.100346 0.7819148936170213\n",
      "2020-07-23 14:02:41.830456 0.7827050997782704\n",
      "2020-07-23 14:03:07.944018 0.7863247863247863\n",
      "2020-07-23 14:03:35.573265 0.7893048128342245\n",
      "2020-07-23 14:04:04.449442 0.7857142857142858\n",
      "2020-07-23 14:04:35.689335 0.7759088442756376\n",
      "2020-07-23 14:05:05.806882 0.7823376623376624\n",
      "2020-07-23 14:05:37.838983 0.7727503812913066\n",
      "2020-07-23 14:06:06.464663 0.7825635703165541\n",
      "2020-07-23 14:06:33.668555 0.7825213460572575\n",
      "2020-07-23 14:07:18.869471 0.7811518324607329\n",
      "2020-07-23 14:08:04.106015 0.7814207650273224\n",
      "2020-07-23 14:08:53.524147 0.7834946510443199\n",
      "2020-07-23 14:09:54.854867 0.7840670859538784\n",
      "2020-07-23 14:10:42.810839 0.7860906217070601\n",
      "2020-07-23 14:11:28.146453 0.7849462365591398\n",
      "2020-07-23 14:12:20.366904 0.779\n",
      "2020-07-23 14:12:35.963996 0.0\n",
      "2020-07-23 14:13:03.577100 0.7839598997493733\n",
      "2020-07-23 14:13:31.491793 0.7859838274932615\n",
      "2020-07-23 14:14:00.297101 0.786206896551724\n",
      "2020-07-23 14:14:00.314760   Min 0.7946949602122017\n",
      "2020-07-23 14:14:00.314818   Max 0.8002159827213823\n",
      "2020-07-23 14:14:00.314833   Avg 0.7953871123736271\n",
      "2020-07-23 14:14:00.314844   Std 0.0010205748234780542\n",
      "2020-07-23 14:14:00.314857   AvgDistance 0.23359548296027413\n",
      "2020-07-23 14:14:00.314869 [56, 42, 'adamax', 0.003722738523755628, 0.9360068577872628, 'glorot_normal', 200, 449, 1, 'linear', 'zeros', 5, 277, 2, 'sigmoid', 'uniform', 3, 195, 4, 'hard_sigmoid', 'lecun_uniform', 1, 326, 3, 'softplus', 'uniform', 5, 224, 5, 'relu', 'glorot_uniform', 5, 12, 'tanh', 'glorot_normal', 5, 20, 'sigmoid', 'glorot_uniform', 1, 10, 'hard_sigmoid', 'lecun_uniform', 1, 24, 'elu', 'lecun_normal', 5, 22, 'selu', 'lecun_uniform', 5, 3, 2, 6, 2, 0.6306174692005132, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-07-23 14:14:02.115370 -- Generation 28 --\n",
      "2020-07-23 14:14:36.707684 0.7810650887573964\n",
      "2020-07-23 14:15:44.983721 0.7870762711864407\n",
      "2020-07-23 14:16:11.212188 0.7843137254901961\n",
      "2020-07-23 14:16:34.867118 0.7840327533265097\n",
      "2020-07-23 14:16:57.842248 0.7652513421181063\n",
      "2020-07-23 14:17:18.910950 0.0\n",
      "2020-07-23 14:17:47.594079 0.79033105622701\n",
      "2020-07-23 14:18:18.881238 0.7812335266209804\n",
      "2020-07-23 14:18:47.968050 0.7852015461071232\n",
      "2020-07-23 14:19:16.005203 0.7909238249594813\n",
      "2020-07-23 14:19:44.211954 0.7770090473656202\n",
      "2020-07-23 14:20:10.929637 0.7833511205976522\n",
      "2020-07-23 14:21:11.204373 0.7809419496166484\n",
      "2020-07-23 14:21:37.890203 0.7801792303637323\n",
      "2020-07-23 14:22:07.715619 0.7789115646258503\n",
      "2020-07-23 14:22:36.728004 0.7817204301075268\n",
      "2020-07-23 14:23:08.713734 0.7904761904761904\n",
      "2020-07-23 14:23:40.920501 0.7686684073107051\n",
      "2020-07-23 14:24:11.382216 0.784796573875803\n",
      "2020-07-23 14:24:55.022827 0.7813504823151126\n",
      "2020-07-23 14:25:08.386462 0.0\n",
      "2020-07-23 14:25:56.601111 0.7843137254901962\n",
      "2020-07-23 14:26:42.666514 0.7791509940891994\n",
      "2020-07-23 14:27:26.587925 0.7840848806366049\n",
      "2020-07-23 14:28:17.618765 0.7854077253218883\n",
      "2020-07-23 14:29:08.285866 0.7857142857142857\n",
      "2020-07-23 14:29:36.483181 0.787847579814624\n",
      "2020-07-23 14:29:54.229785 0.0\n",
      "2020-07-23 14:30:22.434272 0.7896440129449838\n",
      "2020-07-23 14:30:49.649877 0.7796257796257796\n",
      "2020-07-23 14:30:49.668342   Min 0.7946949602122017\n",
      "2020-07-23 14:30:49.668407   Max 0.8002159827213823\n",
      "2020-07-23 14:30:49.668431   Avg 0.7953871123736271\n",
      "2020-07-23 14:30:49.668452   Std 0.0010205748234780542\n",
      "2020-07-23 14:30:49.668474   AvgDistance 0.23359548296027413\n",
      "2020-07-23 14:30:49.668495 [56, 42, 'adamax', 0.003722738523755628, 0.9360068577872628, 'glorot_normal', 200, 449, 1, 'linear', 'zeros', 5, 277, 2, 'sigmoid', 'uniform', 3, 195, 4, 'hard_sigmoid', 'lecun_uniform', 1, 326, 3, 'softplus', 'uniform', 5, 224, 5, 'relu', 'glorot_uniform', 5, 12, 'tanh', 'glorot_normal', 5, 20, 'sigmoid', 'glorot_uniform', 1, 10, 'hard_sigmoid', 'lecun_uniform', 1, 24, 'elu', 'lecun_normal', 5, 22, 'selu', 'lecun_uniform', 5, 3, 2, 6, 2, 0.6306174692005132, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-07-23 14:30:52.174999 -- Generation 29 --\n",
      "2020-07-23 14:31:33.435065 0.7826541274817137\n",
      "2020-07-23 14:32:19.773289 0.7809132888660851\n",
      "2020-07-23 14:32:50.483515 0.7943336831059811\n",
      "2020-07-23 14:33:15.390106 0.7803860198226396\n",
      "2020-07-23 14:33:45.241342 0.7876288659793814\n",
      "2020-07-23 14:34:20.738265 0.7825192802056555\n",
      "2020-07-23 14:34:54.873827 0.783625730994152\n",
      "2020-07-23 14:35:28.526703 0.7811691670977755\n",
      "2020-07-23 14:36:01.417442 0.783625730994152\n",
      "2020-07-23 14:36:42.348229 0.7754475703324808\n",
      "2020-07-23 14:37:02.161380 0.7858310626702997\n",
      "2020-07-23 14:37:36.523388 0.785829307568438\n",
      "2020-07-23 14:38:06.498177 0.783178590933916\n",
      "2020-07-23 14:38:39.701671 0.7844155844155843\n",
      "2020-07-23 14:39:12.262683 0.7892183288409702\n",
      "2020-07-23 14:39:45.769075 0.7786259541984734\n",
      "2020-07-23 14:40:21.378224 0.7746913580246914\n",
      "2020-07-23 14:40:52.749263 0.7744945567651632\n",
      "2020-07-23 14:41:28.354365 0.786505007907222\n",
      "2020-07-23 14:42:23.837023 0.7822757111597374\n",
      "2020-07-23 14:42:56.003250 0.0\n",
      "2020-07-23 14:43:29.177603 0.0\n",
      "2020-07-23 14:44:02.184275 0.7865528281750267\n",
      "2020-07-23 14:44:35.204850 0.7839408346539882\n",
      "2020-07-23 14:44:35.236332   Min 0.7950775815944355\n",
      "2020-07-23 14:44:35.236470   Max 0.8002159827213823\n",
      "2020-07-23 14:44:35.236502   Avg 0.7954236314560941\n",
      "2020-07-23 14:44:35.236528   Std 0.0010029978744651004\n",
      "2020-07-23 14:44:35.236555   AvgDistance 0.2514216575922564\n",
      "2020-07-23 14:44:35.236596 [56, 42, 'adamax', 0.003722738523755628, 0.9360068577872628, 'glorot_normal', 200, 449, 1, 'linear', 'zeros', 5, 277, 2, 'sigmoid', 'uniform', 3, 195, 4, 'hard_sigmoid', 'lecun_uniform', 1, 326, 3, 'softplus', 'uniform', 5, 224, 5, 'relu', 'glorot_uniform', 5, 12, 'tanh', 'glorot_normal', 5, 20, 'sigmoid', 'glorot_uniform', 1, 10, 'hard_sigmoid', 'lecun_uniform', 1, 24, 'elu', 'lecun_normal', 5, 22, 'selu', 'lecun_uniform', 5, 3, 2, 6, 2, 0.6306174692005132, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n",
      "2020-07-23 14:44:38.054604 -- Generation 30 --\n",
      "2020-07-23 14:45:25.474051 0.7796976241900647\n",
      "2020-07-23 14:46:00.471160 0.7911158117398202\n",
      "2020-07-23 14:46:33.949635 0.788657035848047\n",
      "2020-07-23 14:47:06.558397 0.7862356621480708\n",
      "2020-07-23 14:47:32.876569 0.7808807733619764\n",
      "2020-07-23 14:48:09.000494 0.7811663991439272\n",
      "2020-07-23 14:48:47.120729 0.7839472237493128\n",
      "2020-07-23 14:49:16.689634 0.7896950578338592\n",
      "2020-07-23 14:49:46.956563 0.7849293563579278\n",
      "2020-07-23 14:50:20.672164 0.7842301545018647\n",
      "2020-07-23 14:50:54.623679 0.779989241527703\n",
      "2020-07-23 14:51:24.678600 0.7859721505930892\n",
      "2020-07-23 14:51:57.654782 0.7667193259610322\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-07-23 14:52:40.869299 0.785154208050183\n",
      "2020-07-23 14:53:13.772699 0.7852015461071232\n",
      "2020-07-23 14:53:47.713989 0.7814776274713839\n",
      "2020-07-23 14:54:18.644763 0.7780149413020278\n",
      "2020-07-23 14:55:06.703256 0.7939330543933054\n",
      "2020-07-23 14:55:59.549065 0.785751702462022\n",
      "2020-07-23 14:56:45.372690 0.7785234899328858\n",
      "2020-07-23 14:57:40.715411 0.7708225556130368\n",
      "2020-07-23 14:58:14.660233 0.0\n",
      "2020-07-23 14:59:02.857523 0.7889125799573561\n",
      "2020-07-23 15:00:00.311520 0.7751937984496124\n",
      "2020-07-23 15:00:35.345589 0.7755741127348644\n",
      "2020-07-23 15:01:20.919195 0.7794817556848228\n",
      "2020-07-23 15:01:51.909132 0.7842278203723986\n",
      "2020-07-23 15:02:44.806572 0.7886872998932765\n",
      "2020-07-23 15:02:44.825544   Min 0.7950775815944355\n",
      "2020-07-23 15:02:44.825590   Max 0.8002159827213823\n",
      "2020-07-23 15:02:44.825606   Avg 0.7954338386482258\n",
      "2020-07-23 15:02:44.825621   Std 0.0010001994582232907\n",
      "2020-07-23 15:02:44.825641   AvgDistance 0.23561201855212732\n",
      "2020-07-23 15:02:44.825665 [56, 42, 'adamax', 0.003722738523755628, 0.9360068577872628, 'glorot_normal', 200, 449, 1, 'linear', 'zeros', 5, 277, 2, 'sigmoid', 'uniform', 3, 195, 4, 'hard_sigmoid', 'lecun_uniform', 1, 326, 3, 'softplus', 'uniform', 5, 224, 5, 'relu', 'glorot_uniform', 5, 12, 'tanh', 'glorot_normal', 5, 20, 'sigmoid', 'glorot_uniform', 1, 10, 'hard_sigmoid', 'lecun_uniform', 1, 24, 'elu', 'lecun_normal', 5, 22, 'selu', 'lecun_uniform', 5, 3, 2, 6, 2, 0.6306174692005132, 'embedding_layer', 'convolutional_layer', 'global_maxpooling_layer', 'dense_layer', 'dropout_layer', 'output_layer']\n"
     ]
    }
   ],
   "source": [
    "import random\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from deap import base\n",
    "import warnings; warnings.simplefilter('ignore')\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    globalparameters = []\n",
    "    globalparameters.append((\"epochs\", random.randint, 1, 100))\n",
    "    globalparameters.append((\"batch_size\", random.randint, 32, 256))\n",
    "    globalparameters.append((\"optimizer\", random.choice, ['sgd', 'rmsprop', \n",
    "                                                          'adagrad', 'adadelta',\n",
    "                                                          'adam', 'adamax', \n",
    "                                                          'nadam']))\n",
    "    globalparameters.append((\"learning_rate\", random.uniform, 1e-4, 1e-2))\n",
    "    globalparameters.append((\"momentum\", random.uniform, 0, 1))\n",
    "    globalparameters.append((\"output_init_mode\", random.choice,\n",
    "                             ['zeros',\n",
    "                              'ones',\n",
    "                              'uniform',\n",
    "                              'normal',\n",
    "                              'glorot_normal',\n",
    "                              'glorot_uniform',\n",
    "                              'he_normal',\n",
    "                              'he_uniform',\n",
    "                              'lecun_normal',\n",
    "                              'lecun_uniform']))\n",
    "    globalparameters.append((\"output_dim\", random.choice, [50, 100, 200, 300]))\n",
    "\n",
    "    layerparameters = {}\n",
    "    layerparameters[\"num_filters\"] = [random.randint, 32, 512]\n",
    "    layerparameters[\"kernel_size\"] = [random.randint, 1, 5]\n",
    "    layerparameters[\"conv_activation_func\"] = [random.choice,\n",
    "                                               ['relu', 'softmax', 'elu', 'selu',\n",
    "                                                'softplus', 'softsign', 'tanh',\n",
    "                                                'sigmoid', 'hard_sigmoid', 'linear']]\n",
    "    layerparameters[\"conv_init_mode\"] = [random.choice,\n",
    "                                         ['zeros',\n",
    "                                          'ones',\n",
    "                                          'uniform',\n",
    "                                          'normal',\n",
    "                                          'glorot_normal',\n",
    "                                          'glorot_uniform',\n",
    "                                          'he_normal',\n",
    "                                          'he_uniform',\n",
    "                                          'lecun_normal',\n",
    "                                          'lecun_uniform']]\n",
    "    layerparameters[\"conv_weight_constraint\"] = [random.randint, 1, 5]\n",
    "    layerparameters[\"neurons\"] = [random.randint, 1, 30]\n",
    "    layerparameters[\"dense_activation_func\"] = [random.choice,\n",
    "                                                ['relu', 'softmax', 'elu', 'selu',\n",
    "                                                 'softplus', 'softsign', 'tanh',\n",
    "                                                 'sigmoid', 'hard_sigmoid', 'linear']]\n",
    "    layerparameters[\"dense_init_mode\"] = [random.choice,\n",
    "                                          ['zeros',\n",
    "                                           'ones',\n",
    "                                           'uniform',\n",
    "                                           'normal',\n",
    "                                           'glorot_normal',\n",
    "                                           'glorot_uniform',\n",
    "                                           'he_normal',\n",
    "                                           'he_uniform',\n",
    "                                           'lecun_normal',\n",
    "                                           'lecun_uniform']]\n",
    "    layerparameters[\"dense_weight_constraint\"] = [random.randint, 1, 5]\n",
    "    layerparameters[\"pool_size\"] = [random.randint, 2, 6]\n",
    "    layerparameters[\"dropout_rate\"] = [random.uniform, 0, 1]\n",
    "\n",
    "    defaultVal = collections.OrderedDict([\n",
    "        (\"epochs\", 10),\n",
    "        (\"batch_size\", 32),\n",
    "        (\"optimizer\", \"adam\"),\n",
    "        (\"learning_rate\", 1e-4),\n",
    "        (\"momentum\", 0.9),\n",
    "        (\"output_init_mode\", \"glorot_uniform\"),\n",
    "        (\"output_dim\", 100)]\n",
    "    )\n",
    "    \n",
    "    # object class\n",
    "    util = utility()\n",
    "    toolbox = base.Toolbox()\n",
    "    toolboxes = []\n",
    "\n",
    "    # Attribute generator\n",
    "    for hyper in globalparameters:\n",
    "        if len(hyper) == 3:\n",
    "            toolbox.register(hyper[0], hyper[1], hyper[2])\n",
    "        else:\n",
    "            toolbox.register(hyper[0], hyper[1], hyper[2], hyper[3])\n",
    "\n",
    "    toolboxes.append(toolbox.epochs)\n",
    "    toolboxes.append(toolbox.batch_size)\n",
    "    toolboxes.append(toolbox.optimizer)\n",
    "    toolboxes.append(toolbox.learning_rate)\n",
    "    toolboxes.append(toolbox.momentum)\n",
    "    toolboxes.append(toolbox.output_init_mode)\n",
    "    toolboxes.append(toolbox.output_dim)\n",
    "\n",
    "    path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx = generateFSM(n_pop, layerparameters,\n",
    "                                                                                             toolbox, toolboxes,\n",
    "                                                                                             defaultVal)\n",
    "\n",
    "#     # Read population data\n",
    "#     dfPopulation = util.read_CSV(\"{}{}\".format(resultsPath, population_path))\n",
    "\n",
    "#     path_ind, fitnesses, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx = openFSM(dfPopulation, layerparameters,\n",
    "#                                                                                          toolbox, toolboxes, defaultVal)\n",
    "#     dfPopulation = dfPopulation.drop(columns=[col for col in dfPopulation if col not in defaultVal])\n",
    "\n",
    "#     population = dfPopulation.loc[:, ~dfPopulation.columns.str.match('Unnamed')].values.tolist()\n",
    "\n",
    "    # Read data\n",
    "    df = util.read_CSV(\"{}{}\".format(datasetPath, training_path))\n",
    "\n",
    "    from sklearn.model_selection import train_test_split\n",
    "    dfTraining, dfTrial = train_test_split(df, test_size = 0.3, random_state=42)\n",
    "\n",
    "    textsTraining, labelsTraining = util.get_text_label(dfTraining)\n",
    "    textsTrial, labelsTrial = util.get_text_label(dfTrial)\n",
    "    cfold = {}\n",
    "\n",
    "    X_train, X_val, y_train, y_val, vocab_size, maxlen, embedding_matrix = util.get_training_trial_data(\n",
    "        textsTraining, labelsTraining, textsTrial, labelsTrial, glovePath)\n",
    "    cfold= {'X_train': X_train, 'X_val': X_val, 'y_train': y_train, 'y_val': y_val, 'vocab_size': vocab_size,\n",
    "                  'maxlen': maxlen, 'embedding_matrix': embedding_matrix}\n",
    "                  \n",
    "    ga = GeneticAlgorithm(toolbox, toolboxes, cross_rate, mut_rate, n_pop, n_gen, resultsPath, testing_name,\n",
    "                          cfold, globalparameters, layerparameters, defaultVal, path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx)\n",
    "    \n",
    "    del df, dfTraining, dfTrial, textsTraining, labelsTraining, textsTrial, labelsTrial, X_train, X_val, y_train, y_val, vocab_size, maxlen, embedding_matrix, toolbox, toolboxes, n_pop, n_gen, resultsPath, testing_name, cfold, globalparameters, layerparameters, defaultVal, path_ind, max_conv_idx, max_maxpooling_idx, max_dense_idx, max_dropout_idx\n",
    "    gc.collect()\n",
    "    \n",
    "    ga.runGA()\n",
    "#     ga.runGA(population, fitnesses)\n"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "GA-CNN.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}