API fixes, mutation fixes

neuvol/__init__.py

@@ -15,10 +15,9 @@
 from .crossing import Crosser
 from .evaluation import Evaluator
 from .evolution import Evolution
-from .mutation import Mutator
+from .mutation import MutatorBase
 from .probabilty_pool import Distribution
 from .layer import block, layer
 from .individs import cradle
-from .mutation import *
 
-__all__ = ['SESSION', 'Crosser', 'Evaluator', 'Evolution', 'Mutator', 'Distribution', 'block', 'layer', 'cradle']
+__all__ = ['SESSION', 'Crosser', 'Evaluator', 'Evolution', 'MutatorBase', 'Distribution', 'block', 'layer', 'cradle']

neuvol/config.py

@@ -17,13 +17,13 @@
 # it is important if you use
 import tensorflow as tf
 
-config = tf.ConfigProto()
+
+config = tf.compat.v1.ConfigProto()
 # config.gpu_options.visible_device_list = "1"
 # config.gpu_options.per_process_gpu_memory_fraction = 0.9
 # config.allow_soft_placement = True
 # config.gpu_options.allow_growth = True
-
-SESSION = tf.Session(config=config)
+SESSION = tf.compat.v1.Session(config=config)
 
 HANDLER = logging.FileHandler("log.log")
 FORMATTER = logging.Formatter(

neuvol/constants.py

@@ -60,15 +60,15 @@
         'input_rank': [3],
         'units': [i for i in range(1, 32, 1)],
         'recurrent_dropout': [FLOAT32(i / 100) for i in range(5, 95, 1)],
-        'activation': ['tanh', 'relu'],
+        'activation': ['tanh', 'relu', None],
         'implementation': [1, 2],
         'return_sequences': [True, False]},
 
     'lstm': {
         'input_rank': [3],
         'units': [i for i in range(1, 32, 1)],
         'recurrent_dropout': [FLOAT32(i / 100) for i in range(5, 95, 1)],
-        'activation': ['tanh', 'relu'],
+        'activation': ['tanh', 'relu', None],
         'implementation': [1, 2],
         'return_sequences': [True, False]},
 
@@ -78,7 +78,7 @@
         'kernel_size': [i for i in range(1, 11, 2)],
         'strides': [1, 2, 3],
         'padding': ['valid', 'same', 'causal'],
-        'activation': ['tanh', 'relu'],
+        'activation': ['tanh', 'relu', None],
         'dilation_rate': [1, 2, 3]},
 
     'cnn2': {
@@ -87,7 +87,7 @@
         'kernel_size': [i for i in range(1, 11, 2)],
         'strides': [1, 2, 3],
         'padding': ['valid', 'same'],
-        'activation': ['tanh', 'relu'],
+        'activation': ['tanh', 'relu', None],
         'dilation_rate': [1, 2, 3]},
 
     'max_pool': {
@@ -105,10 +105,43 @@
     'dense': {
         'input_rank': [],
         'units': [i for i in range(4, 512, 2)],
-        'activation': ['softmax', 'sigmoid']},
+        'activation': ['softmax', 'sigmoid', None]},
 
     'dropout': {
         'input_rank': [],
-        'rate': [FLOAT32(i / 100) for i in range(5, 95, 1)]}}
+        'rate': [FLOAT32(i / 100) for i in range(5, 95, 1)]},
+
+    'repeatvector': {
+        'input_rank': [2],
+        'n': [i for i in range(2, 32)]},
+
+    'separablecnn': {
+        'input_rank': [3],
+        'filters': [i for i in range(1, 128, 1)],
+        'kernel_size': [i for i in range(1, 11, 2)],
+        'strides': [1, 2, 3],
+        'padding': ['valid', 'same'],
+        'activation': ['tanh', 'relu', None],
+        'dilation_rate': [1, 2, 3]},
+
+    'separablecnn2': {
+        'input_rank': [4],
+        'filters': [i for i in range(1, 128, 1)],
+        'kernel_size': [i for i in range(1, 11, 2)],
+        'strides': [1, 2, 3],
+        'padding': ['valid', 'same'],
+        'activation': ['tanh', 'relu', None],
+        'dilation_rate': [1, 2, 3]},
+
+    'decnn2': {
+        'input_rank': [4],
+        'filters': [i for i in range(1, 128, 1)],
+        'kernel_size': [i for i in range(1, 11, 2)],
+        'strides': [1, 2, 3],
+        'padding': ['valid', 'same'],
+        'output_padding': [i for i in range(1, )] + [None],
+        'activation': ['tanh', 'relu', None],
+        'dilation_rate': [1, 2, 3]}, }
+
 
 POOL_SIZE = len(LAYERS_POOL)

neuvol/individs/cradle.py

@@ -15,7 +15,7 @@
 from .individ_text import IndividText
 
 
-def cradle(epochs, data_type='text', task_type='classification', parents=None, freeze=None, **kwargs):
+def cradle(epochs, options, data_type='text', task_type='classification', parents=None, freeze=None):
     """Factory method for different data types
 
     Attributes:
@@ -26,9 +26,9 @@ def cradle(epochs, data_type='text', task_type='classification', parents=None, f
         freeze (``bool``): ?
     """
     if data_type == 'text':
-        return IndividText(epochs, task_type=task_type, parents=parents, freeze=freeze, **kwargs)
+        return IndividText(epochs, options, task_type=task_type, parents=parents, freeze=freeze)
     elif data_type == 'image':
-        return IndividImage(epochs, task_type=task_type, parents=parents, freeze=freeze, **kwargs)
+        return IndividImage(epochs, options, task_type=task_type, parents=parents, freeze=freeze)
     else:
         raise ValueError("Incorrect \"data_type\" argument."
                          "Available values: \"text\", \"image\"")

neuvol/individs/individ_base.py

@@ -29,7 +29,7 @@ class IndividBase:
     # TODO: add support for different data types
     # TODO: add support for different task types
 
-    def __init__(self, stage, task_type='classification', parents=None, freeze=None, **kwargs):
+    def __init__(self, stage, options, task_type='classification', parents=None, freeze=None):
         """Create individ randomly or with its parents
 
         Attributes:
@@ -43,7 +43,7 @@ def __init__(self, stage, task_type='classification', parents=None, freeze=None,
         # TODO: freeze training or data parameters of individ and set manualy
         self._freeze = freeze
         self._parents = parents
-        self.options = kwargs
+        self.options = options
         self._history = []
         self._name = FAKE.name().replace(' ', '_') + '_' + str(stage)
         self._architecture = []
@@ -79,7 +79,7 @@ def _random_init_architecture(self):
         """
         Init structure of the individ
         """
-        input_layer = Layer('input', **self.options)
+        input_layer = Layer('input', options=self.options)
         architecture = Structure(input_layer)
 
         return architecture
@@ -166,10 +166,11 @@ def init_tf_graph(self):
             raise Exception('Non initialized net')
 
         tails_map = {}
+        last_layer = None
 
         # walk over all layers and connect them between each other
-        for column in range(self.matrix.shape[1]):
-            last_layer = self.rec_imposer(column, tails_map)
+        for column in range(len(self.layers_index_reverse)):
+            last_layer = self.rec_imposer(column, tails_map) or last_layer
 
         network_head = tails_map[0]
         network_tail = tails_map[last_layer]
@@ -439,7 +440,7 @@ def merge_branchs(self, layer, branchs=None):
         Returns:
             str -- return the name of new common ending of the branches
         """
-        self._architecture.merge_branchs(layer, branchs=branchs)
+        return self._architecture.merge_branchs(layer, branchs=branchs)
 
     def split_branch(self, layers, branch):
         """

neuvol/individs/individ_image.py

@@ -20,12 +20,12 @@ class IndividImage(IndividBase):
     """
     Invidiv class for image data types
     """
-    def __init__(self, stage, task_type='classification', parents=None, freeze=None, **kwargs):
-        super().__init__(stage=stage, task_type=task_type, parents=parents, freeze=freeze, **kwargs)
+    def __init__(self, stage, options, task_type='classification', parents=None, freeze=None):
+        super().__init__(stage=stage, options=options, task_type=task_type, parents=parents, freeze=freeze)
         self._data_processing_type = 'image'
 
     def _random_init_architecture(self):
-        input_layer = Layer('input', **self.options)
+        input_layer = Layer('input', self.options)
 
         architecture = StructureImage(input_layer)
 

neuvol/individs/individ_text.py

@@ -21,17 +21,17 @@ class IndividText(IndividBase):
     Invidiv class for text data types
     """
 
-    def __init__(self, stage, task_type='classification', parents=None, freeze=None, **kwargs):
-        super().__init__(stage=stage, task_type=task_type, parents=parents, freeze=freeze, **kwargs)
+    def __init__(self, stage, options, task_type='classification', parents=None, freeze=None):
+        super().__init__(stage=stage, options=options, task_type=task_type, parents=parents, freeze=freeze)
         self._data_processing_type = 'text'
 
     def _random_init_architecture(self):
         """
         At first, we set probabilities pool and the we change
         this uniform distribution according to previous layer
         """
-        input_layer = Layer('input', **self.options)
-        embed = Layer('embedding', **self.options)
+        input_layer = Layer('input', self.options)
+        embed = Layer('embedding', self.options)
 
         architecture = StructureText(input_layer, embed)
 

neuvol/individs/structure/structure.py

@@ -64,7 +64,7 @@ def _register_new_layer(self, matrix, layers_index_reverse, new_layer):
         # self.layers_indexes[new_layer] = len(self.layers_indexes)
         _layers_index_reverse[len(_layers_index_reverse)] = new_layer
 
-        return _matrix, _layers_index_reverse, len(self._layers_index_reverse) - 1
+        return _matrix, _layers_index_reverse, len(_layers_index_reverse) - 1
 
     def _add_layer(self, matrix, layers_index_reverse, branchs_end, layer, branch, branch_out=None):
         """
@@ -182,7 +182,7 @@ def _merge_branchs(self, matrix, layers_index_reverse, branchs_end, branchs_coun
         branchs_counter.append(branch_new)
         branchs_end[branch_new] = index
 
-        return matrix, layers_index_reverse, branchs_end, branchs_counter
+        return matrix, layers_index_reverse, branchs_end, branchs_counter, branch_new
 
     def _split_branch(self, matrix, layers_index_reverse, branchs_end, branchs_counter, layers, branch):
         """
@@ -287,11 +287,12 @@ def merge_branchs(self, layer, branchs=None):
             layer {instance of the Layer} - layer, which will be added after concatenation
             branchs {list{int}} -- list of branchs to concatenate
         """
-        self._matrix, self._layers_index_reverse, self.branchs_end, self.branchs_counter = self._merge_branchs(
+        self._matrix, self._layers_index_reverse, self.branchs_end, self.branchs_counter, branchs_end_new = self._merge_branchs(
             self._matrix, self._layers_index_reverse,
             self.branchs_end, self.branchs_counter, layer, branchs)
         self._matrix_updated = False
         self._layers_index_reverse_updated = False
+        return branchs_end_new
 
     def split_branch(self, layers, branch):
         """
@@ -369,6 +370,9 @@ def mutations_applier(self):
         branchs_counter_copy = list(self.branchs_counter)
 
         for mutation in self.mutations_pool:
+            if mutation.config.get('state', None) == 'broken':
+                continue
+
             if mutation.mutation_type == 'add_layer':
                 layer = mutation.layer
                 before_layer_index = mutation.config['before_layer_index']
@@ -389,26 +393,30 @@ def mutations_applier(self):
                 branchs_counter_copy_tmp = None
 
             elif mutation.mutation_type == 'remove_layer':
-                pass
+                continue
 
             elif mutation.mutation_type == 'remove_connection':
-                pass
+                continue
 
             # its should be False
             if not self._cyclic_check(matrix_copy_tmp):
                 matrix_copy = matrix_copy_tmp
                 layers_index_reverse_copy = layers_index_reverse_copy_tmp or layers_index_reverse_copy
+
                 branchs_end_copy = branchs_end_copy_tmp or branchs_end_copy
                 branchs_counter_copy = branchs_counter_copy_tmp or branchs_counter_copy
-            else:
-                matrix_copy_tmp = None
-                layers_index_reverse_copy_tmp = None
-                branchs_end_copy_tmp = None
-                branchs_counter_copy_tmp = None
+
+            matrix_copy_tmp = None
+            layers_index_reverse_copy_tmp = None
+            branchs_end_copy_tmp = None
+            branchs_counter_copy_tmp = None
 
         return matrix_copy, layers_index_reverse_copy, branchs_end_copy, branchs_counter_copy
 
     def _update_mutated(self):
+        """
+        Update architecture using new mutations
+        """
         matrix, layers_index_reverse, branchs_end, branchs_counter = self.mutations_applier()
 
         self._matrix_updated = True
@@ -419,6 +427,9 @@ def _update_mutated(self):
 
     @property
     def matrix(self):
+        """
+        Return matrix with mutations
+        """
         # apply all mutations before matrix returning
         if not self._matrix_updated:
             self._update_mutated()
@@ -437,7 +448,9 @@ def layers_index_reverse(self):
 class StructureText(Structure):
     def __init__(self, root, embedding):
         """
-        Initialize the architecture of the individual
+        Initialize the architecture of the individual with textual data
+        Can used in case of pure text as the input
+        If own embedding - use pure Structure or 'general' as a type of data in Evolution
 
         Args:
             root {instance of the Layer} - input layer type
@@ -447,11 +460,13 @@ def __init__(self, root, embedding):
 
         self._matrix = np.zeros((2, 2))
 
+        # add root layer - Input layer
         self._matrix, self._layers_index_reverse, root_index = self._register_new_layer(
             self._matrix,
             self._layers_index_reverse,
             root)
 
+        # add embedding layer
         self._matrix, self._layers_index_reverse, embedding_index = self._register_new_layer(
             self._matrix,
             self._layers_index_reverse,

neuvol/layer/block.py

@@ -21,7 +21,7 @@ class Block():
     """
     def __init__(self, layers_type=None, layers_number=1, previous_block=None, next_block=None, **kwargs):
         self.layers = None
-        self.type = layers_type
+        self.layer_type = layers_type
         self.shape = layers_number
         self.previous_block = previous_block
         self.next_block = next_block
@@ -39,24 +39,24 @@ def _init_parameters(self):
         next_layer = self.next_block
 
         tmp_kwargs = self.options
-        self.layers = [Layer(self.type, previous_layers, next_layer, **tmp_kwargs) for _ in range(self.shape)]
+        self.layers = [Layer(self.layer_type, previous_layers, next_layer, **tmp_kwargs) for _ in range(self.shape)]
 
     def _check_compatibility(self):
         """
         For the output shape compatibility we need to freeze padding as the 'same'
         """
         # TODO: layers and block compatibility checker in one place
         if self.shape > 1:
-            if self.type == 'dropout':
+            if self.layer_type == 'dropout':
                 pass
 
-            elif self.type == 'cnn' or self.type == 'cnn2':
+            elif self.layer_type == 'cnn' or self.layer_type == 'cnn2':
                 # note that same padding and strides != 1 is inconsistent in keras
                 for layer in self.layers:
                     layer.config['padding'] = 'same'
                     layer.config['strides'] = 1
 
-            elif self.type == 'max_pool' or self.type == 'max_pool2':
+            elif self.layer_type == 'max_pool' or self.layer_type == 'max_pool2':
                 # note that same padding and strides != 1 is inconsistent in keras
                 for layer in self.layers:
                     layer.config['padding'] = 'same'
@@ -72,7 +72,7 @@ def save(self):
         Serialization of block
         """
         serial = dict()
-        serial['type'] = self.type
+        serial['type'] = self.layer_type
         serial['shape'] = self.shape
         serial['previous_block'] = self.previous_block
         serial['next_block'] = self.next_block