New architecture generator

Allows to create more complex and flexible architectures: multiple branches

neuvol/architecture/individ_base.py

@@ -22,6 +22,7 @@
 from ..layer.block import Block
 from ..layer.layer import init_layer
 from ..probabilty_pool import Distribution
+from .structure import Structure
 from ..utils import dump
 
 
@@ -84,13 +85,7 @@ def _random_init_architecture(self):
         if self._architecture:
             self._architecture = []
 
-        architecture = []
-
-        # initial layer as a baseline
-        layer = Distribution.layer()
-
-        # TODO: architecture class
-        pass
+        return ...
 
     def _random_init_training(self):
         """
@@ -110,32 +105,60 @@ def _random_init_data_processing(self):
         """
         Initialize data processing parameters
         """
-        pass
-
-    def layers_imposer(self, net_tail, net_map, head, layers_map):
-        net = None
-        buffer = None
-        buffer_ids = None
-        state = net_map
-
-        for source, target in net_map:
-            if len(target) > 1:
-                buffer = [self.layers_imposer(net, net_map, branch, layers_map) for branch in target]
-
-            if 'm' in target[0]:
-                if buffer_ids is None:
-                    buffer_ids = [source]
-                else:
-                    buffer_ids.append(source)
-
-                continue
+        return ...
+
+    def layers_imposer(self, net_tail, head, layers_map, arch_map):
+        net = net_tail
+        source = head
+
+        try:
+            target = arch_map[source]
+        except KeyError:
+            return net, ''
+
+        # if the next layer is merger - return its net tail
+        if target[0][0] == 'm':
+            return [net], target[0]
+
+        if len(target) > 1:
+            buffer_tails = {branch: layers_map[branch](net) for branch in target}
+            buffer = [self.layers_imposer(buffer_tails[branch], branch, layers_map, arch_map) for branch in target]
+
+            buffer_tmp = []
+            #unpack
+            for sub_branch in buffer:
+                sub_net_tails = sub_branch[0]
+                for sub_net_tail in sub_net_tails:
+                    buffer_tmp.append((sub_net_tail, sub_branch[1]))
+
+            buffer = buffer_tmp
+            new_head = buffer[0][1]
 
-            if buffer is not None:
-                net = concatenate(buffer)
+            lenghts = int(new_head.split('_')[-1])
+            if len(buffer) < lenghts:
+                return [branch[0] for branch in buffer], new_head
+
+            # check consistency
+            for i in buffer:
+                for j in buffer:
+                    if i[1] != j[1]:
+                        raise ValueError('Inconsistent merger')
+
+            net = concatenate([branch[0] for branch in buffer])
 
+            net, new_head = self.layers_imposer(net, new_head, layers_map, arch_map)
+
+        if 'f' in target[0]:
             net = layers_map[target](net)
+            return net, target[0]
+
+        if len(target) == 1:
+            new_head = target[0]
+            net = layers_map[target[0]](net)
+
+            net, new_head = self.layers_imposer(net, new_head, layers_map, arch_map)
 
-        return net
+        return net, new_head
 
     def init_tf_graph(self):
         """
@@ -150,34 +173,12 @@ def init_tf_graph(self):
             keras_layer_instance = init_layer(layer)
             layers_map[key] = keras_layer_instance
 
-        starter = 'input'
+        starter = 'root'
         network_input = layers_map[starter]
 
+        network_graph = self.layers_imposer(network_input, 'root', layers_map, self._architecture.tree)
 
-
-
-        network_graph_input = init_layer(self._architecture[0])
-        network_graph = network_graph_input
-
-        for block in self._architecture[1:]:
-            if block.shape > 1:
-                # we need to create list of layers and concatenate them
-                tmp_block = []
-                for layer in block.layers:
-                    tmp_block.append(init_layer(layer)(network_graph))
-
-                network_graph = concatenate(tmp_block, axis=-1)
-
-            else:
-                # we need just to add new layer
-                network_graph = init_layer(block)(network_graph)
-            # except ValueError:
-            # in some cases shape of previous output could be less than kernel size of cnn
-            # it leads to a negative dimension size error
-            # add same padding to avoid this problem
-            #    layer.config['padding'] = 'same'
-            #    network_graph.add(init_layer(layer))
-        model = Model(inputs=[network_graph_input], outputs=[network_graph])
+        model = Model(inputs=[network_input], outputs=[network_graph])
 
         if self._training_parameters['optimizer'] == 'adam':
             optimizer = adam(

neuvol/architecture/structure/__init__.py

@@ -11,4 +11,4 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from .structure import StructureText
+from .structure import Structure, StructureText

neuvol/architecture/structure/structure.py

@@ -115,16 +115,16 @@ def merge_branches(self, layer, branches=None):
         add_to = [self.branchs_end[branch] for branch in branches]
         add_to_objects = [self.layers[to] for to in add_to]
 
-        modifier, shape_modifier = merger_mass(add_to_objects)
+        modifier, shape_modifiers = merger_mass(add_to_objects)
 
-        if shape_modifier is not None:
-            add_to = [self.add_layer(shape_modifier, branch) for branch in branches]
+        if shape_modifiers is not None:
+            add_to = [self.add_layer(shape_modifiers[i], branch) for i, branch in enumerate(branches)]
 
         for to in add_to:
             if self.tree.get(to) is None:
                 self.tree[to] = []
 
-        modifier_name =  'm{}_{}'.format(self.current_depth, branches[0])
+        modifier_name =  'm{}_{}_{}'.format(self.current_depth, branches[0], len(branches))
 
         for to in add_to:
             self.tree[to].append(modifier_name)

neuvol/constants.py

@@ -75,7 +75,7 @@
     'cnn': {
         'input_rank': [3],
         'filters': [i for i in range(4, 128, 2)],
-        'kernel_size': [i for i in range(1, 9, 1)],
+        'kernel_size': [i for i in range(1, 11, 2)],
         'strides': [1, 2, 3],
         'padding': ['valid', 'same', 'causal'],
         'activation': ['tanh', 'relu'],
@@ -84,7 +84,7 @@
     'cnn2': {
         'input_rank': [4],
         'filters': [i for i in range(1, 128, 1)],
-        'kernel_size': [i for i in range(1, 11, 1)],
+        'kernel_size': [i for i in range(1, 11, 2)],
         'strides': [1, 2, 3],
         'padding': ['valid', 'same'],
         'activation': ['tanh', 'relu'],

neuvol/layer/layer.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from keras.layers import (Bidirectional, concatenate, Conv1D, Conv2D, Dense, Dropout,
-                          Embedding, Flatten, Input, MaxPool1D, MaxPool2D)
+                          Embedding, Flatten, Input, MaxPool1D, MaxPool2D, Reshape)
 from keras.layers.recurrent import LSTM
 
 from ..constants import LAYERS_POOL, SPECIAL
@@ -211,5 +211,13 @@ def init_layer(layer):
 
     elif layer.type == 'flatten':
         layer_tf = Flatten()
+
+    elif layer.type == 'concat':
+        layer_tf = None
+
+    elif layer.type == 'reshape':
+        layer_tf = Reshape(
+            target_shape=layer.config['target_shape'],
+        )
 
     return layer_tf

neuvol/layer/reshaper.py

@@ -37,18 +37,25 @@ def calculate_shape(prev_layer, layer):
     if layer.type == 'cnn' or layer.type == 'cnn2':
         filters = layer.config['filters']
         kernel_size = layer.config['kernel_size']
+        if kernel_size % 2 == 0:
+            align = 1
+        else:
+            align = 0
         padding = layer.config['padding']
         strides = layer.config['strides']
         dilation_rate = layer.config['dilation_rate']
 
         if padding == 'valid':
-            out = [((i - kernel_size) // strides + 1) for i in prev_shape[1:-1]]
+            if dilation_rate != 1:
+                out = [(i - kernel_size - (kernel_size - 1) * (dilation_rate - 1)) // strides + 1 - align for i in prev_shape[1:-1]]
+            else:
+                out = [((i - kernel_size) // strides + 1 - align) for i in prev_shape[1:-1]]
 
         elif padding == 'same':
-            out = [((i - kernel_size + (2 * (kernel_size // 2))) // strides + 1) for i in prev_shape[1:-1]]
+            out = [((i - kernel_size + (2 * (kernel_size // 2))) // strides + 1 - align) for i in prev_shape[1:-1]]
 
         elif padding == 'causal':
-            out = [(i + (2 * (kernel_size // 2) - kernel_size - (kernel_size - 1) * (dilation_rate - 1))) // strides + 1 for i in prev_shape[1:-1]]
+            out = [(i - kernel_size - (kernel_size - 1) * (dilation_rate - 1)) // strides + 1 - align for i in prev_shape[1:-1]]
 
         for i in out:
             # if some of the layer too small - change the padding
@@ -62,8 +69,12 @@ def calculate_shape(prev_layer, layer):
     elif layer.type == 'max_pool' or layer.type == 'max_pool2':
         kernel_size = layer.config['pool_size']
         strides = layer.config['strides']
+        padding = layer.config['padding']
 
-        out = [((i - kernel_size) // strides + 1) for i in prev_shape[1:-1]]
+        if padding == 'same':
+            out = [((i + 2*(kernel_size // 2) - kernel_size) // strides + 1) for i in prev_shape[1:-1]]
+        else:
+            out = [((i - kernel_size) // strides + 1) for i in prev_shape[1:-1]]
 
         shape = (None, *out, prev_shape[-1])
 
@@ -111,7 +122,7 @@ def reshaper(prev_layer, layer):
         new_shape = (None, *prev_layer.config['shape'][1:-(difference + 2)], np.prod(prev_layer.config['shape'][-(difference + 2): -1]), prev_layer.config['shape'][-1])
 
         modifier = Layer('reshape')
-        modifier.config['target_shape'] = new_shape
+        modifier.config['target_shape'] = new_shape[1:]
         modifier.config['shape'] = new_shape
         modifier.config['input_rank'] = prev_layer.config['rank']
         modifier.config['rank'] = layer.config['input_rank']
@@ -122,7 +133,7 @@ def reshaper(prev_layer, layer):
         new_shape = (None, *prev_layer.config['shape'][1:], *tmp)
 
         modifier = Layer('reshape')
-        modifier.config['target_shape'] = new_shape
+        modifier.config['target_shape'] = new_shape[1:]
         modifier.config['shape'] = new_shape
         modifier.config['input_rank'] = prev_layer.config['rank']
         modifier.config['rank'] = layer.config['input_rank']
@@ -159,13 +170,22 @@ def merger(left_layer, right_layer):
     return modifier, shape_modifier
 
 def merger_mass(layers):
-    shape_modifier = Layer('flatten')
+    shape_modifiers = []
+    for layer in layers:
+        new_shape = (None, np.prod(layer.config['shape'][1:]))
 
-    modifier = Layer('concat')
+        reshape = Layer('reshape')
+        reshape.config['target_shape'] = new_shape[1:]
+        reshape.config['shape'] = new_shape
+        reshape.config['input_rank'] = layer.config['rank']
+        reshape.config['rank'] = 2
+
+        shape_modifiers.append(reshape)
 
-    shapes = [calculate_shape(layer, shape_modifier)[1][1] for layer in layers]
+    modifier = Layer('concat')
 
+    shapes = [shape_modifiers[i].config['shape'][1:] for i, layer in enumerate(layers)]
     modifier.config['shape'] = (None, np.sum(shapes))
     modifier.config['rank'] = 2
 
-    return modifier, shape_modifier
+    return modifier, shape_modifiers