diff --git a/.gitignore b/.gitignore index 5b5f3df..c888b4c 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,7 @@ viz_data_imdb_2\.json \.VSCodeCounter/ neuvol/individs/structure/structure_old\.py + +log\.txt + +stop\.txt diff --git a/neuvol/__init__.py b/neuvol/__init__.py index e34935f..1fc6fd0 100644 --- a/neuvol/__init__.py +++ b/neuvol/__init__.py @@ -11,11 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from .config import SESSION from .crossing import Crosser from .mutation import MutatorBase from .probabilty_pool import Distribution from .layer import layer, capsule_layer from .individs import cradle -__all__ = ['SESSION', 'Crosser', 'MutatorBase', 'Distribution', 'block', 'layer', 'cradle', 'capsule_layer'] +__all__ = ['Crosser', 'MutatorBase', 'Distribution', 'block', 'layer', 'cradle', 'capsule_layer'] diff --git a/neuvol/config.py b/neuvol/config.py index e719392..3f62fee 100644 --- a/neuvol/config.py +++ b/neuvol/config.py @@ -15,16 +15,8 @@ import logging # it is important if you use -import tensorflow as tf -config = tf.compat.v1.ConfigProto() -# config.gpu_options.visible_device_list = "1" -# config.gpu_options.per_process_gpu_memory_fraction = 0.9 -# config.allow_soft_placement = True -# config.gpu_options.allow_growth = True -SESSION = tf.compat.v1.Session(config=config) - HANDLER = logging.FileHandler("log.log") FORMATTER = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s - %(lineno)d') diff --git a/neuvol/constants.py b/neuvol/constants.py index 5ffcb4a..d7a8dec 100644 --- a/neuvol/constants.py +++ b/neuvol/constants.py @@ -32,8 +32,8 @@ 'number_of_splits': [2, 3, 4, 5] }, 'graph_parser': { - 'depth': 7, - 'min_size': 3 + 'depth': 5, + 'min_size': 1 } } @@ -116,7 +116,7 @@ 'input_rank': [4], 'pool_size': [i for i in range(0, 16, 2)][1:], 'strides': [i for i in range(2, 8)], - #'padding': ['valid', 'same'], + 'padding': ['valid', 'same'], 'padding': ['same'],}, 'dense': { @@ -149,7 +149,7 @@ 'kernel_size': [i for i in range(1, 11, 2)], 'strides': [1], # 'strides': [1, 2, 3], - # 'padding': ['valid', 'same'], + 'padding': ['valid', 'same'], 'padding': ['same'], 'activation': ['tanh', 'relu', None], 'dilation_rate': [1, 2, 3]}, @@ -160,7 +160,7 @@ 'kernel_size': [i for i in range(1, 11, 2)], 'strides': [1], # 'strides': [1, 2, 3], - # 'padding': ['valid', 'same'], + 'padding': ['valid', 'same'], 'padding': ['same'], 'output_padding': [i for i in range(1, )] + [None], 'activation': ['tanh', 'relu', None], diff --git a/neuvol/crossing/cross.py b/neuvol/crossing/cross.py index c99a6d5..b203426 100644 --- a/neuvol/crossing/cross.py +++ b/neuvol/crossing/cross.py @@ -11,78 +11,129 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import numpy as np + from ..layer.capsule_layer import detect_best_combination, structure_parser, remove_duplicated_branches from ..mutation import MutationInjector from ..utils import parameters_copy class Crosser: - @parameters_copy - def cross(self, individ1, individ2, size=5): + def cross(self, individ1, individ2, start_point=1, depth=1000): # detect branch in individ1, which will replace branch in individ2 - ind1_complex = structure_parser(individ1.architecture, size) + ind1_complex = structure_parser(individ1.architecture, 1, start_point, depth) + # detect complete path ind1_complex = [detect_best_combination(i) for i in ind1_complex] - ind1_complex = [remove_duplicated_branches(new_chain) for new_chain in ind1_complex if new_chain] + ind1_complex = [remove_duplicated_branches(new_chain) for new_chain in ind1_complex if new_chain][0] if len(ind1_complex) == 0: return None - ind1_complex = [i for i in ind1_complex if i][0] + # select the first detected path + ind1_complex = [i for i in ind1_complex if i] + ind1_complex = [(i, self.calculate_complexity(individ1, i)) for i in ind1_complex] + # select the most complex branch to cut + + max_complexity = max([i[1] for i in ind1_complex]) + ind1_complex = [i[0] for i in ind1_complex if i[1] == max_complexity][0] # detect branch in individ2, which will be dropped in invidid2 - ind2_complex = structure_parser(individ2.architecture, size) + ind2_complex = structure_parser(individ2.architecture, 1, start_point, depth) ind2_complex = [detect_best_combination(i) for i in ind2_complex] - ind2_complex = [remove_duplicated_branches(new_chain) for new_chain in ind2_complex if new_chain] + ind2_complex = [remove_duplicated_branches(new_chain) for new_chain in ind2_complex if new_chain][0] if len(ind2_complex) == 0: return None - ind2_complex = [i for i in ind2_complex if i][0] + ind2_complex = [i for i in ind2_complex if i] + ind2_complex = [(i, self.calculate_complexity(individ2, i)) for i in ind2_complex] + # select the less complex branch to inject + + min_complexity = max([i[1] for i in ind2_complex]) + ind2_complex = [i[0] for i in ind2_complex if i[1] == min_complexity][0] + + # select start node - from which cut the selected graph + # select target node - the end of the selected graph + from_index = np.where(individ2.matrix[:, ind2_complex[0]] == 1)[0] + if len(from_index) == 0: + from_index = None + else: + from_index = from_index[0] + to_index = np.where(individ2.matrix[ind2_complex[-1]] == 1)[0] + if len(to_index) == 0: + to_index = None + else: + to_index = to_index[0] + + if from_index is not None: + ind2_complex.insert(0, from_index) + else: + from_index = ind2_complex[0] + + if to_index is not None: + ind2_complex.insert(-1, to_index) + else: + to_index = ind2_complex[-1] + + # if the target node - finisher layer, replace by the last node in the selected graph + # finisher layer is calculated on the fly and that can break mutations + if to_index == list(individ2.layers_index_reverse.keys())[-1]: + to_index = ind2_complex[-1] + self.cut_branch(individ2, ind2_complex) - self.inject_branch(individ2, individ1, ind1_complex, ind2_complex[0][0], ind2_complex[0][-1]) + self.inject_branch(individ2, individ1, ind1_complex, from_index, to_index) return individ2 + + def calculate_complexity(self, individ, branch): + complexity = 0 + for layer in branch: + if individ.layers_index_reverse[layer].config.get('shape', None) is not None: + complexity += np.prod(individ.layers_index_reverse[layer].shape[1:]) + + return complexity def cut_branch(self, individ, branch): - for chain in branch: - for index in [chain[i: i + 2] for i in range(len(chain[::2]) + 1)]: - if len(index) != 2: - return False - remove_connection_mutation = MutationInjector(None, None, None, None) - remove_connection_mutation.mutation_type = 'remove_connection' - remove_connection_mutation.after_layer_index = index[0] - remove_connection_mutation.before_layer_index = index[1] - individ.add_mutation(remove_connection_mutation) + for index in [branch[i: i + 2] for i in range(len(branch[::2]) + 1)]: + if len(index) != 2: + return False + remove_connection_mutation = MutationInjector(None, None, None, None) + remove_connection_mutation.mutation_type = 'remove_connection' + remove_connection_mutation.after_layer_index = index[0] + remove_connection_mutation.before_layer_index = index[1] + remove_connection_mutation._layer = None + individ.add_mutation(remove_connection_mutation) def inject_branch(self, individ, individ_donor, branch, from_index, to_index): tmp_map = {} # layers_reverse is used to get new layer index after mutation # [:-1] - because of last layer, which is temporary finisher initial_layers_reverse = set(list(individ.layers_index_reverse.keys())[:-1]) - for chain in branch: - # we go through the tail to the head - for index in chain[::-1]: - # if this index was added before - just add required connection withour layer duplication - if tmp_map.get(index, None) is None: - # remove_connection_mutation = MutationInjector(None, None, None, None) - # remove_connection_mutation.mutation_type = 'remove_connection' - # remove_connection_mutation.after_layer_index = from_index - # remove_connection_mutation.before_layer_index = to_index + # we go through the tail to the head + for index in branch[::-1]: + # if this index was added before - just add required connection withour layer duplication + if tmp_map.get(index, None) is None: +# remove_connection_mutation = MutationInjector(None, None, None, None) +# remove_connection_mutation.mutation_type = 'remove_connection' +# remove_connection_mutation.after_layer_index = from_index +# remove_connection_mutation.before_layer_index = to_index + +# individ.add_mutation(remove_connection_mutation) + inject_layer_mutation = MutationInjector(None, None, None, None) + inject_layer_mutation.mutation_type = 'inject_layer' + inject_layer_mutation.layer = individ_donor.layers_index_reverse[index] + inject_layer_mutation.after_layer_index = from_index + inject_layer_mutation.before_layer_index = to_index - # individ.add_mutation(remove_connection_mutation) - inject_layer_mutation = MutationInjector(None, None, None, None) - inject_layer_mutation.mutation_type = 'inject_layer' - inject_layer_mutation.layer = individ_donor.layers_index_reverse[index] - inject_layer_mutation.after_layer_index = from_index - inject_layer_mutation.before_layer_index = to_index + individ.add_mutation(inject_layer_mutation) + new_layers_reverse = set(list(individ.layers_index_reverse.keys())[:-1]) - initial_layers_reverse - individ.add_mutation(inject_layer_mutation) - new_layers_reverse = set(list(individ.layers_index_reverse.keys())[:-1]) - initial_layers_reverse - to_index = list(new_layers_reverse)[0] - initial_layers_reverse = set(list(individ.layers_index_reverse.keys())[:-1]) + to_index = list(new_layers_reverse)[0] + initial_layers_reverse = set(list(individ.layers_index_reverse.keys())[:-1]) - tmp_map[index] = to_index - else: - add_connection_mutation = MutationInjector(None, None, None, None) - add_connection_mutation.mutation_type = 'add_connection' - add_connection_mutation.after_layer_index = tmp_map[index] - add_connection_mutation.before_layer_index = to_index + tmp_map[index] = to_index + else: + add_connection_mutation = MutationInjector(None, None, None, None) + add_connection_mutation.mutation_type = 'add_connection' + add_connection_mutation.after_layer_index = tmp_map[index] + add_connection_mutation.before_layer_index = to_index + add_connection_mutation._layer = None - individ.add_mutation(add_connection_mutation) + individ.add_mutation(add_connection_mutation) diff --git a/neuvol/individs/individ_base.py b/neuvol/individs/individ_base.py index e4782b3..75e9d7f 100644 --- a/neuvol/individs/individ_base.py +++ b/neuvol/individs/individ_base.py @@ -240,6 +240,10 @@ def result(self): Get the result of the efficiency (f1 or AUC) """ return self._result + + @name.setter + def name(self, value): + self._name = value @result.setter def result(self, value): diff --git a/neuvol/individs/structure/structure.py b/neuvol/individs/structure/structure.py index 3a24bbb..a275cfe 100644 --- a/neuvol/individs/structure/structure.py +++ b/neuvol/individs/structure/structure.py @@ -577,7 +577,19 @@ def _update_mutated(self): self._layers_index_reverse_updated = True self._layers_index_reverse_mutated = layers_index_reverse - + + def freeze_state(self): + for mutation in self.mutations_pool: + if mutation.config.get('state', None) == 'broken': + mutation.config['state'] = None + # apply mutations + self._matrix, self._layers_index_reverse, self.branchs_end, self.branchs_counter = self.mutations_applier( + self._matrix, self._layers_index_reverse, + self.branchs_end, self.branchs_counter) + + self.mutations_pool = [] + + @property def matrix(self): """ diff --git a/neuvol/layer/capsule_layer.py b/neuvol/layer/capsule_layer.py index c132c2c..ba7bbe4 100644 --- a/neuvol/layer/capsule_layer.py +++ b/neuvol/layer/capsule_layer.py @@ -38,15 +38,16 @@ def generate_complex_layers(structure, distribution, number_to_generate=5): distribution.register_new_layer(new_layer) -def structure_parser(structure, number_to_generate): +def structure_parser(structure, number_to_generate, start_point=None, depth=None): + depth = depth or GENERAL['graph_parser']['depth'] # remove first two layer - Input and embedder (in case of text) - layer_indexes = list(structure.layers_index_reverse.keys())[1:-2] - layer_indexes_random_sampled = np.random.choice(layer_indexes, number_to_generate, replace=False if len(layer_indexes) >= number_to_generate else True) + layer_indexes = list(structure.layers_index_reverse.keys())[1:-1] + layer_indexes_random_sampled = [start_point] if start_point else np.random.choice(layer_indexes, number_to_generate, replace=False if len(layer_indexes) >= number_to_generate else True) sublayers_chains = [] for index in layer_indexes_random_sampled: - sublayers_chain = sublayer_parser(index, structure.matrix[:-2, :-2], None) + sublayers_chain = sublayer_parser(index, structure.matrix[:-2, :-2], depth, None, 0) flatten_sublayers_chain = flatten(sublayers_chain) @@ -67,10 +68,10 @@ def flatten(chain): return f -def sublayer_parser(start_point, matrix, sub_layer=None, level=0): +def sublayer_parser(start_point, matrix, depth, sub_layer=None, level=0): level += 1 - - if level >= GENERAL['graph_parser']['depth']: + + if level >= depth: return [sub_layer] if sub_layer is None: @@ -79,14 +80,14 @@ def sublayer_parser(start_point, matrix, sub_layer=None, level=0): sub_layer.append(start_point) next_step = np.where(matrix[start_point] == 1)[0] - if level >= GENERAL['graph_parser']['depth']: + if level >= depth: return sub_layer elif len(next_step) == 1: - new_chains = sublayer_parser(next_step[0], matrix, list(sub_layer)) + new_chains = sublayer_parser(next_step[0], matrix, depth, list(sub_layer), level) elif len(next_step) > 1: - new_chains = [sublayer_parser(step, matrix, list(sub_layer)) for step in next_step] + new_chains = [sublayer_parser(step, matrix, depth, list(sub_layer), level) for step in next_step] else: return sub_layer @@ -94,8 +95,15 @@ def sublayer_parser(start_point, matrix, sub_layer=None, level=0): return new_chains -def detect_best_combination(new_chains): - if max([len(chain) for chain in new_chains]) < GENERAL['graph_parser']['min_size']: +def detect_best_combination(new_chains, min_size=None): + min_size = min_size or GENERAL['graph_parser']['min_size'] + + new_chains = [chain for chain in new_chains if len(chain) >= min_size] + # if max([len(chain) for chain in new_chains]) < min_size: + # return None + if len(new_chains) == 1: + return new_chains + elif len(new_chains) == 0: return None last_indexes = [i[-1] for i in new_chains]