[net] weights_reject_freq=1001

zh2angijun · Dec 2, 2020 · 560fda3 · 560fda3
1 parent 6d5addd
commit 560fda3
Show file tree

Hide file tree

Showing 5 changed files with 82 additions and 1 deletion.
diff --git a/build/darknet/x64/cfg/yolov4-tiny.cfg b/build/darknet/x64/cfg/yolov4-tiny.cfg
@@ -28,6 +28,7 @@ scales=.1,.1
 #steps=500000
 #scales=0.01
 
+#weights_reject_freq=1001
 #ema_alpha=0.998
 #equidistant_point=1000
 #num_sigmas_reject_badlabels=3
@@ -242,6 +243,8 @@ random=0
 resize=1.5
 nms_kind=greedynms
 beta_nms=0.6
+#new_coords=1
+#scale_x_y = 2.0
 
 [route]
 layers = -4
@@ -291,3 +294,5 @@ random=0
 resize=1.5
 nms_kind=greedynms
 beta_nms=0.6
+#new_coords=1
+#scale_x_y = 2.0
diff --git a/include/darknet.h b/include/darknet.h
@@ -705,6 +705,7 @@ typedef struct network {
     float *delta_rolling_max;
     float *delta_rolling_avg;
     float *delta_rolling_std;
+    int weights_reject_freq;
     int equidistant_point;
     float badlabels_rejection_percentage;
     float num_sigmas_reject_badlabels;

diff --git a/src/network.c b/src/network.c
@@ -439,7 +439,7 @@ float train_network_waitkey(network net, data d, int wait_key)
             printf(" EMA initialization \n");
         }
 
-        if ((*net.cur_iteration) >= ema_apply_point)
+        if ((*net.cur_iteration) == ema_apply_point)
         {
             ema_apply(net); // apply EMA (BN rolling mean/var recalculation is required)
             printf(" ema_apply() \n");
@@ -453,6 +453,18 @@ float train_network_waitkey(network net, data d, int wait_key)
         }
     }
 
+
+    int reject_stop_point = net.max_batches*3/4;
+
+    if ((*net.cur_iteration) < reject_stop_point &&
+        net.weights_reject_freq &&
+        (*net.cur_iteration) % net.weights_reject_freq == 0)
+    {
+        float sim_threshold = 0.4;
+        reject_similar_weights(net, sim_threshold);
+    }
+
+
     free(X);
     free(y);
     return (float)sum/(n*batch);
@@ -1591,3 +1603,64 @@ void ema_apply(network net)
     }
 }
 
+
+
+void reject_similar_weights(network net, float sim_threshold)
+{
+    int i;
+    for (i = 0; i < net.n; ++i) {
+        layer l = net.layers[i];
+        if (i == 0) continue;
+        if (net.n > i + 1) if (net.layers[i + 1].type == YOLO) continue;
+        if (net.n > i + 2) if (net.layers[i + 2].type == YOLO) continue;
+        if (net.n > i + 3) if (net.layers[i + 3].type == YOLO) continue;
+
+        if (l.type == CONVOLUTIONAL && l.activation != LINEAR) {
+#ifdef GPU
+            if (gpu_index >= 0) {
+                pull_convolutional_layer(l);
+            }
+#endif
+            int k, j;
+            float max_sim = -1000;
+            int max_sim_index = 0;
+            int max_sim_index2 = 0;
+            int filter_size = l.size*l.size*l.c;
+            for (k = 0; k < l.n; ++k)
+            {
+                for (j = k+1; j < l.n; ++j)
+                {
+                    int w1 = k;
+                    int w2 = j;
+
+                    float sim = cosine_similarity(&l.weights[filter_size*w1], &l.weights[filter_size*w2], filter_size);
+                    if (sim > max_sim) {
+                        max_sim = sim;
+                        max_sim_index = w1;
+                        max_sim_index2 = w2;
+                    }
+                }
+            }
+
+            printf(" reject_similar_weights: i = %d, l.n = %d, w1 = %d, w2 = %d, sim = %f, thresh = %f \n",
+                i, l.n, max_sim_index, max_sim_index2, max_sim, sim_threshold);
+
+            if (max_sim > sim_threshold) {
+                printf(" rejecting... \n");
+                float scale = sqrt(2. / (l.size*l.size*l.c / l.groups));
+
+                for (k = 0; k < filter_size; ++k) {
+                    l.weights[max_sim_index*filter_size + k] = scale*rand_uniform(-1, 1);
+                }
+                if (l.biases) l.biases[max_sim_index] = 0.0f;
+                if (l.scales) l.scales[max_sim_index] = 1.0f;
+            }
+
+#ifdef GPU
+            if (gpu_index >= 0) {
+                push_convolutional_layer(l);
+            }
+#endif
+        }
+    }
+}
diff --git a/src/network.h b/src/network.h
@@ -174,6 +174,7 @@ void restore_network_recurrent_state(network net);
 int is_ema_initialized(network net);
 void ema_update(network net, float ema_alpha);
 void ema_apply(network net);
+void reject_similar_weights(network net, float sim_threshold);
 
 
 #ifdef __cplusplus

diff --git a/src/parser.c b/src/parser.c
@@ -1155,6 +1155,7 @@ void parse_net_options(list *options, network *net)
     net->batch *= net->time_steps;  // mini_batch * time_steps
     net->subdivisions = subdivs;    // number of mini_batches
 
+    net->weights_reject_freq = option_find_int_quiet(options, "weights_reject_freq", 0);
     net->equidistant_point = option_find_int_quiet(options, "equidistant_point", 0);
     net->badlabels_rejection_percentage = option_find_float_quiet(options, "badlabels_rejection_percentage", 0);
     net->num_sigmas_reject_badlabels = option_find_float_quiet(options, "num_sigmas_reject_badlabels", 0);