Allow delaying shutdown route flush to prevent blackholes

When shutting down we currently remove routes from the kernel FIB without
waiting for our final hellos to propagate through the network. This could
cause us to either form loops or blackhole traffic using whichever external
routes remain.

Instead let the operator define a suitable delay for their network to
ensure complete propagation of retractions and divert traffic before doing
anything drastic.

babeld.c

@@ -89,6 +89,7 @@ int kernel_socket = -1;
 static int kernel_link_changed = 0;
 static int kernel_addr_changed = 0;
 int kernel_check_interval = 300;
+int shutdown_delay_msec = -1;
 
 struct timeval check_neighbours_timeout, check_interfaces_timeout;
 
@@ -759,31 +760,29 @@ main(int argc, char **argv)
     usleep(roughly(10000));
     gettime(&now);
 
+    for (unsigned retrans=0; retrans < 1; retrans++) {
+        FOR_ALL_INTERFACES(ifp) {
+            if(!if_up(ifp))
+                continue;
+            send_wildcard_retraction(ifp);
+            /* Make sure that we expire quickly from our neighbours'
+               association caches. */
+            send_multicast_hello(ifp, 10, 1);
+            flushbuf(&ifp->buf, ifp);
+            usleep(roughly(1000));
+            gettime(&now);
+        }
+    }
+
+    if (shutdown_delay_msec > 0)
+	    usleep(roughly(shutdown_delay_msec * 1000));
+
     /* We need to flush so interface_updown won't try to reinstall. */
     flush_all_routes();
 
-    FOR_ALL_INTERFACES(ifp) {
-        if(!if_up(ifp))
-            continue;
-        send_wildcard_retraction(ifp);
-        /* Make sure that we expire quickly from our neighbours'
-           association caches. */
-        send_multicast_hello(ifp, 10, 1);
-        flushbuf(&ifp->buf, ifp);
-        usleep(roughly(1000));
-        gettime(&now);
-    }
-    FOR_ALL_INTERFACES(ifp) {
-        if(!if_up(ifp))
-            continue;
-        /* Make sure they got it. */
-        send_wildcard_retraction(ifp);
-        send_multicast_hello(ifp, 1, 1);
-        flushbuf(&ifp->buf, ifp);
-        usleep(roughly(10000));
-        gettime(&now);
+    FOR_ALL_INTERFACES(ifp)
         interface_updown(ifp, 0);
-    }
+
     kernel_setup_socket(0);
     kernel_setup(0);
 

babeld.h

@@ -107,6 +107,7 @@ extern int protocol_socket;
 extern int kernel_socket;
 extern int kernel_check_interval;
 extern int max_request_hopcount;
+extern int shutdown_delay_msec;
 
 void schedule_neighbours_check(int msecs, int override);
 void schedule_interfaces_check(int msecs, int override);

babeld.man

@@ -240,6 +240,13 @@ This specifies the interval between two kernel routing table dumps.  The
 default is 300s (5 minutes).  This may be set to 0 in order to never
 perform periodic kernel dumps.
 .TP
+.BI shutdown-delay-ms " milliseconds"
+During shutdown we first notify neighbours of our imminent shutdown by
+sending route retractions, wait for the specified number of milliseconds
+and then flush kernel routes. This ensures any inflight traffic is still
+properly forwarded. You may want to ensure the delay is appropriate for the
+maximum delay path in your network. Setting this to zero is permissible.
+.TP
 .BR link-detect " {" true | false }
 This specifies whether to use carrier sense for determining interface
 availability, and is equivalent to the command-line option

configuration.c

@@ -979,7 +979,8 @@ parse_option(int c, gnc_t gnc, void *closure, char *token)
        strcmp(token, "local-port-readwrite") == 0 ||
        strcmp(token, "export-table") == 0 ||
        strcmp(token, "import-table") == 0 ||
-       strcmp(token, "kernel-check-interval") == 0) {
+       strcmp(token, "kernel-check-interval") == 0 ||
+       strcmp(token, "shutdown-delay-ms") == 0) {
         int v;
         c = getint(c, &v, gnc, closure);
         if(c < -1 || v <= 0 || v >= 0xFFFF)
@@ -1007,7 +1008,9 @@ parse_option(int c, gnc_t gnc, void *closure, char *token)
             add_import_table(v);
         else if(strcmp(token, "kernel-check-interval") == 0)
             kernel_check_interval = v;
-        else
+        else if(strcmp(token, "shutdown-delay-ms") == 0)
+	    shutdown_delay_msec = v;
+	else
             abort();
     } else if(strcmp(token, "link-detect") == 0 ||
               strcmp(token, "random-id") == 0 ||