From fbd012a78d4dde06bdcb3fb8c522c9a3d0bcb3c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Gr=C3=B6ber?= <dxld@darkboxed.org>
Date: Tue, 21 Feb 2023 02:16:32 +0100
Subject: [PATCH] Allow delaying shutdown route flush to prevent blackholes

When shutting down we currently remove routes from the kernel FIB without
waiting for our final hellos to propagate through the network. This could
cause us to either form loops or blackhole traffic using whichever external
routes remain.

Instead let the operator define a suitable delay for their network to
ensure complete propagation of retractions and divert traffic before doing
anything drastic.
---
 babeld.c        | 41 ++++++++++++++++++++---------------------
 babeld.h        |  1 +
 babeld.man      |  7 +++++++
 configuration.c |  7 +++++--
 4 files changed, 33 insertions(+), 23 deletions(-)

diff --git a/babeld.c b/babeld.c
index d0c7b6e8..b0b7f55d 100644
--- a/babeld.c
+++ b/babeld.c
@@ -89,6 +89,7 @@ int kernel_socket = -1;
 static int kernel_link_changed = 0;
 static int kernel_addr_changed = 0;
 int kernel_check_interval = 300;
+int shutdown_delay_msec = -1;
 
 struct timeval check_neighbours_timeout, check_interfaces_timeout;
 
@@ -759,31 +760,29 @@ main(int argc, char **argv)
     usleep(roughly(10000));
     gettime(&now);
 
+    for (unsigned retrans=0; retrans < 1; retrans++) {
+        FOR_ALL_INTERFACES(ifp) {
+            if(!if_up(ifp))
+                continue;
+            send_wildcard_retraction(ifp);
+            /* Make sure that we expire quickly from our neighbours'
+               association caches. */
+            send_multicast_hello(ifp, 10, 1);
+            flushbuf(&ifp->buf, ifp);
+            usleep(roughly(1000));
+            gettime(&now);
+        }
+    }
+
+    if (shutdown_delay_msec > 0)
+	    usleep(roughly(shutdown_delay_msec * 1000));
+
     /* We need to flush so interface_updown won't try to reinstall. */
     flush_all_routes();
 
-    FOR_ALL_INTERFACES(ifp) {
-        if(!if_up(ifp))
-            continue;
-        send_wildcard_retraction(ifp);
-        /* Make sure that we expire quickly from our neighbours'
-           association caches. */
-        send_multicast_hello(ifp, 10, 1);
-        flushbuf(&ifp->buf, ifp);
-        usleep(roughly(1000));
-        gettime(&now);
-    }
-    FOR_ALL_INTERFACES(ifp) {
-        if(!if_up(ifp))
-            continue;
-        /* Make sure they got it. */
-        send_wildcard_retraction(ifp);
-        send_multicast_hello(ifp, 1, 1);
-        flushbuf(&ifp->buf, ifp);
-        usleep(roughly(10000));
-        gettime(&now);
+    FOR_ALL_INTERFACES(ifp)
         interface_updown(ifp, 0);
-    }
+
     kernel_setup_socket(0);
     kernel_setup(0);
 
diff --git a/babeld.h b/babeld.h
index 1eec06e2..2e1b4722 100644
--- a/babeld.h
+++ b/babeld.h
@@ -107,6 +107,7 @@ extern int protocol_socket;
 extern int kernel_socket;
 extern int kernel_check_interval;
 extern int max_request_hopcount;
+extern int shutdown_delay_msec;
 
 void schedule_neighbours_check(int msecs, int override);
 void schedule_interfaces_check(int msecs, int override);
diff --git a/babeld.man b/babeld.man
index 241bfd2a..1f916900 100644
--- a/babeld.man
+++ b/babeld.man
@@ -240,6 +240,13 @@ This specifies the interval between two kernel routing table dumps.  The
 default is 300s (5 minutes).  This may be set to 0 in order to never
 perform periodic kernel dumps.
 .TP
+.BI shutdown-delay-ms " milliseconds"
+During shutdown we first notify neighbours of our imminent shutdown by
+sending route retractions, wait for the specified number of milliseconds
+and then flush kernel routes. This ensures any inflight traffic is still
+properly forwarded. You may want to ensure the delay is appropriate for the
+maximum delay path in your network. Setting this to zero is permissible.
+.TP
 .BR link-detect " {" true | false }
 This specifies whether to use carrier sense for determining interface
 availability, and is equivalent to the command-line option
diff --git a/configuration.c b/configuration.c
index 4b795111..96b6387e 100644
--- a/configuration.c
+++ b/configuration.c
@@ -979,7 +979,8 @@ parse_option(int c, gnc_t gnc, void *closure, char *token)
        strcmp(token, "local-port-readwrite") == 0 ||
        strcmp(token, "export-table") == 0 ||
        strcmp(token, "import-table") == 0 ||
-       strcmp(token, "kernel-check-interval") == 0) {
+       strcmp(token, "kernel-check-interval") == 0 ||
+       strcmp(token, "shutdown-delay-ms") == 0) {
         int v;
         c = getint(c, &v, gnc, closure);
         if(c < -1 || v <= 0 || v >= 0xFFFF)
@@ -1007,7 +1008,9 @@ parse_option(int c, gnc_t gnc, void *closure, char *token)
             add_import_table(v);
         else if(strcmp(token, "kernel-check-interval") == 0)
             kernel_check_interval = v;
-        else
+        else if(strcmp(token, "shutdown-delay-ms") == 0)
+	    shutdown_delay_msec = v;
+	else
             abort();
     } else if(strcmp(token, "link-detect") == 0 ||
               strcmp(token, "random-id") == 0 ||