ctdb: Add "home_nodes" file to deterministic IP allocation

With a file "home_nodes" next to "public_addresses" you can assign public IPs to specific nodes when using the deterministic allocation algorithm. Whenever the "home node" is up, the IP address will be assigned to that node, independent of any other deterministic calculation. The line 192.168.21.254 2 in the file "home_nodes" assigns the IP address to node 2. Only when node 2 is not able to host IP addresses, 192.168.21.254 undergoes the normal deterministic IP allocation algorithm. Signed-off-by: Volker Lendecke <[email protected]> add home_nodes Reviewed-by: Ralph Boehme <[email protected]> Autobuild-User(master): Volker Lendecke <[email protected]> Autobuild-Date(master): Tue Oct 10 14:17:19 UTC 2023 on atb-devel-224
synarete · Oct 10, 2023 · a6b6666 · a6b6666
1 parent ea9cbbd
commit a6b6666
Show file tree

Hide file tree

Showing 5 changed files with 270 additions and 0 deletions.
diff --git a/ctdb/doc/ctdb-tunables.7.xml b/ctdb/doc/ctdb-tunables.7.xml
@@ -283,6 +283,17 @@ MonitorInterval=20
 	      with care when addresses are defined across multiple
 	      networks.
 	    </para>
+	    <para>
+	      You can override automatic the "home" node allocation by
+	      creating a file "home_nodes" next to the
+	      "public_addresses" file. As an example the following
+	      "home_nodes" file assigns the address 192.168.1.1 to
+	      node 0 and 192.168.1.2 to node 2:
+	    </para>
+	    <screen format="linespecific">
+	      192.168.1.1 0
+	      192.168.1.2 2
+            </screen>
 	  </listitem>
 	</varlistentry>
 	<varlistentry>

diff --git a/ctdb/server/ipalloc_deterministic.c b/ctdb/server/ipalloc_deterministic.c
@@ -24,11 +24,117 @@
 
 #include "lib/util/debug.h"
 #include "common/logging.h"
+#include "common/path.h"
+
+#include "protocol/protocol_util.h"
+#include "lib/util/smb_strtox.h"
+#include "lib/util/memory.h"
 
 #include "server/ipalloc_private.h"
 
+struct home_node {
+	ctdb_sock_addr addr;
+	uint32_t pnn;
+};
+
+static struct home_node *ipalloc_get_home_nodes(TALLOC_CTX *mem_ctx)
+{
+	char *line = NULL;
+	size_t len = 0;
+	char *fname = NULL;
+	FILE *fp = NULL;
+	struct home_node *result = NULL;
+
+	fname = path_etcdir_append(mem_ctx, "home_nodes");
+	if (fname == NULL) {
+		goto fail;
+	}
+
+	fp = fopen(fname, "r");
+	if (fp == NULL) {
+		goto fail;
+	}
+	TALLOC_FREE(fname);
+
+	while (true) {
+		size_t num_nodes = talloc_array_length(result);
+		char *saveptr = NULL, *addrstr = NULL, *nodestr = NULL;
+		struct home_node hn = {
+			.pnn = CTDB_UNKNOWN_PNN,
+		};
+		struct home_node *tmp = NULL;
+		ssize_t n = 0;
+		int ret;
+
+		n = getline(&line, &len, fp);
+		if (n < 0) {
+			if (!feof(fp)) {
+				/* real error */
+				goto fail;
+			}
+			break;
+		}
+		if ((n > 0) && (line[n - 1] == '\n')) {
+			line[n - 1] = '\0';
+		}
+
+		addrstr = strtok_r(line, " \t", &saveptr);
+		if (addrstr == NULL) {
+			continue;
+		}
+		nodestr = strtok_r(NULL, " \t", &saveptr);
+		if (nodestr == NULL) {
+			continue;
+		}
+
+		ret = ctdb_sock_addr_from_string(addrstr, &hn.addr, false);
+		if (ret != 0) {
+			DBG_WARNING("Could not parse %s: %s\n",
+				    addrstr,
+				    strerror(ret));
+			goto fail;
+		}
+
+		hn.pnn = smb_strtoul(nodestr,
+				     NULL,
+				     10,
+				     &ret,
+				     SMB_STR_FULL_STR_CONV);
+		if (ret != 0) {
+			DBG_WARNING("Could not parse \"%s\"\n", nodestr);
+			goto fail;
+		}
+
+		tmp = talloc_realloc(mem_ctx,
+				     result,
+				     struct home_node,
+				     num_nodes + 1);
+		if (tmp == NULL) {
+			goto fail;
+		}
+		result = tmp;
+		result[num_nodes] = hn;
+	}
+
+	fclose(fp);
+	fp = NULL;
+	return result;
+
+fail:
+	if (fp != NULL) {
+		fclose(fp);
+		fp = NULL;
+	}
+	SAFE_FREE(line);
+	TALLOC_FREE(fname);
+	TALLOC_FREE(result);
+	return NULL;
+}
+
 bool ipalloc_deterministic(struct ipalloc_state *ipalloc_state)
 {
+	struct home_node *home_nodes = ipalloc_get_home_nodes(ipalloc_state);
+	size_t num_home_nodes = talloc_array_length(home_nodes);
 	struct public_ip_list *t;
 	int i;
 	uint32_t numnodes;
@@ -42,7 +148,26 @@ bool ipalloc_deterministic(struct ipalloc_state *ipalloc_state)
 	*/
 
 	for (i = 0, t = ipalloc_state->all_ips; t!= NULL; t = t->next, i++) {
+		size_t j;
+
 		t->pnn = i % numnodes;
+
+		for (j = 0; j < num_home_nodes; j++) {
+			struct home_node *hn = &home_nodes[j];
+
+			if (ctdb_sock_addr_same_ip(&t->addr, &hn->addr)) {
+
+				if (hn->pnn >= numnodes) {
+					DBG_WARNING("pnn %" PRIu32
+						    " too large\n",
+						    hn->pnn);
+					break;
+				}
+
+				t->pnn = hn->pnn;
+				break;
+			}
+		}
 	}
 
 	/* IP failback doesn't make sense with deterministic
@@ -60,5 +185,7 @@ bool ipalloc_deterministic(struct ipalloc_state *ipalloc_state)
 
 	/* No failback here! */
 
+	TALLOC_FREE(home_nodes);
+
 	return true;
 }
diff --git a/ctdb/tests/UNIT/takeover/det.004.sh b/ctdb/tests/UNIT/takeover/det.004.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+setup_ctdb_base "$CTDB_TEST_TMP_DIR" "ctdb-etc"
+
+define_test "3 nodes, all healthy with home_nodes"
+
+home_nodes="$CTDB_BASE"/home_nodes
+
+cat > "$home_nodes" <<EOF
+192.168.21.254 2
+192.168.20.251 1
+EOF
+
+required_result <<EOF
+${TEST_DATE_STAMP}Deterministic IPs enabled. Resetting all ip allocations
+192.168.21.254 2
+192.168.21.253 1
+192.168.21.252 2
+192.168.20.254 0
+192.168.20.253 1
+192.168.20.252 2
+192.168.20.251 1
+192.168.20.250 1
+192.168.20.249 2
+EOF
+
+simple_test 0,0,0 <<EOF
+192.168.20.249 1
+192.168.20.250 1
+192.168.20.251 1
+192.168.20.252 1
+192.168.20.253 1
+192.168.20.254 1
+192.168.21.252 1
+192.168.21.253 1
+192.168.21.254 1
+EOF
+
+rm "$home_nodes"
diff --git a/ctdb/tests/UNIT/takeover/det.005.sh b/ctdb/tests/UNIT/takeover/det.005.sh
@@ -0,0 +1,45 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+setup_ctdb_base "$CTDB_TEST_TMP_DIR" "ctdb-etc"
+
+define_test "3 nodes, 2 healthy with home_nodes"
+
+home_nodes="$CTDB_BASE"/home_nodes
+
+cat > "$home_nodes" <<EOF
+192.168.21.254 2
+192.168.20.251 1
+EOF
+
+required_result <<EOF
+${TEST_DATE_STAMP}Deterministic IPs enabled. Resetting all ip allocations
+${TEST_DATE_STAMP}Unassign IP: 192.168.21.253 from 1
+${TEST_DATE_STAMP}Unassign IP: 192.168.20.253 from 1
+${TEST_DATE_STAMP}Unassign IP: 192.168.20.251 from 1
+${TEST_DATE_STAMP}Unassign IP: 192.168.20.250 from 1
+192.168.21.254 2
+192.168.21.253 0
+192.168.21.252 2
+192.168.20.254 0
+192.168.20.253 0
+192.168.20.252 2
+192.168.20.251 0
+192.168.20.250 0
+192.168.20.249 2
+EOF
+
+simple_test 0,2,0 <<EOF
+192.168.20.249 0
+192.168.20.250 1
+192.168.20.251 2
+192.168.20.252 0
+192.168.20.253 1
+192.168.20.254 2
+192.168.21.252 0
+192.168.21.253 1
+192.168.21.254 2
+EOF
+
+rm "$home_nodes"
diff --git a/ctdb/tests/UNIT/takeover/det.006.sh b/ctdb/tests/UNIT/takeover/det.006.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+setup_ctdb_base "$CTDB_TEST_TMP_DIR" "ctdb-etc"
+
+define_test "3 nodes, 1 healthy with home_nodes"
+
+home_nodes="$CTDB_BASE"/home_nodes
+
+cat > "$home_nodes" <<EOF
+192.168.21.254 2
+192.168.20.251 1
+EOF
+
+required_result <<EOF
+${TEST_DATE_STAMP}Deterministic IPs enabled. Resetting all ip allocations
+${TEST_DATE_STAMP}Unassign IP: 192.168.21.253 from 1
+${TEST_DATE_STAMP}Unassign IP: 192.168.20.254 from 0
+${TEST_DATE_STAMP}Unassign IP: 192.168.20.253 from 1
+${TEST_DATE_STAMP}Unassign IP: 192.168.20.251 from 1
+${TEST_DATE_STAMP}Unassign IP: 192.168.20.250 from 1
+192.168.21.254 2
+192.168.21.253 2
+192.168.21.252 2
+192.168.20.254 2
+192.168.20.253 2
+192.168.20.252 2
+192.168.20.251 2
+192.168.20.250 2
+192.168.20.249 2
+EOF
+
+simple_test 2,2,0 <<EOF
+192.168.20.249 0
+192.168.20.250 1
+192.168.20.251 2
+192.168.20.252 0
+192.168.20.253 1
+192.168.20.254 2
+192.168.21.252 0
+192.168.21.253 1
+192.168.21.254 2
+EOF
+
+rm "$home_nodes"