gossipd: make the seeker code use gossmap_manage.

We add a temporary stub gossmap_manage constructor, which simply opens
the gossmap and doesn't do anything else.

Then seeker uses this, rather than routing.c, to probe.

We optimize our "get random node announcements" a bit by traversing a
random set of nodes directly, and seeing if we have no
node_announcement, then querying their first channel.

Signed-off-by: Rusty Russell <[email protected]>

gossipd/Makefile

@@ -5,6 +5,7 @@ GOSSIPD_HEADERS_WSRC := gossipd/gossipd_wiregen.h \
 	gossipd/gossip_store_wiregen.h			\
 	gossipd/gossipd.h				\
 	gossipd/gossip_store.h				\
+	gossipd/gossmap_manage.h			\
 	gossipd/queries.h				\
 	gossipd/routing.h				\
 	gossipd/txout_failures.h			\
@@ -44,6 +45,8 @@ GOSSIPD_COMMON_OBJS :=				\
 	common/dev_disconnect.o			\
 	common/ecdh_hsmd.o			\
 	common/features.o			\
+	common/gossmap.o			\
+	common/fp16.o				\
 	common/status_wiregen.o			\
 	common/key_derive.o			\
 	common/lease_rates.o			\

gossipd/gossipd.c

@@ -30,6 +30,7 @@
 #include <gossipd/gossipd.h>
 #include <gossipd/gossipd_peerd_wiregen.h>
 #include <gossipd/gossipd_wiregen.h>
+#include <gossipd/gossmap_manage.h>
 #include <gossipd/queries.h>
 #include <gossipd/routing.h>
 #include <gossipd/seeker.h>
@@ -699,6 +700,8 @@ static void gossip_init(struct daemon *daemon, const u8 *msg)
 	daemon->gs = gossip_store_new(daemon);
 	daemon->rstate = new_routing_state(daemon, daemon);
 
+	daemon->gm = gossmap_manage_new_gossmap_only(daemon, daemon);
+
 	/* Load stored gossip messages (FIXME: API sucks)*/
 	daemon->gossip_store_populated =
 		(gossip_store_load(daemon->gs) != 0);

gossipd/gossipd.h

@@ -53,6 +53,9 @@ struct daemon {
 	/* Routing information */
 	struct routing_state *rstate;
 
+	/* Manager of writing to the gossip_store */
+	struct gossmap_manage *gm;
+
 	/* Timers: we batch gossip, and also refresh announcements */
 	struct timers timers;
 

gossipd/gossmap_manage.c

@@ -3,19 +3,19 @@
 #include <ccan/mem/mem.h>
 #include <ccan/tal/str/str.h>
 #include <common/daemon_conn.h>
-#include <common/gossmap.h>
 #include <common/gossip_store.h>
+#include <common/gossmap.h>
 #include <common/status.h>
 #include <common/timeout.h>
 #include <common/type_to_string.h>
 #include <common/wire_error.h>
 #include <errno.h>
 #include <fcntl.h>
-#include <gossipd/gossmap_manage.h>
+#include <gossipd/gossip_store.h>
+#include <gossipd/gossip_store_wiregen.h>
 #include <gossipd/gossipd.h>
 #include <gossipd/gossipd_wiregen.h>
-#include <gossipd/gossip_store_wiregen.h>
-#include <gossipd/gossip_store.h>
+#include <gossipd/gossmap_manage.h>
 #include <gossipd/seeker.h>
 #include <gossipd/sigcheck.h>
 #include <gossipd/txout_failures.h>
@@ -421,6 +421,24 @@ static void bad_gossip(const struct node_id *source_peer, const char *str)
 	status_peer_debug(source_peer, "Bad gossip order: %s", str);
 }
 
+/* Minimal gossmap-only transition constructor */
+struct gossmap_manage *gossmap_manage_new_gossmap_only(const tal_t *ctx,
+						       struct daemon *daemon)
+{
+	struct gossmap_manage *gm = tal(ctx, struct gossmap_manage);
+
+	gm->fd = open(GOSSIP_STORE_FILENAME, O_RDWR);
+	if (gm->fd < 0)
+		status_failed(STATUS_FAIL_INTERNAL_ERROR,
+			      "Opening gossip_store store: %s",
+			      strerror(errno));
+	gm->raw_gossmap = gossmap_load_fd(gm, gm->fd, report_bad_update, NULL, gm);
+	assert(gm->raw_gossmap);
+	gm->daemon = daemon;
+
+	return gm;
+}
+
 /* Send peer a warning message, if non-NULL. */
 static void peer_warning(struct gossmap_manage *gm,
 			 const struct node_id *source_peer,

gossipd/gossmap_manage.h

@@ -8,6 +8,10 @@ struct gossmap_manage;
 struct gossmap_manage *gossmap_manage_new(const tal_t *ctx,
 					  struct daemon *daemon);
 
+/* Minimal gossmap-only transition constructor */
+struct gossmap_manage *gossmap_manage_new_gossmap_only(const tal_t *ctx,
+						       struct daemon *daemon);
+
 /**
  * gossmap_manage_channel_announcement: process an incoming channel_announcement
  * @ctx: tal context for return string

gossipd/seeker.c

@@ -3,16 +3,18 @@
 #include <bitcoin/chainparams.h>
 #include <ccan/array_size/array_size.h>
 #include <ccan/asort/asort.h>
+#include <ccan/intmap/intmap.h>
 #include <ccan/tal/str/str.h>
 #include <common/decode_array.h>
+#include <common/gossmap.h>
 #include <common/memleak.h>
 #include <common/pseudorand.h>
 #include <common/random_select.h>
 #include <common/status.h>
 #include <common/timeout.h>
 #include <gossipd/gossipd.h>
+#include <gossipd/gossmap_manage.h>
 #include <gossipd/queries.h>
-#include <gossipd/routing.h>
 #include <gossipd/seeker.h>
 
 #define GOSSIP_SEEKER_INTERVAL(seeker) \
@@ -441,33 +443,38 @@ static int cmp_scid(const struct short_channel_id *a,
 
 /* We can't ask for channels by node_id, so probe at random */
 static bool get_unannounced_nodes(const tal_t *ctx,
-				  struct routing_state *rstate,
+				  struct daemon *daemon,
 				  size_t max,
 				  struct short_channel_id **scids,
 				  u8 **query_flags)
 {
-	size_t num = 0;
-	u64 offset;
-	double total_weight = 0.0;
+	size_t num = 0, off, max_idx;
+	struct gossmap *gossmap = gossmap_manage_get_gossmap(daemon->gm);
 
-	/* Pick an example short_channel_id at random to query.  As a
-	 * side-effect this gets the node. */
+	/* No nodes?  Nothing to ask for */
+	max_idx = gossmap_max_node_idx(gossmap);
+	if (max_idx == 0 || max == 0)
+		return false;
+
+	/* Start at random index */
+	off = pseudorand(max_idx);
 	*scids = tal_arr(ctx, struct short_channel_id, max);
 
-	/* FIXME: This is inefficient!  Reuse next_block_range here! */
-	for (struct chan *c = uintmap_first(&rstate->chanmap, &offset);
-	     c;
-	     c = uintmap_after(&rstate->chanmap, &offset)) {
-		if (c->nodes[0]->bcast.index && c->nodes[1]->bcast.index)
+	for (size_t i = 0; i < max_idx; i++) {
+		const struct gossmap_node *node = gossmap_node_byidx(gossmap, (off + i) % max_idx);
+		const struct gossmap_chan *chan;
+
+		if (!node)
+			continue;
+		if (gossmap_node_announced(node))
 			continue;
 
-		if (num < max) {
-			(*scids)[num++] = c->scid;
-		} else {
-			/* Maybe replace one: approx. reservoir sampling */
-			if (random_select(1.0, &total_weight))
-				(*scids)[pseudorand(max)] = c->scid;
-		}
+		/* Query first chan. */
+		chan = gossmap_nth_chan(gossmap, node, 0, NULL);
+		(*scids)[num++] = gossmap_chan_scid(gossmap, chan);
+
+		if (num == max)
+			break;
 	}
 
 	if (num == 0) {
@@ -478,18 +485,23 @@ static bool get_unannounced_nodes(const tal_t *ctx,
 	if (num < max)
 		tal_resize(scids, num);
 
-	/* Sort them into order. */
+	/* Sort them into order, and remove duplicates! */
 	asort(*scids, num, cmp_scid, NULL);
+	for (size_t i = 1; i < tal_count(*scids); i++) {
+		if (short_channel_id_eq(&(*scids)[i], &(*scids)[i-1])) {
+			tal_arr_remove(scids, i);
+		}
+	}
 
 	/* Now get flags. */
-	*query_flags = tal_arr(ctx, u8, num);
+	*query_flags = tal_arr(ctx, u8, tal_count(*scids));
 	for (size_t i = 0; i < tal_count(*scids); i++) {
-		struct chan *c = get_channel(rstate, &(*scids)[i]);
+		const struct gossmap_chan *chan = gossmap_find_chan(gossmap, &(*scids)[i]);
 
 		(*query_flags)[i] = 0;
-		if (!c->nodes[0]->bcast.index)
+		if (!gossmap_node_announced(gossmap_nth_node(gossmap, chan, 0)))
 			(*query_flags)[i] |= SCID_QF_NODE1;
-		if (!c->nodes[1]->bcast.index)
+		if (!gossmap_node_announced(gossmap_nth_node(gossmap, chan, 1)))
 			(*query_flags)[i] |= SCID_QF_NODE2;
 	}
 	return true;
@@ -500,9 +512,10 @@ static void peer_gossip_probe_nannounces(struct seeker *seeker);
 
 static void nodeannounce_query_done(struct peer *peer, bool complete)
 {
-	struct seeker *seeker = peer->daemon->seeker;
-	struct routing_state *rstate = seeker->daemon->rstate;
+	struct daemon *daemon = peer->daemon;
+	struct seeker *seeker = daemon->seeker;
 	size_t new_nannounce = 0, num_scids;
+	struct gossmap *gossmap = gossmap_manage_get_gossmap(daemon->gm);
 
 	/* We might have given up on them, then they replied. */
 	if (seeker->random_peer != peer) {
@@ -514,16 +527,16 @@ static void nodeannounce_query_done(struct peer *peer, bool complete)
 
 	num_scids = tal_count(seeker->nannounce_scids);
 	for (size_t i = 0; i < num_scids; i++) {
-		struct chan *c = get_channel(rstate,
-					     &seeker->nannounce_scids[i]);
+		struct gossmap_chan *c = gossmap_find_chan(gossmap,
+							   &seeker->nannounce_scids[i]);
 		/* Could have closed since we asked. */
 		if (!c)
 			continue;
 		if ((seeker->nannounce_query_flags[i] & SCID_QF_NODE1)
-		    && c->nodes[0]->bcast.index)
+		    && gossmap_node_announced(gossmap_nth_node(gossmap, c, 0)))
 			new_nannounce++;
 		if ((seeker->nannounce_query_flags[i] & SCID_QF_NODE2)
-		    && c->nodes[1]->bcast.index)
+		    && gossmap_node_announced(gossmap_nth_node(gossmap, c, 1)))
 			new_nannounce++;
 	}
 
@@ -550,7 +563,7 @@ static void nodeannounce_query_done(struct peer *peer, bool complete)
 	if (num_scids > 7000)
 		num_scids = 7000;
 
-	if (!get_unannounced_nodes(seeker, seeker->daemon->rstate, num_scids,
+	if (!get_unannounced_nodes(seeker, seeker->daemon, num_scids,
 				   &seeker->nannounce_scids,
 				   &seeker->nannounce_query_flags)) {
 		/* Nothing unknown at all?  Great, we're done */
@@ -583,26 +596,34 @@ static void peer_gossip_probe_nannounces(struct seeker *seeker)
 }
 
 /* They have update with this timestamp: do we want it? */
-static bool want_update(struct seeker *seeker,
-			u32 timestamp, const struct half_chan *hc)
+static bool want_update(struct gossmap *gossmap,
+			u32 timestamp,
+			const struct gossmap_chan *chan,
+			int dir)
 {
-	if (!is_halfchan_defined(hc))
+	u32 our_timestamp;
+
+	if (!gossmap_chan_set(chan, dir))
 		return timestamp != 0;
 
-	if (timestamp <= hc->bcast.timestamp)
+	gossmap_chan_get_update_details(gossmap, chan, dir, &our_timestamp,
+					NULL, NULL, NULL, NULL, NULL, NULL);
+	if (timestamp <= our_timestamp)
 		return false;
 
 	return true;
 }
 
 /* They gave us timestamps.  Do we want updated versions? */
 static void check_timestamps(struct seeker *seeker,
-			     struct chan *c,
+			     struct gossmap *gossmap,
+			     struct gossmap_chan *c,
 			     const struct channel_update_timestamps *ts,
 			     struct peer *peer)
 {
 	u8 *stale;
 	u8 query_flag = 0;
+	struct short_channel_id scid;
 
 	/* BOLT #7:
 	 * * `timestamp_node_id_1` is the timestamp of the `channel_update`
@@ -612,19 +633,20 @@ static void check_timestamps(struct seeker *seeker,
 	 *    for `node_id_2`, or 0 if there was no `channel_update` from that
 	 *    node.
 	 */
-	if (want_update(seeker, ts->timestamp_node_id_1, &c->half[0]))
+	if (want_update(gossmap, ts->timestamp_node_id_1, c, 0))
 		query_flag |= SCID_QF_UPDATE1;
-	if (want_update(seeker, ts->timestamp_node_id_2, &c->half[1]))
+	if (want_update(gossmap, ts->timestamp_node_id_2, c, 1))
 		query_flag |= SCID_QF_UPDATE2;
 
 	if (!query_flag)
 		return;
 
 	/* Add in flags if we're already getting it. */
-	stale = uintmap_get(&seeker->stale_scids, c->scid.u64);
+	scid = gossmap_chan_scid(gossmap, c);
+	stale = uintmap_get(&seeker->stale_scids, scid.u64);
 	if (!stale) {
 		stale = talz(seeker, u8);
-		uintmap_add(&seeker->stale_scids, c->scid.u64, stale);
+		uintmap_add(&seeker->stale_scids, scid.u64, stale);
 		set_preferred_peer(seeker, peer);
 	}
 	*stale |= query_flag;
@@ -646,20 +668,23 @@ static void process_scid_probe(struct peer *peer,
 			       u32 first_blocknum, u32 number_of_blocks,
 			       const struct range_query_reply *replies)
 {
-	struct seeker *seeker = peer->daemon->seeker;
+	struct daemon *daemon = peer->daemon;
+	struct seeker *seeker = daemon->seeker;
 	bool new_unknown_scids = false;
+	struct gossmap *gossmap;
 
 	/* We might have given up on them, then they replied. */
 	if (seeker->random_peer != peer)
 		return;
 
 	seeker->random_peer = NULL;
+	gossmap = gossmap_manage_get_gossmap(daemon->gm);
 
 	for (size_t i = 0; i < tal_count(replies); i++) {
-		struct chan *c = get_channel(seeker->daemon->rstate,
-					     &replies[i].scid);
+		struct gossmap_chan *c = gossmap_find_chan(gossmap,
+							   &replies[i].scid);
 		if (c) {
-			check_timestamps(seeker, c, &replies[i].ts, peer);
+			check_timestamps(seeker, gossmap, c, &replies[i].ts, peer);
 			continue;
 		}
 
@@ -686,7 +711,7 @@ static void process_scid_probe(struct peer *peer,
 	}
 
 	/* Channel probe finished, try asking for 128 unannounced nodes. */
-	if (!get_unannounced_nodes(seeker, seeker->daemon->rstate, 128,
+	if (!get_unannounced_nodes(seeker, seeker->daemon, 128,
 				   &seeker->nannounce_scids,
 				   &seeker->nannounce_query_flags)) {
 		/* No unknown nodes.  Great! */