bkpr: exclude non-wallet events in the balance snapshot

Anchor outputs are ignored by the clightning wallet, but we keep track
of them in the bookkeeper. This causes problems when we do the balance
checks on restart w/ the balance_snapshot -- it results in us printing
out a journal_entry to 'get rid of' the anchors that the clightning node
doesnt know about.

Instead, we mark some outputs as 'ignored' and exclude these from our
account balance sums when we're comparing to the clightning snapshot.

common/coin_mvt.c

@@ -288,6 +288,19 @@ struct chain_coin_mvt *new_coin_wallet_deposit(const tal_t *ctx,
 				      amount, true);
 }
 
+struct chain_coin_mvt *new_coin_wallet_deposit_tagged(const tal_t *ctx,
+						      const struct bitcoin_outpoint *outpoint,
+						      u32 blockheight,
+						      struct amount_sat amount,
+						      enum mvt_tag *tags TAKES)
+{
+	return new_chain_coin_mvt_sat(ctx, WALLET, NULL,
+				      outpoint, NULL,
+				      blockheight,
+				      take(tags),
+				      amount, true);
+}
+
 struct chain_coin_mvt *new_coin_wallet_withdraw(const tal_t *ctx,
 						const struct bitcoin_txid *spend_txid,
 						const struct bitcoin_outpoint *outpoint,

common/coin_mvt.h

@@ -211,6 +211,13 @@ struct chain_coin_mvt *new_coin_wallet_deposit(const tal_t *ctx,
 					       enum mvt_tag tag)
 	NON_NULL_ARGS(2);
 
+struct chain_coin_mvt *new_coin_wallet_deposit_tagged(const tal_t *ctx,
+						      const struct bitcoin_outpoint *outpoint,
+						      u32 blockheight,
+						      struct amount_sat amount,
+						      enum mvt_tag *tags TAKES)
+	NON_NULL_ARGS(2);
+
 struct chain_coin_mvt *new_coin_wallet_withdraw(const tal_t *ctx,
 						const struct bitcoin_txid *spend_txid,
 						const struct bitcoin_outpoint *outpoint,

onchaind/onchaind.c

@@ -335,10 +335,13 @@ static void record_ignored_wallet_deposit(struct tracked_output *out)
 
 static void record_anchor(struct tracked_output *out)
 {
-	send_coin_mvt(take(new_coin_wallet_deposit(NULL,
+	enum mvt_tag *tags = new_tag_arr(NULL, ANCHOR);
+	tal_arr_expand(&tags, IGNORED);
+	send_coin_mvt(take(new_coin_wallet_deposit_tagged(NULL,
 					&out->outpoint,
 					out->tx_blockheight,
-					out->sat, ANCHOR)));
+					out->sat,
+					tags)));
 }
 
 static void record_coin_movements(struct tracked_output *out,

onchaind/test/run-grind_feerate-bug.c

@@ -140,14 +140,14 @@ struct chain_coin_mvt *new_coin_external_spend(const tal_t *ctx UNNEEDED,
 					       enum mvt_tag tag)
 
 { fprintf(stderr, "new_coin_external_spend called!\n"); abort(); }
-/* Generated stub for new_coin_wallet_deposit */
-struct chain_coin_mvt *new_coin_wallet_deposit(const tal_t *ctx UNNEEDED,
-					       const struct bitcoin_outpoint *outpoint UNNEEDED,
-					       u32 blockheight UNNEEDED,
-					       struct amount_sat amount UNNEEDED,
-					       enum mvt_tag tag)
-
-{ fprintf(stderr, "new_coin_wallet_deposit called!\n"); abort(); }
+/* Generated stub for new_coin_wallet_deposit_tagged */
+struct chain_coin_mvt *new_coin_wallet_deposit_tagged(const tal_t *ctx UNNEEDED,
+						      const struct bitcoin_outpoint *outpoint UNNEEDED,
+						      u32 blockheight UNNEEDED,
+						      struct amount_sat amount UNNEEDED,
+						      enum mvt_tag *tags TAKES)
+
+{ fprintf(stderr, "new_coin_wallet_deposit_tagged called!\n"); abort(); }
 /* Generated stub for new_onchain_htlc_deposit */
 struct chain_coin_mvt *new_onchain_htlc_deposit(const tal_t *ctx UNNEEDED,
 						const struct bitcoin_outpoint *outpoint UNNEEDED,
@@ -181,6 +181,9 @@ struct chain_coin_mvt *new_onchaind_withdraw(const tal_t *ctx UNNEEDED,
 					     enum mvt_tag tag)
 
 { fprintf(stderr, "new_onchaind_withdraw called!\n"); abort(); }
+/* Generated stub for new_tag_arr */
+enum mvt_tag *new_tag_arr(const tal_t *ctx UNNEEDED, enum mvt_tag tag UNNEEDED)
+{ fprintf(stderr, "new_tag_arr called!\n"); abort(); }
 /* Generated stub for notleak_ */
 void *notleak_(void *ptr UNNEEDED, bool plus_children UNNEEDED)
 { fprintf(stderr, "notleak_ called!\n"); abort(); }

onchaind/test/run-grind_feerate.c

@@ -163,14 +163,14 @@ struct chain_coin_mvt *new_coin_external_spend(const tal_t *ctx UNNEEDED,
 					       enum mvt_tag tag)
 
 { fprintf(stderr, "new_coin_external_spend called!\n"); abort(); }
-/* Generated stub for new_coin_wallet_deposit */
-struct chain_coin_mvt *new_coin_wallet_deposit(const tal_t *ctx UNNEEDED,
-					       const struct bitcoin_outpoint *outpoint UNNEEDED,
-					       u32 blockheight UNNEEDED,
-					       struct amount_sat amount UNNEEDED,
-					       enum mvt_tag tag)
+/* Generated stub for new_coin_wallet_deposit_tagged */
+struct chain_coin_mvt *new_coin_wallet_deposit_tagged(const tal_t *ctx UNNEEDED,
+						      const struct bitcoin_outpoint *outpoint UNNEEDED,
+						      u32 blockheight UNNEEDED,
+						      struct amount_sat amount UNNEEDED,
+						      enum mvt_tag *tags TAKES)
 
-{ fprintf(stderr, "new_coin_wallet_deposit called!\n"); abort(); }
+{ fprintf(stderr, "new_coin_wallet_deposit_tagged called!\n"); abort(); }
 /* Generated stub for new_onchain_htlc_deposit */
 struct chain_coin_mvt *new_onchain_htlc_deposit(const tal_t *ctx UNNEEDED,
 						const struct bitcoin_outpoint *outpoint UNNEEDED,
@@ -204,6 +204,9 @@ struct chain_coin_mvt *new_onchaind_withdraw(const tal_t *ctx UNNEEDED,
 					     enum mvt_tag tag)
 
 { fprintf(stderr, "new_onchaind_withdraw called!\n"); abort(); }
+/* Generated stub for new_tag_arr */
+enum mvt_tag *new_tag_arr(const tal_t *ctx UNNEEDED, enum mvt_tag tag UNNEEDED)
+{ fprintf(stderr, "new_tag_arr called!\n"); abort(); }
 /* Generated stub for notleak_ */
 void *notleak_(void *ptr UNNEEDED, bool plus_children UNNEEDED)
 { fprintf(stderr, "notleak_ called!\n"); abort(); }

plugins/bkpr/bookkeeper.c

@@ -471,6 +471,7 @@ static struct command_result *json_list_balances(struct command *cmd,
 		err = account_get_balance(cmd, db,
 					  accts[i]->name,
 					  true,
+					  false, /* don't skip ignored */
 					  &balances);
 
 		if (err)
@@ -596,6 +597,7 @@ static bool new_missed_channel_account(struct command *cmd,
 			chain_ev->outpoint = opt;
 			chain_ev->spending_txid = NULL;
 			chain_ev->payment_id = NULL;
+			chain_ev->ignored = false;
 
 			/* Update the account info too */
 			tags = tal_arr(chain_ev, enum mvt_tag, 1);
@@ -799,7 +801,7 @@ listpeers_multi_done(struct command *cmd,
 
 		db_begin_transaction(db);
 		err = account_get_balance(tmpctx, db, info->acct->name,
-					  false, &balances);
+					  false, false, &balances);
 		db_commit_transaction(db);
 
 		if (err)
@@ -880,7 +882,7 @@ listpeers_done(struct command *cmd, const char *buf,
 					info->ev->timestamp)) {
 		db_begin_transaction(db);
 		err = account_get_balance(tmpctx, db, info->acct->name,
-					  false, &balances);
+					  false, false, &balances);
 		db_commit_transaction(db);
 
 		if (err)
@@ -993,6 +995,9 @@ static struct command_result *json_balance_snapshot(struct command *cmd,
 		err = account_get_balance(cmd, db, acct_name,
 					  /* Don't error if negative */
 					  false,
+					  /* Ignore non-clightning
+					   * balances items */
+					  true,
 					  &balances);
 
 		if (err)
@@ -1193,6 +1198,10 @@ parse_and_log_chain_move(struct command *cmd,
 	e->timestamp = timestamp;
 	e->tag = mvt_tag_str(tags[0]);
 
+	e->ignored = false;
+	for (size_t i = 0; i < tal_count(tags); i++)
+		e->ignored |= tags[i] == IGNORED;
+
 	db_begin_transaction(db);
 	acct = find_account(cmd, db, acct_name);
 

plugins/bkpr/chain_event.h

@@ -27,6 +27,9 @@ struct chain_event {
 	/* Tag describing the event */
 	const char *tag;
 
+	/* Is the node's wallet ignoring this? */
+	bool ignored;
+
 	/* Amount we received in this event */
 	struct amount_msat credit;
 

plugins/bkpr/db.c

@@ -94,6 +94,7 @@ static struct migration db_migrations[] = {
 	NULL},
 	{SQL("ALTER TABLE chain_events ADD origin TEXT;"), NULL},
 	{SQL("ALTER TABLE accounts ADD closed_count INTEGER DEFAULT 0;"), NULL},
+	{SQL("ALTER TABLE chain_events ADD ignored INTEGER;"), NULL},
 };
 
 static bool db_migrate(struct plugin *p, struct db *db)

plugins/bkpr/recorder.c

@@ -55,6 +55,8 @@ static struct chain_event *stmt2chain_event(const tal_t *ctx, struct db_stmt *st
 	} else
 		e->spending_txid = NULL;
 
+	e->ignored = db_col_int(stmt, "e.ignored") == 1;
+
 	return e;
 }
 
@@ -125,6 +127,7 @@ struct chain_event **list_chain_events_timebox(const tal_t *ctx,
 				     ", e.outnum"
 				     ", e.spending_txid"
 				     ", e.payment_id"
+				     ", e.ignored"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON e.account_id = a.id"
@@ -164,6 +167,7 @@ struct chain_event **account_get_chain_events(const tal_t *ctx,
 				     ", e.outnum"
 				     ", e.spending_txid"
 				     ", e.payment_id"
+				     ", e.ignored"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON e.account_id = a.id"
@@ -196,6 +200,7 @@ static struct chain_event **find_txos_for_tx(const tal_t *ctx,
 				     ", e.outnum"
 				     ", e.spending_txid"
 				     ", e.payment_id"
+				     ", e.ignored"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON e.account_id = a.id"
@@ -591,6 +596,7 @@ struct chain_event *find_chain_event_by_id(const tal_t *ctx,
 				     ", e.outnum"
 				     ", e.spending_txid"
 				     ", e.payment_id"
+				     ", e.ignored"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON e.account_id = a.id"
@@ -635,6 +641,7 @@ static struct chain_event *find_chain_event(const tal_t *ctx,
 					     ", e.outnum"
 					     ", e.spending_txid"
 					     ", e.payment_id"
+					     ", e.ignored"
 					     " FROM chain_events e"
 					     " LEFT OUTER JOIN accounts a"
 					     " ON e.account_id = a.id"
@@ -661,6 +668,7 @@ static struct chain_event *find_chain_event(const tal_t *ctx,
 					     ", e.outnum"
 					     ", e.spending_txid"
 					     ", e.payment_id"
+					     ", e.ignored"
 					     " FROM chain_events e"
 					     " LEFT OUTER JOIN accounts a"
 					     " ON e.account_id = a.id"
@@ -689,6 +697,7 @@ char *account_get_balance(const tal_t *ctx,
 			  struct db *db,
 			  const char *acct_name,
 			  bool calc_sum,
+			  bool skip_ignored,
 			  struct acct_balance ***balances)
 {
 	struct db_stmt *stmt;
@@ -701,9 +710,13 @@ char *account_get_balance(const tal_t *ctx,
 				     " LEFT OUTER JOIN accounts a"
 				     " ON a.id = ce.account_id"
 				     " WHERE a.name = ?"
+				     " AND ce.ignored != ?"
 				     " GROUP BY ce.currency"));
 
 	db_bind_text(stmt, 0, acct_name);
+	/* We populate ignored with a 0 or 1,
+	 * if we want both 0+1, we just ignore everything with a 2 */
+	db_bind_int(stmt, 1, skip_ignored ? 1 : 2);
 	db_query_prepared(stmt);
 	*balances = tal_arr(ctx, struct acct_balance *, 0);
 
@@ -1287,6 +1300,7 @@ static struct chain_event **find_chain_events_bytxid(const tal_t *ctx, struct db
 				     ", e.outnum"
 				     ", e.spending_txid"
 				     ", e.payment_id"
+				     ", e.ignored"
 				     " FROM chain_events e"
 				     " LEFT OUTER JOIN accounts a"
 				     " ON a.id = e.account_id"
@@ -1775,9 +1789,10 @@ bool log_chain_event(struct db *db,
 				     ", outnum"
 				     ", payment_id"
 				     ", spending_txid"
+				     ", ignored"
 				     ")"
-				     " VALUES"
-				     " (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"));
+				     " VALUES "
+				     "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"));
 
 	db_bind_u64(stmt, 0, acct->db_id);
 	if (e->origin_acct)
@@ -1804,6 +1819,7 @@ bool log_chain_event(struct db *db,
 	else
 		db_bind_null(stmt, 12);
 
+	db_bind_int(stmt, 13, e->ignored ? 1 : 0);
 	db_exec_prepared_v2(stmt);
 	e->db_id = db_last_insert_id_v2(stmt);
 	e->acct_db_id = acct->db_id;

plugins/bkpr/recorder.h

@@ -91,12 +91,14 @@ struct chain_event **list_chain_events_timebox(const tal_t *ctx,
 
 /* Calculate the balances for an account
  *
- * @calc_sum - compute the total balance. error if negative
+ * @calc_sum     - compute the total balance. error if negative
+ * @skip_ignored - don't include ignored payments in the balance sum
  * */
 char *account_get_balance(const tal_t *ctx,
 			  struct db *db,
 			  const char *acct_name,
 			  bool calc_sum,
+			  bool skip_ignored,
 			  struct acct_balance ***balances);
 
 /* Get chain fees for account */