wallet: fix up PSBTs as a migration.

In the now-misnamed "last_tx" field.

Signed-off-by: Rusty Russell <[email protected]>

gossipd/test/run-check_channel_announcement.c

@@ -86,14 +86,6 @@ const u8 *gossip_store_get_private_update(const tal_t *ctx UNNEEDED,
 void gossip_store_mark_channel_deleted(struct gossip_store *gs UNNEEDED,
 				       const struct short_channel_id *scid UNNEEDED)
 { fprintf(stderr, "gossip_store_mark_channel_deleted called!\n"); abort(); }
-/* Generated stub for gossip_store_mark_channel_zombie */
-void gossip_store_mark_channel_zombie(struct gossip_store *gs UNNEEDED,
-				      struct broadcastable *bcast UNNEEDED)
-{ fprintf(stderr, "gossip_store_mark_channel_zombie called!\n"); abort(); }
-/* Generated stub for gossip_store_mark_cupdate_zombie */
-void gossip_store_mark_cupdate_zombie(struct gossip_store *gs UNNEEDED,
-				      struct broadcastable *bcast UNNEEDED)
-{ fprintf(stderr, "gossip_store_mark_cupdate_zombie called!\n"); abort(); }
 /* Generated stub for gossip_store_new */
 struct gossip_store *gossip_store_new(struct routing_state *rstate UNNEEDED,
 				      struct list_head *peers UNNEEDED)

gossipd/test/run-txout_failure.c

@@ -57,14 +57,6 @@ const u8 *gossip_store_get_private_update(const tal_t *ctx UNNEEDED,
 void gossip_store_mark_channel_deleted(struct gossip_store *gs UNNEEDED,
 				       const struct short_channel_id *scid UNNEEDED)
 { fprintf(stderr, "gossip_store_mark_channel_deleted called!\n"); abort(); }
-/* Generated stub for gossip_store_mark_channel_zombie */
-void gossip_store_mark_channel_zombie(struct gossip_store *gs UNNEEDED,
-				      struct broadcastable *bcast UNNEEDED)
-{ fprintf(stderr, "gossip_store_mark_channel_zombie called!\n"); abort(); }
-/* Generated stub for gossip_store_mark_cupdate_zombie */
-void gossip_store_mark_cupdate_zombie(struct gossip_store *gs UNNEEDED,
-				      struct broadcastable *bcast UNNEEDED)
-{ fprintf(stderr, "gossip_store_mark_cupdate_zombie called!\n"); abort(); }
 /* Generated stub for memleak_add_helper_ */
 void memleak_add_helper_(const tal_t *p UNNEEDED, void (*cb)(struct htable *memtable UNNEEDED,
 						    const tal_t *)){ }

lightningd/test/run-invoice-select-inchan.c

@@ -14,8 +14,7 @@ struct channel *any_channel_by_scid(struct lightningd *ld UNNEEDED,
 				    bool privacy_leak_ok UNNEEDED)
 { fprintf(stderr, "any_channel_by_scid called!\n"); abort(); }
 /* Generated stub for bip32_pubkey */
-void bip32_pubkey(struct lightningd *ld UNNEEDED,
-		  struct pubkey *pubkey UNNEEDED, u32 index UNNEEDED)
+void bip32_pubkey(struct lightningd *ld UNNEEDED, struct pubkey *pubkey UNNEEDED, u32 index UNNEEDED)
 { fprintf(stderr, "bip32_pubkey called!\n"); abort(); }
 /* Generated stub for bitcoind_getutxout_ */
 void bitcoind_getutxout_(struct bitcoind *bitcoind UNNEEDED,

tests/test_db.py

@@ -249,6 +249,8 @@ def test_backfill_scriptpubkeys(node_factory, bitcoind):
 
     # Test the first time, all entries are with option_static_remotekey
     l1 = node_factory.get_node(node_id=3, dbfile='pubkey_regen.sqlite.xz',
+                               # Our db had the old non-DER sig in psbt!
+                               allow_broken_log=True,
                                options={'database-upgrade': True})
     results = l1.db_query('SELECT hex(prev_out_tx) AS txid, hex(scriptpubkey) AS script FROM outputs')
     scripts = [{'txid': x['txid'], 'scriptpubkey': x['script']} for x in results]
@@ -284,6 +286,8 @@ def test_backfill_scriptpubkeys(node_factory, bitcoind):
     l1.stop()
 
     l2 = node_factory.get_node(node_id=3, dbfile='pubkey_regen_commitment_point.sqlite3.xz',
+                               # Our db had the old non-DER sig in psbt!
+                               allow_broken_log=True,
                                options={'database-upgrade': True})
     results = l2.db_query('SELECT hex(prev_out_tx) AS txid, hex(scriptpubkey) AS script FROM outputs')
     scripts = [{'txid': x['txid'], 'scriptpubkey': x['script']} for x in results]
@@ -363,6 +367,8 @@ def test_local_basepoints_cache(bitcoind, node_factory):
     l1 = node_factory.get_node(
         dbfile='no-local-basepoints.sqlite3.xz',
         start=False,
+        # Our db had the old non-DER sig in psbt!
+        allow_broken_log=True,
         options={'database-upgrade': True}
     )
 

wallet/db.c

@@ -16,6 +16,7 @@
 #include <lightningd/hsm_control.h>
 #include <lightningd/plugin_hook.h>
 #include <wallet/db.h>
+#include <wallet/psbt_fixup.h>
 #include <wire/wire_sync.h>
 
 struct migration {
@@ -54,6 +55,9 @@ static void fillin_missing_lease_satoshi(struct lightningd *ld,
 static void fillin_missing_lease_satoshi(struct lightningd *ld,
 					 struct db *db);
 
+static void migrate_invalid_last_tx_psbts(struct lightningd *ld,
+					  struct db *db);
+
 /* Do not reorder or remove elements from this array, it is used to
  * migrate existing databases from a previous state, based on the
  * string indices */
@@ -938,6 +942,7 @@ static struct migration dbmigrations[] = {
     {SQL("ALTER TABLE channels ADD require_confirm_inputs_remote INTEGER DEFAULT 0;"), NULL},
     {SQL("ALTER TABLE channels ADD require_confirm_inputs_local INTEGER DEFAULT 0;"), NULL},
     {NULL, fillin_missing_lease_satoshi},
+    {NULL, migrate_invalid_last_tx_psbts},
 };
 
 /**
@@ -1561,3 +1566,76 @@ static void fillin_missing_lease_satoshi(struct lightningd *ld,
 	db_exec_prepared_v2(stmt);
 	tal_free(stmt);
 }
+
+static void complain_unfixed(struct lightningd *ld,
+			     enum channel_state state,
+			     u64 id,
+			     const u8 *bytes,
+			     const char *why)
+{
+	/* This is OK on closed channels */
+	if (state != CLOSED) {
+		log_broken(ld->log,
+			   "%s channel id %"PRIu64" PSBT hex '%s'",
+			   why, id, tal_hex(tmpctx, bytes));
+	} else {
+		log_debug(ld->log,
+			  "%s on closed channel id %"PRIu64" PSBT hex '%s'",
+			  why, id, tal_hex(tmpctx, bytes));
+	}
+}
+
+static void migrate_invalid_last_tx_psbts(struct lightningd *ld,
+					  struct db *db)
+{
+	struct db_stmt *stmt;
+
+	/* We try all of them, but note that last_tx used to be a tx,
+	 * and migrate_last_tx_to_psbt didn't convert channels which had
+	 * already been closed, so we expect some failures. */
+	stmt = db_prepare_v2(db, SQL("SELECT "
+				     "  id"
+				     ", state"
+				     ", last_tx"
+				     " FROM channels"));
+
+	db_query_prepared(stmt);
+	while (db_step(stmt)) {
+		struct db_stmt *update_stmt;
+		const u8 *bytes, *fixed;
+		enum channel_state state;
+		u64 id;
+		struct wally_psbt *psbt;
+
+		state = db_col_int(stmt, "state");
+		id = db_col_u64(stmt, "id");
+
+		/* Parses fine? */
+		if (db_col_psbt(tmpctx, stmt, "last_tx"))
+			continue;
+
+		/* Can we fix it? */
+		bytes = db_col_arr(tmpctx, stmt, "last_tx", u8);
+		fixed = psbt_fixup(tmpctx, bytes);
+		if (!fixed) {
+			complain_unfixed(ld, state, id, bytes, "Could not fix");
+			continue;
+		}
+		psbt = psbt_from_bytes(tmpctx, fixed, tal_bytelen(fixed));
+		if (!psbt) {
+			complain_unfixed(ld, state, id, fixed, "Fix made invalid psbt");
+			continue;
+		}
+
+		log_broken(ld->log, "Forced database repair of psbt %s -> %s",
+			   tal_hex(tmpctx, bytes), tal_hex(tmpctx, fixed));
+		update_stmt = db_prepare_v2(db, SQL("UPDATE channels"
+						    " SET last_tx = ?"
+						    " WHERE id = ?;"));
+		db_bind_psbt(update_stmt, 0, psbt);
+		db_bind_u64(update_stmt, 1, id);
+		db_exec_prepared_v2(update_stmt);
+		tal_free(update_stmt);
+	}
+	tal_free(stmt);
+}

wallet/test/run-db.c

@@ -21,8 +21,7 @@ static void db_log_(struct log *log UNUSED, enum log_level level UNUSED, const s
 
 /* AUTOGENERATED MOCKS START */
 /* Generated stub for bip32_pubkey */
-void bip32_pubkey(struct lightningd *ld UNNEEDED,
-		  struct pubkey *pubkey UNNEEDED, u32 index UNNEEDED)
+void bip32_pubkey(struct lightningd *ld UNNEEDED, struct pubkey *pubkey UNNEEDED, u32 index UNNEEDED)
 { fprintf(stderr, "bip32_pubkey called!\n"); abort(); }
 /* Generated stub for derive_channel_id */
 void derive_channel_id(struct channel_id *channel_id UNNEEDED,
@@ -44,6 +43,9 @@ void get_channel_basepoints(struct lightningd *ld UNNEEDED,
 			    struct basepoints *local_basepoints UNNEEDED,
 			    struct pubkey *local_funding_pubkey UNNEEDED)
 { fprintf(stderr, "get_channel_basepoints called!\n"); abort(); }
+/* Generated stub for psbt_fixup */
+const u8 *psbt_fixup(const tal_t *ctx UNNEEDED, const u8 *psbtblob UNNEEDED)
+{ fprintf(stderr, "psbt_fixup called!\n"); abort(); }
 /* Generated stub for towire_hsmd_get_channel_basepoints */
 u8 *towire_hsmd_get_channel_basepoints(const tal_t *ctx UNNEEDED, const struct node_id *peerid UNNEEDED, u64 dbid UNNEEDED)
 { fprintf(stderr, "towire_hsmd_get_channel_basepoints called!\n"); abort(); }

wallet/test/run-wallet.c

@@ -721,6 +721,9 @@ struct route_step *process_onionpacket(
 	bool has_realm
 	)
 { fprintf(stderr, "process_onionpacket called!\n"); abort(); }
+/* Generated stub for psbt_fixup */
+const u8 *psbt_fixup(const tal_t *ctx UNNEEDED, const u8 *psbtblob UNNEEDED)
+{ fprintf(stderr, "psbt_fixup called!\n"); abort(); }
 /* Generated stub for report_subd_memleak */
 void report_subd_memleak(struct leak_detect *leak_detect UNNEEDED, struct subd *leaker UNNEEDED)
 { fprintf(stderr, "report_subd_memleak called!\n"); abort(); }