Update libwally to 0.8.8, support PSBTv2

Libwally update breaks compatibility, so
we do this in one large step.

Changelog-Changed: JSON-RPC: elements network PSET now only supports PSETv2.
Changelog-Added: JSON-RPC: PSBTv2 supported for fundchannel_complete, openchannel_update, reserveinputs, sendpsbt, signpsbt, withdraw and unreserveinputs parameter psbt, openchannel_init and openchannel_bump parameter initialpsbt, openchannel_signed parameter signed_psbt and utxopsbt parameter utxopsbt

bitcoin/psbt.h

@@ -29,7 +29,7 @@ struct wally_psbt *create_psbt(const tal_t *ctx, size_t num_inputs, size_t num_o
 
 /*
  * new_psbt - Create a PSBT, using the passed in tx
- * 	      as the global_tx
+ * 	      as the locktime/inputs/output psbt fields
  *
  * @ctx - allocation context
  * @wtx - global_tx starter kit
@@ -228,6 +228,33 @@ struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt);
 bool psbt_has_input(const struct wally_psbt *psbt,
 		    const struct bitcoin_outpoint *outpoint);
 
+/* wally_psbt_input_spends - Returns true if PSBT input spends given outpoint
+ *
+ * @input - psbt input
+ * @outpoint - outpoint
+ */
+bool wally_psbt_input_spends(const struct wally_psbt_input *input,
+               const struct bitcoin_outpoint *outpoint);
+
+void wally_psbt_input_get_outpoint(const struct wally_psbt_input *in,
+                 struct bitcoin_outpoint *outpoint);
+
+const u8 *wally_psbt_output_get_script(const tal_t *ctx,
+                     const struct wally_psbt_output *output);
+
+void wally_psbt_input_get_txid(const struct wally_psbt_input *in,
+                 struct bitcoin_txid *txid);
+
+struct amount_asset
+wally_psbt_output_get_amount(const struct wally_psbt_output *output);
+
+/* psbt_set_version - Returns false if there was any issue with the PSBT.
+ * Returns true if it was a well-formed PSET and treats it as a no-op
+ */
+bool psbt_set_version(struct wally_psbt *psbt, u32 version);
+
+bool elements_psbt_output_is_fee(const struct wally_psbt *psbt, size_t outnum);
+
 struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 const char *b64,
 				 size_t b64len);

bitcoin/test/run-bitcoin_block_from_hex.c

@@ -72,9 +72,6 @@ void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UN
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
-/* Generated stub for script_push_bytes */
-void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
-{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }

bitcoin/test/run-psbt-from-tx.c

@@ -53,9 +53,6 @@ void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UN
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
-/* Generated stub for script_push_bytes */
-void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
-{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }
@@ -105,7 +102,8 @@ int main(int argc, char *argv[])
 
 	/* Witness/scriptsig data is saved down into psbt */
 	assert(tx2->psbt->num_inputs == 1);
-	assert(tx2->psbt->inputs[0].final_scriptsig_len > 0);
+	const struct wally_map_item *final_scriptsig = wally_map_get_integer(&tx2->psbt->inputs[0].psbt_fields, /* PSBT_IN_FINAL_SCRIPTSIG */ 0x07);
+	assert(final_scriptsig->value_len > 0);
 	assert(tx2->psbt->inputs[0].final_witness != NULL);
 
 	common_shutdown();

bitcoin/test/run-tx-bitcoin_tx_2of2_input_witness_weight.c

@@ -81,10 +81,6 @@ struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt UNNEEDED,
 					   const u8 *input_wscript UNNEEDED,
 					   const u8 *redeemscript UNNEEDED)
 { fprintf(stderr, "psbt_append_input called!\n"); abort(); }
-/* Generated stub for psbt_elements_input_set_asset */
-void psbt_elements_input_set_asset(struct wally_psbt *psbt UNNEEDED, size_t in UNNEEDED,
-				   struct amount_asset *asset UNNEEDED)
-{ fprintf(stderr, "psbt_elements_input_set_asset called!\n"); abort(); }
 /* Generated stub for psbt_final_tx */
 struct wally_tx *psbt_final_tx(const tal_t *ctx UNNEEDED, const struct wally_psbt *psbt UNNEEDED)
 { fprintf(stderr, "psbt_final_tx called!\n"); abort(); }

bitcoin/test/run-tx-encode.c

@@ -73,9 +73,6 @@ void pubkey_to_der(u8 der[PUBKEY_CMPR_LEN] UNNEEDED, const struct pubkey *key UN
 /* Generated stub for pubkey_to_hash160 */
 void pubkey_to_hash160(const struct pubkey *pk UNNEEDED, struct ripemd160 *hash UNNEEDED)
 { fprintf(stderr, "pubkey_to_hash160 called!\n"); abort(); }
-/* Generated stub for script_push_bytes */
-void script_push_bytes(u8 **scriptp UNNEEDED, const void *mem UNNEEDED, size_t len UNNEEDED)
-{ fprintf(stderr, "script_push_bytes called!\n"); abort(); }
 /* Generated stub for scriptpubkey_p2wsh */
 u8 *scriptpubkey_p2wsh(const tal_t *ctx UNNEEDED, const u8 *witnessscript UNNEEDED)
 { fprintf(stderr, "scriptpubkey_p2wsh called!\n"); abort(); }

bitcoin/tx.c

@@ -180,7 +180,36 @@ static int elements_tx_add_fee_output(struct bitcoin_tx *tx)
 void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime)
 {
 	tx->wtx->locktime = locktime;
-	tx->psbt->tx->locktime = locktime;
+	tx->psbt->fallback_locktime = locktime;
+	tx->psbt->has_fallback_locktime = true;
+}
+
+/* FIXME Stolen from psbt_append_input; export? */
+static struct wally_tx_input *wally_tx_input_from_outpoint_sequence(const struct bitcoin_outpoint *outpoint,
+			u32 sequence)
+{
+	struct wally_tx_input *tx_in;
+    if (chainparams->is_elements) {
+        if (wally_tx_elements_input_init_alloc(outpoint->txid.shad.sha.u.u8,
+                               sizeof(outpoint->txid.shad.sha.u.u8),
+                               outpoint->n,
+                               sequence, NULL, 0,
+                               NULL,
+                               NULL, 0,
+                               NULL, 0, NULL, 0,
+                               NULL, 0, NULL, 0,
+                               NULL, 0, NULL,
+                               &tx_in) != WALLY_OK)
+            abort();
+    } else {
+        if (wally_tx_input_init_alloc(outpoint->txid.shad.sha.u.u8,
+                          sizeof(outpoint->txid.shad.sha.u.u8),
+                          outpoint->n,
+                          sequence, NULL, 0, NULL,
+                          &tx_in) != WALLY_OK)
+            abort();
+    }
+	return tx_in;
 }
 
 int bitcoin_tx_add_input(struct bitcoin_tx *tx,
@@ -191,6 +220,7 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 {
 	int wally_err;
 	int input_num = tx->wtx->num_inputs;
+	struct wally_tx_input *tx_input;
 
 	psbt_append_input(tx->psbt, outpoint,
 			  sequence, scriptSig,
@@ -205,22 +235,22 @@ int bitcoin_tx_add_input(struct bitcoin_tx *tx,
 				scriptPubkey, amount);
 
 	tal_wally_start();
+	tx_input = wally_tx_input_from_outpoint_sequence(outpoint, sequence);
 	wally_err = wally_tx_add_input(tx->wtx,
-				       &tx->psbt->tx->inputs[input_num]);
+			       tx_input);
 	assert(wally_err == WALLY_OK);
+	wally_tx_input_free(tx_input);
 
 	/* scriptsig isn't actually stored in psbt input, so add that now */
 	wally_tx_set_input_script(tx->wtx, input_num,
 				  scriptSig, tal_bytelen(scriptSig));
 	tal_wally_end(tx->wtx);
 
 	if (is_elements(chainparams)) {
-		struct amount_asset asset;
 		/* FIXME: persist asset tags */
-		asset = amount_sat_to_asset(&amount,
+		amount_sat_to_asset(&amount,
 					    chainparams->fee_asset_tag);
 		/* FIXME: persist nonces */
-		psbt_elements_input_set_asset(tx->psbt, input_num, &asset);
 	}
 	return input_num;
 }
@@ -258,10 +288,6 @@ void bitcoin_tx_output_set_amount(struct bitcoin_tx *tx, int outnum,
 		assert(ret == WALLY_OK);
 	} else {
 		output->satoshi = satoshis;
-
-		/* update the global tx for the psbt also */
-		output = &tx->psbt->tx->outputs[outnum];
-		output->satoshi = satoshis;
 	}
 }
 
@@ -291,14 +317,16 @@ u8 *bitcoin_tx_output_get_witscript(const tal_t *ctx, const struct bitcoin_tx *t
 				    int outnum)
 {
 	struct wally_psbt_output *out;
+	const struct wally_map_item *output_witness_script;
 
 	assert(outnum < tx->psbt->num_outputs);
 	out = &tx->psbt->outputs[outnum];
 
-	if (out->witness_script_len == 0)
+	output_witness_script = wally_map_get_integer(&out->psbt_fields, /* PSBT_OUT_WITNESS_SCRIPT */ 0x01);
+	if (output_witness_script->value_len == 0)
 		return NULL;
 
-	return tal_dup_arr(ctx, u8, out->witness_script, out->witness_script_len, 0);
+	return tal_dup_arr(ctx, u8, output_witness_script->value, output_witness_script->value_len, 0);
 }
 
 struct amount_asset bitcoin_tx_output_get_amount(const struct bitcoin_tx *tx,
@@ -536,18 +564,21 @@ void bitcoin_tx_finalize(struct bitcoin_tx *tx)
 
 struct bitcoin_tx *bitcoin_tx_with_psbt(const tal_t *ctx, struct wally_psbt *psbt STEALS)
 {
+	size_t locktime;
+	wally_psbt_get_locktime(psbt, &locktime);
 	struct bitcoin_tx *tx = bitcoin_tx(ctx, chainparams,
-					   psbt->tx->num_inputs,
-					   psbt->tx->num_outputs,
-					   psbt->tx->locktime);
+					   psbt->num_inputs,
+					   psbt->num_outputs,
+					   locktime);
 	wally_tx_free(tx->wtx);
 
 	psbt_finalize(psbt);
 	tx->wtx = psbt_final_tx(tx, psbt);
 	if (!tx->wtx) {
 		tal_wally_start();
-		if (wally_tx_clone_alloc(psbt->tx, 0, &tx->wtx) != WALLY_OK)
+		if (wally_psbt_extract(psbt, WALLY_PSBT_EXTRACT_NON_FINAL, &tx->wtx) != WALLY_OK) {
 			tx->wtx = NULL;
+		}
 		tal_wally_end_onto(tx, tx->wtx, struct wally_tx);
 		if (!tx->wtx)
 			return tal_free(tx);

common/permute_tx.c

@@ -3,7 +3,6 @@
 #include <wally_psbt.h>
 
 static void swap_wally_outputs(struct wally_tx_output *outputs,
-			       struct wally_tx_output *psbt_global_outs,
 			       struct wally_psbt_output *psbt_outs,
 			       const void **map, u32 *cltvs,
 			       size_t i1, size_t i2)
@@ -18,12 +17,6 @@ static void swap_wally_outputs(struct wally_tx_output *outputs,
 	outputs[i1] = outputs[i2];
 	outputs[i2] = tmpoutput;
 
-	/* For the PSBT, we swap the psbt outputs and
-	 * the global tx's outputs */
-	tmpoutput = psbt_global_outs[i1];
-	psbt_global_outs[i1] = psbt_global_outs[i2];
-	psbt_global_outs[i2] = tmpoutput;
-
 	tmppsbtout = psbt_outs[i1];
 	psbt_outs[i1] = psbt_outs[i2];
 	psbt_outs[i2] = tmppsbtout;
@@ -106,7 +99,6 @@ void permute_outputs(struct bitcoin_tx *tx, u32 *cltvs, const void **map)
 
 		/* Swap best into first place. */
 		swap_wally_outputs(tx->wtx->outputs,
-				   tx->psbt->tx->outputs,
 				   tx->psbt->outputs,
 				   map, cltvs, i, best_pos);
 	}

common/psbt_internal.c

@@ -26,6 +26,7 @@ void psbt_finalize_input(const tal_t *ctx,
 			 struct wally_psbt_input *in,
 			 const struct witness_element **elements)
 {
+	const struct wally_map_item *redeem_script;
 	psbt_input_set_final_witness_stack(ctx, in, elements);
 
 	/* There's this horrible edgecase where we set the final_witnesses
@@ -35,18 +36,16 @@ void psbt_finalize_input(const tal_t *ctx,
 	 * on these just .. ignores it!? Murder. Anyway, here we do a final
 	 * scriptsig check -- if there's a redeemscript field still around we
 	 * just go ahead and mush it into the final_scriptsig field. */
-	if (in->redeem_script) {
+	redeem_script = wally_map_get_integer(&in->psbt_fields, /* PSBT_IN_REDEEM_SCRIPT */ 0x04);
+	if (redeem_script) {
 		u8 *redeemscript = tal_dup_arr(NULL, u8,
-					       in->redeem_script,
-					       in->redeem_script_len, 0);
-		in->final_scriptsig =
+					       redeem_script->value,
+					       redeem_script->value_len, 0);
+		u8 *final_scriptsig =
 			bitcoin_scriptsig_redeem(NULL,
 						 take(redeemscript));
-		in->final_scriptsig_len =
-			tal_bytelen(in->final_scriptsig);
-
-		in->redeem_script = tal_free(in->redeem_script);
-		in->redeem_script_len = 0;
+		wally_psbt_input_set_final_scriptsig(in, final_scriptsig, tal_bytelen(final_scriptsig));
+		wally_psbt_input_set_redeem_script(in, tal_arr(NULL, u8, 0), 0);
 	}
 }
 

common/psbt_keypath.c

@@ -14,7 +14,7 @@ void psbt_set_keypath(u32 index, const struct ext_key *ext, struct wally_map *ma
 	u32 path[1];
 	path[0] = index;
 
-	if (wally_map_add_keypath_item(map_in,
+	if (wally_map_keypath_add(map_in,
 				       ext->pub_key, sizeof(ext->pub_key),
 				       fingerprint, sizeof(fingerprint),
 				       path, 1) != WALLY_OK)