Merge bitcoin#21470: BIP 350: Implement Bech32m and use it for v1+ se…

…gwit addresses (0.20 backport) c0f85fd Backport invalid address tests (Pieter Wuille) c670986 naming nits (Fabian Jahr) 1a4e88e Use Bech32m encoding for v1+ segwit addresses (Pieter Wuille) cf18ac9 Add Bech32m test vectors (Pieter Wuille) 5f9537b Implement Bech32m encoding/decoding (Pieter Wuille) Pull request description: Backport of bitcoin#20861 for 0.20. Also includes bitcoin#21471. ACKs for top commit: jnewbery: utACK c0f85fd MarcoFalke: range-diff-only ACK c0f85fd 🐔 Tree-SHA512: e87e52995cb9b555109bc500dba1e844993d881821d2001443b3de9e3ef9050ddb73deefa0c1af732418fa7355a86b875789887c9611c340713b3ad26809d58e
hrodrig · Apr 23, 2021 · 5631198 · 5631198
2 parents f1c3c53 + c0f85fd
commit 5631198
Show file tree

Hide file tree

Showing 15 changed files with 566 additions and 484 deletions.
diff --git a/contrib/testgen/gen_key_io_test_vectors.py b/contrib/testgen/gen_key_io_test_vectors.py
@@ -16,7 +16,7 @@
 from base58 import b58encode_chk, b58decode_chk, b58chars
 import random
 from binascii import b2a_hex
-from segwit_addr import bech32_encode, decode, convertbits, CHARSET
+from segwit_addr import bech32_encode, decode, convertbits, CHARSET, Encoding
 
 # key types
 PUBKEY_ADDRESS = 0
@@ -33,6 +33,7 @@
 OP_0 = 0x00
 OP_1 = 0x51
 OP_2 = 0x52
+OP_3 = 0x53
 OP_16 = 0x60
 OP_DUP = 0x76
 OP_EQUAL = 0x87
@@ -45,6 +46,7 @@
 script_suffix = (OP_EQUAL,)
 p2wpkh_prefix = (OP_0, 20)
 p2wsh_prefix = (OP_0, 32)
+p2tr_prefix = (OP_1, 32)
 
 metadata_keys = ['isPrivkey', 'chain', 'isCompressed', 'tryCaseFlip']
 # templates for valid sequences
@@ -66,29 +68,39 @@
 ]
 # templates for valid bech32 sequences
 bech32_templates = [
-  # hrp, version, witprog_size, metadata, output_prefix
-  ('bc',    0, 20, (False, 'main',    None, True), p2wpkh_prefix),
-  ('bc',    0, 32, (False, 'main',    None, True), p2wsh_prefix),
-  ('bc',    1,  2, (False, 'main',    None, True), (OP_1, 2)),
-  ('tb',    0, 20, (False, 'test',    None, True), p2wpkh_prefix),
-  ('tb',    0, 32, (False, 'test',    None, True), p2wsh_prefix),
-  ('tb',    2, 16, (False, 'test',    None, True), (OP_2, 16)),
-  ('bcrt',  0, 20, (False, 'regtest', None, True), p2wpkh_prefix),
-  ('bcrt',  0, 32, (False, 'regtest', None, True), p2wsh_prefix),
-  ('bcrt', 16, 40, (False, 'regtest', None, True), (OP_16, 40))
+  # hrp, version, witprog_size, metadata, encoding, output_prefix
+  ('bc',    0, 20, (False, 'main',    None, True), Encoding.BECH32,  p2wpkh_prefix),
+  ('bc',    0, 32, (False, 'main',    None, True), Encoding.BECH32,  p2wsh_prefix),
+  ('bc',    1, 32, (False, 'main',    None, True), Encoding.BECH32M, p2tr_prefix),
+  ('bc',    2,  2, (False, 'main',    None, True), Encoding.BECH32M, (OP_2, 2)),
+  ('tb',    0, 20, (False, 'test',    None, True), Encoding.BECH32,  p2wpkh_prefix),
+  ('tb',    0, 32, (False, 'test',    None, True), Encoding.BECH32,  p2wsh_prefix),
+  ('tb',    1, 32, (False, 'test',    None, True), Encoding.BECH32M, p2tr_prefix),
+  ('tb',    3, 16, (False, 'test',    None, True), Encoding.BECH32M, (OP_3, 16)),
+  ('bcrt',  0, 20, (False, 'regtest', None, True), Encoding.BECH32,  p2wpkh_prefix),
+  ('bcrt',  0, 32, (False, 'regtest', None, True), Encoding.BECH32,  p2wsh_prefix),
+  ('bcrt',  1, 32, (False, 'regtest', None, True), Encoding.BECH32M, p2tr_prefix),
+  ('bcrt', 16, 40, (False, 'regtest', None, True), Encoding.BECH32M, (OP_16, 40))
 ]
 # templates for invalid bech32 sequences
 bech32_ng_templates = [
-  # hrp, version, witprog_size, invalid_bech32, invalid_checksum, invalid_char
-  ('tc',    0, 20, False, False, False),
-  ('tb',   17, 32, False, False, False),
-  ('bcrt',  3,  1, False, False, False),
-  ('bc',   15, 41, False, False, False),
-  ('tb',    0, 16, False, False, False),
-  ('bcrt',  0, 32, True,  False, False),
-  ('bc',    0, 16, True,  False, False),
-  ('tb',    0, 32, False, True,  False),
-  ('bcrt',  0, 20, False, False, True)
+  # hrp, version, witprog_size, encoding, invalid_bech32, invalid_checksum, invalid_char
+  ('tc',    0, 20, Encoding.BECH32,  False, False, False),
+  ('bt',    1, 32, Encoding.BECH32M, False, False, False),
+  ('tb',   17, 32, Encoding.BECH32M, False, False, False),
+  ('bcrt',  3,  1, Encoding.BECH32M, False, False, False),
+  ('bc',   15, 41, Encoding.BECH32M, False, False, False),
+  ('tb',    0, 16, Encoding.BECH32,  False, False, False),
+  ('bcrt',  0, 32, Encoding.BECH32,  True,  False, False),
+  ('bc',    0, 16, Encoding.BECH32,  True,  False, False),
+  ('tb',    0, 32, Encoding.BECH32,  False, True,  False),
+  ('bcrt',  0, 20, Encoding.BECH32,  False, False, True),
+  ('bc',    0, 20, Encoding.BECH32M, False, False, False),
+  ('tb',    0, 32, Encoding.BECH32M, False, False, False),
+  ('bcrt',  0, 20, Encoding.BECH32M, False, False, False),
+  ('bc',    1, 32, Encoding.BECH32,  False, False, False),
+  ('tb',    2, 16, Encoding.BECH32,  False, False, False),
+  ('bcrt', 16, 20, Encoding.BECH32,  False, False, False),
 ]
 
 def is_valid(v):
@@ -128,8 +140,9 @@ def gen_valid_bech32_vector(template):
     hrp = template[0]
     witver = template[1]
     witprog = bytearray(os.urandom(template[2]))
-    dst_prefix = bytearray(template[4])
-    rv = bech32_encode(hrp, [witver] + convertbits(witprog, 8, 5))
+    encoding = template[4]
+    dst_prefix = bytearray(template[5])
+    rv = bech32_encode(encoding, hrp, [witver] + convertbits(witprog, 8, 5))
     return rv, dst_prefix + witprog
 
 def gen_valid_vectors():
@@ -189,22 +202,23 @@ def gen_invalid_bech32_vector(template):
     hrp = template[0]
     witver = template[1]
     witprog = bytearray(os.urandom(template[2]))
+    encoding = template[3]
 
     if no_data:
-        rv = bech32_encode(hrp, [])
+        rv = bech32_encode(encoding, hrp, [])
     else:
         data = [witver] + convertbits(witprog, 8, 5)
-        if template[3] and not no_data:
+        if template[4] and not no_data:
             if template[2] % 5 in {2, 4}:
                 data[-1] |= 1
             else:
                 data.append(0)
-        rv = bech32_encode(hrp, data)
+        rv = bech32_encode(encoding, hrp, data)
 
-    if template[4]:
+    if template[5]:
         i = len(rv) - random.randrange(1, 7)
         rv = rv[:i] + random.choice(CHARSET.replace(rv[i], '')) + rv[i + 1:]
-    if template[5]:
+    if template[6]:
         i = len(hrp) + 1 + random.randrange(0, len(rv) - len(hrp) - 4)
         rv = rv[:i] + rv[i:i + 4].upper() + rv[i + 4:]
 

diff --git a/doc/bips.md b/doc/bips.md
@@ -1,4 +1,4 @@
-BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.19.0**):
+BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.20.2**):
 
 * [`BIP 9`](https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki): The changes allowing multiple soft-forks to be deployed in parallel have been implemented since **v0.12.1**  ([PR #7575](https://github.com/bitcoin/bitcoin/pull/7575))
 * [`BIP 11`](https://github.com/bitcoin/bips/blob/master/bip-0011.mediawiki): Multisig outputs are standard since **v0.6.0** ([PR #669](https://github.com/bitcoin/bitcoin/pull/669)).
@@ -42,3 +42,4 @@ BIPs that are implemented by Bitcoin Core (up-to-date up to **v0.19.0**):
 * [`BIP 173`](https://github.com/bitcoin/bips/blob/master/bip-0173.mediawiki): Bech32 addresses for native Segregated Witness outputs are supported as of **v0.16.0** ([PR 11167](https://github.com/bitcoin/bitcoin/pull/11167)). Bech32 addresses are generated by default as of **v0.20.0** ([PR 16884](https://github.com/bitcoin/bitcoin/pull/16884)).
 * [`BIP 174`](https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki): RPCs to operate on Partially Signed Bitcoin Transactions (PSBT) are present as of **v0.17.0** ([PR 13557](https://github.com/bitcoin/bitcoin/pull/13557)).
 * [`BIP 176`](https://github.com/bitcoin/bips/blob/master/bip-0176.mediawiki): Bits Denomination [QT only] is supported as of **v0.16.0** ([PR 12035](https://github.com/bitcoin/bitcoin/pull/12035)).
+* [`BIP 350`](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki): Addresses for native v1+ segregated Witness outputs use Bech32m instead of Bech32 as of **v0.20.2** ([PR 20861](https://github.com/bitcoin/bitcoin/pull/20861)).
diff --git a/doc/release-notes-20861.md b/doc/release-notes-20861.md
@@ -0,0 +1,12 @@
+Updated RPCs
+------------
+
+- Due to [BIP 350](https://github.com/bitcoin/bips/blob/master/bip-0350.mediawiki)
+  being implemented, behavior for all RPCs that accept addresses is changed when
+  a native witness version 1 (or higher) is passed. These now require a Bech32m
+  encoding instead of a Bech32 one, and Bech32m encoding will be used for such
+  addresses in RPC output as well. No version 1 addresses should be created
+  for mainnet until consensus rules are adopted that give them meaning
+  (e.g. through [BIP 341](https://github.com/bitcoin/bips/blob/master/bip-0341.mediawiki)).
+  Once that happens, Bech32m is expected to be used for them, so this shouldn't
+  affect any production systems, but may be observed on other networks.
diff --git a/src/bech32.cpp b/src/bech32.cpp
@@ -1,4 +1,4 @@
-// Copyright (c) 2017 Pieter Wuille
+// Copyright (c) 2017, 2021 Pieter Wuille
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
@@ -7,15 +7,18 @@
 
 #include <assert.h>
 
+namespace bech32
+{
+
 namespace
 {
 
 typedef std::vector<uint8_t> data;
 
-/** The Bech32 character set for encoding. */
+/** The Bech32 and Bech32m character set for encoding. */
 const char* CHARSET = "qpzry9x8gf2tvdw0s3jn54khce6mua7l";
 
-/** The Bech32 character set for decoding. */
+/** The Bech32 and Bech32m character set for decoding. */
 const int8_t CHARSET_REV[128] = {
     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
     -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
@@ -27,6 +30,12 @@ const int8_t CHARSET_REV[128] = {
      1,  0,  3, 16, 11, 28, 12, 14,  6,  4,  2, -1, -1, -1, -1, -1
 };
 
+/* Determine the final constant to use for the specified encoding. */
+uint32_t EncodingConstant(Encoding encoding) {
+    assert(encoding == Encoding::BECH32 || encoding == Encoding::BECH32M);
+    return encoding == Encoding::BECH32 ? 1 : 0x2bc830a3;
+}
+
 /** This function will compute what 6 5-bit values to XOR into the last 6 input values, in order to
  *  make the checksum 0. These 6 values are packed together in a single 30-bit integer. The higher
  *  bits correspond to earlier values. */
@@ -111,21 +120,24 @@ data ExpandHRP(const std::string& hrp)
 }
 
 /** Verify a checksum. */
-bool VerifyChecksum(const std::string& hrp, const data& values)
+Encoding VerifyChecksum(const std::string& hrp, const data& values)
 {
     // PolyMod computes what value to xor into the final values to make the checksum 0. However,
     // if we required that the checksum was 0, it would be the case that appending a 0 to a valid
     // list of values would result in a new valid list. For that reason, Bech32 requires the
-    // resulting checksum to be 1 instead.
-    return PolyMod(Cat(ExpandHRP(hrp), values)) == 1;
+    // resulting checksum to be 1 instead. In Bech32m, this constant was amended.
+    const uint32_t check = PolyMod(Cat(ExpandHRP(hrp), values));
+    if (check == EncodingConstant(Encoding::BECH32)) return Encoding::BECH32;
+    if (check == EncodingConstant(Encoding::BECH32M)) return Encoding::BECH32M;
+    return Encoding::INVALID;
 }
 
 /** Create a checksum. */
-data CreateChecksum(const std::string& hrp, const data& values)
+data CreateChecksum(Encoding encoding, const std::string& hrp, const data& values)
 {
     data enc = Cat(ExpandHRP(hrp), values);
     enc.resize(enc.size() + 6); // Append 6 zeroes
-    uint32_t mod = PolyMod(enc) ^ 1; // Determine what to XOR into those 6 zeroes.
+    uint32_t mod = PolyMod(enc) ^ EncodingConstant(encoding); // Determine what to XOR into those 6 zeroes.
     data ret(6);
     for (size_t i = 0; i < 6; ++i) {
         // Convert the 5-bit groups in mod to checksum values.
@@ -136,16 +148,13 @@ data CreateChecksum(const std::string& hrp, const data& values)
 
 } // namespace
 
-namespace bech32
-{
-
-/** Encode a Bech32 string. */
-std::string Encode(const std::string& hrp, const data& values) {
-    // First ensure that the HRP is all lowercase. BIP-173 requires an encoder
-    // to return a lowercase Bech32 string, but if given an uppercase HRP, the
+/** Encode a Bech32 or Bech32m string. */
+std::string Encode(Encoding encoding, const std::string& hrp, const data& values) {
+    // First ensure that the HRP is all lowercase. BIP-173 and BIP350 require an encoder
+    // to return a lowercase Bech32/Bech32m string, but if given an uppercase HRP, the
     // result will always be invalid.
     for (const char& c : hrp) assert(c < 'A' || c > 'Z');
-    data checksum = CreateChecksum(hrp, values);
+    data checksum = CreateChecksum(encoding, hrp, values);
     data combined = Cat(values, checksum);
     std::string ret = hrp + '1';
     ret.reserve(ret.size() + combined.size());
@@ -155,8 +164,8 @@ std::string Encode(const std::string& hrp, const data& values) {
     return ret;
 }
 
-/** Decode a Bech32 string. */
-std::pair<std::string, data> Decode(const std::string& str) {
+/** Decode a Bech32 or Bech32m string. */
+DecodeResult Decode(const std::string& str) {
     bool lower = false, upper = false;
     for (size_t i = 0; i < str.size(); ++i) {
         unsigned char c = str[i];
@@ -183,10 +192,9 @@ std::pair<std::string, data> Decode(const std::string& str) {
     for (size_t i = 0; i < pos; ++i) {
         hrp += LowerCase(str[i]);
     }
-    if (!VerifyChecksum(hrp, values)) {
-        return {};
-    }
-    return {hrp, data(values.begin(), values.end() - 6)};
+    Encoding result = VerifyChecksum(hrp, values);
+    if (result == Encoding::INVALID) return {};
+    return {result, std::move(hrp), data(values.begin(), values.end() - 6)};
 }
 
 } // namespace bech32
diff --git a/src/bech32.h b/src/bech32.h
@@ -1,13 +1,14 @@
-// Copyright (c) 2017 Pieter Wuille
+// Copyright (c) 2017, 2021 Pieter Wuille
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
-// Bech32 is a string encoding format used in newer address types.
-// The output consists of a human-readable part (alphanumeric), a
-// separator character (1), and a base32 data section, the last
-// 6 characters of which are a checksum.
+// Bech32 and Bech32m are string encoding formats used in newer
+// address types. The outputs consist of a human-readable part
+// (alphanumeric), a separator character (1), and a base32 data
+// section, the last 6 characters of which are a checksum. The
+// module is namespaced under bech32 for historical reasons.
 //
-// For more information, see BIP 173.
+// For more information, see BIP 173 and BIP 350.
 
 #ifndef BITCOIN_BECH32_H
 #define BITCOIN_BECH32_H
@@ -19,11 +20,29 @@
 namespace bech32
 {
 
-/** Encode a Bech32 string. If hrp contains uppercase characters, this will cause an assertion error. */
-std::string Encode(const std::string& hrp, const std::vector<uint8_t>& values);
+enum class Encoding {
+    INVALID, //!< Failed decoding
 
-/** Decode a Bech32 string. Returns (hrp, data). Empty hrp means failure. */
-std::pair<std::string, std::vector<uint8_t>> Decode(const std::string& str);
+    BECH32,  //!< Bech32 encoding as defined in BIP173
+    BECH32M, //!< Bech32m encoding as defined in BIP350
+};
+
+/** Encode a Bech32 or Bech32m string. If hrp contains uppercase characters, this will cause an
+ *  assertion error. Encoding must be one of BECH32 or BECH32M. */
+std::string Encode(Encoding encoding, const std::string& hrp, const std::vector<uint8_t>& values);
+
+struct DecodeResult
+{
+    Encoding encoding;         //!< What encoding was detected in the result; Encoding::INVALID if failed.
+    std::string hrp;           //!< The human readable part
+    std::vector<uint8_t> data; //!< The payload (excluding checksum)
+
+    DecodeResult() : encoding(Encoding::INVALID) {}
+    DecodeResult(Encoding enc, std::string&& h, std::vector<uint8_t>&& d) : encoding(enc), hrp(std::move(h)), data(std::move(d)) {}
+};
+
+/** Decode a Bech32 or Bech32m string. */
+DecodeResult Decode(const std::string& str);
 
 } // namespace bech32
 

diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp
@@ -18,7 +18,7 @@ static void Bech32Encode(benchmark::State& state)
     tmp.reserve(1 + 32 * 8 / 5);
     ConvertBits<8, 5, true>([&](unsigned char c) { tmp.push_back(c); }, v.begin(), v.end());
     while (state.KeepRunning()) {
-        bech32::Encode("bc", tmp);
+        bech32::Encode(bech32::Encoding::BECH32, "bc", tmp);
     }
 }
 

diff --git a/src/key_io.cpp b/src/key_io.cpp
@@ -44,15 +44,15 @@ class DestinationEncoder : public boost::static_visitor<std::string>
         std::vector<unsigned char> data = {0};
         data.reserve(33);
         ConvertBits<8, 5, true>([&](unsigned char c) { data.push_back(c); }, id.begin(), id.end());
-        return bech32::Encode(m_params.Bech32HRP(), data);
+        return bech32::Encode(bech32::Encoding::BECH32, m_params.Bech32HRP(), data);
     }
 
     std::string operator()(const WitnessV0ScriptHash& id) const
     {
         std::vector<unsigned char> data = {0};
         data.reserve(53);
         ConvertBits<8, 5, true>([&](unsigned char c) { data.push_back(c); }, id.begin(), id.end());
-        return bech32::Encode(m_params.Bech32HRP(), data);
+        return bech32::Encode(bech32::Encoding::BECH32, m_params.Bech32HRP(), data);
     }
 
     std::string operator()(const WitnessUnknown& id) const
@@ -63,7 +63,7 @@ class DestinationEncoder : public boost::static_visitor<std::string>
         std::vector<unsigned char> data = {(unsigned char)id.version};
         data.reserve(1 + (id.length * 8 + 4) / 5);
         ConvertBits<8, 5, true>([&](unsigned char c) { data.push_back(c); }, id.program, id.program + id.length);
-        return bech32::Encode(m_params.Bech32HRP(), data);
+        return bech32::Encode(bech32::Encoding::BECH32M, m_params.Bech32HRP(), data);
     }
 
     std::string operator()(const CNoDestination& no) const { return {}; }
@@ -91,13 +91,19 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par
         }
     }
     data.clear();
-    auto bech = bech32::Decode(str);
-    if (bech.second.size() > 0 && bech.first == params.Bech32HRP()) {
+    const auto dec = bech32::Decode(str);
+    if ((dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) && dec.data.size() > 0 && dec.hrp == params.Bech32HRP()) {
         // Bech32 decoding
-        int version = bech.second[0]; // The first 5 bit symbol is the witness version (0-16)
+        int version = dec.data[0]; // The first 5 bit symbol is the witness version (0-16)
+        if (version == 0 && dec.encoding != bech32::Encoding::BECH32) {
+            return CNoDestination();
+        }
+        if (version != 0 && dec.encoding != bech32::Encoding::BECH32M) {
+            return CNoDestination();
+        }
         // The rest of the symbols are converted witness program bytes.
-        data.reserve(((bech.second.size() - 1) * 5) / 8);
-        if (ConvertBits<5, 8, false>([&](unsigned char c) { data.push_back(c); }, bech.second.begin() + 1, bech.second.end())) {
+        data.reserve(((dec.data.size() - 1) * 5) / 8);
+        if (ConvertBits<5, 8, false>([&](unsigned char c) { data.push_back(c); }, dec.data.begin() + 1, dec.data.end())) {
             if (version == 0) {
                 {
                     WitnessV0KeyHash keyid;