Merge bitcoin#30657: test: add functional test for XORed block/undo files (-blocksxor option)

faa1b9b test: add functional test for XORed block/undo files (`-blocksxor`) (Sebastian Falbesoner)
6b3676b test: refactor: move `read_xor_key`/`util_xor` helpers to util module (Sebastian Falbesoner)

Pull request description:

  This PR adds a dedicated functional test for XORed block data/undo file support (bitcoind option `-blocksxor`, see PR bitcoin#28052). In order to verify that the XOR pattern has been applied, the {blk,rev}*.dat files are rewritten un-XORed manually by the test while the node is shut down; the node is then started again with `-blocksxor=0`, and both the data and undo files are verified via the `verifychain` RPC (with checklevel=2). Note that starting bitcoind with `-blocksxor=0` fails if a xor key is present already, which is also tested explicitly.

  Fixes bitcoin#30599.

ACKs for top commit:
  glozow:
    ACK faa1b9b
  maflcko:
    ACK faa1b9b
  ismaelsadeeq:
    Tested ACK faa1b9b

Tree-SHA512: e1df106f6b4e3ba67eca108e36d762f1b991673b881934b84cd36946496a09ce9c329c1363c36aa29409137ae4881e2d177e651359686511632ddf2870f7ca8e

Author: glozow

test/functional/feature_blocksxor.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+# Copyright (c) 2024 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test support for XORed block data and undo files (`-blocksxor` option)."""
+import os
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.test_node import ErrorMatch
+from test_framework.util import (
+    assert_equal,
+    assert_greater_than,
+    read_xor_key,
+    util_xor,
+)
+from test_framework.wallet import MiniWallet
+
+
+class BlocksXORTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+        self.extra_args = [[
+            '-blocksxor=1',
+            '-fastprune=1',             # use smaller block files
+            '-datacarriersize=100000',  # needed to pad transaction with MiniWallet
+        ]]
+
+    def run_test(self):
+        self.log.info("Mine some blocks, to create multiple blk*.dat/rev*.dat files")
+        node = self.nodes[0]
+        wallet = MiniWallet(node)
+        for _ in range(5):
+            wallet.send_self_transfer(from_node=node, target_weight=80000)
+            self.generate(wallet, 1)
+
+        block_files = list(node.blocks_path.glob('blk[0-9][0-9][0-9][0-9][0-9].dat'))
+        undo_files  = list(node.blocks_path.glob('rev[0-9][0-9][0-9][0-9][0-9].dat'))
+        assert_equal(len(block_files), len(undo_files))
+        assert_greater_than(len(block_files), 1)  # we want at least one full block file
+
+        self.log.info("Shut down node and un-XOR block/undo files manually")
+        self.stop_node(0)
+        xor_key = read_xor_key(node=node)
+        for data_file in sorted(block_files + undo_files):
+            self.log.debug(f"Rewriting file {data_file}...")
+            with open(data_file, 'rb+') as f:
+                xored_data = f.read()
+                f.seek(0)
+                f.write(util_xor(xored_data, xor_key, offset=0))
+
+        self.log.info("Check that restarting with 'blocksxor=0' fails if XOR key is present")
+        node.assert_start_raises_init_error(['-blocksxor=0'],
+            'The blocksdir XOR-key can not be disabled when a random key was already stored!',
+            match=ErrorMatch.PARTIAL_REGEX)
+
+        self.log.info("Delete XOR key, restart node with '-blocksxor=0', check blk*.dat/rev*.dat file integrity")
+        os.remove(node.blocks_path / 'xor.dat')
+        self.start_node(0, extra_args=['-blocksxor=0'])
+        # checklevel=2 -> verify block validity + undo data
+        # nblocks=0    -> verify all blocks
+        node.verifychain(checklevel=2, nblocks=0)
+
+
+if __name__ == '__main__':
+    BlocksXORTest(__file__).main()

test/functional/feature_reindex.py

@@ -12,7 +12,11 @@
 
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.messages import MAGIC_BYTES
-from test_framework.util import assert_equal
+from test_framework.util import (
+    assert_equal,
+    read_xor_key,
+    util_xor,
+)
 
 
 class ReindexTest(BitcoinTestFramework):
@@ -39,15 +43,7 @@ def out_of_order(self):
         # we're generating them rather than getting them from peers), so to
         # test out-of-order handling, swap blocks 1 and 2 on disk.
         blk0 = self.nodes[0].blocks_path / "blk00000.dat"
-        with open(self.nodes[0].blocks_path / "xor.dat", "rb") as xor_f:
-            NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size()
-            xor_dat = xor_f.read(NUM_XOR_BYTES)
-
-        def util_xor(data, key, *, offset):
-            data = bytearray(data)
-            for i in range(len(data)):
-                data[i] ^= key[(i + offset) % len(key)]
-            return bytes(data)
+        xor_dat = read_xor_key(node=self.nodes[0])
 
         with open(blk0, 'r+b') as bf:
             # Read at least the first few blocks (including genesis)

test/functional/test_framework/util.py

@@ -311,6 +311,13 @@ def sha256sum_file(filename):
     return h.digest()
 
 
+def util_xor(data, key, *, offset):
+    data = bytearray(data)
+    for i in range(len(data)):
+        data[i] ^= key[(i + offset) % len(key)]
+    return bytes(data)
+
+
 # RPC/P2P connection constants and functions
 ############################################
 
@@ -508,6 +515,12 @@ def check_node_connections(*, node, num_in, num_out):
     assert_equal(info["connections_out"], num_out)
 
 
+def read_xor_key(*, node):
+    with open(node.blocks_path / "xor.dat", "rb") as xor_f:
+        NUM_XOR_BYTES = 8 # From InitBlocksdirXorKey::xor_key.size()
+        return xor_f.read(NUM_XOR_BYTES)
+
+
 # Transaction/Block functions
 #############################
 

test/functional/test_runner.py

@@ -284,6 +284,7 @@
     'mempool_package_limits.py',
     'mempool_package_rbf.py',
     'feature_versionbits_warning.py',
+    'feature_blocksxor.py',
     'rpc_preciousblock.py',
     'wallet_importprunedfunds.py --legacy-wallet',
     'wallet_importprunedfunds.py --descriptors',