Merge bitcoin/bitcoin#29939: test: add MiniWallet tagging support to avoid UTXO mixing, use in fill_mempool

dd8fa86 test: use tagged ephemeral MiniWallet instance in fill_mempool (Sebastian Falbesoner)
b2037ad test: add MiniWallet tagging support to avoid UTXO mixing (Sebastian Falbesoner)
c8e6d08 test: refactor: eliminate COINBASE_MATURITY magic number in fill_mempool (Sebastian Falbesoner)
4f34714 test: refactor: move fill_mempool to new module mempool_util (Sebastian Falbesoner)

Pull request description:

  Different MiniWallet instances using the same mode (either ADDRESS_OP_TRUE, RAW_OP_TRUE or RAW_P2PK) currently always create and spend UTXOs with identical output scripts, which can cause unintentional tx dependencies (see e.g. the discussion in bitcoin/bitcoin#29827 (comment)). In order to avoid mixing of UTXOs between instances, this PR introduces the possibility to provide a MiniWallet tag name, that is used to derive a different internal key for the taproot construction, leading to a different P2TR output script. Note that since we use script-path spending and only the key-path is changed here, no changes in the MiniWallet spending logic are needed.

  The new tagging option is then used in the `fill_mempool` helper to create an ephemeral wallet for the filling txs, as suggested in bitcoin/bitcoin#29827 (comment). To avoid circular dependencies, `fill_mempool` is moved to a new module `mempool_util.py` first.

  I'm still not sure if a generic word like "tag" is the right term for what this tries to achieve, happy to pick up better suggestions. Also, maybe passing a tag name is overkill and a boolean flag like "random_output_script" is sufficient?

ACKs for top commit:
  glozow:
    ACK dd8fa86
  achow101:
    ACK dd8fa86
  rkrux:
    tACK [dd8fa86](bitcoin/bitcoin@dd8fa86)
  brunoerg:
    utACK dd8fa86

Tree-SHA512: 5ef3558c3ef5ac32cfa79c8f751972ca6bceaa332cd7daac7e93412a88e30dec472cb041c0845b04abf8a317036d31ebddfc3234e609ed442417894c2bdeeac9

Author: achow101

test/functional/mempool_limit.py

@@ -6,14 +6,16 @@
 
 from decimal import Decimal
 
+from test_framework.mempool_util import (
+    fill_mempool,
+)
 from test_framework.p2p import P2PTxInvStore
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
     assert_fee_amount,
     assert_greater_than,
     assert_raises_rpc_error,
-    fill_mempool,
 )
 from test_framework.wallet import (
     COIN,
@@ -93,7 +95,7 @@ def test_mid_package_eviction(self):
         assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000'))
         assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000'))
 
-        fill_mempool(self, node, self.wallet)
+        fill_mempool(self, node)
         current_info = node.getmempoolinfo()
         mempoolmin_feerate = current_info["mempoolminfee"]
 
@@ -183,7 +185,7 @@ def test_mid_package_replacement(self):
         assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000'))
         assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000'))
 
-        fill_mempool(self, node, self.wallet)
+        fill_mempool(self, node)
         current_info = node.getmempoolinfo()
         mempoolmin_feerate = current_info["mempoolminfee"]
 
@@ -257,7 +259,7 @@ def run_test(self):
         assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000'))
         assert_equal(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000'))
 
-        fill_mempool(self, node, self.wallet)
+        fill_mempool(self, node)
 
         # Deliberately try to create a tx with a fee less than the minimum mempool fee to assert that it does not get added to the mempool
         self.log.info('Create a mempool tx that will not pass mempoolminfee')

test/functional/p2p_1p1c_network.py

@@ -12,6 +12,9 @@
 from decimal import Decimal
 from math import ceil
 
+from test_framework.mempool_util import (
+    fill_mempool,
+)
 from test_framework.messages import (
     msg_tx,
 )
@@ -22,7 +25,6 @@
 from test_framework.util import (
     assert_equal,
     assert_greater_than,
-    fill_mempool,
 )
 from test_framework.wallet import (
     MiniWallet,
@@ -45,8 +47,7 @@ def set_test_params(self):
         self.supports_cli = False
 
     def raise_network_minfee(self):
-        filler_wallet = MiniWallet(self.nodes[0])
-        fill_mempool(self, self.nodes[0], filler_wallet)
+        fill_mempool(self, self.nodes[0])
 
         self.log.debug("Wait for the network to sync mempools")
         self.sync_mempools()

test/functional/p2p_opportunistic_1p1c.py

@@ -8,6 +8,9 @@
 
 from decimal import Decimal
 import time
+from test_framework.mempool_util import (
+    fill_mempool,
+)
 from test_framework.messages import (
     CInv,
     CTxInWitness,
@@ -24,7 +27,6 @@
 from test_framework.util import (
     assert_equal,
     assert_greater_than,
-    fill_mempool,
 )
 from test_framework.wallet import (
     MiniWallet,
@@ -386,8 +388,7 @@ def run_test(self):
         self.generate(self.wallet_nonsegwit, 10)
         self.generate(self.wallet, 20)
 
-        filler_wallet = MiniWallet(node)
-        fill_mempool(self, node, filler_wallet)
+        fill_mempool(self, node)
 
         self.log.info("Check opportunistic 1p1c logic when parent (txid != wtxid) is received before child")
         self.test_basic_parent_then_child(self.wallet)

test/functional/p2p_tx_download.py

@@ -8,6 +8,9 @@
 from decimal import Decimal
 import time
 
+from test_framework.mempool_util import (
+    fill_mempool,
+)
 from test_framework.messages import (
     CInv,
     MSG_TX,
@@ -24,7 +27,6 @@
 from test_framework.test_framework import BitcoinTestFramework
 from test_framework.util import (
     assert_equal,
-    fill_mempool,
 )
 from test_framework.wallet import MiniWallet
 
@@ -248,7 +250,7 @@ def test_spurious_notfound(self):
     def test_rejects_filter_reset(self):
         self.log.info('Check that rejected tx is not requested again')
         node = self.nodes[0]
-        fill_mempool(self, node, self.wallet)
+        fill_mempool(self, node)
         self.wallet.rescan_utxos()
         mempoolminfee = node.getmempoolinfo()['mempoolminfee']
         peer = node.add_p2p_connection(TestP2PConn())

test/functional/rpc_packages.py

@@ -8,6 +8,9 @@
 import random
 
 from test_framework.blocktools import COINBASE_MATURITY
+from test_framework.mempool_util import (
+    fill_mempool,
+)
 from test_framework.messages import (
     MAX_BIP125_RBF_SEQUENCE,
     tx_from_hex,
@@ -18,7 +21,6 @@
     assert_equal,
     assert_fee_amount,
     assert_raises_rpc_error,
-    fill_mempool,
 )
 from test_framework.wallet import (
     DEFAULT_FEE,
@@ -398,7 +400,7 @@ def test_maxfeerate_submitpackage(self):
         ])
         self.wallet.rescan_utxos()
 
-        fill_mempool(self, node, self.wallet)
+        fill_mempool(self, node)
 
         minrelay = node.getmempoolinfo()["minrelaytxfee"]
         parent = self.wallet.create_self_transfer(

test/functional/test_framework/address.py

@@ -47,17 +47,18 @@ class AddressType(enum.Enum):
 b58chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'
 
 
-def create_deterministic_address_bcrt1_p2tr_op_true():
+def create_deterministic_address_bcrt1_p2tr_op_true(explicit_internal_key=None):
     """
     Generates a deterministic bech32m address (segwit v1 output) that
     can be spent with a witness stack of OP_TRUE and the control block
     with internal public key (script-path spending).
 
     Returns a tuple with the generated address and the internal key.
     """
-    internal_key = (1).to_bytes(32, 'big')
+    internal_key = explicit_internal_key or (1).to_bytes(32, 'big')
     address = output_key_to_p2tr(taproot_construct(internal_key, [(None, CScript([OP_TRUE]))]).output_pubkey)
-    assert_equal(address, 'bcrt1p9yfmy5h72durp7zrhlw9lf7jpwjgvwdg0jr0lqmmjtgg83266lqsekaqka')
+    if explicit_internal_key is None:
+        assert_equal(address, 'bcrt1p9yfmy5h72durp7zrhlw9lf7jpwjgvwdg0jr0lqmmjtgg83266lqsekaqka')
     return (address, internal_key)
 
 

test/functional/test_framework/mempool_util.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# Copyright (c) 2024 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Helpful routines for mempool testing."""
+from decimal import Decimal
+
+from .blocktools import (
+    COINBASE_MATURITY,
+)
+from .util import (
+    assert_equal,
+    assert_greater_than,
+    create_lots_of_big_transactions,
+    gen_return_txouts,
+)
+from .wallet import (
+    MiniWallet,
+)
+
+
+def fill_mempool(test_framework, node):
+    """Fill mempool until eviction.
+
+    Allows for simpler testing of scenarios with floating mempoolminfee > minrelay
+    Requires -datacarriersize=100000 and
+   -maxmempool=5.
+    It will not ensure mempools become synced as it
+    is based on a single node and assumes -minrelaytxfee
+    is 1 sat/vbyte.
+    To avoid unintentional tx dependencies, the mempool filling txs are created with a
+    tagged ephemeral miniwallet instance.
+    """
+    test_framework.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises")
+    txouts = gen_return_txouts()
+    relayfee = node.getnetworkinfo()['relayfee']
+
+    assert_equal(relayfee, Decimal('0.00001000'))
+
+    tx_batch_size = 1
+    num_of_batches = 75
+    # Generate UTXOs to flood the mempool
+    # 1 to create a tx initially that will be evicted from the mempool later
+    # 75 transactions each with a fee rate higher than the previous one
+    ephemeral_miniwallet = MiniWallet(node, tag_name="fill_mempool_ephemeral_wallet")
+    test_framework.generate(ephemeral_miniwallet, 1 + num_of_batches * tx_batch_size)
+
+    # Mine enough blocks so that the UTXOs are allowed to be spent
+    test_framework.generate(node, COINBASE_MATURITY - 1)
+
+    # Get all UTXOs up front to ensure none of the transactions spend from each other, as that may
+    # change their effective feerate and thus the order in which they are selected for eviction.
+    confirmed_utxos = [ephemeral_miniwallet.get_utxo(confirmed_only=True) for _ in range(num_of_batches * tx_batch_size + 1)]
+    assert_equal(len(confirmed_utxos), num_of_batches * tx_batch_size + 1)
+
+    test_framework.log.debug("Create a mempool tx that will be evicted")
+    tx_to_be_evicted_id = ephemeral_miniwallet.send_self_transfer(
+        from_node=node, utxo_to_spend=confirmed_utxos.pop(0), fee_rate=relayfee)["txid"]
+
+    # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool
+    # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB)
+    # by 130 should result in a fee that corresponds to 2x of that fee rate
+    base_fee = relayfee * 130
+
+    test_framework.log.debug("Fill up the mempool with txs with higher fee rate")
+    with node.assert_debug_log(["rolling minimum fee bumped"]):
+        for batch_of_txid in range(num_of_batches):
+            fee = (batch_of_txid + 1) * base_fee
+            utxos = confirmed_utxos[:tx_batch_size]
+            create_lots_of_big_transactions(ephemeral_miniwallet, node, fee, tx_batch_size, txouts, utxos)
+            del confirmed_utxos[:tx_batch_size]
+
+    test_framework.log.debug("The tx should be evicted by now")
+    # The number of transactions created should be greater than the ones present in the mempool
+    assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool()))
+    # Initial tx created should not be present in the mempool anymore as it had a lower fee rate
+    assert tx_to_be_evicted_id not in node.getrawmempool()
+
+    test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee")
+    assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000'))
+    assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000'))

test/functional/test_framework/util.py

@@ -496,65 +496,6 @@ def check_node_connections(*, node, num_in, num_out):
     assert_equal(info["connections_in"], num_in)
     assert_equal(info["connections_out"], num_out)
 
-def fill_mempool(test_framework, node, miniwallet):
-    """Fill mempool until eviction.
-
-    Allows for simpler testing of scenarios with floating mempoolminfee > minrelay
-    Requires -datacarriersize=100000 and
-   -maxmempool=5.
-    It will not ensure mempools become synced as it
-    is based on a single node and assumes -minrelaytxfee
-    is 1 sat/vbyte.
-    To avoid unintentional tx dependencies, it is recommended to use separate miniwallets for
-    mempool filling vs transactions in tests.
-    """
-    test_framework.log.info("Fill the mempool until eviction is triggered and the mempoolminfee rises")
-    txouts = gen_return_txouts()
-    relayfee = node.getnetworkinfo()['relayfee']
-
-    assert_equal(relayfee, Decimal('0.00001000'))
-
-    tx_batch_size = 1
-    num_of_batches = 75
-    # Generate UTXOs to flood the mempool
-    # 1 to create a tx initially that will be evicted from the mempool later
-    # 75 transactions each with a fee rate higher than the previous one
-    test_framework.generate(miniwallet, 1 + (num_of_batches * tx_batch_size))
-
-    # Mine COINBASE_MATURITY - 1 blocks so that the UTXOs are allowed to be spent
-    test_framework.generate(node, 100 - 1)
-
-    # Get all UTXOs up front to ensure none of the transactions spend from each other, as that may
-    # change their effective feerate and thus the order in which they are selected for eviction.
-    confirmed_utxos = [miniwallet.get_utxo(confirmed_only=True) for _ in range(num_of_batches * tx_batch_size + 1)]
-    assert_equal(len(confirmed_utxos), num_of_batches * tx_batch_size + 1)
-
-    test_framework.log.debug("Create a mempool tx that will be evicted")
-    tx_to_be_evicted_id = miniwallet.send_self_transfer(from_node=node, utxo_to_spend=confirmed_utxos[0], fee_rate=relayfee)["txid"]
-    del confirmed_utxos[0]
-
-    # Increase the tx fee rate to give the subsequent transactions a higher priority in the mempool
-    # The tx has an approx. vsize of 65k, i.e. multiplying the previous fee rate (in sats/kvB)
-    # by 130 should result in a fee that corresponds to 2x of that fee rate
-    base_fee = relayfee * 130
-
-    test_framework.log.debug("Fill up the mempool with txs with higher fee rate")
-    with node.assert_debug_log(["rolling minimum fee bumped"]):
-        for batch_of_txid in range(num_of_batches):
-            fee = (batch_of_txid + 1) * base_fee
-            utxos = confirmed_utxos[:tx_batch_size]
-            create_lots_of_big_transactions(miniwallet, node, fee, tx_batch_size, txouts, utxos)
-            del confirmed_utxos[:tx_batch_size]
-
-    test_framework.log.debug("The tx should be evicted by now")
-    # The number of transactions created should be greater than the ones present in the mempool
-    assert_greater_than(tx_batch_size * num_of_batches, len(node.getrawmempool()))
-    # Initial tx created should not be present in the mempool anymore as it had a lower fee rate
-    assert tx_to_be_evicted_id not in node.getrawmempool()
-
-    test_framework.log.debug("Check that mempoolminfee is larger than minrelaytxfee")
-    assert_equal(node.getmempoolinfo()['minrelaytxfee'], Decimal('0.00001000'))
-    assert_greater_than(node.getmempoolinfo()['mempoolminfee'], Decimal('0.00001000'))
 
 # Transaction/Block functions
 #############################

test/functional/test_framework/wallet.py

@@ -32,6 +32,7 @@
     CTxIn,
     CTxInWitness,
     CTxOut,
+    hash256,
 )
 from test_framework.script import (
     CScript,
@@ -65,7 +66,10 @@ class MiniWalletMode(Enum):
     However, if the transactions need to be modified by the user (e.g. prepending
     scriptSig for testing opcodes that are activated by a soft-fork), or the txs
     should contain an actual signature, the raw modes RAW_OP_TRUE and RAW_P2PK
-    can be useful. Summary of modes:
+    can be useful. In order to avoid mixing of UTXOs between different MiniWallet
+    instances, a tag name can be passed to the default mode, to create different
+    output scripts. Note that the UTXOs from the pre-generated test chain can
+    only be spent if no tag is passed. Summary of modes:
 
                     |      output       |           |  tx is   | can modify |  needs
          mode       |    description    |  address  | standard | scriptSig  | signing
@@ -80,22 +84,25 @@ class MiniWalletMode(Enum):
 
 
 class MiniWallet:
-    def __init__(self, test_node, *, mode=MiniWalletMode.ADDRESS_OP_TRUE):
+    def __init__(self, test_node, *, mode=MiniWalletMode.ADDRESS_OP_TRUE, tag_name=None):
         self._test_node = test_node
         self._utxos = []
         self._mode = mode
 
         assert isinstance(mode, MiniWalletMode)
         if mode == MiniWalletMode.RAW_OP_TRUE:
+            assert tag_name is None
             self._scriptPubKey = bytes(CScript([OP_TRUE]))
         elif mode == MiniWalletMode.RAW_P2PK:
             # use simple deterministic private key (k=1)
+            assert tag_name is None
             self._priv_key = ECKey()
             self._priv_key.set((1).to_bytes(32, 'big'), True)
             pub_key = self._priv_key.get_pubkey()
             self._scriptPubKey = key_to_p2pk_script(pub_key.get_bytes())
         elif mode == MiniWalletMode.ADDRESS_OP_TRUE:
-            self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true()
+            internal_key = None if tag_name is None else hash256(tag_name.encode())
+            self._address, self._internal_key = create_deterministic_address_bcrt1_p2tr_op_true(internal_key)
             self._scriptPubKey = address_to_scriptpubkey(self._address)
 
         # When the pre-mined test framework chain is used, it contains coinbase