pytest: enhance tests to test anchor and htlc tx feerates match targets.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Author: rustyrussell

tests/test_closing.py

@@ -3777,9 +3777,9 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind):
     fundsats = int(Millisatoshi(only_one(l1.rpc.listfunds()['outputs'])['amount_msat']).to_satoshi())
     psbt = l1.rpc.fundpsbt("all", "1000perkw", 1000)['psbt']
     # Pay 5k sats in fees, send most to l2
-    psbt = l1.rpc.addpsbtoutput(fundsats - 20000 - 5000, psbt, destination=l2.rpc.newaddr()['bech32'])['psbt']
-    # 10x2000 sat outputs for l1 to use.
-    for i in range(10):
+    psbt = l1.rpc.addpsbtoutput(fundsats - 24000 - 5000, psbt, destination=l2.rpc.newaddr()['bech32'])['psbt']
+    # 12x2000 sat outputs for l1 to use.
+    for i in range(12):
         psbt = l1.rpc.addpsbtoutput(2000, psbt)['psbt']
     l1.rpc.sendpsbt(l1.rpc.signpsbt(psbt)['signed_psbt'])
     bitcoind.generate_block(1, wait_for_mempool=1)
@@ -3811,6 +3811,13 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind):
 
     wait_for(lambda: len(bitcoind.rpc.getrawmempool()) == 2)
 
+    # Expect package feerate of 2000
+    details = bitcoind.rpc.getrawmempool(True).values()
+    total_weight = sum([d['weight'] for d in details])
+    total_fees = sum([float(d['fees']['base']) * 100_000_000 for d in details])
+    total_feerate_perkw = total_fees / total_weight * 1000
+    assert 2000 - 1 < total_feerate_perkw < 2000 + 1
+
     # But we don't mine it!  And fees go up again!
     l1.set_feerates((3000, 3000, 3000, 3000))
     bitcoind.generate_block(1, needfeerate=5000)
@@ -3819,6 +3826,13 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind):
     # We actually resubmit the commit tx, then the RBF:
     l1.daemon.wait_for_logs(['sendrawtx exit 0'] * 2)
 
+    # Expect package feerate of 3000
+    details = bitcoind.rpc.getrawmempool(True).values()
+    total_weight = sum([d['weight'] for d in details])
+    total_fees = sum([float(d['fees']['base']) * 100_000_000 for d in details])
+    total_feerate_perkw = total_fees / total_weight * 1000
+    assert 3000 - 1 < total_feerate_perkw < 3000 + 1
+
     # And now we'll get it in (there's some rounding, so feerate a bit lower!)
     bitcoind.generate_block(1, needfeerate=2990)
 
@@ -3840,8 +3854,15 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind):
     # It will enter the mempool
     wait_for(lambda: txid in bitcoind.rpc.getrawmempool())
 
+    tx = bitcoind.rpc.getrawmempool(True)[txid]
+    feerate_perkw = float(tx['fees']['base']) * 100_000_000 / tx['weight'] * 1000
+    # It actually has no change output, so it exceeds the fee quite a bit.
+    assert len(bitcoind.rpc.decoderawtransaction(bitcoind.rpc.getrawtransaction(txid))['vout']) == 1
+    assert feerate_perkw > 5000
+
     # And this will mine it!
-    bitcoind.generate_block(1, needfeerate=4990)
+    bitcoind.generate_block(1, needfeerate=5000)
+    assert bitcoind.rpc.getrawmempool() == []
 
 
 @pytest.mark.parametrize("anchors", [False, True])
@@ -4019,6 +4040,12 @@ def test_peer_anchor_push(node_factory, bitcoind, executor, chainparams):
     # As blocks pass, we will use anchor to boost l3's tx.
     for block, feerate in zip(range(120, 124), (12000, 13000, 14000, 15000)):
         l2.daemon.wait_for_log(fr"Worth fee [0-9]*sat for remote commit tx to get 100000000msat at block 125 \(\+{125 - block}\) at feerate {feerate}perkw")
+        # Check feerate for entire package (commitment tx + anchor) is ~ correct
+        details = bitcoind.rpc.getrawmempool(True).values()
+        total_weight = sum([d['weight'] for d in details])
+        total_fees = sum([float(d['fees']['base']) * 100_000_000 for d in details])
+        total_feerate_perkw = total_fees / total_weight * 1000
+        assert feerate - 1 < total_feerate_perkw < feerate + 1
         bitcoind.generate_block(1, needfeerate=16000)
         sync_blockheight(bitcoind, [l2])
         assert len(bitcoind.rpc.getrawmempool()) == 2

tests/test_wallet.py

@@ -287,6 +287,19 @@ def test_txprepare_multi(node_factory, bitcoind):
     l1.rpc.txdiscard(prep['txid'])
 
 
+def feerate_from_psbt(bitcoind, node, psbt):
+    # signpsbt insists they are reserved!
+    node.rpc.reserveinputs(psbt, exclusive=False)
+    final = node.rpc.dev_finalizepsbt(node.rpc.signpsbt(psbt)['signed_psbt'])
+    node.rpc.unreserveinputs(psbt)
+    psbt = node.rpc.setpsbtversion(final['psbt'], 0)['psbt']
+    # analyzepsbt gives a vsize, but not a weight!
+    # e.g. 'estimated_vsize': 356, 'estimated_feerate': Decimal('0.00030042'), 'fee': Decimal('0.00010695')
+    fee = int(bitcoind.rpc.analyzepsbt(psbt)['fee'] * 100_000_000)
+    weight = bitcoind.rpc.decoderawtransaction(final['tx'])['weight']
+    return fee / weight * 1000
+
+
 def test_txprepare(node_factory, bitcoind, chainparams):
     amount = 1000000
     l1 = node_factory.get_node(random_hsm=True)
@@ -299,13 +312,18 @@ def test_txprepare(node_factory, bitcoind, chainparams):
 
     bitcoind.generate_block(1)
     wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 10)
+    for est in l1.rpc.feerates('perkw')['perkw']['estimates']:
+        if est['blockcount'] == 12:
+            normal_feerate_perkw = est['feerate']
 
     prep = l1.rpc.txprepare(outputs=[{addr: Millisatoshi(amount * 3 * 1000)}])
     decode = bitcoind.rpc.decoderawtransaction(prep['unsigned_tx'])
     assert decode['txid'] == prep['txid']
     # 4 inputs, 2 outputs (3 if we have a fee output).
     assert len(decode['vin']) == 4
     assert len(decode['vout']) == 2 if not chainparams['feeoutput'] else 3
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 2 < feerate_from_psbt(bitcoind, l1, prep['psbt']) < normal_feerate_perkw + 2
 
     # One output will be correct.
     outnum = [i for i, o in enumerate(decode['vout']) if o['value'] == Decimal(amount * 3) / 10**8][0]
@@ -333,6 +351,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['value'] > Decimal(amount * 6) / 10**8 - Decimal(0.0002)
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 2 < feerate_from_psbt(bitcoind, l1, prep2['psbt']) < normal_feerate_perkw + 2
 
     # If I cancel the first one, I can get those first 4 outputs.
     discard = l1.rpc.txdiscard(prep['txid'])
@@ -351,6 +371,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['value'] > Decimal(amount * 4) / 10**8 - Decimal(0.0002)
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep3['psbt']) < normal_feerate_perkw + 1
 
     # Cannot discard twice.
     with pytest.raises(RpcError, match=r'not an unreleased txid'):
@@ -371,20 +393,28 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['value'] > Decimal(amount * 10) / 10**8 - Decimal(0.0003)
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 2 < feerate_from_psbt(bitcoind, l1, prep4['psbt']) < normal_feerate_perkw + 2
     l1.rpc.txdiscard(prep4['txid'])
 
     # Try passing in a utxo set
     utxos = [utxo["txid"] + ":" + str(utxo["output"])
              for utxo in l1.rpc.listfunds()["outputs"]][:4]
     prep5 = l1.rpc.txprepare([{addr:
                              Millisatoshi(amount * 3.5 * 1000)}], utxos=utxos)
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 2 < feerate_from_psbt(bitcoind, l1, prep3['psbt']) < normal_feerate_perkw + 2
 
     # Try passing unconfirmed utxos
     unconfirmed_utxo = l1.rpc.withdraw(l1.rpc.newaddr()["bech32"], 10**5)
     uutxos = [unconfirmed_utxo["txid"] + ":0"]
     with pytest.raises(RpcError, match=r"Could not afford"):
         l1.rpc.txprepare([{addr: Millisatoshi(amount * 3.5 * 1000)}],
                          utxos=uutxos)
+    # Feerate should be ~ as we asked for
+    unconfirmed_tx = bitcoind.rpc.getrawmempool(True)[unconfirmed_utxo["txid"]]
+    feerate_perkw = int(unconfirmed_tx['fees']['base'] * 100_000_000) * 1000 / unconfirmed_tx['weight']
+    assert normal_feerate_perkw - 1 < feerate_perkw < normal_feerate_perkw + 1
 
     decode = bitcoind.rpc.decoderawtransaction(prep5['unsigned_tx'])
     assert decode['txid'] == prep5['txid']
@@ -413,12 +443,16 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     # You can have one which is all, but not two.
     prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 1000)},
                               {addr: 'all'}])
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 2 < feerate_from_psbt(bitcoind, l1, prep5['psbt']) < normal_feerate_perkw + 2
     l1.rpc.txdiscard(prep5['txid'])
     with pytest.raises(RpcError, match=r"'all'"):
         prep5 = l1.rpc.txprepare([{addr: 'all'}, {addr: 'all'}])
 
     prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 500 + 100000)},
                               {addr: Millisatoshi(amount * 3 * 500 - 100000)}])
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 2 < feerate_from_psbt(bitcoind, l1, prep5['psbt']) < normal_feerate_perkw + 2
     decode = bitcoind.rpc.decoderawtransaction(prep5['unsigned_tx'])
     assert decode['txid'] == prep5['txid']
     # 4 inputs, 3 outputs(include change).
@@ -445,6 +479,91 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     else:
         assert decode['vout'][changenum]['scriptPubKey']['type'] == 'witness_v1_taproot'
 
+    l1.rpc.txdiscard(prep5['txid'])
+
+
+def test_txprepare_feerate(node_factory, bitcoind, chainparams):
+    # Make sure it works at different feerates!
+    l1, l2 = node_factory.get_nodes(2)
+
+    # Add some funds to withdraw later
+    for i in range(20):
+        bitcoind.rpc.sendtoaddress(l1.rpc.newaddr()['bech32'],
+                                   1000 / 10**8)
+
+    bitcoind.generate_block(1)
+    out_addrs = l2.rpc.newaddr('all')
+    wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 20)
+
+    for addrtype in ('bech32', 'p2tr'):
+        for feerate in range(255, 1000, 250):
+            prep = l1.rpc.txprepare([{out_addrs[addrtype]: Millisatoshi(9000)}], f"{feerate}perkw")
+            actual_feerate = feerate_from_psbt(bitcoind, l1, prep['psbt'])
+            assert feerate - 2 < actual_feerate
+            # Feerate can be larger, if it chose not to give change output.
+            if chainparams['elements']:
+                fee_output = 1
+            else:
+                fee_output = 0
+            if len(bitcoind.rpc.decoderawtransaction(prep['unsigned_tx'])['vout']) == 1 + 1 + fee_output:
+                assert actual_feerate < feerate + 2
+            l1.rpc.txdiscard(prep['txid'])
+
+
+@pytest.mark.parametrize("addrtype", ["bech32", "p2tr"])
+def test_fundpsbt_feerates(node_factory, bitcoind, chainparams, addrtype):
+    if chainparams['elements'] and addrtype == 'p2tr':
+        pytest.skip('No p2tr for elements')
+
+    l1 = node_factory.get_node()
+
+    # Add some funds to withdraw later
+    for i in range(20):
+        bitcoind.rpc.sendtoaddress(l1.rpc.newaddr(addrtype)[addrtype],
+                                   5000 / 10**8)
+
+    # See utxo_spend_weight()
+    if addrtype == 'bech32':
+        witness_weight = 1 + 71 + 1 + 33
+    elif addrtype == 'p2tr':
+        witness_weight = 1 + 64
+    else:
+        assert False
+
+    input_weight = 1 + witness_weight + (32 + 4 + 4 + 1) * 4
+    if chainparams['elements']:
+        input_weight += 6
+
+    bitcoind.generate_block(1)
+    wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 20)
+
+    # version, input count, output count, locktime, segwit marker, flag
+    base_weight = (4 + 1 + 1 + 4) * 4 + 1 + 1
+    if chainparams['elements']:
+        # Elements has empty surjection and rangeproof, and fee output
+        base_weight += 2 * 4 + (32 + 1 + 1 + 1) * 4
+        # Bech32 change output
+        change_weight = (8 + 1 + (1 + 1 + 20) + (32 + 1 + 1 + 1)) * 4
+    else:
+        # P2TR output
+        change_weight = (8 + 1 + (1 + 1 + 32)) * 4
+
+    # Both minimal and higher feerate
+    for feerate in (253, 1000):
+        # Try with both 1 and 2 inputs
+        for amount, num_inputs in ((260, 1), (5000, 2)):
+            prep = l1.rpc.fundpsbt(amount, f"{feerate}perkw", base_weight, excess_as_change=True)
+            assert prep['estimated_final_weight'] == base_weight + change_weight + input_weight * num_inputs
+            signed = l1.rpc.signpsbt(prep['psbt'])['signed_psbt']
+            sent = l1.rpc.sendpsbt(signed)
+            txinfo = bitcoind.rpc.getmempoolentry(sent['txid'])
+            assert txinfo['weight'] == prep['estimated_final_weight']
+            # We never actually added that `amount` output to PSBT, so that appears as "fee"
+            fee = int(txinfo['fees']['base'] * 100_000_000) - amount
+            actual_feerate = fee / (txinfo['weight'] / 1000)
+            # Out by one errors (due to rounding) change feerate by 2.
+            assert feerate - 2 < actual_feerate < feerate + 2
+
 
 def test_reserveinputs(node_factory, bitcoind, chainparams):
     amount = 1000000