pytest: enhance tests to test anchor and htlc tx feerates match targets.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Author: rustyrussell

tests/test_closing.py

@@ -3769,9 +3769,9 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind):
     fundsats = int(Millisatoshi(only_one(l1.rpc.listfunds()['outputs'])['amount_msat']).to_satoshi())
     psbt = l1.rpc.fundpsbt("all", "1000perkw", 1000)['psbt']
     # Pay 5k sats in fees, send most to l2
-    psbt = l1.rpc.addpsbtoutput(fundsats - 20000 - 5000, psbt, destination=l2.rpc.newaddr()['bech32'])['psbt']
-    # 10x2000 sat outputs for l1 to use.
-    for i in range(10):
+    psbt = l1.rpc.addpsbtoutput(fundsats - 24000 - 5000, psbt, destination=l2.rpc.newaddr()['bech32'])['psbt']
+    # 12x2000 sat outputs for l1 to use.
+    for i in range(12):
         psbt = l1.rpc.addpsbtoutput(2000, psbt)['psbt']
     l1.rpc.sendpsbt(l1.rpc.signpsbt(psbt)['signed_psbt'])
     bitcoind.generate_block(1, wait_for_mempool=1)
@@ -3803,6 +3803,13 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind):
 
     wait_for(lambda: len(bitcoind.rpc.getrawmempool()) == 2)
 
+    # Expect package feerate of 2000
+    details = bitcoind.rpc.getrawmempool(True).values()
+    total_weight = sum([d['weight'] for d in details])
+    total_fees = sum([float(d['fees']['base'])* 100_000_000 for d in details])
+    total_feerate_perkw = total_fees / total_weight * 1000
+    assert 2000 - 1 < total_feerate_perkw < 2000 + 1
+
     # But we don't mine it!  And fees go up again!
     l1.set_feerates((3000, 3000, 3000, 3000))
     bitcoind.generate_block(1, needfeerate=5000)
@@ -3811,6 +3818,13 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind):
     # We actually resubmit the commit tx, then the RBF:
     l1.daemon.wait_for_logs(['sendrawtx exit 0'] * 2)
 
+    # Expect package feerate of 3000
+    details = bitcoind.rpc.getrawmempool(True).values()
+    total_weight = sum([d['weight'] for d in details])
+    total_fees = sum([float(d['fees']['base'])* 100_000_000 for d in details])
+    total_feerate_perkw = total_fees / total_weight * 1000
+    assert 3000 - 1 < total_feerate_perkw < 3000 + 1
+
     # And now we'll get it in (there's some rounding, so feerate a bit lower!)
     bitcoind.generate_block(1, needfeerate=2990)
 
@@ -3832,8 +3846,15 @@ def test_closing_anchorspend_htlc_tx_rbf(node_factory, bitcoind):
     # It will enter the mempool
     wait_for(lambda: txid in bitcoind.rpc.getrawmempool())
 
+    tx = bitcoind.rpc.getrawmempool(True)[txid]
+    feerate_perkw = float(tx['fees']['base']) * 100_000_000 / tx['weight'] * 1000
+    # It actually has no change output, so it exceeds the fee quite a bit.
+    assert len(bitcoind.rpc.decoderawtransaction(bitcoind.rpc.getrawtransaction(txid))['vout']) == 1
+    assert 5000 - 1 < feerate_perkw < 5060 + 1
+
     # And this will mine it!
-    bitcoind.generate_block(1, needfeerate=4990)
+    bitcoind.generate_block(1, needfeerate=5000)
+    assert bitcoind.rpc.getrawmempool() == []
 
 
 @pytest.mark.parametrize("anchors", [False, True])
@@ -4011,6 +4032,12 @@ def test_peer_anchor_push(node_factory, bitcoind, executor, chainparams):
     # As blocks pass, we will use anchor to boost l3's tx.
     for block, feerate in zip(range(120, 124), (12000, 13000, 14000, 15000)): 
         l2.daemon.wait_for_log(fr"Worth fee [0-9]*sat for remote commit tx to get 100000000msat at block 125 \(\+{125 - block}\) at feerate {feerate}perkw")
+        # Check feerate for entire package (commitment tx + anchor) is ~ correct
+        details = bitcoind.rpc.getrawmempool(True).values()
+        total_weight = sum([d['weight'] for d in details])
+        total_fees = sum([float(d['fees']['base']) * 100_000_000 for d in details])
+        total_feerate_perkw = total_fees / total_weight * 1000
+        assert feerate - 1 < total_feerate_perkw < feerate + 1
         bitcoind.generate_block(1, needfeerate=16000)
         sync_blockheight(bitcoind, [l2])
         assert len(bitcoind.rpc.getrawmempool()) == 2

tests/test_wallet.py

@@ -287,6 +287,20 @@ def test_txprepare_multi(node_factory, bitcoind):
     l1.rpc.txdiscard(prep['txid'])
 
 
+def feerate_from_psbt(bitcoind, node, psbt):
+    # signpsbt insists they are reserved!
+    node.rpc.reserveinputs(psbt, exclusive=False)
+    final = node.rpc.dev_finalizepsbt(node.rpc.signpsbt(psbt)['signed_psbt'])
+    node.rpc.unreserveinputs(psbt)
+    psbt = node.rpc.setpsbtversion(final['psbt'], 0)['psbt']
+    # analyzepsbt gives a vsize, but not a weight!
+    # e.g. 'estimated_vsize': 356, 'estimated_feerate': Decimal('0.00030042'), 'fee': Decimal('0.00010695')
+    fee = int(bitcoind.rpc.analyzepsbt(psbt)['fee'] * 100_000_000)
+    weight = bitcoind.rpc.decoderawtransaction(final['tx'])['weight']
+    print(f"XXX actual weight = {weight}")
+    return fee / weight * 1000
+
+
 def test_txprepare(node_factory, bitcoind, chainparams):
     amount = 1000000
     l1 = node_factory.get_node(random_hsm=True)
@@ -299,13 +313,18 @@ def test_txprepare(node_factory, bitcoind, chainparams):
 
     bitcoind.generate_block(1)
     wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 10)
+    for est in l1.rpc.feerates('perkw')['perkw']['estimates']:
+        if est['blockcount'] == 12:
+            normal_feerate_perkw = est['feerate']
 
     prep = l1.rpc.txprepare(outputs=[{addr: Millisatoshi(amount * 3 * 1000)}])
     decode = bitcoind.rpc.decoderawtransaction(prep['unsigned_tx'])
     assert decode['txid'] == prep['txid']
     # 4 inputs, 2 outputs (3 if we have a fee output).
     assert len(decode['vin']) == 4
     assert len(decode['vout']) == 2 if not chainparams['feeoutput'] else 3
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep['psbt']) < normal_feerate_perkw + 1
 
     # One output will be correct.
     outnum = [i for i, o in enumerate(decode['vout']) if o['value'] == Decimal(amount * 3) / 10**8][0]
@@ -333,6 +352,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['value'] > Decimal(amount * 6) / 10**8 - Decimal(0.0002)
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep2['psbt']) < normal_feerate_perkw + 1
 
     # If I cancel the first one, I can get those first 4 outputs.
     discard = l1.rpc.txdiscard(prep['txid'])
@@ -351,6 +372,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['value'] > Decimal(amount * 4) / 10**8 - Decimal(0.0002)
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep3['psbt']) < normal_feerate_perkw + 1
 
     # Cannot discard twice.
     with pytest.raises(RpcError, match=r'not an unreleased txid'):
@@ -371,20 +394,28 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['value'] > Decimal(amount * 10) / 10**8 - Decimal(0.0003)
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep4['psbt']) < normal_feerate_perkw + 1
     l1.rpc.txdiscard(prep4['txid'])
 
     # Try passing in a utxo set
     utxos = [utxo["txid"] + ":" + str(utxo["output"])
              for utxo in l1.rpc.listfunds()["outputs"]][:4]
     prep5 = l1.rpc.txprepare([{addr:
                              Millisatoshi(amount * 3.5 * 1000)}], utxos=utxos)
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep3['psbt']) < normal_feerate_perkw + 1
 
     # Try passing unconfirmed utxos
     unconfirmed_utxo = l1.rpc.withdraw(l1.rpc.newaddr()["bech32"], 10**5)
     uutxos = [unconfirmed_utxo["txid"] + ":0"]
     with pytest.raises(RpcError, match=r"Could not afford"):
         l1.rpc.txprepare([{addr: Millisatoshi(amount * 3.5 * 1000)}],
                          utxos=uutxos)
+    # Feerate should be ~ as we asked for
+    unconfirmed_tx = bitcoind.rpc.getrawmempool(True)[unconfirmed_utxo["txid"]]
+    feerate_perkw = int(unconfirmed_tx['fees']['base'] * 100_000_000) * 1000 / unconfirmed_tx['weight']
+    assert normal_feerate_perkw - 1 < feerate_perkw < normal_feerate_perkw + 1
 
     decode = bitcoind.rpc.decoderawtransaction(prep5['unsigned_tx'])
     assert decode['txid'] == prep5['txid']
@@ -413,12 +444,16 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     # You can have one which is all, but not two.
     prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 1000)},
                               {addr: 'all'}])
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep5['psbt']) < normal_feerate_perkw + 1
     l1.rpc.txdiscard(prep5['txid'])
     with pytest.raises(RpcError, match=r"'all'"):
         prep5 = l1.rpc.txprepare([{addr: 'all'}, {addr: 'all'}])
 
     prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 500 + 100000)},
                               {addr: Millisatoshi(amount * 3 * 500 - 100000)}])
+    # Feerate should be ~ as we asked for
+    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep5['psbt']) < normal_feerate_perkw + 1
     decode = bitcoind.rpc.decoderawtransaction(prep5['unsigned_tx'])
     assert decode['txid'] == prep5['txid']
     # 4 inputs, 3 outputs(include change).
@@ -445,6 +480,80 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     else:
         assert decode['vout'][changenum]['scriptPubKey']['type'] == 'witness_v1_taproot'
 
+    l1.rpc.txdiscard(prep5['txid'])
+
+
+def test_txprepare_feerate(node_factory, bitcoind):
+    # Make sure it works at different feerates!
+    l1, l2 = node_factory.get_nodes(2)
+
+    # Add some funds to withdraw later
+    for i in range(20):
+        bitcoind.rpc.sendtoaddress(l1.rpc.newaddr()['bech32'],
+                                   1000 / 10**8)
+
+    bitcoind.generate_block(1)
+    out_addrs = l2.rpc.newaddr('all')
+    wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 20)
+
+    for addrtype in ('bech32', 'p2tr'):
+        for feerate in range(255, 10000, 250):
+            prep = l1.rpc.txprepare([{out_addrs[addrtype]: Millisatoshi(9000)}], f"{feerate}perkw")
+            assert feerate - 1 < feerate_from_psbt(bitcoind, l1, prep['psbt']) < feerate + 1
+            l1.rpc.txdiscard(prep6['txid'])
+
+
+@pytest.mark.parametrize("addrtype", ["bech32", "p2tr"])
+def test_fundpsbt_feerates(node_factory, bitcoind, chainparams, addrtype):
+    if chainparams['elements'] and addrtype == 'p2tr':
+        pytest.skip('No p2tr for elements')
+
+    l1 = node_factory.get_node()
+
+    # Add some funds to withdraw later
+    for i in range(20):
+        bitcoind.rpc.sendtoaddress(l1.rpc.newaddr()[addrtype],
+                                   1000 / 10**8)
+
+    # See utxo_spend_weight()
+    if addrtype == 'bech32':
+        witness_weight = 1 + 71 + 1 + 33
+    elif addrtype == 'p2tr':
+        witness_weight = 1 + 64
+    else:
+        assert False
+
+    input_weight = 1 + witness_weight + (32 + 4 + 4 + 1) * 4
+    if chainparams['elements']:
+        input_weight += 6
+
+    bitcoind.generate_block(1)
+    wait_for(lambda: len(l1.rpc.listfunds()['outputs']) == 20)
+
+    # version, input count, output count, locktime, segwit marker, flag
+    base_weight = (4 + 1 + 1 + 4) * 4 + 1 + 1
+    if chainparams['elements']:
+        # Elements has empty surjection and rangeproof, and fee output
+        base_weight += 2 * 4 + (32 + 1 + 1 + 1) * 4
+        # Bech32 change output
+        change_weight = (8 + 1 + (1 + 1 + 20) + (32 + 1 + 1 + 1)) * 4
+    else:
+        # P2TR output
+        change_weight = (8 + 1 + (1 + 1 + 32)) * 4
+
+    # Both minimal and higher feerate
+    for feerate in (253, 1000):
+        # Try with both 1 and 2 inputs
+        for amount, num_inputs in ((260, 1), (1000, 2)):
+            prep = l1.rpc.fundpsbt(amount, f"{feerate}perkw", base_weight, excess_as_change=True)
+            assert prep['estimated_final_weight'] == base_weight + change_weight + input_weight * num_inputs
+            signed = l1.rpc.signpsbt(prep['psbt'])['signed_psbt']
+            sent = l1.rpc.sendpsbt(signed)
+            txinfo = bitcoind.rpc.getmempoolentry(sent['txid'])
+            assert txinfo['weight'] == prep['estimated_final_weight']
+            actual_feerate = txinfo['fees']['base'] / txinfo['weight']
+            assert feerate - 1 < actual_feerate < feerate + 1
+
 
 def test_reserveinputs(node_factory, bitcoind, chainparams):
     amount = 1000000

wallet/reservation.c

@@ -397,6 +397,13 @@ static struct command_result *finish_psbt(struct command *cmd,
 		change_outnum = psbt->num_outputs;
 		psbt_append_output(psbt, b32script, change);
 		/* Add additional weight of output */
+		log_debug(cmd->ld->log, "Adding change %s: weight %zu + change %zu = %zu",
+			  fmt_amount_sat(tmpctx, change),
+			  weight,
+			  bitcoin_tx_output_weight(
+				  chainparams->is_elements ? BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN : BITCOIN_SCRIPTPUBKEY_P2TR_LEN),
+			  weight + bitcoin_tx_output_weight(
+				  chainparams->is_elements ? BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN : BITCOIN_SCRIPTPUBKEY_P2TR_LEN));
 		weight += bitcoin_tx_output_weight(
 				chainparams->is_elements ? BITCOIN_SCRIPTPUBKEY_P2WPKH_LEN : BITCOIN_SCRIPTPUBKEY_P2TR_LEN);
 	} else {
@@ -525,6 +532,8 @@ static struct command_result *json_fundpsbt(struct command *cmd,
 			 NULL))
 		return command_param_failed();
 
+	log_debug(cmd->ld->log, "fundpsbt feerate = %uperkw", *feerate_per_kw);
+
 	/* If we have anchor channels, we definitely need to keep
 	 * emergency funds.  */
 	if (have_anchor_channel(cmd->ld))
@@ -574,6 +583,9 @@ static struct command_result *json_fundpsbt(struct command *cmd,
 						    "impossible UTXO value");
 
 			/* But also adds weight */
+			log_debug(cmd->ld->log, "Adding utxo %s: weight %u + utxo_weight %u = %u",
+				  fmt_amount_sat(tmpctx, utxo->amount),
+				  *weight, utxo_weight, *weight + utxo_weight);
 			*weight += utxo_weight;
 			continue;
 		}