pytest: create warning if we grind signature shorter than 71 bytes, don't fail.

One in 256 times, we will grind a signature to 70 bytes (or shorter).  This breaks
our feerate tests.  Unfortunately the grinding is deterministic, so there doesn't
seem to be a way to avoid it.  So we add a log message, and then we skip the
feerate test if it happens.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

Author: rustyrussell

hsmd/hsmd.c

@@ -442,8 +442,11 @@ static struct io_plan *preinit_hsm(struct io_conn *conn,
 	if (tlv->no_preapprove_check)
 		dev_no_preapprove_check = *tlv->no_preapprove_check;
 
-	status_debug("preinit: dev_fail_preapprove = %u, dev_no_preapprove_check = %u",
-		     dev_fail_preapprove, dev_no_preapprove_check);
+	if (tlv->warn_on_overgrind)
+		dev_warn_on_overgrind = *tlv->warn_on_overgrind;
+
+	status_debug("preinit: dev_fail_preapprove = %u, dev_no_preapprove_check = %u, dev_warn_on_overgrind = %u",
+		     dev_fail_preapprove, dev_no_preapprove_check, dev_warn_on_overgrind);
 	/* We don't send a reply, just read next */
 	return client_read_next(conn, c);
 }

hsmd/hsmd_wire.csv

@@ -14,6 +14,8 @@ tlvtype,hsmd_dev_preinit_tlvs,fail_preapprove,1
 tlvdata,hsmd_dev_preinit_tlvs,fail_preapprove,fail,bool,
 tlvtype,hsmd_dev_preinit_tlvs,no_preapprove_check,3
 tlvdata,hsmd_dev_preinit_tlvs,no_preapprove_check,disable,bool,
+tlvtype,hsmd_dev_preinit_tlvs,warn_on_overgrind,5
+tlvdata,hsmd_dev_preinit_tlvs,warn_on_overgrind,enable,bool,
 
 #include <bitcoin/chainparams.h>
 # Start the HSM.

hsmd/libhsmd.c

@@ -40,6 +40,7 @@ bool initialized = false;
 /* Do we fail all preapprove requests? */
 bool dev_fail_preapprove = false;
 bool dev_no_preapprove_check = false;
+bool dev_warn_on_overgrind = false;
 
 struct hsmd_client *hsmd_client_new_main(const tal_t *ctx, u64 capabilities,
 					 void *extra)
@@ -553,6 +554,7 @@ static void sign_our_inputs(struct utxo **utxos, struct wally_psbt *psbt)
 		for (size_t j = 0; j < psbt->num_inputs; j++) {
 			struct privkey privkey;
 			struct pubkey pubkey;
+			bool needed_sig;
 
 			if (!wally_psbt_input_spends(&psbt->inputs[j],
 						   &utxo->outpoint))
@@ -590,6 +592,10 @@ static void sign_our_inputs(struct utxo **utxos, struct wally_psbt *psbt)
 				wally_psbt_signing_cache_enable(psbt, 0);
 				is_cache_enabled = true;
 			}
+
+			/* We watch for pre-taproot variable-length sigs */
+			needed_sig = (psbt->inputs[j].signatures.num_items == 0);
+
 			if (wally_psbt_sign(psbt, privkey.secret.data,
 					    sizeof(privkey.secret.data),
 					    EC_FLAG_GRIND_R) != WALLY_OK) {
@@ -602,6 +608,14 @@ static void sign_our_inputs(struct utxo **utxos, struct wally_psbt *psbt)
 				    j, fmt_pubkey(tmpctx, &pubkey),
 				    fmt_wally_psbt(tmpctx, psbt));
 			}
+			if (dev_warn_on_overgrind
+			    && needed_sig
+			    && psbt->inputs[j].signatures.num_items == 1
+			    && psbt->inputs[j].signatures.items[0].value_len < 71) {
+				hsmd_status_fmt(LOG_BROKEN, NULL,
+						"overgrind: short signature length %zu",
+						psbt->inputs[j].signatures.items[0].value_len);
+			}
 			tal_wally_end(psbt);
 		}
 	}

hsmd/libhsmd.h

@@ -100,4 +100,6 @@ extern struct secret *dev_force_bip32_seed;
 extern bool dev_fail_preapprove;
 /* If they specify --dev-no-preapprove-check it ends up in here. */
 extern bool dev_no_preapprove_check;
+/* If they specify --dev-warn-on-overgrind it ends up in here. */
+extern bool dev_warn_on_overgrind;
 #endif /* LIGHTNING_HSMD_LIBHSMD_H */

lightningd/hsm_control.c

@@ -119,6 +119,8 @@ struct ext_key *hsm_init(struct lightningd *ld)
 					       &ld->dev_hsmd_fail_preapprove);
 		tlv->no_preapprove_check = tal_dup(tlv, bool,
 						   &ld->dev_hsmd_no_preapprove_check);
+		tlv->warn_on_overgrind = tal_dup(tlv, bool,
+						 &ld->dev_hsmd_warn_on_overgrind);
 
 		msg = towire_hsmd_dev_preinit(tmpctx, tlv);
 		if (!wire_sync_write(ld->hsm_fd, msg))

lightningd/lightningd.c

@@ -151,6 +151,7 @@ static struct lightningd *new_lightningd(const tal_t *ctx)
 	ld->dev_allow_shutdown_destination_change = false;
 	ld->dev_hsmd_no_preapprove_check = false;
 	ld->dev_hsmd_fail_preapprove = false;
+	ld->dev_hsmd_warn_on_overgrind = false;
 	ld->dev_handshake_no_reply = false;
 	ld->dev_strict_forwarding = false;
 	ld->dev_limit_connections_inflight = false;

lightningd/lightningd.h

@@ -353,6 +353,7 @@ struct lightningd {
 	/* hsmd characteristic tweaks */
 	bool dev_hsmd_no_preapprove_check;
 	bool dev_hsmd_fail_preapprove;
+	bool dev_hsmd_warn_on_overgrind;
 
 	/* Tell connectd not to talk after handshake */
 	bool dev_handshake_no_reply;

lightningd/options.c

@@ -936,6 +936,10 @@ static void dev_register_opts(struct lightningd *ld)
 		       opt_set_crash_timeout, NULL,
 		       ld,
 		       "Crash if we are still going after this long.");
+	clnopt_noarg("--dev-warn-on-overgrind", OPT_DEV,
+		       opt_set_bool,
+		       &ld->dev_hsmd_warn_on_overgrind,
+		       "Warn if we create signatures that are not exactly 71 bytes.");
 	/* This is handled directly in daemon_developer_mode(), so we ignore it here */
 	clnopt_noarg("--dev-debug-self", OPT_DEV,
 		     opt_ignore,

tests/test_wallet.py

@@ -301,9 +301,18 @@ def feerate_from_psbt(bitcoind, node, psbt):
     return fee / weight * 1000
 
 
+# I wish we could force libwally to use different entropy and thus force it to
+# create 71-byte sigs always!
+def did_short_sig(node):
+    # This can take a moment to appear in the log!
+    time.sleep(1)
+    return node.daemon.is_in_log('overgrind: short signature length')
+
+
 def test_txprepare(node_factory, bitcoind, chainparams):
     amount = 1000000
-    l1 = node_factory.get_node(random_hsm=True)
+    l1 = node_factory.get_node(random_hsm=True, options={'dev-warn-on-overgrind': None},
+                               broken_log='overgrind: short signature length')
     addr = chainparams['example_addr']
 
     # Add some funds to withdraw later
@@ -324,7 +333,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert len(decode['vin']) == 4
     assert len(decode['vout']) == 2 if not chainparams['feeoutput'] else 3
     # Feerate should be ~ as we asked for
-    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep['psbt']) < normal_feerate_perkw + 1
+    if not did_short_sig(l1):
+        assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep['psbt']) < normal_feerate_perkw + 1
 
     # One output will be correct.
     outnum = [i for i, o in enumerate(decode['vout']) if o['value'] == Decimal(amount * 3) / 10**8][0]
@@ -353,7 +363,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
     # Feerate should be ~ as we asked for
-    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep2['psbt']) < normal_feerate_perkw + 1
+    if not did_short_sig(l1):
+        assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep2['psbt']) < normal_feerate_perkw + 1
 
     # If I cancel the first one, I can get those first 4 outputs.
     discard = l1.rpc.txdiscard(prep['txid'])
@@ -373,7 +384,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
     # Feerate should be ~ as we asked for
-    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep3['psbt']) < normal_feerate_perkw + 1
+    if not did_short_sig(l1):
+        assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep3['psbt']) < normal_feerate_perkw + 1
 
     # Cannot discard twice.
     with pytest.raises(RpcError, match=r'not an unreleased txid'):
@@ -395,7 +407,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     assert decode['vout'][0]['scriptPubKey']['type'] == 'witness_v0_keyhash'
     assert scriptpubkey_addr(decode['vout'][0]['scriptPubKey']) == addr
     # Feerate should be ~ as we asked for
-    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep4['psbt']) < normal_feerate_perkw + 1
+    if not did_short_sig(l1):
+        assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep4['psbt']) < normal_feerate_perkw + 1
     l1.rpc.txdiscard(prep4['txid'])
 
     # Try passing in a utxo set
@@ -404,7 +417,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     prep5 = l1.rpc.txprepare([{addr:
                              Millisatoshi(amount * 3.5 * 1000)}], utxos=utxos)
     # Feerate should be ~ as we asked for
-    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep3['psbt']) < normal_feerate_perkw + 1
+    if not did_short_sig(l1):
+        assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep3['psbt']) < normal_feerate_perkw + 1
 
     # Try passing unconfirmed utxos
     unconfirmed_utxo = l1.rpc.withdraw(l1.rpc.newaddr()["bech32"], 10**5)
@@ -445,15 +459,17 @@ def test_txprepare(node_factory, bitcoind, chainparams):
     prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 1000)},
                               {addr: 'all'}])
     # Feerate should be ~ as we asked for
-    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep5['psbt']) < normal_feerate_perkw + 1
+    if not did_short_sig(l1):
+        assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep5['psbt']) < normal_feerate_perkw + 1
     l1.rpc.txdiscard(prep5['txid'])
     with pytest.raises(RpcError, match=r"'all'"):
         prep5 = l1.rpc.txprepare([{addr: 'all'}, {addr: 'all'}])
 
     prep5 = l1.rpc.txprepare([{addr: Millisatoshi(amount * 3 * 500 + 100000)},
                               {addr: Millisatoshi(amount * 3 * 500 - 100000)}])
     # Feerate should be ~ as we asked for
-    assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep5['psbt']) < normal_feerate_perkw + 1
+    if not did_short_sig(l1):
+        assert normal_feerate_perkw - 1 < feerate_from_psbt(bitcoind, l1, prep5['psbt']) < normal_feerate_perkw + 1
     decode = bitcoind.rpc.decoderawtransaction(prep5['unsigned_tx'])
     assert decode['txid'] == prep5['txid']
     # 4 inputs, 3 outputs(include change).
@@ -485,7 +501,8 @@ def test_txprepare(node_factory, bitcoind, chainparams):
 
 def test_txprepare_feerate(node_factory, bitcoind):
     # Make sure it works at different feerates!
-    l1, l2 = node_factory.get_nodes(2)
+    l1, l2 = node_factory.get_nodes(2, opts={'dev-warn-on-overgrind': None,
+                                             'broken_log': 'overgrind: short signature length'})
 
     # Add some funds to withdraw later
     for i in range(20):
@@ -499,7 +516,8 @@ def test_txprepare_feerate(node_factory, bitcoind):
     for addrtype in ('bech32', 'p2tr'):
         for feerate in range(255, 10000, 250):
             prep = l1.rpc.txprepare([{out_addrs[addrtype]: Millisatoshi(9000)}], f"{feerate}perkw")
-            assert feerate - 1 < feerate_from_psbt(bitcoind, l1, prep['psbt']) < feerate + 1
+            if not did_short_sig(l1):
+                assert feerate - 1 < feerate_from_psbt(bitcoind, l1, prep['psbt']) < feerate + 1
             l1.rpc.txdiscard(prep6['txid'])