From 983f9cecb804e26989811294f7392fa7b4b32120 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Thu, 19 Jun 2025 15:19:21 -0700 Subject: [PATCH 01/21] feat: add failsafe to transaction replay --- stacks-signer/src/v0/signer_state.rs | 89 ++++++++- stackslib/src/net/api/postblock_proposal.rs | 23 +++ testnet/stacks-node/src/tests/signer/v0.rs | 194 +++++++++++++++++++- 3 files changed, 299 insertions(+), 7 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index fb5e13776a..d35512bf4b 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -21,6 +21,7 @@ use std::time::{Duration, UNIX_EPOCH}; use blockstack_lib::chainstate::burn::ConsensusHashExtensions; use blockstack_lib::chainstate::nakamoto::{NakamotoBlock, NakamotoBlockHeader}; use blockstack_lib::chainstate::stacks::{StacksTransaction, TransactionPayload}; +use blockstack_lib::net::api::get_tenures_fork_info::TenureForkingInfo; use blockstack_lib::net::api::postblock_proposal::NakamotoBlockProposal; use clarity::types::chainstate::StacksAddress; #[cfg(any(test, feature = "testing"))] @@ -620,7 +621,7 @@ impl LocalStateMachine { client, &expected_burn_block, &prior_state_machine, - replay_state, + &replay_state, )? { match new_replay_state { ReplayState::Unset => { @@ -632,6 +633,18 @@ impl LocalStateMachine { *tx_replay_scope = Some(new_scope); } } + } else { + if Self::handle_possible_replay_failsafe( + &replay_state, + &expected_burn_block, + client, + )? { + info!( + "Signer state: replay set is stalled after 2 tenures. Clearing the replay set." + ); + tx_replay_set = ReplayTransactionSet::none(); + *tx_replay_scope = None; + } } } @@ -981,7 +994,7 @@ impl LocalStateMachine { client: &StacksClient, expected_burn_block: &NewBurnBlock, prior_state_machine: &SignerStateMachine, - replay_state: ReplayState, + replay_state: &ReplayState, ) -> Result, SignerChainstateError> { if expected_burn_block.burn_block_height > prior_state_machine.burn_block_height { // no bitcoin fork, because we're advancing the burn block height @@ -1088,7 +1101,7 @@ impl LocalStateMachine { client: &StacksClient, expected_burn_block: &NewBurnBlock, prior_state_machine: &SignerStateMachine, - scope: ReplayScope, + scope: &ReplayScope, ) -> Result, SignerChainstateError> { info!("Tx Replay: detected bitcoin fork while in replay mode. Tryng to handle the fork"; "expected_burn_block.height" => expected_burn_block.burn_block_height, @@ -1182,6 +1195,10 @@ impl LocalStateMachine { return Ok(None); } + Ok(Some(Self::get_forked_txs_from_fork_info(&fork_info))) + } + + fn get_forked_txs_from_fork_info(fork_info: &Vec) -> Vec { // Collect transactions to be replayed across the forked blocks let mut forked_blocks = fork_info .iter() @@ -1201,6 +1218,70 @@ impl LocalStateMachine { )) .cloned() .collect::>(); - Ok(Some(forked_txs)) + forked_txs + } + + /// If it has been 2 burn blocks since the origin of our replay set, and + /// we haven't produced any replay blocks since then, we should reset our replay set + /// + /// Returns a `bool` indicating whether the replay set should be reset. + fn handle_possible_replay_failsafe( + replay_state: &ReplayState, + new_burn_block: &NewBurnBlock, + client: &StacksClient, + ) -> Result { + let ReplayState::InProgress(_, replay_scope) = replay_state else { + // Not in replay - skip + return Ok(false); + }; + + // if replay_scope.fork_origin.burn_block_height + 2 >= new_burn_block.burn_block_height { + if new_burn_block.burn_block_height < replay_scope.fork_origin.burn_block_height + 2 { + // We havent' had two burn blocks yet - skip + return Ok(false); + } + + info!("Signer state: checking for replay set failsafe"; + "replay_scope.fork_origin.burn_block_height" => replay_scope.fork_origin.burn_block_height, + "new_burn_block.burn_block_height" => new_burn_block.burn_block_height, + ); + let Ok(fork_info) = client.get_tenure_forking_info( + &replay_scope.fork_origin.consensus_hash, + &new_burn_block.consensus_hash, + ) else { + warn!("Signer state: failed to get fork info"); + return Ok(false); + }; + + let tenures_with_sortition = fork_info + .iter() + .filter(|fork_info| { + fork_info.was_sortition + && fork_info + .nakamoto_blocks + .as_ref() + .map(|b| b.len()) + .unwrap_or(0) + > 0 + }) + .count(); + + info!("Signer state: fork info in failsafe check"; + "tenures_with_sortition" => tenures_with_sortition, + "fork_info" => ?fork_info, + ); + + if tenures_with_sortition < 2 { + // We might have had 2 burn blocks, but not 2 tenures. + return Ok(false); + } + + let forked_txs = Self::get_forked_txs_from_fork_info(&fork_info); + + info!("Signer state: forked txs in failsafe check"; + "forked_txs_len" => forked_txs.len(), + ); + + Ok(forked_txs.is_empty()) } } diff --git a/stackslib/src/net/api/postblock_proposal.rs b/stackslib/src/net/api/postblock_proposal.rs index 9cddbbedf0..c95d9e92c4 100644 --- a/stackslib/src/net/api/postblock_proposal.rs +++ b/stackslib/src/net/api/postblock_proposal.rs @@ -68,6 +68,10 @@ pub static TEST_REPLAY_TRANSACTIONS: LazyLock< TestFlag>, > = LazyLock::new(TestFlag::default); +#[cfg(any(test, feature = "testing"))] +/// Whether to reject any transaction while we're in a replay set. +pub static TEST_REJECT_REPLAY_TXS: LazyLock> = LazyLock::new(TestFlag::default); + // This enum is used to supply a `reason_code` for validation // rejection responses. This is serialized as an enum with string // type (in jsonschema terminology). @@ -200,6 +204,24 @@ fn fault_injection_validation_delay() { #[cfg(not(any(test, feature = "testing")))] fn fault_injection_validation_delay() {} +#[cfg(any(test, feature = "testing"))] +fn fault_injection_reject_replay_txs() -> Result<(), BlockValidateRejectReason> { + let reject = TEST_REJECT_REPLAY_TXS.get(); + if reject { + Err(BlockValidateRejectReason { + reason_code: ValidateRejectCode::InvalidTransactionReplay, + reason: "Rejected by test flag".into(), + }) + } else { + Ok(()) + } +} + +#[cfg(not(any(test, feature = "testing")))] +fn fault_injection_reject_replay_txs() -> Result<(), BlockValidateRejectReason> { + Ok(()) +} + /// Represents a block proposed to the `v3/block_proposal` endpoint for validation #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct NakamotoBlockProposal { @@ -723,6 +745,7 @@ impl NakamotoBlockProposal { // Allow this to happen, tenure extend checks happen elsewhere. break; } + fault_injection_reject_replay_txs()?; let Some(replay_tx) = replay_txs.pop_front() else { // During transaction replay, we expect that the block only // contains transactions from the replay set. Thus, if we're here, diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 54e1afdac4..bee662697d 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -49,7 +49,6 @@ use stacks::chainstate::stacks::boot::MINERS_NAME; use stacks::chainstate::stacks::db::{StacksBlockHeaderTypes, StacksChainState, StacksHeaderInfo}; use stacks::chainstate::stacks::miner::{ TransactionEvent, TransactionSuccessEvent, TEST_EXCLUDE_REPLAY_TXS, - TEST_MINE_ALLOWED_REPLAY_TXS, }; use stacks::chainstate::stacks::{ StacksTransaction, TenureChangeCause, TenureChangePayload, TransactionPayload, @@ -65,8 +64,8 @@ use stacks::core::{StacksEpochId, CHAIN_ID_TESTNET, HELIUM_BLOCK_LIMIT_20}; use stacks::libstackerdb::StackerDBChunkData; use stacks::net::api::getsigner::GetSignerResponse; use stacks::net::api::postblock_proposal::{ - BlockValidateResponse, ValidateRejectCode, TEST_VALIDATE_DELAY_DURATION_SECS, - TEST_VALIDATE_STALL, + BlockValidateResponse, ValidateRejectCode, TEST_REJECT_REPLAY_TXS, + TEST_VALIDATE_DELAY_DURATION_SECS, TEST_VALIDATE_STALL, }; use stacks::net::relay::fault_injection::{clear_ignore_block, set_ignore_block}; use stacks::types::chainstate::{ @@ -3802,6 +3801,195 @@ fn tx_replay_btc_on_stx_invalidation() { signer_test.shutdown(); } +/// Test scenario to ensure that the replay set is cleared +/// if there have been multiple tenures with a stalled replay set. +/// +/// This test is executed by triggering a fork, and then using +/// a test flag to reject any transaction replay blocks. +/// +/// The test mines a number of burn blocks during replay before +/// validating that the replay set is eventually cleared. +#[ignore] +#[test] +fn tx_replay_failsafe() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let num_signers = 5; + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); + let sender_addr = tests::to_addr(&sender_sk); + let send_amt = 100; + let send_fee = 180; + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender_addr, (send_amt + send_fee) * 10)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); + + let conf = &signer_test.running_nodes.conf; + let _http_origin = format!("http://{}", &conf.node.rpc_bind); + let btc_controller = &signer_test.running_nodes.btc_regtest_controller; + + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); + + let burnchain = conf.get_burnchain(); + + let tip = signer_test.get_peer_info(); + let pox_info = signer_test.get_pox_data(); + + info!("---- Burnchain ----"; + // "burnchain" => ?conf.burnchain, + "pox_constants" => ?burnchain.pox_constants, + "cycle" => burnchain.pox_constants.reward_cycle_index(0, tip.burn_block_height), + "pox_info" => ?pox_info, + ); + + let pre_fork_tenures = 11; + for i in 0..pre_fork_tenures { + info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + } + + info!("---- Submitting STX transfer ----"); + + let tip = get_chain_info(&conf); + // Make a transfer tx (this will get forked) + let (txid, nonce) = signer_test + .submit_transfer_tx(&sender_sk, send_fee, send_amt) + .unwrap(); + + // Ensure we got a new block with this tx + signer_test + .wait_for_nonce_increase(&sender_addr, nonce) + .expect("Timed out waiting for transfer tx to be mined"); + + wait_for(30, || { + let new_tip = get_chain_info(&conf); + Ok(new_tip.stacks_tip_height > tip.stacks_tip_height) + }) + .expect("Timed out waiting for transfer tx to be mined"); + + let tip = get_chain_info(&conf); + + info!("---- Triggering Bitcoin fork ----"; + "tip.stacks_tip_height" => tip.stacks_tip_height, + "tip.burn_block_height" => tip.burn_block_height, + ); + + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + btc_controller.invalidate_block(&burn_header_hash_to_fork); + btc_controller.build_next_block(3); + + TEST_MINE_STALL.set(true); + + let submitted_commits = signer_test + .running_nodes + .counters + .naka_submitted_commits + .clone(); + + // we need to mine some blocks to get back to being considered a frequent miner + for i in 0..3 { + let current_burn_height = get_chain_info(&conf).burn_block_height; + info!( + "Mining block #{i} to be considered a frequent miner"; + "current_burn_height" => current_burn_height, + ); + let commits_count = submitted_commits.load(Ordering::SeqCst); + next_block_and(&btc_controller, 60, || { + Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) + }) + .unwrap(); + } + + info!("---- Wait for tx replay set to be updated ----"); + + signer_test + .wait_for_signer_state_check(30, |state| { + let Some(tx_replay_set) = state.get_tx_replay_set() else { + return Ok(false); + }; + let len_ok = tx_replay_set.len() == 1; + let txid_ok = tx_replay_set[0].txid().to_hex() == txid; + info!("---- Signer state check ----"; + "tx_replay_set" => ?tx_replay_set, + "len_ok" => len_ok, + "txid_ok" => txid_ok, + ); + Ok(len_ok && txid_ok) + }) + .expect("Timed out waiting for tx replay set to be updated"); + + let tip_after_fork = get_chain_info(&conf); + + info!("---- Waiting for two tenures, without replay set cleared ----"; + "tip_after_fork.stacks_tip_height" => tip_after_fork.stacks_tip_height, + "tip_after_fork.burn_block_height" => tip_after_fork.burn_block_height + ); + + TEST_REJECT_REPLAY_TXS.set(true); + TEST_MINE_STALL.set(false); + + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height) + }) + .expect("Timed out waiting for one TenureChange block to be mined"); + + signer_test + .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_some())) + .expect("Expected replay set to still be set"); + + info!("---- Mining a second tenure ----"); + + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + + signer_test + .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_some())) + .expect("Expected replay set to still be set"); + + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 1) + }) + .expect("Timed out waiting for a TenureChange block to be mined"); + + info!("---- Mining a third tenure ----"); + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 1) + }) + .expect("Timed out waiting for a TenureChange block to be mined"); + + info!("---- Waiting for tx replay set to be cleared ----"); + + signer_test + .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_none())) + .expect("Expected replay set to be cleared"); + + signer_test.shutdown(); +} + /// Test scenario where two signers disagree on the tx replay set, /// which means there is no consensus on the tx replay set. #[test] From 4fa34995af3abfb552780c0e00f3a5d893982dd1 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Fri, 20 Jun 2025 06:42:46 -0700 Subject: [PATCH 02/21] fix: clippy --- stacks-signer/src/v0/signer_state.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index d35512bf4b..38bb420f6a 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -633,18 +633,16 @@ impl LocalStateMachine { *tx_replay_scope = Some(new_scope); } } - } else { - if Self::handle_possible_replay_failsafe( - &replay_state, - &expected_burn_block, - client, - )? { - info!( + } else if Self::handle_possible_replay_failsafe( + &replay_state, + &expected_burn_block, + client, + )? { + info!( "Signer state: replay set is stalled after 2 tenures. Clearing the replay set." ); - tx_replay_set = ReplayTransactionSet::none(); - *tx_replay_scope = None; - } + tx_replay_set = ReplayTransactionSet::none(); + *tx_replay_scope = None; } } @@ -1198,7 +1196,7 @@ impl LocalStateMachine { Ok(Some(Self::get_forked_txs_from_fork_info(&fork_info))) } - fn get_forked_txs_from_fork_info(fork_info: &Vec) -> Vec { + fn get_forked_txs_from_fork_info(fork_info: &[TenureForkingInfo]) -> Vec { // Collect transactions to be replayed across the forked blocks let mut forked_blocks = fork_info .iter() From d422eae9b1166740751226c906fb4e6cce69126a Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Sat, 21 Jun 2025 11:21:24 -0700 Subject: [PATCH 03/21] feat: wait for +2 blocks after previous fork tip to reset --- stacks-signer/src/chainstate.rs | 4 + stacks-signer/src/client/mod.rs | 1 + stacks-signer/src/config.rs | 19 ++++ stacks-signer/src/runloop.rs | 1 + stacks-signer/src/tests/chainstate.rs | 1 + stacks-signer/src/v0/signer.rs | 3 + stacks-signer/src/v0/signer_state.rs | 54 +-------- .../src/tests/nakamoto_integrations.rs | 3 + testnet/stacks-node/src/tests/signer/v0.rs | 105 +++++++++++++----- 9 files changed, 117 insertions(+), 74 deletions(-) diff --git a/stacks-signer/src/chainstate.rs b/stacks-signer/src/chainstate.rs index 28173daa4c..d2cfc86e64 100644 --- a/stacks-signer/src/chainstate.rs +++ b/stacks-signer/src/chainstate.rs @@ -143,6 +143,9 @@ pub struct ProposalEvalConfig { pub reorg_attempts_activity_timeout: Duration, /// Time to wait before submitting a block proposal to the stacks-node pub proposal_wait_for_parent_time: Duration, + /// How many blocks after a fork should we reset the replay set, + /// as a failsafe mechanism? + pub reset_replay_set_after_fork_blocks: u64, } impl From<&SignerConfig> for ProposalEvalConfig { @@ -155,6 +158,7 @@ impl From<&SignerConfig> for ProposalEvalConfig { reorg_attempts_activity_timeout: value.reorg_attempts_activity_timeout, tenure_idle_timeout_buffer: value.tenure_idle_timeout_buffer, proposal_wait_for_parent_time: value.proposal_wait_for_parent_time, + reset_replay_set_after_fork_blocks: value.reset_replay_set_after_fork_blocks, } } } diff --git a/stacks-signer/src/client/mod.rs b/stacks-signer/src/client/mod.rs index ee4d3c7f1b..7cf47f28e1 100644 --- a/stacks-signer/src/client/mod.rs +++ b/stacks-signer/src/client/mod.rs @@ -432,6 +432,7 @@ pub(crate) mod tests { reorg_attempts_activity_timeout: config.reorg_attempts_activity_timeout, proposal_wait_for_parent_time: config.proposal_wait_for_parent_time, validate_with_replay_tx: config.validate_with_replay_tx, + reset_replay_set_after_fork_blocks: config.reset_replay_set_after_fork_blocks, } } diff --git a/stacks-signer/src/config.rs b/stacks-signer/src/config.rs index c7a7cd8ecc..37b0b32e69 100644 --- a/stacks-signer/src/config.rs +++ b/stacks-signer/src/config.rs @@ -49,6 +49,9 @@ const DEFAULT_TENURE_IDLE_TIMEOUT_BUFFER_SECS: u64 = 2; /// cannot determine that our stacks-node has processed the parent /// block const DEFAULT_PROPOSAL_WAIT_TIME_FOR_PARENT_SECS: u64 = 15; +/// Default number of blocks after a fork to reset the replay set, +/// as a failsafe mechanism +const DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS: u64 = 2; #[derive(thiserror::Error, Debug)] /// An error occurred parsing the provided configuration @@ -184,6 +187,9 @@ pub struct SignerConfig { pub proposal_wait_for_parent_time: Duration, /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: bool, + /// How many blocks after a fork should we reset the replay set, + /// as a failsafe mechanism? + pub reset_replay_set_after_fork_blocks: u64, } /// The parsed configuration for the signer @@ -237,6 +243,9 @@ pub struct GlobalConfig { pub dry_run: bool, /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: bool, + /// How many blocks after a fork should we reset the replay set, + /// as a failsafe mechanism? + pub reset_replay_set_after_fork_blocks: u64, } /// Internal struct for loading up the config file @@ -288,6 +297,9 @@ struct RawConfigFile { pub dry_run: Option, /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: Option, + /// How many blocks after a fork should we reset the replay set, + /// as a failsafe mechanism? + pub reset_replay_set_after_fork_blocks: Option, } impl RawConfigFile { @@ -413,6 +425,10 @@ impl TryFrom for GlobalConfig { // https://github.com/stacks-network/stacks-core/issues/6087 let validate_with_replay_tx = raw_data.validate_with_replay_tx.unwrap_or(false); + let reset_replay_set_after_fork_blocks = raw_data + .reset_replay_set_after_fork_blocks + .unwrap_or(DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS); + Ok(Self { node_host: raw_data.node_host, endpoint, @@ -435,6 +451,7 @@ impl TryFrom for GlobalConfig { tenure_idle_timeout_buffer, proposal_wait_for_parent_time, validate_with_replay_tx, + reset_replay_set_after_fork_blocks, }) } } @@ -714,12 +731,14 @@ network = "mainnet" auth_password = "abcd" db_path = ":memory:" validate_with_replay_tx = true +reset_replay_set_after_fork_blocks = 100 "# ); let config = GlobalConfig::load_from_str(&config_toml).unwrap(); assert_eq!(config.stacks_address.to_string(), expected_addr); assert_eq!(config.to_chain_id(), CHAIN_ID_MAINNET); assert!(config.validate_with_replay_tx); + assert_eq!(config.reset_replay_set_after_fork_blocks, 100); } #[test] diff --git a/stacks-signer/src/runloop.rs b/stacks-signer/src/runloop.rs index c6f3cf3cfe..0c4da280ec 100644 --- a/stacks-signer/src/runloop.rs +++ b/stacks-signer/src/runloop.rs @@ -329,6 +329,7 @@ impl, T: StacksMessageCodec + Clone + Send + Debug> RunLo reorg_attempts_activity_timeout: self.config.reorg_attempts_activity_timeout, proposal_wait_for_parent_time: self.config.proposal_wait_for_parent_time, validate_with_replay_tx: self.config.validate_with_replay_tx, + reset_replay_set_after_fork_blocks: self.config.reset_replay_set_after_fork_blocks, })) } diff --git a/stacks-signer/src/tests/chainstate.rs b/stacks-signer/src/tests/chainstate.rs index 4569343ea1..aa8ebd72e8 100644 --- a/stacks-signer/src/tests/chainstate.rs +++ b/stacks-signer/src/tests/chainstate.rs @@ -92,6 +92,7 @@ fn setup_test_environment( tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(3), proposal_wait_for_parent_time: Duration::from_secs(0), + reset_replay_set_after_fork_blocks: 2, }, }; diff --git a/stacks-signer/src/v0/signer.rs b/stacks-signer/src/v0/signer.rs index 4db68aa349..42cd6facd3 100644 --- a/stacks-signer/src/v0/signer.rs +++ b/stacks-signer/src/v0/signer.rs @@ -127,6 +127,8 @@ pub struct Signer { pub validate_with_replay_tx: bool, /// Scope of Tx Replay in terms of Burn block boundaries pub tx_replay_scope: ReplayScopeOpt, + /// The number of blocks after the past tip to reset the replay set + pub reset_replay_set_after_fork_blocks: u64, } impl std::fmt::Display for SignerMode { @@ -244,6 +246,7 @@ impl SignerTrait for Signer { global_state_evaluator, validate_with_replay_tx: signer_config.validate_with_replay_tx, tx_replay_scope: None, + reset_replay_set_after_fork_blocks: signer_config.reset_replay_set_after_fork_blocks, } } diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index 38bb420f6a..5378db12fe 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -636,7 +636,7 @@ impl LocalStateMachine { } else if Self::handle_possible_replay_failsafe( &replay_state, &expected_burn_block, - client, + proposal_config.reset_replay_set_after_fork_blocks, )? { info!( "Signer state: replay set is stalled after 2 tenures. Clearing the replay set." @@ -1226,60 +1226,16 @@ impl LocalStateMachine { fn handle_possible_replay_failsafe( replay_state: &ReplayState, new_burn_block: &NewBurnBlock, - client: &StacksClient, + reset_replay_set_after_fork_blocks: u64, ) -> Result { let ReplayState::InProgress(_, replay_scope) = replay_state else { // Not in replay - skip return Ok(false); }; - // if replay_scope.fork_origin.burn_block_height + 2 >= new_burn_block.burn_block_height { - if new_burn_block.burn_block_height < replay_scope.fork_origin.burn_block_height + 2 { - // We havent' had two burn blocks yet - skip - return Ok(false); - } - - info!("Signer state: checking for replay set failsafe"; - "replay_scope.fork_origin.burn_block_height" => replay_scope.fork_origin.burn_block_height, - "new_burn_block.burn_block_height" => new_burn_block.burn_block_height, - ); - let Ok(fork_info) = client.get_tenure_forking_info( - &replay_scope.fork_origin.consensus_hash, - &new_burn_block.consensus_hash, - ) else { - warn!("Signer state: failed to get fork info"); - return Ok(false); - }; - - let tenures_with_sortition = fork_info - .iter() - .filter(|fork_info| { - fork_info.was_sortition - && fork_info - .nakamoto_blocks - .as_ref() - .map(|b| b.len()) - .unwrap_or(0) - > 0 - }) - .count(); - - info!("Signer state: fork info in failsafe check"; - "tenures_with_sortition" => tenures_with_sortition, - "fork_info" => ?fork_info, - ); - - if tenures_with_sortition < 2 { - // We might have had 2 burn blocks, but not 2 tenures. - return Ok(false); - } - - let forked_txs = Self::get_forked_txs_from_fork_info(&fork_info); - - info!("Signer state: forked txs in failsafe check"; - "forked_txs_len" => forked_txs.len(), - ); + let failsafe_height = + replay_scope.past_tip.burn_block_height + reset_replay_set_after_fork_blocks; - Ok(forked_txs.is_empty()) + Ok(new_burn_block.burn_block_height > failsafe_height) } } diff --git a/testnet/stacks-node/src/tests/nakamoto_integrations.rs b/testnet/stacks-node/src/tests/nakamoto_integrations.rs index f886a568d2..1bb251a4a1 100644 --- a/testnet/stacks-node/src/tests/nakamoto_integrations.rs +++ b/testnet/stacks-node/src/tests/nakamoto_integrations.rs @@ -6589,6 +6589,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: 2, }; let mut sortitions_view = SortitionsView::fetch_view(proposal_conf, &signer_client).unwrap(); @@ -6716,6 +6717,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: 2, }; let burn_block_height = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) .unwrap() @@ -6794,6 +6796,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: 2, }; let mut sortitions_view = SortitionsView::fetch_view(proposal_conf, &signer_client).unwrap(); sortitions_view diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index bee662697d..5ae8566bd1 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -74,13 +74,14 @@ use stacks::types::chainstate::{ }; use stacks::types::PublicKey; use stacks::util::get_epoch_time_secs; -use stacks::util::hash::{hex_bytes, Hash160, MerkleHashFunc, Sha512Trunc256Sum}; +use stacks::util::hash::{hex_bytes, to_hex, Hash160, MerkleHashFunc, Sha512Trunc256Sum}; use stacks::util::secp256k1::{Secp256k1PrivateKey, Secp256k1PublicKey}; use stacks::util_lib::boot::boot_code_id; use stacks::util_lib::signed_structured_data::pox4::{ make_pox_4_signer_key_signature, Pox4SignatureTopic, }; use stacks_common::bitvec::BitVec; +use stacks_common::deps_common::bitcoin::network::serialize::serialize; use stacks_common::types::chainstate::TrieHash; use stacks_common::util::sleep_ms; use stacks_signer::chainstate::{ProposalEvalConfig, SortitionsView}; @@ -101,6 +102,7 @@ use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; use super::SignerTest; +use crate::burnchains::bitcoin_regtest_controller::BitcoinRPCRequest; use crate::event_dispatcher::{ EventObserver, MinedNakamotoBlockEvent, TEST_SKIP_BLOCK_ANNOUNCEMENT, }; @@ -1602,6 +1604,7 @@ fn block_proposal_rejection() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: 2, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -3843,6 +3846,15 @@ fn tx_replay_failsafe() { let _http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; + let miner_pk = signer_test + .running_nodes + .btc_regtest_controller + .get_mining_pubkey() + .as_deref() + .map(Secp256k1PublicKey::from_hex) + .unwrap() + .unwrap(); + if signer_test.bootstrap_snapshot() { signer_test.shutdown_and_snapshot(); return; @@ -3862,7 +3874,7 @@ fn tx_replay_failsafe() { "pox_info" => ?pox_info, ); - let pre_fork_tenures = 11; + let pre_fork_tenures = 3; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -3887,38 +3899,77 @@ fn tx_replay_failsafe() { }) .expect("Timed out waiting for transfer tx to be mined"); - let tip = get_chain_info(&conf); + let tip_before = get_chain_info(&conf); info!("---- Triggering Bitcoin fork ----"; - "tip.stacks_tip_height" => tip.stacks_tip_height, - "tip.burn_block_height" => tip.burn_block_height, + "tip.stacks_tip_height" => tip_before.stacks_tip_height, + "tip.burn_block_height" => tip_before.burn_block_height, ); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let mut tx_hex = String::new(); + let mut commit_txid: Option = None; + + wait_for(30, || { + let Some(confirmed_utxo) = btc_controller + .get_all_utxos(&miner_pk) + .into_iter() + .find(|utxo| utxo.confirmations == 0) + else { + return Ok(false); + }; + let unconfirmed_txid = Txid::from_bitcoin_tx_hash(&confirmed_utxo.txid); + let unconfirmed_tx = btc_controller.get_raw_transaction(&unconfirmed_txid); + info!("---- Unconfirmed tx ----"; + "unconfirmed_tx.input" => ?unconfirmed_tx.input, + ); + let parent_txid = unconfirmed_tx.input[0].previous_output.txid; + let parent_tx = + btc_controller.get_raw_transaction(&Txid::from_bitcoin_tx_hash(&parent_txid)); + let parent_tx_opreturn_bytes = parent_tx.output[0].script_pubkey.as_bytes(); + // info!( + // "Parent tx bytes: {}", + // stacks::util::hash::to_hex(parent_tx_opreturn_bytes) + // ); + let data = LeaderBlockCommitOp::parse_data( + &parent_tx_opreturn_bytes[parent_tx_opreturn_bytes.len() - 77..], + ) + .unwrap(); + info!("---- Parent tx data ----"; + "data" => ?data, + ); + tx_hex = to_hex(serialize(&parent_tx).unwrap().as_slice()); + commit_txid = Some(Txid::from_bitcoin_tx_hash(&parent_txid)); + Ok(true) + }) + .expect("Failed to get unconfirmed tx"); + + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_before.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(1); TEST_MINE_STALL.set(true); - let submitted_commits = signer_test - .running_nodes - .counters - .naka_submitted_commits - .clone(); + // Wait for the block commit re-broadcast to be confirmed + wait_for(10, || { + let is_confirmed = + BitcoinRPCRequest::check_transaction_confirmed(&conf, &commit_txid.unwrap()).unwrap(); + Ok(is_confirmed) + }) + .expect("Timed out waiting for transaction to be confirmed"); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(&btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + let tip_before = get_chain_info(&conf); + + info!("---- Building next block ----"; + "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, + "tip_before.burn_block_height" => tip_before.burn_block_height, + ); + + btc_controller.build_next_block(1); + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for next block to be mined"); info!("---- Wait for tx replay set to be updated ----"); @@ -10310,6 +10361,7 @@ fn block_validation_response_timeout() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: 2, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -10599,6 +10651,7 @@ fn block_validation_pending_table() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: 2, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -11882,6 +11935,7 @@ fn incoming_signers_ignore_block_proposals() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: 2, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -12057,6 +12111,7 @@ fn outgoing_signers_ignore_block_proposals() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), + reset_replay_set_after_fork_blocks: 2, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), From 772798b098665ffbb21c2d0e16474d35c1d7f767 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Wed, 25 Jun 2025 07:14:39 -0700 Subject: [PATCH 04/21] fix: use pending burn block in bitcoin_block_arrival --- stacks-signer/src/v0/signer_state.rs | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index 5378db12fe..8d673626ec 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -564,15 +564,7 @@ impl LocalStateMachine { // to be changed. match update { StateMachineUpdate::BurnBlock(pending_burn_block) => { - match expected_burn_block { - None => expected_burn_block = Some(pending_burn_block), - Some(ref expected) => { - if pending_burn_block.burn_block_height > expected.burn_block_height - { - expected_burn_block = Some(pending_burn_block); - } - } - } + expected_burn_block = Some(pending_burn_block); } } @@ -595,9 +587,11 @@ impl LocalStateMachine { && next_burn_block_hash != expected_burn_block.consensus_hash; if node_behind_expected || node_on_equal_fork { let err_msg = format!( - "Node has not processed the next burn block yet. Expected height = {}, Expected consensus hash = {}", + "Node has not processed the next burn block yet. Expected height = {}, Expected consensus hash = {}, Node height = {}, Node consensus hash = {}", expected_burn_block.burn_block_height, expected_burn_block.consensus_hash, + next_burn_block_height, + next_burn_block_hash, ); *self = Self::Pending { update: StateMachineUpdate::BurnBlock(expected_burn_block), From bbea1c7cfc3bf93de320a7a3e84bc096e9bb4e63 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Wed, 25 Jun 2025 07:15:04 -0700 Subject: [PATCH 05/21] wip: update tx replay tests to work with failsafe --- .../burnchains/bitcoin_regtest_controller.rs | 11 + testnet/stacks-node/src/tests/signer/mod.rs | 43 + testnet/stacks-node/src/tests/signer/v0.rs | 1208 ++++++----------- 3 files changed, 467 insertions(+), 795 deletions(-) diff --git a/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs b/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs index d8a47e8f32..c860f21eff 100644 --- a/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs +++ b/testnet/stacks-node/src/burnchains/bitcoin_regtest_controller.rs @@ -2833,6 +2833,17 @@ impl BitcoinRPCRequest { BitcoinRPCRequest::send(config, payload) } + pub fn get_chain_tips(config: &Config) -> RPCResult { + let payload = BitcoinRPCRequest { + method: "getchaintips".to_string(), + params: vec![], + id: "stacks".to_string(), + jsonrpc: "2.0".to_string(), + }; + + BitcoinRPCRequest::send(config, payload) + } + pub fn send(config: &Config, payload: BitcoinRPCRequest) -> RPCResult { let request = BitcoinRPCRequest::build_rpc_request(config, &payload); let timeout = Duration::from_secs(u64::from(config.burnchain.timeout)); diff --git a/testnet/stacks-node/src/tests/signer/mod.rs b/testnet/stacks-node/src/tests/signer/mod.rs index 1729d6ca39..80e656f5b9 100644 --- a/testnet/stacks-node/src/tests/signer/mod.rs +++ b/testnet/stacks-node/src/tests/signer/mod.rs @@ -32,6 +32,7 @@ use libsigner::v0::messages::{ use libsigner::v0::signer_state::MinerState; use libsigner::{BlockProposal, SignerEntries, SignerEventTrait}; use serde::{Deserialize, Serialize}; +use stacks::burnchains::Txid; use stacks::chainstate::coordinator::comm::CoordinatorChannels; use stacks::chainstate::nakamoto::signer_set::NakamotoSigners; use stacks::chainstate::nakamoto::NakamotoBlock; @@ -192,6 +193,9 @@ impl + Send + 'static, T: SignerEventTrait + 'static> SignerTest + Send + 'static, T: SignerEventTrait + 'static> SignerTest) { + self.wait_for_signer_state_check(timeout, |state| { + let Some(replay_set) = state.get_tx_replay_set() else { + return Ok(false); + }; + let txids = replay_set + .iter() + .map(|tx| tx.txid().to_hex()) + .collect::>(); + Ok(txids == expected_txids) + }) + .expect("Timed out waiting for replay set to be equal to expected txids"); + } + /// Replace the test's configured signer st pub fn replace_signers( &mut self, @@ -1509,6 +1527,31 @@ impl + Send + 'static, T: SignerEventTrait + 'static> SignerTest(accepted.into()) .expect("Failed to send accept signature"); } + + /// Get the txid of the parent block commit transaction for the given miner + pub fn get_parent_block_commit_txid(&self, miner_pk: &StacksPublicKey) -> Option { + let Some(confirmed_utxo) = self + .running_nodes + .btc_regtest_controller + .get_all_utxos(&miner_pk) + .into_iter() + .find(|utxo| utxo.confirmations == 0) + else { + return None; + }; + let unconfirmed_txid = Txid::from_bitcoin_tx_hash(&confirmed_utxo.txid); + let unconfirmed_tx = self + .running_nodes + .btc_regtest_controller + .get_raw_transaction(&unconfirmed_txid); + let parent_txid = unconfirmed_tx + .input + .get(0) + .expect("First input should exist") + .previous_output + .txid; + Some(Txid::from_bitcoin_tx_hash(&parent_txid)) + } } fn setup_stx_btc_node( diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 5ae8566bd1..8e74a46d88 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -74,14 +74,13 @@ use stacks::types::chainstate::{ }; use stacks::types::PublicKey; use stacks::util::get_epoch_time_secs; -use stacks::util::hash::{hex_bytes, to_hex, Hash160, MerkleHashFunc, Sha512Trunc256Sum}; +use stacks::util::hash::{hex_bytes, Hash160, MerkleHashFunc, Sha512Trunc256Sum}; use stacks::util::secp256k1::{Secp256k1PrivateKey, Secp256k1PublicKey}; use stacks::util_lib::boot::boot_code_id; use stacks::util_lib::signed_structured_data::pox4::{ make_pox_4_signer_key_signature, Pox4SignatureTopic, }; use stacks_common::bitvec::BitVec; -use stacks_common::deps_common::bitcoin::network::serialize::serialize; use stacks_common::types::chainstate::TrieHash; use stacks_common::util::sleep_ms; use stacks_signer::chainstate::{ProposalEvalConfig, SortitionsView}; @@ -3109,37 +3108,46 @@ fn tx_replay_forking_test() { } let num_signers = 5; - let sender_sk = Secp256k1PrivateKey::random(); + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender_addr = tests::to_addr(&sender_sk); let send_amt = 100; let send_fee = 180; let deploy_fee = 1000000; let call_fee = 1000; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![( - sender_addr, - (send_amt + send_fee) * 10 + deploy_fee + call_fee, - )], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![( + sender_addr, + (send_amt + send_fee) * 10 + deploy_fee + call_fee, + )], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let http_origin = format!("http://{}", &conf.node.rpc_bind); let stacks_miner_pk = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); - let pre_fork_tenures = 10; + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); + + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); @@ -3165,49 +3173,24 @@ fn tx_replay_forking_test() { info!("------------------------- Triggering Bitcoin Fork -------------------------"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let tip_before = signer_test.get_peer_info(); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); - let submitted_commits = signer_test - .running_nodes - .counters - .naka_submitted_commits - .clone(); + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + btc_controller.build_next_block(1); + wait_for(30, || { + let tip = signer_test.get_peer_info(); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for stacks tip to decrease"); let post_fork_1_nonce = get_account(&http_origin, &sender_addr).nonce; - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == txid; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); // We should have forked 1 tx assert_eq!(post_fork_1_nonce, pre_fork_1_nonce - 1); @@ -3267,39 +3250,23 @@ fn tx_replay_forking_test() { TEST_MINE_STALL.set(true); + info!("---- Triggering deeper fork ----"); + + let tip_before = signer_test.get_peer_info(); + let burn_header_hash_to_fork = btc_controller.get_block_hash(pre_fork_2_tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(4); - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and( - &signer_test.running_nodes.btc_regtest_controller, - 60, - || Ok(submitted_commits.load(Ordering::SeqCst) > commits_count), - ) - .unwrap(); - } + wait_for(30, || { + let tip = signer_test.get_peer_info(); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for stacks tip to decrease"); let expected_tx_replay_txids = vec![transfer_txid, contract_deploy_txid, contract_call_txid]; - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let tx_replay_set_txids = tx_replay_set - .iter() - .map(|tx| tx.txid().to_hex()) - .collect::>(); - Ok(tx_replay_set_txids == expected_tx_replay_txids) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, expected_tx_replay_txids.clone()); info!("---- Mining post-fork block to clear tx replay set ----"); let tip_after_fork = get_chain_info(&conf); @@ -3375,37 +3342,46 @@ fn tx_replay_reject_invalid_proposals_during_replay() { } let num_signers = 5; - let sender_sk = Secp256k1PrivateKey::random(); + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender_addr = tests::to_addr(&sender_sk); - let sender_sk2 = Secp256k1PrivateKey::random(); + let sender_sk2 = Secp256k1PrivateKey::from_seed("sender_2".as_bytes()); let sender_addr2 = tests::to_addr(&sender_sk2); let send_amt = 100; let send_fee = 180; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![ - (sender_addr, send_amt + send_fee), - (sender_addr2, send_amt + send_fee), - ], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![ + (sender_addr, send_amt + send_fee), + (sender_addr2, send_amt + send_fee), + ], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; let stacks_miner_pk = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); - let pre_fork_tenures = 10; + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); + + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); @@ -3429,50 +3405,15 @@ fn tx_replay_reject_invalid_proposals_during_replay() { info!("------------------------- Triggering Bitcoin Fork -------------------------"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); - let submitted_commits = signer_test - .running_nodes - .counters - .naka_submitted_commits - .clone(); - - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); let post_fork_1_nonce = get_account(&http_origin, &sender_addr).nonce; - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == txid; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); - // We should have forked 1 tx assert_eq!(post_fork_1_nonce, pre_fork_1_nonce - 1); @@ -3631,11 +3572,13 @@ fn tx_replay_btc_on_stx_invalidation() { let mut miner_keychain = Keychain::default(conf.node.seed.clone()).generate_op_signer(); let _http_origin = format!("http://{}", &conf.node.rpc_bind); let mut btc_controller = BitcoinRegtestController::new(conf.clone(), None); - let submitted_commits = signer_test - .running_nodes - .counters - .naka_submitted_commits - .clone(); + + let miner_pk = btc_controller + .get_mining_pubkey() + .as_deref() + .map(Secp256k1PublicKey::from_hex) + .unwrap() + .unwrap(); if signer_test.bootstrap_snapshot() { signer_test.shutdown_and_snapshot(); @@ -3644,18 +3587,6 @@ fn tx_replay_btc_on_stx_invalidation() { info!("------------------------- Beginning test -------------------------"); - let burnchain = conf.get_burnchain(); - - let tip = signer_test.get_peer_info(); - let pox_info = signer_test.get_pox_data(); - - info!("---- Burnchain ----"; - // "burnchain" => ?conf.burnchain, - "pox_constants" => ?burnchain.pox_constants, - "cycle" => burnchain.pox_constants.reward_cycle_index(0, tip.burn_block_height), - "pox_info" => ?pox_info, - ); - info!("Submitting first pre-stx op"); let pre_stx_op = PreStxOp { output: sender_addr, @@ -3678,7 +3609,7 @@ fn tx_replay_btc_on_stx_invalidation() { "Pre-stx operation should submit successfully" ); - let pre_fork_tenures = 9; + let pre_fork_tenures = 1; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -3728,45 +3659,51 @@ fn tx_replay_btc_on_stx_invalidation() { info!("---- Triggering Bitcoin fork ----"); - let tip = signer_test.get_peer_info(); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); - btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + let mut commit_txid: Option = None; + wait_for(30, || { + let Some(txid) = signer_test.get_parent_block_commit_txid(&miner_pk) else { + return Ok(false); + }; + commit_txid = Some(txid); + Ok(true) + }) + .expect("Failed to get unconfirmed tx"); + let tip_before = signer_test.get_peer_info(); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_before.burn_block_height - 1); TEST_MINE_STALL.set(true); + btc_controller.invalidate_block(&burn_header_hash_to_fork); + btc_controller.build_next_block(2); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, + let tip_before = get_chain_info(&conf); + + info!("---- Building next block ----"; + "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, + "tip_before.burn_block_height" => tip_before.burn_block_height, + ); + + let chain_tips = BitcoinRPCRequest::get_chain_tips(&conf).unwrap(); + info!("---- chain_tips -----"; + "chain_tips" => ?chain_tips, + ); + + btc_controller.build_next_block(1); + + wait_for(30, || { + let tip = get_chain_info(&conf); + info!("----- tip -----"; + "tip.stacks_tip_height" => tip.stacks_tip_height, + "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, + "tip.burn_block_height" => tip.burn_block_height, + "tip_before.burn_block_height" => tip_before.burn_block_height, ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(&btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for next block to be mined"); info!("---- Wait for tx replay set to be updated ----"); - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - info!("---- No tx replay set"); - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == txid; - info!("---- Signer state check ----"; - "tx_replay_set" => ?tx_replay_set, - "len_ok" => len_ok, - "txid_ok" => txid_ok, - ); - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); info!("---- Waiting for tx replay set to be cleared ----"); @@ -3778,6 +3715,9 @@ fn tx_replay_btc_on_stx_invalidation() { .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_none())) .expect("Timed out waiting for tx replay set to be cleared"); + let account = get_account(&_http_origin, &recipient_addr); + assert_eq!(account.nonce, 0, "Expected recipient nonce to be 0"); + // Ensure that only one block was mined wait_for(30, || { let new_tip = get_chain_info(&conf).stacks_tip_height; @@ -3792,7 +3732,7 @@ fn tx_replay_btc_on_stx_invalidation() { assert!(matches!( block.transactions[0].payload, TransactionPayload::TenureChange(TenureChangePayload { - cause: TenureChangeCause::BlockFound, + cause: TenureChangeCause::Extended, .. }) )); @@ -3846,9 +3786,7 @@ fn tx_replay_failsafe() { let _http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let miner_pk = signer_test - .running_nodes - .btc_regtest_controller + let miner_pk = btc_controller .get_mining_pubkey() .as_deref() .map(Secp256k1PublicKey::from_hex) @@ -3906,39 +3844,12 @@ fn tx_replay_failsafe() { "tip.burn_block_height" => tip_before.burn_block_height, ); - let mut tx_hex = String::new(); let mut commit_txid: Option = None; - wait_for(30, || { - let Some(confirmed_utxo) = btc_controller - .get_all_utxos(&miner_pk) - .into_iter() - .find(|utxo| utxo.confirmations == 0) - else { + let Some(txid) = signer_test.get_parent_block_commit_txid(&miner_pk) else { return Ok(false); }; - let unconfirmed_txid = Txid::from_bitcoin_tx_hash(&confirmed_utxo.txid); - let unconfirmed_tx = btc_controller.get_raw_transaction(&unconfirmed_txid); - info!("---- Unconfirmed tx ----"; - "unconfirmed_tx.input" => ?unconfirmed_tx.input, - ); - let parent_txid = unconfirmed_tx.input[0].previous_output.txid; - let parent_tx = - btc_controller.get_raw_transaction(&Txid::from_bitcoin_tx_hash(&parent_txid)); - let parent_tx_opreturn_bytes = parent_tx.output[0].script_pubkey.as_bytes(); - // info!( - // "Parent tx bytes: {}", - // stacks::util::hash::to_hex(parent_tx_opreturn_bytes) - // ); - let data = LeaderBlockCommitOp::parse_data( - &parent_tx_opreturn_bytes[parent_tx_opreturn_bytes.len() - 77..], - ) - .unwrap(); - info!("---- Parent tx data ----"; - "data" => ?data, - ); - tx_hex = to_hex(serialize(&parent_tx).unwrap().as_slice()); - commit_txid = Some(Txid::from_bitcoin_tx_hash(&parent_txid)); + commit_txid = Some(txid); Ok(true) }) .expect("Failed to get unconfirmed tx"); @@ -3973,21 +3884,7 @@ fn tx_replay_failsafe() { info!("---- Wait for tx replay set to be updated ----"); - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == txid; - info!("---- Signer state check ----"; - "tx_replay_set" => ?tx_replay_set, - "len_ok" => len_ok, - "txid_ok" => txid_ok, - ); - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); let tip_after_fork = get_chain_info(&conf); @@ -4086,7 +3983,14 @@ fn tx_replay_disagreement() { miners.boot_to_epoch_3(); let btc_controller = &miners.signer_test.running_nodes.btc_regtest_controller; - let pre_fork_tenures = 10; + let miner_pk = btc_controller + .get_mining_pubkey() + .as_deref() + .map(Secp256k1PublicKey::from_hex) + .unwrap() + .unwrap(); + + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); @@ -4121,9 +4025,41 @@ fn tx_replay_disagreement() { }) .expect("Timed out waiting for transfer tx to be mined"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let mut commit_txid: Option = None; + wait_for(30, || { + let Some(txid) = miners.signer_test.get_parent_block_commit_txid(&miner_pk) else { + return Ok(false); + }; + commit_txid = Some(txid); + Ok(true) + }) + .expect("Failed to get unconfirmed tx"); + + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(1); + + // Wait for the block commit re-broadcast to be confirmed + wait_for(10, || { + let is_confirmed = + BitcoinRPCRequest::check_transaction_confirmed(&conf_1, &commit_txid.unwrap()).unwrap(); + Ok(is_confirmed) + }) + .expect("Timed out waiting for transaction to be confirmed"); + + let tip_before = get_chain_info(&conf_1); + + info!("---- Building next block ----"; + "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, + "tip_before.burn_block_height" => tip_before.burn_block_height, + ); + + btc_controller.build_next_block(1); + wait_for(30, || { + let tip = get_chain_info(&conf_1); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for next block to be mined"); // note, we should still have normal signer states! miners.signer_test.check_signer_states_normal(); @@ -4214,33 +4150,39 @@ fn tx_replay_solved_by_mempool_txs() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 2; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); - let pre_fork_tenures = 3; + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -4267,41 +4209,14 @@ fn tx_replay_solved_by_mempool_txs() { info!("------------------------- Triggering Bitcoin Fork -------------------------"); let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1 - && tx_replay_set[1].txid().to_hex() == sender1_tx2; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone(), sender1_tx2.clone()]); // We should have forked 2 txs let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; @@ -4474,31 +4389,37 @@ fn tx_replay_with_fork_occured_before_starting_replaying_txs() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 1; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); let pre_fork_tenures = 12; //go to 2nd tenure of 12th cycle for i in 0..pre_fork_tenures { @@ -4519,41 +4440,15 @@ fn tx_replay_with_fork_occured_before_starting_replaying_txs() { info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); // Signers move in Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); // We should have forked 1 tx let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; @@ -4563,37 +4458,13 @@ fn tx_replay_with_fork_occured_before_starting_replaying_txs() { let tip = get_chain_info(&conf); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - info!("Wait for block off of shallow fork"); TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); //Signers still are in the initial state of Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); info!("----------- Solve TX Replay ------------"); TEST_MINE_STALL.set(false); @@ -4631,31 +4502,35 @@ fn tx_replay_with_fork_after_empty_tenures_before_starting_replaying_txs() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 1; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); let pre_fork_tenures = 10; //go to Tenure #4 in Cycle #12 for i in 0..pre_fork_tenures { @@ -4677,110 +4552,41 @@ fn tx_replay_with_fork_after_empty_tenures_before_starting_replaying_txs() { info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); // Signers moved in Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); // We should have forked tx1 let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; assert_eq!(0, sender1_nonce_post_fork); info!("------------------- Produce Empty Tenuree -------------------------"); - TEST_MINE_STALL.set(false); let tip = get_chain_info(&conf); + TEST_MINE_STALL.set(false); _ = wait_for_tenure_change_tx(30, TenureChangeCause::BlockFound, tip.stacks_tip_height + 1); TEST_MINE_STALL.set(true); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); + + info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); + test_observer::clear(); - TEST_MINE_STALL.set(false); let tip = get_chain_info(&conf); - _ = wait_for_tenure_change_tx(30, TenureChangeCause::BlockFound, tip.stacks_tip_height + 1); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); + btc_controller.invalidate_block(&burn_header_hash_to_fork); TEST_MINE_STALL.set(true); - - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); - - info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); - test_observer::clear(); - - let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); - btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } // Signers still are in Tx Replay mode (as the initial replay state) - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); info!("------------------------- Mine Tx Replay Set -------------------------"); TEST_MINE_STALL.set(false); @@ -4814,37 +4620,40 @@ fn tx_replay_with_fork_causing_replay_set_to_be_updated() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 2; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); let pre_fork_tenures = 10; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.check_signer_states_normal(); } // Make 2 transfer txs, each in its own tenure so that can be forked in different forks @@ -4858,11 +4667,9 @@ fn tx_replay_with_fork_causing_replay_set_to_be_updated() { .expect("Expect sender1 nonce increased"); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.mine_nakamoto_block(Duration::from_secs(30), true); let tip_at_tx2 = get_chain_info(&conf); - assert_eq!(244, tip_at_tx2.burn_block_height); + assert_eq!(242, tip_at_tx2.burn_block_height); let (sender1_tx2, sender1_nonce) = signer_test .submit_transfer_tx(&sender1_sk, send_fee, send_amt) .unwrap(); @@ -4874,82 +4681,45 @@ fn tx_replay_with_fork_causing_replay_set_to_be_updated() { assert_eq!(2, sender1_nonce); info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_tx2.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_tx2.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); + btc_controller.build_next_block(1); info!("Wait for block off of shallow fork"); TEST_MINE_STALL.set(true); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - assert_eq!(247, get_chain_info(&conf).burn_block_height); + btc_controller.build_next_block(1); + + wait_for(10, || { + let tip = get_chain_info(&conf); + Ok(tip.burn_block_height == 243) + }) + .expect("Timed out waiting for burn block height to be 243"); // Signers move in Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx2; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx2.clone()]); // We should have forked one tx (Tx2) let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; assert_eq!(1, sender1_nonce_post_fork); - info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); + info!( + "------------------------- Triggering Bitcoin Fork #2 from {} -------------------------", + tip_at_tx1.burn_block_height + ); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_tx1.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(7); + btc_controller.build_next_block(4); + wait_for(10, || { + let tip = get_chain_info(&conf); + info!("Burn block height: {}", tip.burn_block_height); + Ok(tip.burn_block_height == 244) + }) + .expect("Timed out waiting for burn block height to be 244"); info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); - - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - assert_eq!(250, get_chain_info(&conf).burn_block_height); //Signers should update the Tx Replay Set - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1 - && tx_replay_set[1].txid().to_hex() == sender1_tx2; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone(), sender1_tx2.clone()]); info!("----------- Solve TX Replay ------------"); TEST_MINE_STALL.set(false); @@ -4986,31 +4756,35 @@ fn tx_replay_with_fork_causing_replay_to_be_cleared_due_to_cycle() { } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send_amt = 100; let send_fee = 180; let num_txs = 2; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender1_addr, (send_amt + send_fee) * num_txs)], - |c| { - c.validate_with_replay_tx = true; - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender1_addr, (send_amt + send_fee) * num_txs)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let http_origin = format!("http://{}", &conf.node.rpc_bind); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); let pre_fork_tenures = 8; for i in 0..pre_fork_tenures { @@ -5043,40 +4817,15 @@ fn tx_replay_with_fork_causing_replay_to_be_cleared_due_to_cycle() { assert_eq!(1, sender1_nonce); info!("------------------------- Triggering Bitcoin Fork #1 -------------------------"); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_rc12.burn_block_height - 2); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_rc12.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - // note, we should still have normal signer states! - signer_test.check_signer_states_normal(); + TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } // Signers move in Tx Replay mode - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 1; - let txid_ok = tx_replay_set[0].txid().to_hex() == sender1_tx1; - Ok(len_ok && txid_ok) - }) - .expect("Timed out waiting for tx replay set to be updated"); + signer_test.wait_for_replay_set_eq(30, vec![sender1_tx1.clone()]); // We should have forked one tx (Tx2) let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; @@ -5085,25 +4834,9 @@ fn tx_replay_with_fork_causing_replay_to_be_cleared_due_to_cycle() { info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_at_rc11.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(7); + btc_controller.build_next_block(6); info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); - - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } //Signers should clear the Tx Replay Set signer_test @@ -5139,35 +4872,39 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { } let num_signers = 5; - let sender_sk = Secp256k1PrivateKey::random(); + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender_addr = tests::to_addr(&sender_sk); let deploy_fee = 1000000; let call_fee = 1000; let call_num = 2; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![(sender_addr, deploy_fee + call_fee * call_num)], - |c| { - c.validate_with_replay_tx = true; - c.tenure_idle_timeout = Duration::from_secs(10); - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender_addr, deploy_fee + call_fee * call_num)], + |c| { + c.validate_with_replay_tx = true; + c.tenure_idle_timeout = Duration::from_secs(10); + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let stacks_miner_pk = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); - let pre_fork_tenures = 10; + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -5190,7 +4927,6 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { .expect("Timed out waiting for nonce to increase"); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.mine_nakamoto_block(Duration::from_secs(30), true); // Then, sumbmit 2 Contract Calls that require Tenure Extension to be addressed. info!("---- Submit big tx1 to be mined ----"); @@ -5202,12 +4938,13 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { .expect("Timed out waiting for nonce to increase"); info!("---- Submit big tx2 to be mined ----"); + let tip = get_chain_info(conf); + let (txid2, txid2_nonce) = signer_test .submit_contract_call(&sender_sk, call_fee, "big-contract", "big-tx", &vec![]) .unwrap(); // Tenure Extend happen because of tenure budget exceeded - let tip = get_chain_info(conf); _ = wait_for_tenure_change_tx(30, TenureChangeCause::Extended, tip.stacks_tip_height + 1); signer_test @@ -5219,39 +4956,14 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { info!("------------------------- Triggering Bitcoin Fork -------------------------"); let tip = get_chain_info(conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 1); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - info!("Wait for block off of shallow fork"); TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); let post_fork_nonce = get_account(&http_origin, &sender_addr).nonce; assert_eq!(1, post_fork_nonce); //due to contract deploy tx @@ -5265,54 +4977,18 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { TEST_MINE_STALL.set(true); // Signers still waiting for the Tx Replay set to be completed - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); //Fork in the middle of Tx Replay let tip = get_chain_info(&conf); let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 1); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); let post_fork_nonce = get_account(&http_origin, &sender_addr).nonce; assert_eq!(1, post_fork_nonce); //due to contract deploy tx @@ -5363,45 +5039,49 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted } let num_signers = 5; - let sender1_sk = Secp256k1PrivateKey::random(); + let sender1_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); let sender1_addr = tests::to_addr(&sender1_sk); let send1_deploy_fee = 1000000; let send1_call_fee = 1000; let send1_call_num = 2; - let sender2_sk = Secp256k1PrivateKey::random(); + let sender2_sk = Secp256k1PrivateKey::from_seed("sender_2".as_bytes()); let sender2_addr = tests::to_addr(&sender2_sk); let send2_amt = 100; let send2_fee = 180; let send2_txs = 1; - let signer_test: SignerTest = SignerTest::new_with_config_modifications( - num_signers, - vec![ - ( - sender1_addr, - send1_deploy_fee + send1_call_fee * send1_call_num, - ), - (sender2_addr, (send2_amt + send2_fee) * send2_txs), - ], - |c| { - c.validate_with_replay_tx = true; - c.tenure_idle_timeout = Duration::from_secs(10); - }, - |node_config| { - node_config.miner.block_commit_delay = Duration::from_secs(1); - node_config.miner.replay_transactions = true; - }, - None, - None, - ); + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![ + ( + sender1_addr, + send1_deploy_fee + send1_call_fee * send1_call_num, + ), + (sender2_addr, (send2_amt + send2_fee) * send2_txs), + ], + |c| { + c.validate_with_replay_tx = true; + c.tenure_idle_timeout = Duration::from_secs(10); + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); let conf = &signer_test.running_nodes.conf; let http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let counters = &signer_test.running_nodes.counters; let stacks_miner_pk = StacksPublicKey::from_private(&conf.miner.mining_key.unwrap()); - signer_test.boot_to_epoch_3(); - info!("------------------------- Reached Epoch 3.0 -------------------------"); - let pre_fork_tenures = 10; + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + info!("------------------------- Beginning test -------------------------"); + let pre_fork_tenures = 2; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -5424,7 +5104,6 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted .expect("Timed out waiting for nonce to increase"); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test.mine_nakamoto_block(Duration::from_secs(30), true); // Then, sumbmit 2 Contract Calls that require Tenure Extension to be addressed. info!("---- Waiting for first big tx to be mined ----"); @@ -5466,39 +5145,14 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted info!("------------------------- Triggering Bitcoin Fork -------------------------"); let tip = get_chain_info(conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 1); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); - - info!("Wait for block off of shallow fork"); TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + info!("Wait for block off of shallow fork"); - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); let post_fork_nonce = get_account(&http_origin, &sender1_addr).nonce; assert_eq!(1, post_fork_nonce); //due to contract deploy tx @@ -5512,17 +5166,7 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted TEST_MINE_STALL.set(true); // Signers still waiting for the Tx Replay set to be completed - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); info!("---- New Transaction is Submitted ----"); // Tx3 reach the mempool, meanwhile mining is stalled @@ -5533,39 +5177,13 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending_and_new_tx_submitted info!("------------------------- Triggering Bitcoin Fork #2 -------------------------"); //Fork in the middle of Tx Replay let tip = get_chain_info(&conf); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 1); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(3); + btc_controller.build_next_block(2); info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); - let submitted_commits = counters.naka_submitted_commits.clone(); - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } - - signer_test - .wait_for_signer_state_check(30, |state| { - let Some(tx_replay_set) = state.get_tx_replay_set() else { - return Ok(false); - }; - let len_ok = tx_replay_set.len() == 2; - let txid1_ok = tx_replay_set[0].txid().to_hex() == txid1; - let txid2_ok = tx_replay_set[1].txid().to_hex() == txid2; - Ok(len_ok && txid1_ok && txid2_ok) - }) - .expect("Timed out waiting for tx replay set"); + signer_test.wait_for_replay_set_eq(30, vec![txid1.clone(), txid2.clone()]); let sender1_nonce_post_fork = get_account(&http_origin, &sender1_addr).nonce; assert_eq!(1, sender1_nonce_post_fork); //due to contract deploy tx From 991f0102c1cfba686fdc923029b5ddc58237812c Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Wed, 25 Jun 2025 14:21:24 -0700 Subject: [PATCH 06/21] fix: build warnings in test commands --- testnet/stacks-node/src/tests/signer/commands/block_verify.rs | 2 ++ testnet/stacks-node/src/tests/signer/commands/block_wait.rs | 1 + testnet/stacks-node/src/tests/signer/commands/context.rs | 1 + 3 files changed, 4 insertions(+) diff --git a/testnet/stacks-node/src/tests/signer/commands/block_verify.rs b/testnet/stacks-node/src/tests/signer/commands/block_verify.rs index 053ac30dc0..236a75bae6 100644 --- a/testnet/stacks-node/src/tests/signer/commands/block_verify.rs +++ b/testnet/stacks-node/src/tests/signer/commands/block_verify.rs @@ -17,6 +17,7 @@ pub struct ChainVerifyMinerNakaBlockCount { #[derive(Debug)] enum HeightStrategy { AfterBootToEpoch3, + #[allow(dead_code)] AfterSpecificHeight(u64), } @@ -48,6 +49,7 @@ impl ChainVerifyMinerNakaBlockCount { ) } + #[allow(dead_code)] pub fn after_specific_height( ctx: Arc, miner_index: usize, diff --git a/testnet/stacks-node/src/tests/signer/commands/block_wait.rs b/testnet/stacks-node/src/tests/signer/commands/block_wait.rs index 87de4ab7e7..1d4d9af2bf 100644 --- a/testnet/stacks-node/src/tests/signer/commands/block_wait.rs +++ b/testnet/stacks-node/src/tests/signer/commands/block_wait.rs @@ -222,6 +222,7 @@ impl ChainExpectNakaBlockProposal { } } + #[allow(dead_code)] pub fn with_ok(ctx: Arc, miner_index: usize) -> Self { Self { ctx, diff --git a/testnet/stacks-node/src/tests/signer/commands/context.rs b/testnet/stacks-node/src/tests/signer/commands/context.rs index baf061f680..16ba94e756 100644 --- a/testnet/stacks-node/src/tests/signer/commands/context.rs +++ b/testnet/stacks-node/src/tests/signer/commands/context.rs @@ -61,6 +61,7 @@ impl SignerTestContext { } // Getter for num_transfer_txs + #[allow(dead_code)] pub fn get_num_transfer_txs(&self) -> u64 { self.num_transfer_txs } From de8b6e65ca72906648b695f3eee8a322335b65f1 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Wed, 25 Jun 2025 16:56:51 -0700 Subject: [PATCH 07/21] fix: tx_replay_disagreement --- testnet/stacks-node/src/tests/signer/v0.rs | 120 ++++++++------------- 1 file changed, 45 insertions(+), 75 deletions(-) diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 8e74a46d88..65e6f1166c 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -3948,40 +3948,36 @@ fn tx_replay_disagreement() { } let num_signers = 5; - let mut miners = MultipleMinerTest::new_with_config_modifications( - num_signers, - 10, - |config| { - config.validate_with_replay_tx = true; - }, - |config| { - config.burnchain.pox_reward_length = Some(30); - config.miner.block_commit_delay = Duration::from_secs(0); - config.miner.tenure_cost_limit_per_block_percentage = None; - config.miner.replay_transactions = true; - }, - |config| { - config.burnchain.pox_reward_length = Some(30); - config.miner.block_commit_delay = Duration::from_secs(0); - config.miner.tenure_cost_limit_per_block_percentage = None; - config.miner.replay_transactions = true; - }, - ); + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); + let sender_addr = tests::to_addr(&sender_sk); + let send_amt = 100; + let send_fee = 180; + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender_addr, (send_amt + send_fee) * 10)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + }, + None, + None, + Some(function_name!()), + ); - let (conf_1, _conf_2) = miners.get_node_configs(); - let _skip_commit_op_rl1 = miners - .signer_test - .running_nodes - .counters - .naka_skip_commit_op - .clone(); - let skip_commit_op_rl2 = miners.rl2_counters.naka_skip_commit_op.clone(); + let conf = &signer_test.running_nodes.conf; + let _http_origin = format!("http://{}", &conf.node.rpc_bind); + let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - // Make sure that the first miner wins the first sortition. - info!("Pausing miner 2's block commit submissions"); - skip_commit_op_rl2.set(true); - miners.boot_to_epoch_3(); - let btc_controller = &miners.signer_test.running_nodes.btc_regtest_controller; + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); let miner_pk = btc_controller .get_mining_pubkey() @@ -3994,13 +3990,10 @@ fn tx_replay_disagreement() { for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); - miners - .signer_test - .mine_nakamoto_block(Duration::from_secs(30), false); + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); } - let ignore_bitcoin_fork_keys = miners - .signer_test + let ignore_bitcoin_fork_keys = signer_test .signer_stacks_private_keys .iter() .enumerate() @@ -4015,19 +4008,21 @@ fn tx_replay_disagreement() { TEST_IGNORE_BITCOIN_FORK_PUBKEYS.set(ignore_bitcoin_fork_keys); info!("------------------------- Triggering Bitcoin Fork -------------------------"); - let tip = get_chain_info(&conf_1); + let tip = get_chain_info(&conf); // Make a transfer tx (this will get forked) - let (txid, _) = miners.send_transfer_tx(); + let (txid, _) = signer_test + .submit_transfer_tx(&sender_sk, send_fee, send_amt) + .unwrap(); wait_for(30, || { - let new_tip = get_chain_info(&conf_1); + let new_tip = get_chain_info(&conf); Ok(new_tip.stacks_tip_height > tip.stacks_tip_height) }) .expect("Timed out waiting for transfer tx to be mined"); let mut commit_txid: Option = None; wait_for(30, || { - let Some(txid) = miners.signer_test.get_parent_block_commit_txid(&miner_pk) else { + let Some(txid) = signer_test.get_parent_block_commit_txid(&miner_pk) else { return Ok(false); }; commit_txid = Some(txid); @@ -4042,12 +4037,12 @@ fn tx_replay_disagreement() { // Wait for the block commit re-broadcast to be confirmed wait_for(10, || { let is_confirmed = - BitcoinRPCRequest::check_transaction_confirmed(&conf_1, &commit_txid.unwrap()).unwrap(); + BitcoinRPCRequest::check_transaction_confirmed(&conf, &commit_txid.unwrap()).unwrap(); Ok(is_confirmed) }) .expect("Timed out waiting for transaction to be confirmed"); - let tip_before = get_chain_info(&conf_1); + let tip_before = get_chain_info(&conf); info!("---- Building next block ----"; "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, @@ -4056,43 +4051,19 @@ fn tx_replay_disagreement() { btc_controller.build_next_block(1); wait_for(30, || { - let tip = get_chain_info(&conf_1); + let tip = get_chain_info(&conf); Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) }) .expect("Timed out waiting for next block to be mined"); - // note, we should still have normal signer states! - miners.signer_test.check_signer_states_normal(); - - info!("Wait for block off of shallow fork"); - TEST_MINE_STALL.set(true); - let submitted_commits = miners - .signer_test - .running_nodes - .counters - .naka_submitted_commits - .clone(); - - // we need to mine some blocks to get back to being considered a frequent miner - for i in 0..3 { - let current_burn_height = get_chain_info(&conf_1).burn_block_height; - info!( - "Mining block #{i} to be considered a frequent miner"; - "current_burn_height" => current_burn_height, - ); - let commits_count = submitted_commits.load(Ordering::SeqCst); - next_block_and(btc_controller, 60, || { - Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) - }) - .unwrap(); - } + btc_controller.build_next_block(1); // Wait for the signer states to be updated. Odd indexed signers // should not have a replay set. wait_for(30, || { - let (signer_states, _) = miners.signer_test.get_burn_updated_states(); + let (signer_states, _) = signer_test.get_burn_updated_states(); let all_pass = signer_states.iter().enumerate().all(|(i, state)| { if i % 2 == 0 { let Some(tx_replay_set) = state.get_tx_replay_set() else { @@ -4107,27 +4078,26 @@ fn tx_replay_disagreement() { }) .expect("Timed out waiting for signer states to be updated"); - let tip = get_chain_info(&conf_1); + let tip = get_chain_info(&conf); TEST_MINE_STALL.set(false); // Now, wait for the tx replay set to be cleared wait_for(30, || { - let new_tip = get_chain_info(&conf_1); + let new_tip = get_chain_info(&conf); Ok(new_tip.stacks_tip_height >= tip.stacks_tip_height + 2) }) .expect("Timed out waiting for transfer tx to be mined"); - miners - .signer_test + signer_test .wait_for_signer_state_check(30, |state| { let tx_replay_set = state.get_tx_replay_set(); Ok(tx_replay_set.is_none()) }) .expect("Timed out waiting for tx replay set to be cleared"); - miners.shutdown(); + signer_test.shutdown(); } #[test] From 8f790dc63f0f96fac258204311c45f17fe2ed429 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Wed, 25 Jun 2025 17:14:22 -0700 Subject: [PATCH 08/21] fix: btc_on_stx test --- testnet/stacks-node/src/tests/signer/v0.rs | 105 +++++++++++---------- 1 file changed, 55 insertions(+), 50 deletions(-) diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 65e6f1166c..39a17fcb20 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -3556,6 +3556,7 @@ fn tx_replay_btc_on_stx_invalidation() { vec![(sender_addr, (send_amt + send_fee) * 10)], |c| { c.validate_with_replay_tx = true; + c.reset_replay_set_after_fork_blocks = 5; }, |node_config| { node_config.miner.block_commit_delay = Duration::from_secs(1); @@ -3572,13 +3573,11 @@ fn tx_replay_btc_on_stx_invalidation() { let mut miner_keychain = Keychain::default(conf.node.seed.clone()).generate_op_signer(); let _http_origin = format!("http://{}", &conf.node.rpc_bind); let mut btc_controller = BitcoinRegtestController::new(conf.clone(), None); - - let miner_pk = btc_controller - .get_mining_pubkey() - .as_deref() - .map(Secp256k1PublicKey::from_hex) - .unwrap() - .unwrap(); + let submitted_commits = signer_test + .running_nodes + .counters + .naka_submitted_commits + .clone(); if signer_test.bootstrap_snapshot() { signer_test.shutdown_and_snapshot(); @@ -3587,6 +3586,18 @@ fn tx_replay_btc_on_stx_invalidation() { info!("------------------------- Beginning test -------------------------"); + let burnchain = conf.get_burnchain(); + + let tip = signer_test.get_peer_info(); + let pox_info = signer_test.get_pox_data(); + + info!("---- Burnchain ----"; + // "burnchain" => ?conf.burnchain, + "pox_constants" => ?burnchain.pox_constants, + "cycle" => burnchain.pox_constants.reward_cycle_index(0, tip.burn_block_height), + "pox_info" => ?pox_info, + ); + info!("Submitting first pre-stx op"); let pre_stx_op = PreStxOp { output: sender_addr, @@ -3609,7 +3620,7 @@ fn tx_replay_btc_on_stx_invalidation() { "Pre-stx operation should submit successfully" ); - let pre_fork_tenures = 1; + let pre_fork_tenures = 10; for i in 0..pre_fork_tenures { info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); @@ -3659,51 +3670,45 @@ fn tx_replay_btc_on_stx_invalidation() { info!("---- Triggering Bitcoin fork ----"); - let mut commit_txid: Option = None; - wait_for(30, || { - let Some(txid) = signer_test.get_parent_block_commit_txid(&miner_pk) else { - return Ok(false); - }; - commit_txid = Some(txid); - Ok(true) - }) - .expect("Failed to get unconfirmed tx"); - - let tip_before = signer_test.get_peer_info(); - let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_before.burn_block_height - 1); - TEST_MINE_STALL.set(true); + let tip = signer_test.get_peer_info(); + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip.burn_block_height - 2); btc_controller.invalidate_block(&burn_header_hash_to_fork); - btc_controller.build_next_block(2); - - let tip_before = get_chain_info(&conf); - - info!("---- Building next block ----"; - "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, - "tip_before.burn_block_height" => tip_before.burn_block_height, - ); - - let chain_tips = BitcoinRPCRequest::get_chain_tips(&conf).unwrap(); - info!("---- chain_tips -----"; - "chain_tips" => ?chain_tips, - ); + btc_controller.build_next_block(3); - btc_controller.build_next_block(1); + TEST_MINE_STALL.set(true); - wait_for(30, || { - let tip = get_chain_info(&conf); - info!("----- tip -----"; - "tip.stacks_tip_height" => tip.stacks_tip_height, - "tip_before.stacks_tip_height" => tip_before.stacks_tip_height, - "tip.burn_block_height" => tip.burn_block_height, - "tip_before.burn_block_height" => tip_before.burn_block_height, + // we need to mine some blocks to get back to being considered a frequent miner + for i in 0..3 { + let current_burn_height = get_chain_info(&conf).burn_block_height; + info!( + "Mining block #{i} to be considered a frequent miner"; + "current_burn_height" => current_burn_height, ); - Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) - }) - .expect("Timed out waiting for next block to be mined"); + let commits_count = submitted_commits.load(Ordering::SeqCst); + next_block_and(&btc_controller, 60, || { + Ok(submitted_commits.load(Ordering::SeqCst) > commits_count) + }) + .unwrap(); + } info!("---- Wait for tx replay set to be updated ----"); - signer_test.wait_for_replay_set_eq(30, vec![txid.clone()]); + signer_test + .wait_for_signer_state_check(30, |state| { + let Some(tx_replay_set) = state.get_tx_replay_set() else { + info!("---- No tx replay set"); + return Ok(false); + }; + let len_ok = tx_replay_set.len() == 1; + let txid_ok = tx_replay_set[0].txid().to_hex() == txid; + info!("---- Signer state check ----"; + "tx_replay_set" => ?tx_replay_set, + "len_ok" => len_ok, + "txid_ok" => txid_ok, + ); + Ok(len_ok && txid_ok) + }) + .expect("Timed out waiting for tx replay set to be updated"); info!("---- Waiting for tx replay set to be cleared ----"); @@ -3715,9 +3720,6 @@ fn tx_replay_btc_on_stx_invalidation() { .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_none())) .expect("Timed out waiting for tx replay set to be cleared"); - let account = get_account(&_http_origin, &recipient_addr); - assert_eq!(account.nonce, 0, "Expected recipient nonce to be 0"); - // Ensure that only one block was mined wait_for(30, || { let new_tip = get_chain_info(&conf).stacks_tip_height; @@ -3725,6 +3727,9 @@ fn tx_replay_btc_on_stx_invalidation() { }) .expect("Timed out waiting for block to advance by 1"); + let account = get_account(&_http_origin, &recipient_addr); + assert_eq!(account.nonce, 0, "Expected recipient nonce to be 0"); + let blocks = test_observer::get_blocks(); let block: StacksBlockEvent = serde_json::from_value(blocks.last().unwrap().clone()).expect("Failed to parse block"); @@ -3732,7 +3737,7 @@ fn tx_replay_btc_on_stx_invalidation() { assert!(matches!( block.transactions[0].payload, TransactionPayload::TenureChange(TenureChangePayload { - cause: TenureChangeCause::Extended, + cause: TenureChangeCause::BlockFound, .. }) )); From d4d391797a04db0fbe65d571bc0cf1dbb94777a9 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Thu, 26 Jun 2025 14:40:20 -0700 Subject: [PATCH 09/21] fix: revert logic for setting `expected_burn_height` --- stacks-signer/src/v0/signer_state.rs | 47 ++++++++- testnet/stacks-node/src/tests/signer/v0.rs | 114 +++++++++++++++++++++ 2 files changed, 158 insertions(+), 3 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index 8d673626ec..973afbbabf 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -564,7 +564,15 @@ impl LocalStateMachine { // to be changed. match update { StateMachineUpdate::BurnBlock(pending_burn_block) => { - expected_burn_block = Some(pending_burn_block); + match expected_burn_block { + None => expected_burn_block = Some(pending_burn_block), + Some(ref expected) => { + if pending_burn_block.burn_block_height > expected.burn_block_height + { + expected_burn_block = Some(pending_burn_block); + } + } + } } } @@ -989,8 +997,41 @@ impl LocalStateMachine { replay_state: &ReplayState, ) -> Result, SignerChainstateError> { if expected_burn_block.burn_block_height > prior_state_machine.burn_block_height { - // no bitcoin fork, because we're advancing the burn block height - return Ok(None); + // prevent too large of a loop + if expected_burn_block + .burn_block_height + .saturating_sub(prior_state_machine.burn_block_height) + > 10 + { + return Ok(None); + } + // are we building on top of this prior tip? + let mut parent_burn_block_info = + db.get_burn_block_by_ch(&expected_burn_block.consensus_hash)?; + + while parent_burn_block_info.block_height > prior_state_machine.burn_block_height { + let Ok(parent_info) = + db.get_burn_block_by_hash(&parent_burn_block_info.parent_burn_block_hash) + else { + warn!( + "Failed to get parent burn block info for {}", + parent_burn_block_info.parent_burn_block_hash + ); + return Ok(None); + }; + parent_burn_block_info = parent_info; + } + if parent_burn_block_info.consensus_hash == prior_state_machine.burn_block { + // no bitcoin fork, because we're building on the parent + return Ok(None); + } else { + info!("Detected bitcoin fork - prior tip is not parent of new tip."; + "new_tip.burn_block_height" => expected_burn_block.burn_block_height, + "new_tip.consensus_hash" => %expected_burn_block.consensus_hash, + "prior_tip.burn_block_height" => prior_state_machine.burn_block_height, + "prior_tip.consensus_hash" => %prior_state_machine.burn_block, + ); + } } if expected_burn_block.consensus_hash == prior_state_machine.burn_block { // no bitcoin fork, because we're at the same burn block hash as before diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 39a17fcb20..552e967f77 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -3943,6 +3943,118 @@ fn tx_replay_failsafe() { signer_test.shutdown(); } +/// Simple/fast test scenario for transaction replay. +/// +/// We fork one tenure, which has a STX transfer. The test +/// verifies that the replay set is updated correctly, and then +/// exits. +#[ignore] +#[test] +fn tx_replay_simple() { + if env::var("BITCOIND_TEST") != Ok("1".into()) { + return; + } + + let num_signers = 5; + let sender_sk = Secp256k1PrivateKey::from_seed("sender_1".as_bytes()); + let sender_addr = tests::to_addr(&sender_sk); + let send_amt = 100; + let send_fee = 180; + let signer_test: SignerTest = + SignerTest::new_with_config_modifications_and_snapshot( + num_signers, + vec![(sender_addr, (send_amt + send_fee) * 10)], + |c| { + c.validate_with_replay_tx = true; + }, + |node_config| { + node_config.miner.block_commit_delay = Duration::from_secs(1); + node_config.miner.replay_transactions = true; + node_config.miner.activated_vrf_key_path = + Some(format!("{}/vrf_key", node_config.node.working_dir)); + }, + None, + None, + Some(function_name!()), + ); + + let conf = &signer_test.running_nodes.conf; + let _http_origin = format!("http://{}", &conf.node.rpc_bind); + let btc_controller = &signer_test.running_nodes.btc_regtest_controller; + + let miner_pk = btc_controller + .get_mining_pubkey() + .as_deref() + .map(Secp256k1PublicKey::from_hex) + .unwrap() + .unwrap(); + + if signer_test.bootstrap_snapshot() { + signer_test.shutdown_and_snapshot(); + return; + } + + info!("------------------------- Beginning test -------------------------"); + + let tip = signer_test.get_peer_info(); + + info!("---- Tip ----"; + "tip.stacks_tip_height" => tip.stacks_tip_height, + "tip.burn_block_height" => tip.burn_block_height, + ); + + let pre_fork_tenures = 1; + for i in 0..pre_fork_tenures { + info!("Mining pre-fork tenure {} of {pre_fork_tenures}", i + 1); + signer_test.mine_nakamoto_block(Duration::from_secs(30), true); + } + + info!("---- Submitting STX transfer ----"); + + // let tip = get_chain_info(&conf); + // Make a transfer tx (this will get forked) + let (txid, nonce) = signer_test + .submit_transfer_tx(&sender_sk, send_fee, send_amt) + .unwrap(); + + // Ensure we got a new block with this tx + signer_test + .wait_for_nonce_increase(&sender_addr, nonce) + .expect("Timed out waiting for transfer tx to be mined"); + + let tip_before = get_chain_info(&conf); + + info!("---- Triggering Bitcoin fork ----"; + "tip.stacks_tip_height" => tip_before.stacks_tip_height, + "tip.burn_block_height" => tip_before.burn_block_height, + "tip.consensus_hash" => %tip_before.pox_consensus, + ); + + let burn_header_hash_to_fork = btc_controller.get_block_hash(tip_before.burn_block_height); + btc_controller.invalidate_block(&burn_header_hash_to_fork); + TEST_MINE_STALL.set(true); + btc_controller.build_next_block(2); + + wait_for(30, || { + let tip = get_chain_info(&conf); + Ok(tip.stacks_tip_height < tip_before.stacks_tip_height) + }) + .expect("Timed out waiting for next block to be mined"); + + let tip = get_chain_info(&conf); + + info!("---- Tip after fork ----"; + "tip.stacks_tip_height" => tip.stacks_tip_height, + "tip.burn_block_height" => tip.burn_block_height, + ); + + info!("---- Wait for tx replay set to be updated ----"); + + signer_test.wait_for_replay_set_eq(5, vec![txid.clone()]); + + signer_test.shutdown(); +} + /// Test scenario where two signers disagree on the tx replay set, /// which means there is no consensus on the tx replay set. #[test] @@ -4488,6 +4600,7 @@ fn tx_replay_with_fork_after_empty_tenures_before_starting_replaying_txs() { vec![(sender1_addr, (send_amt + send_fee) * num_txs)], |c| { c.validate_with_replay_tx = true; + c.reset_replay_set_after_fork_blocks = 5; }, |node_config| { node_config.miner.block_commit_delay = Duration::from_secs(1); @@ -4859,6 +4972,7 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { |c| { c.validate_with_replay_tx = true; c.tenure_idle_timeout = Duration::from_secs(10); + c.reset_replay_set_after_fork_blocks = 5; }, |node_config| { node_config.miner.block_commit_delay = Duration::from_secs(1); From c6ca6b9c90439ee49f02d5e9e0aa731bffe5e747 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Fri, 27 Jun 2025 07:05:03 -0700 Subject: [PATCH 10/21] fix: dont rely on node burn block to be processed --- stacks-signer/src/v0/signer_state.rs | 50 ++++++---------------- testnet/stacks-node/src/tests/signer/v0.rs | 7 --- 2 files changed, 12 insertions(+), 45 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index 973afbbabf..c5ffec8b47 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -580,9 +580,16 @@ impl LocalStateMachine { } }; - let peer_info = client.get_peer_info()?; - let next_burn_block_height = peer_info.burn_block_height; - let next_burn_block_hash = peer_info.pox_consensus; + let (next_burn_block_height, next_burn_block_hash) = match expected_burn_block.clone() { + Some(expected_burn_block) => ( + expected_burn_block.burn_block_height, + expected_burn_block.consensus_hash, + ), + None => { + let peer_info = client.get_peer_info()?; + (peer_info.burn_block_height, peer_info.pox_consensus) + } + }; let mut tx_replay_set = prior_state_machine.tx_replay_set.clone(); if let Some(expected_burn_block) = expected_burn_block { @@ -997,41 +1004,8 @@ impl LocalStateMachine { replay_state: &ReplayState, ) -> Result, SignerChainstateError> { if expected_burn_block.burn_block_height > prior_state_machine.burn_block_height { - // prevent too large of a loop - if expected_burn_block - .burn_block_height - .saturating_sub(prior_state_machine.burn_block_height) - > 10 - { - return Ok(None); - } - // are we building on top of this prior tip? - let mut parent_burn_block_info = - db.get_burn_block_by_ch(&expected_burn_block.consensus_hash)?; - - while parent_burn_block_info.block_height > prior_state_machine.burn_block_height { - let Ok(parent_info) = - db.get_burn_block_by_hash(&parent_burn_block_info.parent_burn_block_hash) - else { - warn!( - "Failed to get parent burn block info for {}", - parent_burn_block_info.parent_burn_block_hash - ); - return Ok(None); - }; - parent_burn_block_info = parent_info; - } - if parent_burn_block_info.consensus_hash == prior_state_machine.burn_block { - // no bitcoin fork, because we're building on the parent - return Ok(None); - } else { - info!("Detected bitcoin fork - prior tip is not parent of new tip."; - "new_tip.burn_block_height" => expected_burn_block.burn_block_height, - "new_tip.consensus_hash" => %expected_burn_block.consensus_hash, - "prior_tip.burn_block_height" => prior_state_machine.burn_block_height, - "prior_tip.consensus_hash" => %prior_state_machine.burn_block, - ); - } + // no bitcoin fork, because we're higher than the previous tip + return Ok(None); } if expected_burn_block.consensus_hash == prior_state_machine.burn_block { // no bitcoin fork, because we're at the same burn block hash as before diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 552e967f77..49e7a236bd 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -3982,13 +3982,6 @@ fn tx_replay_simple() { let _http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let miner_pk = btc_controller - .get_mining_pubkey() - .as_deref() - .map(Secp256k1PublicKey::from_hex) - .unwrap() - .unwrap(); - if signer_test.bootstrap_snapshot() { signer_test.shutdown_and_snapshot(); return; From 795b4a7d04e3c8fff4c3a5d4b3095d46cc73a8ae Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Mon, 30 Jun 2025 05:26:13 -0700 Subject: [PATCH 11/21] Revert "fix: dont rely on node burn block to be processed" This reverts commit c6ca6b9c90439ee49f02d5e9e0aa731bffe5e747. --- stacks-signer/src/v0/signer_state.rs | 50 ++++++++++++++++------ testnet/stacks-node/src/tests/signer/v0.rs | 7 +++ 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index c5ffec8b47..973afbbabf 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -580,16 +580,9 @@ impl LocalStateMachine { } }; - let (next_burn_block_height, next_burn_block_hash) = match expected_burn_block.clone() { - Some(expected_burn_block) => ( - expected_burn_block.burn_block_height, - expected_burn_block.consensus_hash, - ), - None => { - let peer_info = client.get_peer_info()?; - (peer_info.burn_block_height, peer_info.pox_consensus) - } - }; + let peer_info = client.get_peer_info()?; + let next_burn_block_height = peer_info.burn_block_height; + let next_burn_block_hash = peer_info.pox_consensus; let mut tx_replay_set = prior_state_machine.tx_replay_set.clone(); if let Some(expected_burn_block) = expected_burn_block { @@ -1004,8 +997,41 @@ impl LocalStateMachine { replay_state: &ReplayState, ) -> Result, SignerChainstateError> { if expected_burn_block.burn_block_height > prior_state_machine.burn_block_height { - // no bitcoin fork, because we're higher than the previous tip - return Ok(None); + // prevent too large of a loop + if expected_burn_block + .burn_block_height + .saturating_sub(prior_state_machine.burn_block_height) + > 10 + { + return Ok(None); + } + // are we building on top of this prior tip? + let mut parent_burn_block_info = + db.get_burn_block_by_ch(&expected_burn_block.consensus_hash)?; + + while parent_burn_block_info.block_height > prior_state_machine.burn_block_height { + let Ok(parent_info) = + db.get_burn_block_by_hash(&parent_burn_block_info.parent_burn_block_hash) + else { + warn!( + "Failed to get parent burn block info for {}", + parent_burn_block_info.parent_burn_block_hash + ); + return Ok(None); + }; + parent_burn_block_info = parent_info; + } + if parent_burn_block_info.consensus_hash == prior_state_machine.burn_block { + // no bitcoin fork, because we're building on the parent + return Ok(None); + } else { + info!("Detected bitcoin fork - prior tip is not parent of new tip."; + "new_tip.burn_block_height" => expected_burn_block.burn_block_height, + "new_tip.consensus_hash" => %expected_burn_block.consensus_hash, + "prior_tip.burn_block_height" => prior_state_machine.burn_block_height, + "prior_tip.consensus_hash" => %prior_state_machine.burn_block, + ); + } } if expected_burn_block.consensus_hash == prior_state_machine.burn_block { // no bitcoin fork, because we're at the same burn block hash as before diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 49e7a236bd..552e967f77 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -3982,6 +3982,13 @@ fn tx_replay_simple() { let _http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; + let miner_pk = btc_controller + .get_mining_pubkey() + .as_deref() + .map(Secp256k1PublicKey::from_hex) + .unwrap() + .unwrap(); + if signer_test.bootstrap_snapshot() { signer_test.shutdown_and_snapshot(); return; From 4cc0758ba359336a0d55ec4b7781df62133f2ce3 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Mon, 30 Jun 2025 13:38:12 -0700 Subject: [PATCH 12/21] fix: better descendency check --- stacks-signer/src/v0/signer_state.rs | 85 ++++++++++++++------- testnet/stacks-node/src/tests/signer/mod.rs | 6 +- testnet/stacks-node/src/tests/signer/v0.rs | 7 -- 3 files changed, 62 insertions(+), 36 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index 973afbbabf..ff1590fd18 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -997,40 +997,20 @@ impl LocalStateMachine { replay_state: &ReplayState, ) -> Result, SignerChainstateError> { if expected_burn_block.burn_block_height > prior_state_machine.burn_block_height { - // prevent too large of a loop - if expected_burn_block - .burn_block_height - .saturating_sub(prior_state_machine.burn_block_height) - > 10 - { - return Ok(None); - } - // are we building on top of this prior tip? - let mut parent_burn_block_info = - db.get_burn_block_by_ch(&expected_burn_block.consensus_hash)?; - - while parent_burn_block_info.block_height > prior_state_machine.burn_block_height { - let Ok(parent_info) = - db.get_burn_block_by_hash(&parent_burn_block_info.parent_burn_block_hash) - else { - warn!( - "Failed to get parent burn block info for {}", - parent_burn_block_info.parent_burn_block_hash - ); - return Ok(None); - }; - parent_burn_block_info = parent_info; - } - if parent_burn_block_info.consensus_hash == prior_state_machine.burn_block { - // no bitcoin fork, because we're building on the parent - return Ok(None); - } else { + if Self::new_burn_block_fork_descendency_check( + db, + expected_burn_block, + prior_state_machine.burn_block_height, + prior_state_machine.burn_block, + )? { info!("Detected bitcoin fork - prior tip is not parent of new tip."; "new_tip.burn_block_height" => expected_burn_block.burn_block_height, "new_tip.consensus_hash" => %expected_burn_block.consensus_hash, "prior_tip.burn_block_height" => prior_state_machine.burn_block_height, "prior_tip.consensus_hash" => %prior_state_machine.burn_block, ); + } else { + return Ok(None); } } if expected_burn_block.consensus_hash == prior_state_machine.burn_block { @@ -1273,4 +1253,53 @@ impl LocalStateMachine { Ok(new_burn_block.burn_block_height > failsafe_height) } + + /// Check if the new burn block is a fork, by checking if the new burn block + /// is a descendant of the prior burn block + fn new_burn_block_fork_descendency_check( + db: &SignerDb, + new_burn_block: &NewBurnBlock, + prior_burn_block_height: u64, + prior_burn_block_ch: ConsensusHash, + ) -> Result { + let max_height_delta = 10; + let height_delta = match new_burn_block + .burn_block_height + .checked_sub(prior_burn_block_height) + { + None | Some(0) => return Ok(false), // same height or older + Some(d) if d > max_height_delta => return Ok(false), // too far apart + Some(d) => d, + }; + + let Ok(mut parent_burn_block_info) = + db.get_burn_block_by_ch(&new_burn_block.consensus_hash) + else { + warn!( + "Failed to get parent burn block info for {}", + new_burn_block.consensus_hash + ); + return Ok(false); + }; + + for _ in 0..height_delta { + if parent_burn_block_info.block_height == prior_burn_block_height { + return Ok(parent_burn_block_info.consensus_hash != prior_burn_block_ch); + } + + parent_burn_block_info = + match db.get_burn_block_by_hash(&parent_burn_block_info.parent_burn_block_hash) { + Ok(bi) => bi, + Err(e) => { + warn!( + "Failed to get parent burn block info for {}. Error: {e}", + parent_burn_block_info.parent_burn_block_hash + ); + return Ok(false); + } + }; + } + + Ok(false) + } } diff --git a/testnet/stacks-node/src/tests/signer/mod.rs b/testnet/stacks-node/src/tests/signer/mod.rs index 80e656f5b9..65045cf7eb 100644 --- a/testnet/stacks-node/src/tests/signer/mod.rs +++ b/testnet/stacks-node/src/tests/signer/mod.rs @@ -325,7 +325,11 @@ impl + Send + 'static, T: SignerEventTrait + 'static> SignerTest( File::open(metadata_path.clone()).unwrap(), diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 552e967f77..49e7a236bd 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -3982,13 +3982,6 @@ fn tx_replay_simple() { let _http_origin = format!("http://{}", &conf.node.rpc_bind); let btc_controller = &signer_test.running_nodes.btc_regtest_controller; - let miner_pk = btc_controller - .get_mining_pubkey() - .as_deref() - .map(Secp256k1PublicKey::from_hex) - .unwrap() - .unwrap(); - if signer_test.bootstrap_snapshot() { signer_test.shutdown_and_snapshot(); return; From fbce54a737e7635071f6f51f8d907a81c82f8b3c Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Tue, 1 Jul 2025 11:05:43 -0700 Subject: [PATCH 13/21] fix: off-by-one in failsafe descendency check --- stacks-signer/src/v0/signer_state.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index ff1590fd18..1ab9da8df8 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -1272,14 +1272,20 @@ impl LocalStateMachine { Some(d) => d, }; - let Ok(mut parent_burn_block_info) = - db.get_burn_block_by_ch(&new_burn_block.consensus_hash) - else { - warn!( - "Failed to get parent burn block info for {}", - new_burn_block.consensus_hash - ); - return Ok(false); + let mut parent_burn_block_info = match db + .get_burn_block_by_ch(&new_burn_block.consensus_hash) + .and_then(|burn_block_info| { + db.get_burn_block_by_hash(&burn_block_info.parent_burn_block_hash) + }) { + Ok(info) => info, + Err(e) => { + warn!( + "Failed to get parent burn block info for {}", + new_burn_block.consensus_hash; + "error" => ?e, + ); + return Ok(false); + } }; for _ in 0..height_delta { From 66bfc13bd4ecfb012b3d89057fc0c50cfe9a5313 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Tue, 1 Jul 2025 16:22:40 -0700 Subject: [PATCH 14/21] fix: prevent panic in test setup --- testnet/stacks-node/src/tests/signer/v0.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index fb424e2e0a..6ce3ed105c 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -292,11 +292,11 @@ impl SignerTest { Ok(self .stacks_client .get_reward_set_signers(reward_cycle) - .expect("Failed to check if reward set is calculated") - .map(|reward_set| { + .and_then(|reward_set| { debug!("Signer set: {reward_set:?}"); + Ok(reward_set.is_some()) }) - .is_some()) + .unwrap_or(false)) }) .expect("Timed out waiting for reward set calculation"); info!("Signer set calculated"); From 78ac7cc2f7dd48e3db64874a8be11ab9f11afc7d Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Wed, 2 Jul 2025 07:16:38 -0700 Subject: [PATCH 15/21] crc: review comments --- stacks-signer/src/config.rs | 6 +++--- stacks-signer/src/v0/signer_state.rs | 22 ++++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/stacks-signer/src/config.rs b/stacks-signer/src/config.rs index 37b0b32e69..36a163b84e 100644 --- a/stacks-signer/src/config.rs +++ b/stacks-signer/src/config.rs @@ -188,7 +188,7 @@ pub struct SignerConfig { /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: bool, /// How many blocks after a fork should we reset the replay set, - /// as a failsafe mechanism? + /// as a failsafe mechanism pub reset_replay_set_after_fork_blocks: u64, } @@ -244,7 +244,7 @@ pub struct GlobalConfig { /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: bool, /// How many blocks after a fork should we reset the replay set, - /// as a failsafe mechanism? + /// as a failsafe mechanism pub reset_replay_set_after_fork_blocks: u64, } @@ -298,7 +298,7 @@ struct RawConfigFile { /// Whether or not to validate blocks with replay transactions pub validate_with_replay_tx: Option, /// How many blocks after a fork should we reset the replay set, - /// as a failsafe mechanism? + /// as a failsafe mechanism pub reset_replay_set_after_fork_blocks: Option, } diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index 1ab9da8df8..e37ec03425 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -1234,7 +1234,7 @@ impl LocalStateMachine { forked_txs } - /// If it has been 2 burn blocks since the origin of our replay set, and + /// If it has been `reset_replay_set_after_fork_blocks` burn blocks since the origin of our replay set, and /// we haven't produced any replay blocks since then, we should reset our replay set /// /// Returns a `bool` indicating whether the replay set should be reset. @@ -1243,15 +1243,17 @@ impl LocalStateMachine { new_burn_block: &NewBurnBlock, reset_replay_set_after_fork_blocks: u64, ) -> Result { - let ReplayState::InProgress(_, replay_scope) = replay_state else { - // Not in replay - skip - return Ok(false); - }; - - let failsafe_height = - replay_scope.past_tip.burn_block_height + reset_replay_set_after_fork_blocks; - - Ok(new_burn_block.burn_block_height > failsafe_height) + match replay_state { + ReplayState::Unset => { + // not in replay - skip + return Ok(false); + } + ReplayState::InProgress(_, replay_scope) => { + let failsafe_height = + replay_scope.past_tip.burn_block_height + reset_replay_set_after_fork_blocks; + Ok(new_burn_block.burn_block_height > failsafe_height) + } + } } /// Check if the new burn block is a fork, by checking if the new burn block From 13a71b2d248411646a8d331f9ecb8cf14f219549 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Thu, 3 Jul 2025 07:29:18 -0700 Subject: [PATCH 16/21] crc: code improvements from feedback --- stacks-signer/src/chainstate.rs | 2 +- stacks-signer/src/config.rs | 2 +- stacks-signer/src/v0/signer_state.rs | 5 +++-- testnet/stacks-node/src/tests/nakamoto_integrations.rs | 7 ++++--- testnet/stacks-node/src/tests/signer/v0.rs | 3 --- 5 files changed, 9 insertions(+), 10 deletions(-) diff --git a/stacks-signer/src/chainstate.rs b/stacks-signer/src/chainstate.rs index d2cfc86e64..a94434dcbd 100644 --- a/stacks-signer/src/chainstate.rs +++ b/stacks-signer/src/chainstate.rs @@ -144,7 +144,7 @@ pub struct ProposalEvalConfig { /// Time to wait before submitting a block proposal to the stacks-node pub proposal_wait_for_parent_time: Duration, /// How many blocks after a fork should we reset the replay set, - /// as a failsafe mechanism? + /// as a failsafe mechanism pub reset_replay_set_after_fork_blocks: u64, } diff --git a/stacks-signer/src/config.rs b/stacks-signer/src/config.rs index 36a163b84e..6de3cd160d 100644 --- a/stacks-signer/src/config.rs +++ b/stacks-signer/src/config.rs @@ -51,7 +51,7 @@ const DEFAULT_TENURE_IDLE_TIMEOUT_BUFFER_SECS: u64 = 2; const DEFAULT_PROPOSAL_WAIT_TIME_FOR_PARENT_SECS: u64 = 15; /// Default number of blocks after a fork to reset the replay set, /// as a failsafe mechanism -const DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS: u64 = 2; +pub const DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS: u64 = 2; #[derive(thiserror::Error, Debug)] /// An error occurred parsing the provided configuration diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index e37ec03425..02ed22186e 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -641,7 +641,8 @@ impl LocalStateMachine { proposal_config.reset_replay_set_after_fork_blocks, )? { info!( - "Signer state: replay set is stalled after 2 tenures. Clearing the replay set." + "Signer state: replay set is stalled after {} tenures. Clearing the replay set.", + proposal_config.reset_replay_set_after_fork_blocks ); tx_replay_set = ReplayTransactionSet::none(); *tx_replay_scope = None; @@ -1246,7 +1247,7 @@ impl LocalStateMachine { match replay_state { ReplayState::Unset => { // not in replay - skip - return Ok(false); + Ok(false) } ReplayState::InProgress(_, replay_scope) => { let failsafe_height = diff --git a/testnet/stacks-node/src/tests/nakamoto_integrations.rs b/testnet/stacks-node/src/tests/nakamoto_integrations.rs index ba95631434..95276619ab 100644 --- a/testnet/stacks-node/src/tests/nakamoto_integrations.rs +++ b/testnet/stacks-node/src/tests/nakamoto_integrations.rs @@ -103,6 +103,7 @@ use stacks_common::util::hash::{to_hex, Hash160, Sha512Trunc256Sum}; use stacks_common::util::secp256k1::{MessageSignature, Secp256k1PrivateKey, Secp256k1PublicKey}; use stacks_common::util::{get_epoch_time_secs, sleep_ms}; use stacks_signer::chainstate::{ProposalEvalConfig, SortitionsView}; +use stacks_signer::config::DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS; use stacks_signer::signerdb::{BlockInfo, BlockState, ExtraBlockInfo, SignerDb}; use stacks_signer::v0::SpawnedSigner; @@ -6589,7 +6590,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut sortitions_view = SortitionsView::fetch_view(proposal_conf, &signer_client).unwrap(); @@ -6717,7 +6718,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let burn_block_height = SortitionDB::get_canonical_burn_chain_tip(sortdb.conn()) .unwrap() @@ -6796,7 +6797,7 @@ fn signer_chainstate() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut sortitions_view = SortitionsView::fetch_view(proposal_conf, &signer_client).unwrap(); sortitions_view diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 6ce3ed105c..0daafcc891 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -4219,7 +4219,6 @@ fn tx_replay_disagreement() { /// The test flow is: /// /// - Boot to Epoch 3 -/// - Mine 3 tenures /// - Submit 2 STX Transfer txs (Tx1, Tx2) in the last tenure /// - Trigger a Bitcoin fork (3 blocks) /// - Verify that signers move into tx replay state [Tx1, Tx2] @@ -4937,7 +4936,6 @@ fn tx_replay_with_fork_causing_replay_to_be_cleared_due_to_cycle() { /// The test flow is: /// /// - Boot to Epoch 3 -/// - Mine 10 tenures (to handle multiple fork in Cycle 12) /// - Deploy 1 Big Contract and mine 2 tenures (to escape fork) /// - Submit 2 Contract Call txs (Tx1, Tx2) in the last tenure, /// requiring Tenure Extend due to Tenure Budget exceeded @@ -5102,7 +5100,6 @@ fn tx_replay_with_fork_middle_replay_while_tenure_extending() { /// The test flow is: /// /// - Boot to Epoch 3 -/// - Mine 10 tenures (to handle multiple fork in Cycle 12) /// - Deploy 1 Big Contract and mine 2 tenures (to escape fork) /// - Submit 2 Contract Call txs (Tx1, Tx2) in the last tenure, /// requiring Tenure Extend due to Tenure Budget exceeded From 3130e83a2ad0f49234a899f347017cafcfcb9d3d Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Thu, 3 Jul 2025 07:37:19 -0700 Subject: [PATCH 17/21] fix: return `bool` instead of `Result` --- stacks-signer/src/v0/signer_state.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stacks-signer/src/v0/signer_state.rs b/stacks-signer/src/v0/signer_state.rs index 02ed22186e..1083d897e2 100644 --- a/stacks-signer/src/v0/signer_state.rs +++ b/stacks-signer/src/v0/signer_state.rs @@ -639,7 +639,7 @@ impl LocalStateMachine { &replay_state, &expected_burn_block, proposal_config.reset_replay_set_after_fork_blocks, - )? { + ) { info!( "Signer state: replay set is stalled after {} tenures. Clearing the replay set.", proposal_config.reset_replay_set_after_fork_blocks @@ -1243,16 +1243,16 @@ impl LocalStateMachine { replay_state: &ReplayState, new_burn_block: &NewBurnBlock, reset_replay_set_after_fork_blocks: u64, - ) -> Result { + ) -> bool { match replay_state { ReplayState::Unset => { // not in replay - skip - Ok(false) + false } ReplayState::InProgress(_, replay_scope) => { let failsafe_height = replay_scope.past_tip.burn_block_height + reset_replay_set_after_fork_blocks; - Ok(new_burn_block.burn_block_height > failsafe_height) + new_burn_block.burn_block_height > failsafe_height } } } From 8f4d08e105b46475bfe520e30e7d0fc11e6ba1c5 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Mon, 7 Jul 2025 07:33:51 -0700 Subject: [PATCH 18/21] fix: use `DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS` in tests --- stacks-signer/src/tests/chainstate.rs | 3 ++- testnet/stacks-node/src/tests/signer/v0.rs | 15 +++++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/stacks-signer/src/tests/chainstate.rs b/stacks-signer/src/tests/chainstate.rs index aa8ebd72e8..f3520e6b66 100644 --- a/stacks-signer/src/tests/chainstate.rs +++ b/stacks-signer/src/tests/chainstate.rs @@ -44,6 +44,7 @@ use stacks_common::util::secp256k1::MessageSignature; use crate::chainstate::{ProposalEvalConfig, SortitionMinerStatus, SortitionState, SortitionsView}; use crate::client::tests::MockServerClient; use crate::client::StacksClient; +use crate::config::DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS; use crate::signerdb::{BlockInfo, SignerDb}; fn setup_test_environment( @@ -92,7 +93,7 @@ fn setup_test_environment( tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(3), proposal_wait_for_parent_time: Duration::from_secs(0), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }, }; diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index 0daafcc891..fb46622654 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -85,7 +85,10 @@ use stacks_common::types::chainstate::TrieHash; use stacks_common::util::sleep_ms; use stacks_signer::chainstate::{ProposalEvalConfig, SortitionsView}; use stacks_signer::client::StackerDB; -use stacks_signer::config::{build_signer_config_tomls, GlobalConfig as SignerConfig, Network}; +use stacks_signer::config::{ + build_signer_config_tomls, GlobalConfig as SignerConfig, Network, + DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, +}; use stacks_signer::signerdb::SignerDb; use stacks_signer::v0::signer::TEST_REPEAT_PROPOSAL_RESPONSE; use stacks_signer::v0::signer_state::{ @@ -1603,7 +1606,7 @@ fn block_proposal_rejection() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -10058,7 +10061,7 @@ fn block_validation_response_timeout() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -10348,7 +10351,7 @@ fn block_validation_pending_table() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -11632,7 +11635,7 @@ fn incoming_signers_ignore_block_proposals() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), @@ -11808,7 +11811,7 @@ fn outgoing_signers_ignore_block_proposals() { tenure_idle_timeout: Duration::from_secs(300), tenure_idle_timeout_buffer: Duration::from_secs(2), reorg_attempts_activity_timeout: Duration::from_secs(30), - reset_replay_set_after_fork_blocks: 2, + reset_replay_set_after_fork_blocks: DEFAULT_RESET_REPLAY_SET_AFTER_FORK_BLOCKS, }; let mut block = NakamotoBlock { header: NakamotoBlockHeader::empty(), From 89920fea34e84d070a48002d45e3d8857a4c0108 Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Mon, 7 Jul 2025 07:38:04 -0700 Subject: [PATCH 19/21] fix: incorrect block wait logic, test logic ordering --- testnet/stacks-node/src/tests/signer/v0.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index fb46622654..c32e1212c6 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -3918,22 +3918,22 @@ fn tx_replay_failsafe() { signer_test.mine_nakamoto_block(Duration::from_secs(30), true); - signer_test - .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_some())) - .expect("Expected replay set to still be set"); - wait_for(30, || { let tip = get_chain_info(&conf); Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 1) }) .expect("Timed out waiting for a TenureChange block to be mined"); + signer_test + .wait_for_signer_state_check(30, |state| Ok(state.get_tx_replay_set().is_some())) + .expect("Expected replay set to still be set"); + info!("---- Mining a third tenure ----"); signer_test.mine_nakamoto_block(Duration::from_secs(30), true); wait_for(30, || { let tip = get_chain_info(&conf); - Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 1) + Ok(tip.stacks_tip_height > tip_after_fork.stacks_tip_height + 2) }) .expect("Timed out waiting for a TenureChange block to be mined"); From d63fb5deaf54e5aae45d6aa788afe97b9cd15dac Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Mon, 7 Jul 2025 07:39:21 -0700 Subject: [PATCH 20/21] fix: rename integration test name --- testnet/stacks-node/src/tests/signer/v0.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testnet/stacks-node/src/tests/signer/v0.rs b/testnet/stacks-node/src/tests/signer/v0.rs index c32e1212c6..fdd921b72f 100644 --- a/testnet/stacks-node/src/tests/signer/v0.rs +++ b/testnet/stacks-node/src/tests/signer/v0.rs @@ -3953,7 +3953,7 @@ fn tx_replay_failsafe() { /// exits. #[ignore] #[test] -fn tx_replay_simple() { +fn tx_replay_starts_correctly() { if env::var("BITCOIND_TEST") != Ok("1".into()) { return; } From a9c9efa8f52112e8f676036d2218f55aca9c35fb Mon Sep 17 00:00:00 2001 From: Hank Stoever Date: Mon, 7 Jul 2025 07:55:25 -0700 Subject: [PATCH 21/21] feat: changelog for failsafe --- stacks-signer/CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stacks-signer/CHANGELOG.md b/stacks-signer/CHANGELOG.md index b0abb74a2d..45352dfad0 100644 --- a/stacks-signer/CHANGELOG.md +++ b/stacks-signer/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to the versioning scheme outlined in the [README.md](README.md). +## Unreleased + +### Added + +- When a transaction replay set has been active for a configurable number of burn blocks (which defaults to `2`), and the replay set still hasn't been cleared, the replay set is automatically cleared. This is provided as a "failsafe" to ensure chain liveness as transaction replay is rolled out. + ## [3.1.0.0.13.0] ### Changed