Merge branch 'develop' into feat/propagate-vm-error

Author: obycode

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 ### Added
 
 - Added new `ValidateRejectCode` values to the `/v3/block_proposal` endpoint
+- Added `StateMachineUpdateContent::V1` to support a vector of `StacksTransaction` expected to be replayed in subsequent Stacks blocks
 - Include a reason string in the transaction receipt when a transaction is rolled back due to a post-condition. This should help users in understanding what went wrong.
 
 ### Changed

CONTRIBUTING.md

@@ -361,17 +361,31 @@ A non-exhaustive list of examples of consensus-critical changes include:
 
 - Every consensus-critical change needs an integration test to verify that the feature activates only when the hard fork activates.
 
-PRs must include test coverage. However, if your PR includes large tests or tests which cannot run in parallel
+- PRs must include test coverage. However, if your PR includes large tests or tests which cannot run in parallel
 (which is the default operation of the `cargo test` command), these tests should be decorated with `#[ignore]`.
-
 A test should be marked `#[ignore]` if:
 
-1. It does not _always_ pass `cargo test` in a vanilla environment
+  1. It does not _always_ pass `cargo test` in a vanilla environment
    (i.e., it does not need to run with `--test-threads 1`).
 
-2. Or, it runs for over a minute via a normal `cargo test` execution
+  2. Or, it runs for over a minute via a normal `cargo test` execution
    (the `cargo test` command will warn if this is not the case).
 
+- **Integration tests need to be properly tagged** using [pinny-rs](https://github.com/BitcoinL2-Labs/pinny-rs/) crate. Tagging requires two fundamental steps:
+  1. Define allowed tags in the package `Cargo.toml` file (if needed).
+  2. Apply relevant tags to the tests, picking from the allowed set.
+  
+  Then it will be possible to run tests with filtering based on the tags using `cargo test` and `cargo nextest` runner.
+  > For more information and examples on how tagging works, refer to the [pinny-rs](https://github.com/BitcoinL2-Labs/pinny-rs/) readme.
+  
+  Below the tag set currently defined with related purpose:
+
+  | Tag             | Description                                  |
+  |-----------------|----------------------------------------------|
+  | `slow`          | tests running over a minute                  |
+  | `bitcoind`      | tests requiring bitcoin daemon               |
+  | `flaky`         | tests that exhibit flaky behavior            |
+
 ## Formatting
 
 PRs will be checked against `rustfmt` and will _fail_ if not properly formatted.

Cargo.lock

@@ -254,16 +254,13 @@ impl SqliteConnection {
 
         Ok(())
     }
+
     pub fn memory() -> Result<Connection> {
         let contract_db = SqliteConnection::inner_open(":memory:")?;
         SqliteConnection::initialize_conn(&contract_db)?;
         Ok(contract_db)
     }
-    pub fn open(filename: &str) -> Result<Connection> {
-        let contract_db = SqliteConnection::inner_open(filename)?;
-        SqliteConnection::check_schema(&contract_db)?;
-        Ok(contract_db)
-    }
+
     pub fn check_schema(conn: &Connection) -> Result<()> {
         let sql = "SELECT sql FROM sqlite_master WHERE name=?";
         let _: String = conn
@@ -272,10 +269,13 @@ impl SqliteConnection {
         let _: String = conn
             .query_row(sql, params!["metadata_table"], |row| row.get(0))
             .map_err(|x| InterpreterError::SqliteError(IncomparableError { err: x }))?;
+        let _: String = conn
+            .query_row(sql, params!["md_blockhashes"], |row| row.get(0))
+            .map_err(|x| InterpreterError::SqliteError(IncomparableError { err: x }))?;
         Ok(())
     }
 
-    pub fn inner_open(filename: &str) -> Result<Connection> {
+    fn inner_open(filename: &str) -> Result<Connection> {
         let conn = Connection::open(filename)
             .map_err(|x| InterpreterError::SqliteError(IncomparableError { err: x }))?;
 

clarity/src/vm/database/sqlite.rs

@@ -605,6 +605,7 @@ struct BlockEvent {
     #[serde(with = "prefix_hex")]
     index_block_hash: StacksBlockId,
     #[serde(with = "prefix_opt_hex")]
+    #[serde(default)]
     signer_signature_hash: Option<Sha512Trunc256Sum>,
     #[serde(with = "prefix_hex")]
     consensus_hash: ConsensusHash,

libsigner/src/events.rs

@@ -571,6 +571,17 @@ pub enum StateMachineUpdateContent {
         /// The signer's view of who the current miner should be (and their tenure building info)
         current_miner: StateMachineUpdateMinerState,
     },
+    /// Version 1
+    V1 {
+        /// The tip burn block (i.e., the latest bitcoin block) seen by this signer
+        burn_block: ConsensusHash,
+        /// The tip burn block height (i.e., the latest bitcoin block) seen by this signer
+        burn_block_height: u64,
+        /// The signer's view of who the current miner should be (and their tenure building info)
+        current_miner: StateMachineUpdateMinerState,
+        /// The replay transactions
+        replay_transactions: Vec<StacksTransaction>,
+    },
 }
 
 /// Message for update the Signer State infos
@@ -676,6 +687,7 @@ impl StateMachineUpdateContent {
     fn is_protocol_version_compatible(&self, version: u64) -> bool {
         match self {
             Self::V0 { .. } => version == 0,
+            Self::V1 { .. } => version == 1,
         }
     }
 
@@ -690,6 +702,17 @@ impl StateMachineUpdateContent {
                 burn_block_height.consensus_serialize(fd)?;
                 current_miner.consensus_serialize(fd)?;
             }
+            Self::V1 {
+                burn_block,
+                burn_block_height,
+                current_miner,
+                replay_transactions,
+            } => {
+                burn_block.consensus_serialize(fd)?;
+                burn_block_height.consensus_serialize(fd)?;
+                current_miner.consensus_serialize(fd)?;
+                replay_transactions.consensus_serialize(fd)?;
+            }
         }
         Ok(())
     }
@@ -705,6 +728,18 @@ impl StateMachineUpdateContent {
                     current_miner,
                 })
             }
+            1 => {
+                let burn_block = read_next(fd)?;
+                let burn_block_height = read_next(fd)?;
+                let current_miner = read_next(fd)?;
+                let replay_transactions = read_next(fd)?;
+                Ok(Self::V1 {
+                    burn_block,
+                    burn_block_height,
+                    current_miner,
+                    replay_transactions,
+                })
+            }
             other => Err(CodecError::DeserializeError(format!(
                 "Unknown state machine update version: {other}"
             ))),
@@ -1716,15 +1751,17 @@ impl From<StateMachineUpdate> for SignerMessage {
 mod test {
     use blockstack_lib::chainstate::nakamoto::NakamotoBlockHeader;
     use blockstack_lib::chainstate::stacks::{
-        TransactionAnchorMode, TransactionAuth, TransactionPayload, TransactionPostConditionMode,
-        TransactionSmartContract, TransactionVersion,
+        TransactionAnchorMode, TransactionAuth, TransactionContractCall, TransactionPayload,
+        TransactionPostConditionMode, TransactionSmartContract, TransactionSpendingCondition,
+        TransactionVersion,
     };
     use blockstack_lib::util_lib::strings::StacksString;
     use clarity::consts::CHAIN_ID_MAINNET;
-    use clarity::types::chainstate::{ConsensusHash, StacksBlockId, TrieHash};
+    use clarity::types::chainstate::{ConsensusHash, StacksAddress, StacksBlockId, TrieHash};
     use clarity::types::PrivateKey;
     use clarity::util::hash::{hex_bytes, MerkleTree};
     use clarity::util::secp256k1::MessageSignature;
+    use clarity::vm::{ClarityName, ContractName};
     use rand::rngs::mock;
     use rand::{thread_rng, Rng, RngCore};
     use rand_core::OsRng;
@@ -2316,4 +2353,84 @@ mod test {
 
         assert_eq!(signer_message, signer_message_deserialized);
     }
+
+    #[test]
+    fn deserialize_state_machine_update_v1() {
+        let signer_message = StateMachineUpdate::new(
+            1,
+            3,
+            StateMachineUpdateContent::V1 {
+                burn_block: ConsensusHash([0x55; 20]),
+                burn_block_height: 100,
+                current_miner: StateMachineUpdateMinerState::ActiveMiner {
+                    current_miner_pkh: Hash160([0xab; 20]),
+                    tenure_id: ConsensusHash([0x44; 20]),
+                    parent_tenure_id: ConsensusHash([0x22; 20]),
+                    parent_tenure_last_block: StacksBlockId([0x33; 32]),
+                    parent_tenure_last_block_height: 1,
+                },
+                replay_transactions: vec![],
+            },
+        )
+        .unwrap();
+
+        let mut bytes = vec![];
+        signer_message.consensus_serialize(&mut bytes).unwrap();
+
+        // check for raw content for avoiding regressions when structure changes
+        let raw_signer_message: Vec<&[u8]> = vec![
+            /* active_signer_protocol_version*/ &[0, 0, 0, 0, 0, 0, 0, 1],
+            /* local_supported_signer_protocol_version*/ &[0, 0, 0, 0, 0, 0, 0, 3],
+            /* content_len*/ &[0, 0, 0, 133],
+            /* burn_block*/ &[0x55; 20],
+            /* burn_block_height*/ &[0, 0, 0, 0, 0, 0, 0, 100],
+            /* current_miner_variant */ &[0x01],
+            /* current_miner_pkh */ &[0xab; 20],
+            /* tenure_id*/ &[0x44; 20],
+            /* parent_tenure_id*/ &[0x22; 20],
+            /* parent_tenure_last_block */ &[0x33; 32],
+            /* parent_tenure_last_block_height*/ &[0, 0, 0, 0, 0, 0, 0, 1],
+            /* replay_transactions */ &[0, 0, 0, 0],
+        ];
+
+        assert_eq!(bytes, raw_signer_message.concat());
+
+        let signer_message_deserialized =
+            StateMachineUpdate::consensus_deserialize(&mut &bytes[..]).unwrap();
+
+        assert_eq!(signer_message, signer_message_deserialized);
+
+        let signer_message = StateMachineUpdate::new(
+            1,
+            4,
+            StateMachineUpdateContent::V1 {
+                burn_block: ConsensusHash([0x55; 20]),
+                burn_block_height: 100,
+                current_miner: StateMachineUpdateMinerState::NoValidMiner,
+                replay_transactions: vec![],
+            },
+        )
+        .unwrap();
+
+        let mut bytes = vec![];
+        signer_message.consensus_serialize(&mut bytes).unwrap();
+
+        // check for raw content for avoiding regressions when structure changes
+        let raw_signer_message: Vec<&[u8]> = vec![
+            /* active_signer_protocol_version*/ &[0, 0, 0, 0, 0, 0, 0, 1],
+            /* local_supported_signer_protocol_version*/ &[0, 0, 0, 0, 0, 0, 0, 4],
+            /* content_len*/ &[0, 0, 0, 33],
+            /* burn_block*/ &[0x55; 20],
+            /* burn_block_height*/ &[0, 0, 0, 0, 0, 0, 0, 100],
+            /* current_miner_variant */ &[0x00],
+            /* replay_transactions */ &[0, 0, 0, 0],
+        ];
+
+        assert_eq!(bytes, raw_signer_message.concat());
+
+        let signer_message_deserialized =
+            StateMachineUpdate::consensus_deserialize(&mut &bytes[..]).unwrap();
+
+        assert_eq!(signer_message, signer_message_deserialized);
+    }
 }

libsigner/src/v0/messages.rs

@@ -5,6 +5,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
+## [Unreleased]
+
+### Changed
+
+- Upgraded `SUPPORTED_SIGNER_PROTOCOL_VERSION` to 1
+
+## [3.1.0.0.8.1]
+
+### Added
+
+- The signer will now check if their associated stacks-node has processed the parent block for a block proposal before submitting that block proposal. If it cannot confirm that the parent block has been processed, it waits a default time of 15s before submitting, configurable via `proposal_wait_for_parent_time_secs` in the signer config.toml.
+
+
 ## [3.1.0.0.8.0]
 
 ### Changed

stacks-signer/CHANGELOG.md

@@ -141,6 +141,8 @@ pub struct ProposalEvalConfig {
     /// Time following the last block of the previous tenure's global acceptance that a signer will consider an attempt by
     /// the new miner to reorg it as valid towards miner activity
     pub reorg_attempts_activity_timeout: Duration,
+    /// Time to wait before submitting a block proposal to the stacks-node
+    pub proposal_wait_for_parent_time: Duration,
 }
 
 impl From<&SignerConfig> for ProposalEvalConfig {
@@ -152,6 +154,7 @@ impl From<&SignerConfig> for ProposalEvalConfig {
             tenure_idle_timeout: value.tenure_idle_timeout,
             reorg_attempts_activity_timeout: value.reorg_attempts_activity_timeout,
             tenure_idle_timeout_buffer: value.tenure_idle_timeout_buffer,
+            proposal_wait_for_parent_time: value.proposal_wait_for_parent_time,
         }
     }
 }

stacks-signer/src/chainstate.rs

@@ -418,6 +418,7 @@ pub(crate) mod tests {
             tenure_idle_timeout_buffer: config.tenure_idle_timeout_buffer,
             block_proposal_max_age_secs: config.block_proposal_max_age_secs,
             reorg_attempts_activity_timeout: config.reorg_attempts_activity_timeout,
+            proposal_wait_for_parent_time: config.proposal_wait_for_parent_time,
         }
     }
 

stacks-signer/src/client/mod.rs

@@ -45,6 +45,10 @@ const DEFAULT_REORG_ATTEMPTS_ACTIVITY_TIMEOUT_MS: u64 = 200_000;
 /// Default number of seconds to add to the tenure extend time, after computing the idle timeout,
 /// to allow for clock skew between the signer and the miner
 const DEFAULT_TENURE_IDLE_TIMEOUT_BUFFER_SECS: u64 = 2;
+/// Default time (in ms) to wait before submitting a proposal if we
+///  cannot determine that our stacks-node has processed the parent
+///  block
+const DEFAULT_PROPOSAL_WAIT_TIME_FOR_PARENT_SECS: u64 = 15;
 
 #[derive(thiserror::Error, Debug)]
 /// An error occurred parsing the provided configuration
@@ -175,6 +179,9 @@ pub struct SignerConfig {
     pub reorg_attempts_activity_timeout: Duration,
     /// The running mode for the signer (dry-run or normal)
     pub signer_mode: SignerConfigMode,
+    /// Time to wait before submitting a block proposal to the stacks-node if we cannot
+    ///  determine that the stacks-node has processed the parent
+    pub proposal_wait_for_parent_time: Duration,
 }
 
 /// The parsed configuration for the signer
@@ -221,6 +228,9 @@ pub struct GlobalConfig {
     /// Time following the last block of the previous tenure's global acceptance that a signer will consider an attempt by
     /// the new miner to reorg it as valid towards miner activity
     pub reorg_attempts_activity_timeout: Duration,
+    /// Time to wait before submitting a block proposal to the stacks-node if we cannot
+    ///  determine that the stacks-node has processed the parent
+    pub proposal_wait_for_parent_time: Duration,
     /// Is this signer binary going to be running in dry-run mode?
     pub dry_run: bool,
 }
@@ -268,6 +278,8 @@ struct RawConfigFile {
     /// Time (in millisecs) following a block's global acceptance that a signer will consider an attempt by a miner
     /// to reorg the block as valid towards miner activity
     pub reorg_attempts_activity_timeout_ms: Option<u64>,
+    /// Time to wait (in millisecs) before submitting a block proposal to the stacks-node
+    pub proposal_wait_for_parent_time_secs: Option<u64>,
     /// Is this signer binary going to be running in dry-run mode?
     pub dry_run: Option<bool>,
 }
@@ -385,6 +397,12 @@ impl TryFrom<RawConfigFile> for GlobalConfig {
                 .unwrap_or(DEFAULT_TENURE_IDLE_TIMEOUT_BUFFER_SECS),
         );
 
+        let proposal_wait_for_parent_time = Duration::from_secs(
+            raw_data
+                .proposal_wait_for_parent_time_secs
+                .unwrap_or(DEFAULT_PROPOSAL_WAIT_TIME_FOR_PARENT_SECS),
+        );
+
         Ok(Self {
             node_host: raw_data.node_host,
             endpoint,
@@ -405,6 +423,7 @@ impl TryFrom<RawConfigFile> for GlobalConfig {
             reorg_attempts_activity_timeout,
             dry_run,
             tenure_idle_timeout_buffer,
+            proposal_wait_for_parent_time,
         })
     }
 }