Merge branch 'develop' of https://github.com/stacks-network/stacks-core into fix/reconsider-invalid-parent-block-proposals

Author: jferrant

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 
 ### Changed
 
+- Added new `ValidateRejectCode` values to the `/v3/block_proposal` endpoint
 - Reduce the default `block_rejection_timeout_steps` configuration so that miners will retry faster when blocks fail to reach 70% approved or 30% rejected.
 
 ## [3.1.0.0.8]

stacks-signer/src/client/stacks_client.rs

@@ -313,6 +313,7 @@ impl StacksClient {
         let block_proposal = NakamotoBlockProposal {
             block,
             chain_id: self.chain_id,
+            replay_txs: None,
         };
         let timer = crate::monitoring::actions::new_rpc_call_timer(
             &self.block_proposal_path(),

stackslib/src/core/test_util.rs

@@ -5,6 +5,7 @@ use clarity::codec::StacksMessageCodec;
 use clarity::types::chainstate::{
     BlockHeaderHash, ConsensusHash, StacksAddress, StacksPrivateKey, StacksPublicKey,
 };
+use clarity::vm::costs::ExecutionCost;
 use clarity::vm::tests::BurnStateDB;
 use clarity::vm::types::PrincipalData;
 use clarity::vm::{ClarityName, ClarityVersion, ContractName, Value};
@@ -269,17 +270,29 @@ pub fn to_addr(sk: &StacksPrivateKey) -> StacksAddress {
     StacksAddress::p2pkh(false, &StacksPublicKey::from_private(sk))
 }
 
-pub fn make_stacks_transfer(
+pub fn make_stacks_transfer_tx(
     sender: &StacksPrivateKey,
     nonce: u64,
     tx_fee: u64,
     chain_id: u32,
     recipient: &PrincipalData,
     amount: u64,
-) -> Vec<u8> {
+) -> StacksTransaction {
     let payload =
         TransactionPayload::TokenTransfer(recipient.clone(), amount, TokenTransferMemo([0; 34]));
-    let tx = sign_standard_single_sig_tx(payload, sender, nonce, tx_fee, chain_id);
+    sign_standard_single_sig_tx(payload, sender, nonce, tx_fee, chain_id)
+}
+
+/// Make a stacks transfer transaction, returning the serialized transaction bytes
+pub fn make_stacks_transfer_serialized(
+    sender: &StacksPrivateKey,
+    nonce: u64,
+    tx_fee: u64,
+    chain_id: u32,
+    recipient: &PrincipalData,
+    amount: u64,
+) -> Vec<u8> {
+    let tx = make_stacks_transfer_tx(sender, nonce, tx_fee, chain_id, recipient, amount);
     let mut tx_bytes = vec![];
     tx.consensus_serialize(&mut tx_bytes).unwrap();
     tx_bytes
@@ -512,3 +525,25 @@ pub fn insert_tx_in_mempool(
         .execute(sql, args)
         .expect("Failed to insert transaction into mempool");
 }
+
+/// Generate source code for a contract that exposes a public function
+/// `big-tx`. This function uses `proportion` of read_count when called
+pub fn make_big_read_count_contract(limit: ExecutionCost, proportion: u64) -> String {
+    let read_count = (limit.read_count * proportion) / 100;
+
+    let read_lines = (0..read_count)
+        .map(|_| format!("(var-get my-var)"))
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    format!(
+        "
+(define-data-var my-var uint u0)
+(define-public (big-tx)
+(begin
+{}
+(ok true)))
+        ",
+        read_lines
+    )
+}

stackslib/src/core/tests/mod.rs

@@ -65,7 +65,7 @@ use crate::core::mempool::{
     db_get_all_nonces, MemPoolSyncData, MemPoolWalkSettings, MemPoolWalkTxTypes, TxTag,
     BLOOM_COUNTER_DEPTH, BLOOM_COUNTER_ERROR_RATE, MAX_BLOOM_COUNTER_TXS,
 };
-use crate::core::test_util::{insert_tx_in_mempool, make_stacks_transfer, to_addr};
+use crate::core::test_util::{insert_tx_in_mempool, make_stacks_transfer_serialized, to_addr};
 use crate::core::{FIRST_BURNCHAIN_CONSENSUS_HASH, FIRST_STACKS_BLOCK_HASH};
 use crate::net::Error as NetError;
 use crate::util_lib::bloom::test::setup_bloom_counter;
@@ -2795,7 +2795,7 @@ fn large_mempool() {
             let sender_addr = to_addr(sender_sk);
             let fee = thread_rng().gen_range(180..2000);
             let transfer_tx =
-                make_stacks_transfer(sender_sk, *nonce, fee, 0x80000000, &recipient, 1);
+                make_stacks_transfer_serialized(sender_sk, *nonce, fee, 0x80000000, &recipient, 1);
             insert_tx_in_mempool(
                 &mempool_tx,
                 transfer_tx,

stackslib/src/net/api/postblock_proposal.rs

@@ -14,6 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::collections::VecDeque;
 use std::io::{Read, Write};
 #[cfg(any(test, feature = "testing"))]
 use std::sync::LazyLock;
@@ -48,10 +49,14 @@ use crate::chainstate::nakamoto::miner::NakamotoBlockBuilder;
 use crate::chainstate::nakamoto::{NakamotoBlock, NakamotoChainState, NAKAMOTO_BLOCK_VERSION};
 use crate::chainstate::stacks::db::blocks::MINIMUM_TX_FEE_RATE_PER_BYTE;
 use crate::chainstate::stacks::db::{StacksBlockHeaderTypes, StacksChainState};
-use crate::chainstate::stacks::miner::{BlockBuilder, BlockLimitFunction, TransactionResult};
+use crate::chainstate::stacks::miner::{
+    BlockBuilder, BlockLimitFunction, TransactionError, TransactionProblematic, TransactionResult,
+    TransactionSkipped,
+};
 use crate::chainstate::stacks::{
     Error as ChainError, StacksBlock, StacksBlockHeader, StacksTransaction, TransactionPayload,
 };
+use crate::clarity_vm::clarity::Error as ClarityError;
 use crate::core::mempool::{MemPoolDB, ProposalCallbackReceiver};
 use crate::cost_estimates::FeeRateEstimate;
 use crate::net::http::{
@@ -76,6 +81,11 @@ pub static TEST_VALIDATE_STALL: LazyLock<TestFlag<bool>> = LazyLock::new(TestFla
 /// Artificial delay to add to block validation.
 pub static TEST_VALIDATE_DELAY_DURATION_SECS: LazyLock<TestFlag<u64>> =
     LazyLock::new(TestFlag::default);
+#[cfg(any(test, feature = "testing"))]
+/// Mock for the set of transactions that must be replayed
+pub static TEST_REPLAY_TRANSACTIONS: LazyLock<
+    TestFlag<std::collections::VecDeque<StacksTransaction>>,
+> = LazyLock::new(TestFlag::default);
 
 // This enum is used to supply a `reason_code` for validation
 //  rejection responses. This is serialized as an enum with string
@@ -88,9 +98,10 @@ define_u8_enum![ValidateRejectCode {
     UnknownParent = 4,
     NonCanonicalTenure = 5,
     NoSuchTenure = 6,
-    InvalidParentBlock = 7,
-    InvalidTimestamp = 8,
-    NetworkChainMismatch = 9
+    InvalidTransactionReplay = 7,
+    InvalidParentBlock = 8,
+    InvalidTimestamp = 9,
+    NetworkChainMismatch = 10
 }];
 
 pub static TOO_MANY_REQUESTS_STATUS: u16 = 429;
@@ -210,6 +221,8 @@ pub struct NakamotoBlockProposal {
     pub block: NakamotoBlock,
     /// Identifies which chain block is for (Mainnet, Testnet, etc.)
     pub chain_id: u32,
+    /// Optional transaction replay set
+    pub replay_txs: Option<Vec<StacksTransaction>>,
 }
 
 impl NakamotoBlockProposal {
@@ -375,6 +388,11 @@ impl NakamotoBlockProposal {
     ///   - Miner signature is valid
     /// - Validation of transactions by executing them agains current chainstate.
     ///   This is resource intensive, and therefore done only if previous checks pass
+    ///
+    /// During transaction replay, we also check that the block only contains the unmined
+    /// transactions that need to be replayed, up until either:
+    /// - The set of transactions that must be replayed is exhausted
+    /// - A cost limit is hit
     pub fn validate(
         &self,
         sortdb: &SortitionDB,
@@ -544,8 +562,88 @@ impl NakamotoBlockProposal {
             builder.load_tenure_info(chainstate, &burn_dbconn, tenure_cause)?;
         let mut tenure_tx = builder.tenure_begin(&burn_dbconn, &mut miner_tenure_info)?;
 
+        let mut replay_txs_maybe: Option<VecDeque<StacksTransaction>> =
+            self.replay_txs.clone().map(|txs| txs.into());
+
         for (i, tx) in self.block.txs.iter().enumerate() {
             let tx_len = tx.tx_len();
+
+            // If a list of replay transactions is set, this transaction must be the next
+            // mineable transaction from this list.
+            if let Some(ref mut replay_txs) = replay_txs_maybe {
+                loop {
+                    let Some(replay_tx) = replay_txs.pop_front() else {
+                        // During transaction replay, we expect that the block only
+                        // contains transactions from the replay set. Thus, if we're here,
+                        // the block contains a transaction that is not in the replay set,
+                        // and we should reject the block.
+                        return Err(BlockValidateRejectReason {
+                            reason_code: ValidateRejectCode::InvalidTransactionReplay,
+                            reason: "Transaction is not in the replay set".into(),
+                        });
+                    };
+                    if replay_tx.txid() == tx.txid() {
+                        break;
+                    }
+
+                    // The included tx doesn't match the next tx in the
+                    // replay set. Check to see if the tx is skipped because
+                    // it was unmineable.
+                    let tx_result = builder.try_mine_tx_with_len(
+                        &mut tenure_tx,
+                        &replay_tx,
+                        replay_tx.tx_len(),
+                        &BlockLimitFunction::NO_LIMIT_HIT,
+                        ASTRules::PrecheckSize,
+                        None,
+                    );
+                    match tx_result {
+                        TransactionResult::Skipped(TransactionSkipped { error, .. })
+                        | TransactionResult::ProcessingError(TransactionError { error, .. })
+                        | TransactionResult::Problematic(TransactionProblematic {
+                            error, ..
+                        }) => {
+                            // The tx wasn't able to be mined. Check the underlying error, to
+                            // see if we should reject the block or allow the tx to be
+                            // dropped from the replay set.
+
+                            match error {
+                                ChainError::CostOverflowError(..)
+                                | ChainError::BlockTooBigError
+                                | ChainError::ClarityError(ClarityError::CostError(..)) => {
+                                    // block limit reached; add tx back to replay set.
+                                    // BUT we know that the block should have ended at this point, so
+                                    // return an error.
+                                    let txid = replay_tx.txid();
+                                    replay_txs.push_front(replay_tx);
+
+                                    warn!("Rejecting block proposal. Next replay tx exceeds cost limits, so should have been in the next block.";
+                                        "error" => %error,
+                                        "txid" => %txid,
+                                    );
+
+                                    return Err(BlockValidateRejectReason {
+                                        reason_code: ValidateRejectCode::InvalidTransactionReplay,
+                                        reason: "Transaction is not in the replay set".into(),
+                                    });
+                                }
+                                _ => {
+                                    // it's ok, drop it
+                                    continue;
+                                }
+                            }
+                        }
+                        TransactionResult::Success(_) => {
+                            // Tx should have been included
+                            return Err(BlockValidateRejectReason {
+                                reason_code: ValidateRejectCode::InvalidTransactionReplay,
+                                reason: "Transaction is not in the replay set".into(),
+                            });
+                        }
+                    };
+                }
+            }
+
             let tx_result = builder.try_mine_tx_with_len(
                 &mut tenure_tx,
                 tx,