Merge remote-tracking branch 'origin/develop' into feat/fork-detection-state-machine

Author: hstove

.github/CODEOWNERS

@@ -0,0 +1,16 @@
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+#
+# require both blockchain-team-codeowners and blockchain-team to review all PR's not specifically matched below
+* @stacks-network/blockchain-team-codeowners @stacks-network/blockchain-team
+
+
+# Signer code
+# require both blockchain-team-codeowners and blockchain-team-signer to review PR's for the signer folder(s)
+libsigner/**/*.rs @stacks-network/blockchain-team-codeowners @stacks-network/blockchain-team-signer
+stacks-signer/**/*.rs @stacks-network/blockchain-team-codeowners @stacks-network/blockchain-team-signer
+
+# CI workflows
+#  require both blockchain-team and blockchain-team-ci teams to review PR's modifying CI workflows
+/.github/workflows/ @stacks-network/blockchain-team @stacks-network/blockchain-team-ci
+/.github/actions/ @stacks-network/blockchain-team @stacks-network/blockchain-team-ci

CODEOWNERS

@@ -409,13 +409,8 @@ impl Signer {
                             block_response,
                             sortition_state,
                         ),
-                        SignerMessage::StateMachineUpdate(update) => {
-                            self.handle_state_machine_update(
-                                signer_public_key,
-                                update,
-                                received_time,
-                            );
-                        }
+                        SignerMessage::StateMachineUpdate(update) => self
+                            .handle_state_machine_update(signer_public_key, update, received_time),
                         _ => {}
                     }
                 }
@@ -676,22 +671,43 @@ impl Signer {
 
     /// The actual `send_block_response` implementation. Declared so that we do
     /// not need to duplicate in testing.
-    fn impl_send_block_response(&mut self, block_response: BlockResponse) {
-        let res = self
+    fn impl_send_block_response(
+        &mut self,
+        block: Option<&NakamotoBlock>,
+        block_response: BlockResponse,
+    ) {
+        info!(
+            "{self}: Broadcasting a block response to stacks node: {block_response:?}";
+        );
+        let accepted = matches!(block_response, BlockResponse::Accepted(..));
+        match self
             .stackerdb
-            .send_message_with_retry::<SignerMessage>(block_response.clone().into());
-        match res {
-            Err(e) => warn!("{self}: Failed to send block rejection to stacker-db: {e:?}"),
-            Ok(ack) if !ack.accepted => warn!(
-                "{self}: Block rejection not accepted by stacker-db: {:?}",
-                ack.reason
-            ),
-            Ok(_) => debug!("{self}: Block rejection accepted by stacker-db"),
+            .send_message_with_retry::<SignerMessage>(block_response.into())
+        {
+            Ok(ack) => {
+                if !ack.accepted {
+                    warn!(
+                        "{self}: Block response not accepted by stacker-db: {:?}",
+                        ack.reason
+                    );
+                }
+                crate::monitoring::actions::increment_block_responses_sent(accepted);
+                if let Some(block) = block {
+                    crate::monitoring::actions::record_block_response_latency(block);
+                }
+            }
+            Err(e) => {
+                warn!("{self}: Failed to send block response to stacker-db: {e:?}",);
+            }
         }
     }
 
     #[cfg(any(test, feature = "testing"))]
-    fn send_block_response(&mut self, block_response: BlockResponse) {
+    fn send_block_response(
+        &mut self,
+        block: Option<&NakamotoBlock>,
+        block_response: BlockResponse,
+    ) {
         const NUM_REPEATS: usize = 1;
         let mut count = 0;
         let public_keys = TEST_REPEAT_PROPOSAL_RESPONSE.get();
@@ -701,16 +717,20 @@ impl Signer {
             count = NUM_REPEATS;
         }
         while count <= NUM_REPEATS {
-            self.impl_send_block_response(block_response.clone());
+            self.impl_send_block_response(block, block_response.clone());
 
             count += 1;
             sleep_ms(1000);
         }
     }
 
     #[cfg(not(any(test, feature = "testing")))]
-    fn send_block_response(&mut self, block_response: BlockResponse) {
-        self.impl_send_block_response(block_response)
+    fn send_block_response(
+        &mut self,
+        block: Option<&NakamotoBlock>,
+        block_response: BlockResponse,
+    ) {
+        self.impl_send_block_response(block, block_response)
     }
 
     /// Handle signer state update message
@@ -840,8 +860,7 @@ impl Signer {
 
         if let Some(block_response) = block_response {
             // We know proposal is invalid. Send rejection message, do not do further validation and do not store it.
-            debug!("{self}: Broadcasting a block response to stacks node: {block_response:?}");
-            self.send_block_response(block_response);
+            self.send_block_response(Some(&block_info.block), block_response);
         } else {
             // Just in case check if the last block validation submission timed out.
             self.check_submitted_block_proposal();
@@ -895,19 +914,7 @@ impl Signer {
             return;
         };
 
-        // Submit a proposal response to the .signers contract for miners
-        debug!("{self}: Broadcasting a block response to stacks node: {block_response:?}");
-
-        let accepted = matches!(block_response, BlockResponse::Accepted(..));
-        if let Err(e) = self
-            .stackerdb
-            .send_message_with_retry::<SignerMessage>(block_response.into())
-        {
-            warn!("{self}: Failed to send block response to stacker-db: {e:?}");
-        } else {
-            crate::monitoring::actions::increment_block_responses_sent(accepted);
-            crate::monitoring::actions::record_block_response_latency(&block_info.block);
-        }
+        self.impl_send_block_response(Some(&block_info.block), block_response);
     }
 
     /// Handle block response messages from a signer
@@ -1043,21 +1050,7 @@ impl Signer {
                     warn!("{self}: Failed to mark block as locally rejected: {e:?}");
                 }
             };
-            debug!("{self}: Broadcasting a block response to stacks node: {block_response:?}");
-            let res = self
-                .stackerdb
-                .send_message_with_retry::<SignerMessage>(block_response.into());
-
-            crate::monitoring::actions::record_block_response_latency(&block_info.block);
-
-            match res {
-                Err(e) => warn!("{self}: Failed to send block rejection to stacker-db: {e:?}"),
-                Ok(ack) if !ack.accepted => warn!(
-                    "{self}: Block rejection not accepted by stacker-db: {:?}",
-                    ack.reason
-                ),
-                Ok(_) => debug!("{self}: Block rejection accepted by stacker-db"),
-            }
+            self.impl_send_block_response(Some(&block_info.block), block_response);
             self.signer_db
                 .insert_block(&block_info)
                 .unwrap_or_else(|e| self.handle_insert_block_error(e));
@@ -1164,30 +1157,12 @@ impl Signer {
             .unwrap_or_else(|e| warn!("{self}: Failed to remove pending block validation: {e:?}"));
 
         if let Some(response) = block_response {
-            // Submit a proposal response to the .signers contract for miners
-            info!(
-                "{self}: Broadcasting a block response to stacks node: {response:?}";
-            );
-            let accepted = matches!(response, BlockResponse::Accepted(..));
-            match self
-                .stackerdb
-                .send_message_with_retry::<SignerMessage>(response.into())
-            {
-                Ok(_) => {
-                    crate::monitoring::actions::increment_block_responses_sent(accepted);
-                    if let Ok(Some(block_info)) = self
-                        .signer_db
-                        .block_lookup(&block_validate_response.signer_signature_hash())
-                    {
-                        crate::monitoring::actions::record_block_response_latency(
-                            &block_info.block,
-                        );
-                    }
-                }
-                Err(e) => {
-                    warn!("{self}: Failed to send block rejection to stacker-db: {e:?}",);
-                }
-            }
+            let block = self
+                .signer_db
+                .block_lookup(&signer_sig_hash)
+                .unwrap_or_default()
+                .map(|info| info.block);
+            self.impl_send_block_response(block.as_ref(), response);
         };
 
         // Check if there is a pending block validation that we need to submit to the node
@@ -1280,21 +1255,7 @@ impl Signer {
                 warn!("{self}: Failed to mark block as locally rejected: {e:?}");
             }
         };
-        debug!("{self}: Broadcasting a block response to stacks node: {rejection:?}");
-        let res = self
-            .stackerdb
-            .send_message_with_retry::<SignerMessage>(rejection.into());
-
-        crate::monitoring::actions::record_block_response_latency(&block_info.block);
-
-        match res {
-            Err(e) => warn!("{self}: Failed to send block rejection to stacker-db: {e:?}"),
-            Ok(ack) if !ack.accepted => warn!(
-                "{self}: Block rejection not accepted by stacker-db: {:?}",
-                ack.reason
-            ),
-            Ok(_) => debug!("{self}: Block rejection accepted by stacker-db"),
-        }
+        self.impl_send_block_response(Some(&block_info.block), rejection);
 
         self.signer_db
             .insert_block(&block_info)

stacks-signer/src/v0/signer.rs

@@ -1264,13 +1264,25 @@ impl<'a> SortitionHandleTx<'a> {
         burn_header_hash: &BurnchainHeaderHash,
         chain_tip: &SortitionId,
     ) -> Result<Option<BlockSnapshot>, db_error> {
+        let Some(sortition_id) = self.get_sortition_id_for_bhh(burn_header_hash, chain_tip)? else {
+            return Ok(None);
+        };
+
+        SortitionDB::get_block_snapshot(self.tx(), &sortition_id)
+    }
+
+    fn get_sortition_id_for_bhh(
+        &mut self,
+        burn_header_hash: &BurnchainHeaderHash,
+        chain_tip: &SortitionId,
+    ) -> Result<Option<SortitionId>, db_error> {
         let sortition_identifier_key = db_keys::sortition_id_for_bhh(burn_header_hash);
         let sortition_id = match self.get_indexed(chain_tip, &sortition_identifier_key)? {
             None => return Ok(None),
             Some(x) => SortitionId::from_hex(&x).expect("FATAL: bad Sortition ID stored in DB"),
         };
 
-        SortitionDB::get_block_snapshot(self.tx(), &sortition_id)
+        Ok(Some(sortition_id))
     }
 
     /// Get a leader key at a specific location in the burn chain's fork history, given the
@@ -6528,25 +6540,25 @@ impl SortitionHandleTx<'_> {
             }
 
             // must be an ancestor of this tip, or must be this tip
-            if let Some(sn) =
-                self.get_block_snapshot(&arrival_sn.burn_header_hash, &parent_tip.sortition_id)?
+            if let Some(sortition_id) = self
+                .get_sortition_id_for_bhh(&arrival_sn.burn_header_hash, &parent_tip.sortition_id)?
             {
-                if !sn.pox_valid || sn != arrival_sn {
+                if sortition_id != arrival_sn.sortition_id {
                     continue;
                 }
 
                 debug!(
                     "New Stacks anchored block arrived: block {}/{} ({}) ari={} tip={}",
-                    &sn.consensus_hash,
-                    &sn.winning_stacks_block_hash,
-                    sn.stacks_block_height,
+                    &arrival_sn.consensus_hash,
+                    &arrival_sn.winning_stacks_block_hash,
+                    arrival_sn.stacks_block_height,
                     ari,
                     &parent_tip.burn_header_hash
                 );
                 new_block_arrivals.push((
-                    sn.consensus_hash,
-                    sn.winning_stacks_block_hash,
-                    sn.stacks_block_height,
+                    arrival_sn.consensus_hash,
+                    arrival_sn.winning_stacks_block_hash,
+                    arrival_sn.stacks_block_height,
                 ));
             } else {
                 // this block did not arrive on an ancestor block

stackslib/src/chainstate/burn/db/sortdb.rs

@@ -294,15 +294,15 @@ impl DBConfig {
         });
         match epoch_id {
             StacksEpochId::Epoch10 => true,
-            StacksEpochId::Epoch20 => version_u32 >= 1 && version_u32 <= 8,
-            StacksEpochId::Epoch2_05 => version_u32 >= 2 && version_u32 <= 8,
-            StacksEpochId::Epoch21 => version_u32 >= 3 && version_u32 <= 8,
-            StacksEpochId::Epoch22 => version_u32 >= 3 && version_u32 <= 8,
-            StacksEpochId::Epoch23 => version_u32 >= 3 && version_u32 <= 8,
-            StacksEpochId::Epoch24 => version_u32 >= 3 && version_u32 <= 8,
-            StacksEpochId::Epoch25 => version_u32 >= 3 && version_u32 <= 8,
-            StacksEpochId::Epoch30 => version_u32 >= 3 && version_u32 <= 8,
-            StacksEpochId::Epoch31 => version_u32 >= 3 && version_u32 <= 8,
+            StacksEpochId::Epoch20 => version_u32 >= 1 && version_u32 <= 9,
+            StacksEpochId::Epoch2_05 => version_u32 >= 2 && version_u32 <= 9,
+            StacksEpochId::Epoch21 => version_u32 >= 3 && version_u32 <= 9,
+            StacksEpochId::Epoch22 => version_u32 >= 3 && version_u32 <= 9,
+            StacksEpochId::Epoch23 => version_u32 >= 3 && version_u32 <= 9,
+            StacksEpochId::Epoch24 => version_u32 >= 3 && version_u32 <= 9,
+            StacksEpochId::Epoch25 => version_u32 >= 3 && version_u32 <= 9,
+            StacksEpochId::Epoch30 => version_u32 >= 3 && version_u32 <= 9,
+            StacksEpochId::Epoch31 => version_u32 >= 3 && version_u32 <= 9,
         }
     }
 }
@@ -654,7 +654,7 @@ impl<'a> DerefMut for ChainstateTx<'a> {
     }
 }
 
-pub const CHAINSTATE_VERSION: &str = "8";
+pub const CHAINSTATE_VERSION: &str = "9";
 
 const CHAINSTATE_INITIAL_SCHEMA: &[&str] = &[
     "PRAGMA foreign_keys = ON;",
@@ -853,6 +853,15 @@ const CHAINSTATE_SCHEMA_3: &[&str] = &[
     "#,
 ];
 
+const CHAINSTATE_SCHEMA_4: &[&str] = &[
+    // schema change is JUST a new index, so just bump db_config.version
+    //   and add the index to `CHAINSTATE_INDEXES` (which gets re-execed
+    //   on every schema change)
+    r#"
+    UPDATE db_config SET version = "9";
+    "#,
+];
+
 const CHAINSTATE_INDEXES: &[&str] = &[
     "CREATE INDEX IF NOT EXISTS index_block_hash_to_primary_key ON block_headers(index_block_hash,consensus_hash,block_hash);",
     "CREATE INDEX IF NOT EXISTS block_headers_hash_index ON block_headers(block_hash,block_height);",
@@ -877,6 +886,7 @@ const CHAINSTATE_INDEXES: &[&str] = &[
     "CREATE INDEX IF NOT EXISTS index_block_header_by_affirmation_weight ON block_headers(affirmation_weight);",
     "CREATE INDEX IF NOT EXISTS index_block_header_by_height_and_affirmation_weight ON block_headers(block_height,affirmation_weight);",
     "CREATE INDEX IF NOT EXISTS index_headers_by_consensus_hash ON block_headers(consensus_hash);",
+    "CREATE INDEX IF NOT EXISTS processable_block ON staging_blocks(processed, orphaned, attachable);",
 ];
 
 pub use stacks_common::consts::MINER_REWARD_MATURITY;
@@ -1104,6 +1114,14 @@ impl StacksChainState {
                         tx.execute_batch(cmd)?;
                     }
                 }
+                "8" => {
+                    info!(
+                        "Migrating chainstate schema from version 8 to 9: add index for staging_blocks"
+                    );
+                    for cmd in CHAINSTATE_SCHEMA_4.iter() {
+                        tx.execute_batch(cmd)?;
+                    }
+                }
                 _ => {
                     error!(
                         "Invalid chain state database: expected version = {}, got {}",