Merge branch 'develop' into feat/tx-replay-handle-forks

Author: fdefelici

.cargo/config.toml

@@ -1,7 +1,7 @@
 [alias]
 stacks-node = "run --package stacks-node --"
 fmt-stacks = "fmt -- --config group_imports=StdExternalCrate,imports_granularity=Module"
-clippy-stacks = "clippy -p libstackerdb -p stacks-signer -p pox-locking -p clarity -p libsigner -p stacks-common --no-deps --tests --all-features -- -D warnings"
+clippy-stacks = "clippy -p stx-genesis -p libstackerdb -p stacks-signer -p pox-locking -p clarity -p libsigner -p stacks-common --no-deps --tests --all-features -- -D warnings"
 
 # Uncomment to improve performance slightly, at the cost of portability
 #   * Note that native binaries may not run on CPUs that are different from the build machine

CHANGELOG.md

@@ -11,6 +11,27 @@ and this project adheres to the versioning scheme outlined in the [README.md](RE
 
 - Added a new RPC endpoint `/v3/health` to query the node's health status. The endpoint returns a 200 status code with relevant synchronization information (including the node's current Stacks tip height, the maximum Stacks tip height among its neighbors, and the difference between these two). A user can use the `difference_from_max_peer` value to decide what is a good threshold for them before considering the node out of sync. The endpoint returns a 500 status code if the query cannot retrieve viable data.
 
+## [3.1.0.0.12]
+
+### Added
+
+- Document missing config structs
+- Document MinerConfig parameters
+- Document BurnchainConfig parameters
+- Document NodeConfig parameters
+
+### Changed
+
+- `get_fresh_random_neighbors` to include allowed neigbors
+- Logging improvements and cleanup
+- Move serde serializers to stacks_common
+- Depend on clarity backing store interface
+- Updated `./docs/event-dispacher.md`
+
+### Fixed
+
+- Handle Bitcoin reorgs during Stacks tenure extend
+
 ## [3.1.0.0.11]
 
 - Hotfix for p2p stack misbehavior in mempool syncing conditions

Cargo.lock

@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to the versioning scheme outlined in the [README.md](README.md).
 
+## [3.1.0.0.12.0]
+
+### Changed
+
+- Refactor / cleanup signerDB migrations code
+- Signers should not infinitely loop when pushing a block to stacks-node
+- Logging improvements and cleanup
+
+### Fixed
+
+- Fix `capitulate_miner_view` so stacks-node won't swap between multiple miners
+- Mark current miner as invalid on capitulation
+- Fix flaky `miner_recovers_when_broadcast_block_delay_across_tenures_occurs` test
+
 ## [3.1.0.0.10.0]
 
 ### Added

stacks-signer/CHANGELOG.md

@@ -231,33 +231,18 @@
 /// bootstrapping nodes.
 ///
 use std::cmp;
-use std::collections::{BTreeMap, HashMap, HashSet, VecDeque};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::fmt;
 use std::fmt::Write;
-use std::sync::mpsc::SyncSender;
-use std::time::Duration;
 
 use serde::de::Error as de_Error;
-use serde::ser::Error as ser_Error;
 use serde::{Deserialize, Serialize};
-use stacks_common::types::chainstate::{
-    BlockHeaderHash, BurnchainHeaderHash, PoxId, SortitionId, StacksAddress, StacksBlockId,
-};
-
-use crate::burnchains::db::{
-    BurnchainBlockData, BurnchainDB, BurnchainDBTransaction, BurnchainHeaderReader,
-};
-use crate::burnchains::{Address, Burnchain, BurnchainBlockHeader, Error, PoxConstants, Txid};
-use crate::chainstate::burn::db::sortdb::SortitionDB;
-use crate::chainstate::burn::operations::leader_block_commit::{
-    RewardSetInfo, BURN_BLOCK_MINED_AT_MODULUS,
-};
+
+use crate::burnchains::db::{BurnchainDB, BurnchainDBTransaction, BurnchainHeaderReader};
+use crate::burnchains::{Burnchain, Error, PoxConstants};
+use crate::chainstate::burn::operations::leader_block_commit::BURN_BLOCK_MINED_AT_MODULUS;
 use crate::chainstate::burn::operations::{BlockstackOperationType, LeaderBlockCommitOp};
-use crate::chainstate::burn::{BlockSnapshot, ConsensusHash};
-use crate::chainstate::stacks::StacksBlockHeader;
-use crate::core::StacksEpochId;
-use crate::util_lib::boot::boot_code_id;
-use crate::util_lib::db::{DBConn, Error as DBError};
+use crate::util_lib::db::Error as DBError;
 
 /// Affirmation map entries.  By building on a PoX-mined block,
 /// a PoB-mined block (in a PoX reward cycle),

stackslib/src/burnchains/affirmation.rs

@@ -15,14 +15,14 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use stacks_common::address::b58 as base58;
-use stacks_common::address::c32::c32_address;
 use stacks_common::deps_common::bech32;
 use stacks_common::deps_common::bech32::{u5, FromBase32, ToBase32};
 use stacks_common::deps_common::bitcoin::blockdata::opcodes::All as BtcOp;
 use stacks_common::deps_common::bitcoin::blockdata::script::Builder as BtcScriptBuilder;
 use stacks_common::deps_common::bitcoin::blockdata::transaction::TxOut;
-use stacks_common::util::hash::{hex_bytes, to_hex, Hash160};
-use stacks_common::util::log;
+#[cfg(test)]
+use stacks_common::util::hash::hex_bytes;
+use stacks_common::util::hash::Hash160;
 
 use crate::burnchains::bitcoin::{BitcoinNetworkType, Error as btc_error};
 use crate::burnchains::Address;
@@ -686,7 +686,6 @@ impl Address for BitcoinAddress {
 mod tests {
     use stacks_common::types::Address;
     use stacks_common::util::hash::{hex_bytes, Hash160};
-    use stacks_common::util::log;
 
     use super::{
         BitcoinAddress, LegacyBitcoinAddress, LegacyBitcoinAddressType, SegwitBitcoinAddress,

stackslib/src/burnchains/bitcoin/address.rs

@@ -13,30 +13,21 @@
 //
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-use sha2::{Digest, Sha256};
-use stacks_common::address::{public_keys_to_address_hash, AddressHashMode};
 use stacks_common::deps_common::bitcoin::blockdata::opcodes::{All as btc_opcodes, Class};
-use stacks_common::deps_common::bitcoin::blockdata::script::{Builder, Instruction, Script};
+use stacks_common::deps_common::bitcoin::blockdata::script::{Instruction, Script};
 use stacks_common::deps_common::bitcoin::blockdata::transaction::{
     TxIn as BtcTxIn, TxOut as BtcTxOut,
 };
-use stacks_common::deps_common::bitcoin::util::hash::Sha256dHash;
-use stacks_common::types::chainstate::BurnchainHeaderHash;
-use stacks_common::util::hash::{hex_bytes, Hash160};
-use stacks_common::util::log;
+#[cfg(test)]
+use stacks_common::util::hash::hex_bytes;
 
 use crate::burnchains::bitcoin::address::{BitcoinAddress, LegacyBitcoinAddressType};
 use crate::burnchains::bitcoin::keys::BitcoinPublicKey;
 use crate::burnchains::bitcoin::{
     BitcoinInputType, BitcoinNetworkType, BitcoinTxInput, BitcoinTxInputRaw,
     BitcoinTxInputStructured, BitcoinTxOutput, Error as btc_error,
 };
-use crate::burnchains::{PublicKey, Txid};
-use crate::chainstate::stacks::{
-    C32_ADDRESS_VERSION_MAINNET_MULTISIG, C32_ADDRESS_VERSION_MAINNET_SINGLESIG,
-    C32_ADDRESS_VERSION_TESTNET_MULTISIG, C32_ADDRESS_VERSION_TESTNET_SINGLESIG,
-};
+use crate::burnchains::Txid;
 
 /// Parse a script into its structured constituant opcodes and data and collect them
 pub fn parse_script(script: &Script) -> Vec<Instruction<'_>> {
@@ -624,14 +615,12 @@ mod tests {
     use stacks_common::deps_common::bitcoin::blockdata::transaction::Transaction;
     use stacks_common::deps_common::bitcoin::network::serialize::deserialize as bitcoinlib_deserialize;
     use stacks_common::util::hash::hex_bytes;
-    use stacks_common::util::log;
 
     use super::{
-        parse_script, to_txid, BitcoinTxInput, BitcoinTxInputRaw, BitcoinTxInputStructured,
-        BitcoinTxOutput,
+        parse_script, to_txid, BitcoinTxInputRaw, BitcoinTxInputStructured, BitcoinTxOutput,
     };
     use crate::burnchains::bitcoin::address::{
-        BitcoinAddress, LegacyBitcoinAddress, LegacyBitcoinAddressType, SegwitBitcoinAddress,
+        BitcoinAddress, LegacyBitcoinAddressType, SegwitBitcoinAddress,
     };
     use crate::burnchains::bitcoin::keys::BitcoinPublicKey;
     use crate::burnchains::bitcoin::{BitcoinInputType, BitcoinNetworkType};

stackslib/src/burnchains/bitcoin/bits.rs

@@ -14,8 +14,6 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-use std::ops::Deref;
-
 use stacks_common::deps_common::bitcoin::blockdata::block::{Block, LoneBlockHeader};
 use stacks_common::deps_common::bitcoin::blockdata::opcodes::All as btc_opcodes;
 use stacks_common::deps_common::bitcoin::blockdata::script::{Instruction, Script};
@@ -25,25 +23,21 @@ use stacks_common::deps_common::bitcoin::network::serialize::BitcoinHash;
 use stacks_common::deps_common::bitcoin::util::hash::bitcoin_merkle_root;
 use stacks_common::types::chainstate::BurnchainHeaderHash;
 use stacks_common::util::hash::to_hex;
-use stacks_common::util::log;
 
 use crate::burnchains::bitcoin::address::BitcoinAddress;
 use crate::burnchains::bitcoin::indexer::BitcoinIndexer;
-use crate::burnchains::bitcoin::keys::BitcoinPublicKey;
 use crate::burnchains::bitcoin::messages::BitcoinMessageHandler;
 use crate::burnchains::bitcoin::{
-    bits, BitcoinBlock, BitcoinInputType, BitcoinNetworkType, BitcoinTransaction, BitcoinTxInput,
-    BitcoinTxOutput, Error as btc_error, PeerMessage,
+    bits, BitcoinBlock, BitcoinNetworkType, BitcoinTransaction, BitcoinTxInput, BitcoinTxOutput,
+    Error as btc_error, PeerMessage,
 };
 use crate::burnchains::indexer::{
     BurnBlockIPC, BurnHeaderIPC, BurnchainBlockDownloader, BurnchainBlockParser,
 };
 use crate::burnchains::{
-    BurnchainBlock, BurnchainTransaction, Error as burnchain_error, MagicBytes, Txid,
-    MAGIC_BYTES_LENGTH,
+    BurnchainBlock, Error as burnchain_error, MagicBytes, Txid, MAGIC_BYTES_LENGTH,
 };
 use crate::core::StacksEpochId;
-use crate::deps;
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct BitcoinHeaderIPC {
@@ -550,16 +544,15 @@ mod tests {
     use stacks_common::types::chainstate::BurnchainHeaderHash;
     use stacks_common::types::Address;
     use stacks_common::util::hash::hex_bytes;
-    use stacks_common::util::log;
 
     use super::BitcoinBlockParser;
     use crate::burnchains::bitcoin::address::{BitcoinAddress, LegacyBitcoinAddressType};
     use crate::burnchains::bitcoin::keys::BitcoinPublicKey;
     use crate::burnchains::bitcoin::{
-        BitcoinBlock, BitcoinInputType, BitcoinNetworkType, BitcoinTransaction, BitcoinTxInput,
-        BitcoinTxInputRaw, BitcoinTxInputStructured, BitcoinTxOutput,
+        BitcoinBlock, BitcoinInputType, BitcoinNetworkType, BitcoinTransaction, BitcoinTxInputRaw,
+        BitcoinTxInputStructured, BitcoinTxOutput,
     };
-    use crate::burnchains::{BurnchainBlock, BurnchainTransaction, MagicBytes, Txid};
+    use crate::burnchains::{MagicBytes, Txid};
     use crate::core::StacksEpochId;
 
     struct TxFixture {

stackslib/src/burnchains/bitcoin/blocks.rs

@@ -15,23 +15,24 @@
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 use std::net::Shutdown;
-use std::ops::{Deref, DerefMut};
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
-use std::{cmp, fs, net, path, time};
+use std::{cmp, fs, net};
 
 use rand::{thread_rng, Rng};
+#[cfg(test)]
 use stacks_common::deps_common::bitcoin::blockdata::block::{BlockHeader, LoneBlockHeader};
+#[cfg(test)]
 use stacks_common::deps_common::bitcoin::network::encodable::VarInt;
 use stacks_common::deps_common::bitcoin::network::message::NetworkMessage;
 use stacks_common::deps_common::bitcoin::network::serialize::{
     BitcoinHash, Error as btc_serialization_err,
 };
 use stacks_common::deps_common::bitcoin::util::hash::Sha256dHash;
 use stacks_common::types::chainstate::BurnchainHeaderHash;
-use stacks_common::util::{get_epoch_time_secs, log};
+use stacks_common::util::get_epoch_time_secs;
 
 use crate::burnchains::bitcoin::blocks::{
     BitcoinBlockDownloader, BitcoinBlockParser, BitcoinHeaderIPC,
@@ -42,7 +43,7 @@ use crate::burnchains::bitcoin::{BitcoinNetworkType, Error as btc_error};
 use crate::burnchains::db::BurnchainHeaderReader;
 use crate::burnchains::indexer::{BurnchainIndexer, *};
 use crate::burnchains::{
-    Burnchain, BurnchainBlockHeader, Error as burnchain_error, MagicBytes, BLOCKSTACK_MAGIC_MAINNET,
+    BurnchainBlockHeader, Error as burnchain_error, MagicBytes, BLOCKSTACK_MAGIC_MAINNET,
 };
 use crate::core::{
     EpochList, StacksEpoch, StacksEpochExtension, STACKS_EPOCHS_MAINNET, STACKS_EPOCHS_REGTEST,
@@ -1181,15 +1182,12 @@ mod test {
 
     use stacks_common::deps_common::bitcoin::blockdata::block::{BlockHeader, LoneBlockHeader};
     use stacks_common::deps_common::bitcoin::network::encodable::VarInt;
-    use stacks_common::deps_common::bitcoin::network::serialize::{
-        deserialize, serialize, BitcoinHash,
-    };
     use stacks_common::deps_common::bitcoin::util::hash::Sha256dHash;
     use stacks_common::util::get_epoch_time_secs;
     use stacks_common::util::uint::Uint256;
 
     use super::*;
-    use crate::burnchains::bitcoin::{Error as btc_error, *};
+    use crate::burnchains::bitcoin::*;
     use crate::burnchains::{Error as burnchain_error, *};
 
     #[test]

stackslib/src/burnchains/bitcoin/indexer.rs

@@ -17,7 +17,6 @@
 // This module is concerned with the implementation of the BitcoinIndexer
 // structure and its methods and traits.
 
-use std::sync::Arc;
 use std::{error, fmt, io};
 
 use stacks_common::deps_common::bitcoin::network::serialize::Error as btc_serialize_error;
@@ -27,8 +26,6 @@ use stacks_common::util::HexError as btc_hex_error;
 use crate::burnchains::bitcoin::address::BitcoinAddress;
 use crate::burnchains::bitcoin::keys::BitcoinPublicKey;
 use crate::burnchains::Txid;
-use crate::chainstate::burn::operations::BlockstackOperationType;
-use crate::deps;
 use crate::util_lib::db::Error as db_error;
 
 pub mod address;