Merge branch 'develop' into feat/fork-detection-state-machine

Author: hstove

clarity/src/vm/database/sqlite.rs

@@ -254,16 +254,13 @@ impl SqliteConnection {
 
         Ok(())
     }
+
     pub fn memory() -> Result<Connection> {
         let contract_db = SqliteConnection::inner_open(":memory:")?;
         SqliteConnection::initialize_conn(&contract_db)?;
         Ok(contract_db)
     }
-    pub fn open(filename: &str) -> Result<Connection> {
-        let contract_db = SqliteConnection::inner_open(filename)?;
-        SqliteConnection::check_schema(&contract_db)?;
-        Ok(contract_db)
-    }
+
     pub fn check_schema(conn: &Connection) -> Result<()> {
         let sql = "SELECT sql FROM sqlite_master WHERE name=?";
         let _: String = conn
@@ -272,10 +269,13 @@ impl SqliteConnection {
         let _: String = conn
             .query_row(sql, params!["metadata_table"], |row| row.get(0))
             .map_err(|x| InterpreterError::SqliteError(IncomparableError { err: x }))?;
+        let _: String = conn
+            .query_row(sql, params!["md_blockhashes"], |row| row.get(0))
+            .map_err(|x| InterpreterError::SqliteError(IncomparableError { err: x }))?;
         Ok(())
     }
 
-    pub fn inner_open(filename: &str) -> Result<Connection> {
+    fn inner_open(filename: &str) -> Result<Connection> {
         let conn = Connection::open(filename)
             .map_err(|x| InterpreterError::SqliteError(IncomparableError { err: x }))?;
 

stackslib/src/chainstate/burn/db/sortdb.rs

@@ -14,13 +14,16 @@
 // You should have received a copy of the GNU General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+use std::cell::RefCell;
 use std::cmp::Ordering;
 use std::collections::{HashMap, HashSet};
 use std::io::{ErrorKind, Write};
 use std::ops::{Deref, DerefMut};
 use std::str::FromStr;
+use std::sync::{Arc, LazyLock, Mutex, MutexGuard};
 use std::{cmp, fmt, fs};
 
+use clarity::util::lru_cache::LruCache;
 use clarity::vm::ast::ASTRules;
 use clarity::vm::costs::ExecutionCost;
 use clarity::vm::representations::{ClarityName, ContractName};
@@ -95,6 +98,10 @@ pub const REWARD_WINDOW_END: u64 = 144 * 90 + REWARD_WINDOW_START;
 
 pub type BlockHeaderCache = HashMap<ConsensusHash, (Option<BlockHeaderHash>, ConsensusHash)>;
 
+const DESCENDANCY_CACHE_SIZE: usize = 2000;
+static DESCENDANCY_CACHE: LazyLock<Arc<Mutex<LruCache<(SortitionId, BlockHeaderHash), bool>>>> =
+    LazyLock::new(|| Arc::new(Mutex::new(LruCache::new(DESCENDANCY_CACHE_SIZE))));
+
 pub enum FindIter<R> {
     Found(R),
     Continue,
@@ -1085,6 +1092,38 @@ pub trait SortitionHandle {
         Ok(Some(StacksBlockId::new(&ch, &bhh)))
     }
 
+    /// Check if the descendancy cache has an entry for whether or not the winning block in `key.0`
+    ///  descends from `key.1`
+    ///
+    /// If it does, return the cached entry
+    fn descendancy_cache_get(
+        cache: &mut MutexGuard<'_, LruCache<(SortitionId, BlockHeaderHash), bool>>,
+        key: &(SortitionId, BlockHeaderHash),
+    ) -> Option<bool> {
+        match cache.get(key) {
+            Ok(result) => result,
+            // cache is broken, create a new one
+            Err(e) => {
+                error!("SortitionDB's descendant cache errored. Will continue operation with cleared cache"; "err" => %e);
+                **cache = LruCache::new(DESCENDANCY_CACHE_SIZE);
+                None
+            }
+        }
+    }
+
+    /// Cache the result of the descendancy check on whether or not the winning block in `key.0`
+    ///  descends from `key.1`
+    fn descendancy_cache_put(
+        cache: &mut MutexGuard<'_, LruCache<(SortitionId, BlockHeaderHash), bool>>,
+        key: (SortitionId, BlockHeaderHash),
+        is_descended: bool,
+    ) {
+        if let Err(e) = cache.insert_clean(key, is_descended) {
+            error!("SortitionDB's descendant cache errored. Will continue operation with cleared cache"; "err" => %e);
+            **cache = LruCache::new(DESCENDANCY_CACHE_SIZE);
+        }
+    }
+
     /// is the given block a descendant of `potential_ancestor`?
     ///  * block_at_burn_height: the burn height of the sortition that chose the stacks block to check
     ///  * potential_ancestor: the stacks block hash of the potential ancestor
@@ -1112,12 +1151,43 @@ pub trait SortitionHandle {
                 test_debug!("No snapshot at height {}", block_at_burn_height);
                 db_error::NotFoundError
             })?;
+        let top_sortition_id = sn.sortition_id;
+
+        let mut cache = DESCENDANCY_CACHE
+            .lock()
+            .expect("FATAL: lock poisoned in SortitionDB");
 
         while sn.block_height >= earliest_block_height {
+            let cache_check_key = (sn.sortition_id, potential_ancestor.clone());
+            match Self::descendancy_cache_get(&mut cache, &cache_check_key) {
+                Some(result) => {
+                    if sn.sortition_id != top_sortition_id {
+                        Self::descendancy_cache_put(
+                            &mut cache,
+                            (top_sortition_id, cache_check_key.1),
+                            result,
+                        );
+                    }
+                    return Ok(result);
+                }
+                // not cached, don't need to do anything.
+                None => {}
+            }
+
             if !sn.sortition {
+                Self::descendancy_cache_put(
+                    &mut cache,
+                    (top_sortition_id, cache_check_key.1),
+                    false,
+                );
                 return Ok(false);
             }
             if &sn.winning_stacks_block_hash == potential_ancestor {
+                Self::descendancy_cache_put(
+                    &mut cache,
+                    (top_sortition_id, cache_check_key.1),
+                    true,
+                );
                 return Ok(true);
             }
 
@@ -1153,6 +1223,11 @@ pub trait SortitionHandle {
                 }
             }
         }
+        Self::descendancy_cache_put(
+            &mut cache,
+            (top_sortition_id, potential_ancestor.clone()),
+            false,
+        );
         return Ok(false);
     }
 }
@@ -2028,15 +2103,15 @@ impl<'a> SortitionHandleConn<'a> {
         connection: &'a SortitionDBConn<'a>,
         chain_tip: &SortitionId,
     ) -> Result<SortitionHandleConn<'a>, db_error> {
-        Ok(SortitionHandleConn {
-            context: SortitionHandleContext {
+        Ok(SortitionHandleConn::new(
+            &connection.index,
+            SortitionHandleContext {
                 chain_tip: chain_tip.clone(),
                 first_block_height: connection.context.first_block_height,
                 pox_constants: connection.context.pox_constants.clone(),
                 dryrun: connection.context.dryrun,
             },
-            index: connection.index,
-        })
+        ))
     }
 
     fn get_tip_indexed(&self, key: &str) -> Result<Option<String>, db_error> {
@@ -3723,15 +3798,15 @@ impl SortitionDBTx<'_> {
 
 impl SortitionDBConn<'_> {
     pub fn as_handle<'b>(&'b self, chain_tip: &SortitionId) -> SortitionHandleConn<'b> {
-        SortitionHandleConn {
-            index: self.index,
-            context: SortitionHandleContext {
+        SortitionHandleConn::new(
+            &self.index,
+            SortitionHandleContext {
                 first_block_height: self.context.first_block_height.clone(),
                 chain_tip: chain_tip.clone(),
                 pox_constants: self.context.pox_constants.clone(),
                 dryrun: self.context.dryrun,
             },
-        }
+        )
     }
 
     /// Given a burnchain consensus hash,

stackslib/src/chainstate/stacks/index/marf.rs

@@ -25,6 +25,7 @@ use stacks_common::types::chainstate::{BlockHeaderHash, TrieHash, TRIEHASH_ENCOD
 use stacks_common::util::hash::Sha512Trunc256Sum;
 use stacks_common::util::log;
 
+use super::storage::ReopenedTrieStorageConnection;
 use crate::chainstate::stacks::index::bits::{get_leaf_hash, get_node_hash, read_root_hash};
 use crate::chainstate::stacks::index::node::{
     clear_backptr, is_backptr, set_backptr, CursorError, TrieCursor, TrieNode, TrieNode16,
@@ -251,6 +252,20 @@ impl<T: MarfTrieId> MarfConnection<T> for MarfTransaction<'_, T> {
     }
 }
 
+impl<T: MarfTrieId> MarfConnection<T> for ReopenedTrieStorageConnection<'_, T> {
+    fn with_conn<F, R>(&mut self, exec: F) -> R
+    where
+        F: FnOnce(&mut TrieStorageConnection<T>) -> R,
+    {
+        let mut conn = self.connection();
+        exec(&mut conn)
+    }
+
+    fn sqlite_conn(&self) -> &Connection {
+        self.db_conn()
+    }
+}
+
 impl<T: MarfTrieId> MarfConnection<T> for MARF<T> {
     fn with_conn<F, R>(&mut self, exec: F) -> R
     where
@@ -1620,6 +1635,21 @@ impl<T: MarfTrieId> MARF<T> {
         })
     }
 
+    /// Build a read-only storage connection which can be used for reads without modifying the
+    ///  calling MARF struct (i.e., the tip pointer is only changed in the connection)
+    ///  but reusing self's existing SQLite Connection (avoiding the overhead of
+    ///   `reopen_readonly`).
+    pub fn reopen_connection(&self) -> Result<ReopenedTrieStorageConnection<'_, T>, Error> {
+        if self.open_chain_tip.is_some() {
+            error!(
+                "MARF at {} is already in the process of writing",
+                &self.storage.db_path
+            );
+            return Err(Error::InProgressError);
+        }
+        self.storage.reopen_connection()
+    }
+
     /// Get the root trie hash at a particular block
     pub fn get_root_hash_at(&mut self, block_hash: &T) -> Result<TrieHash, Error> {
         self.storage.connection().get_root_hash_at(block_hash)

stackslib/src/chainstate/stacks/index/storage.rs

@@ -1310,6 +1310,50 @@ impl<T: MarfTrieId> TrieStorageTransientData<T> {
     }
 }
 
+pub struct ReopenedTrieStorageConnection<'a, T: MarfTrieId> {
+    pub db_path: &'a str,
+    db: &'a Connection,
+    blobs: Option<TrieFile>,
+    data: TrieStorageTransientData<T>,
+    cache: TrieCache<T>,
+    bench: TrieBenchmark,
+    pub hash_calculation_mode: TrieHashCalculationMode,
+
+    /// row ID of a trie that represents unconfirmed state (i.e. trie state that will never become
+    /// part of the MARF, but nevertheless represents a persistent scratch space).  If this field
+    /// is Some(..), then the storage connection here was used to (re-)open an unconfirmed trie
+    /// (via `open_unconfirmed()` or `open_block()` when `self.unconfirmed()` is `true`), or used
+    /// to create an unconfirmed trie (via `extend_to_unconfirmed_block()`).
+    unconfirmed_block_id: Option<u32>,
+
+    // used in testing in order to short-circuit block-height lookups
+    //   when the trie struct is tested outside of marf.rs usage
+    #[cfg(test)]
+    pub test_genesis_block: Option<T>,
+}
+
+impl<'a, T: MarfTrieId> ReopenedTrieStorageConnection<'a, T> {
+    pub fn db_conn(&self) -> &Connection {
+        self.db
+    }
+
+    pub fn connection(&mut self) -> TrieStorageConnection<'_, T> {
+        TrieStorageConnection {
+            db: SqliteConnection::ConnRef(&self.db),
+            db_path: self.db_path,
+            data: &mut self.data,
+            blobs: self.blobs.as_mut(),
+            cache: &mut self.cache,
+            bench: &mut self.bench,
+            hash_calculation_mode: self.hash_calculation_mode,
+            unconfirmed_block_id: None,
+
+            #[cfg(test)]
+            test_genesis_block: &mut self.test_genesis_block,
+        }
+    }
+}
+
 impl<T: MarfTrieId> TrieFileStorage<T> {
     pub fn connection(&mut self) -> TrieStorageConnection<'_, T> {
         TrieStorageConnection {
@@ -1327,6 +1371,54 @@ impl<T: MarfTrieId> TrieFileStorage<T> {
         }
     }
 
+    /// Build a read-only storage connection which can be used for reads without modifying the
+    ///  calling TrieFileStorage struct (i.e., the tip pointer is only changed in the connection)
+    ///  but reusing the TrieFileStorage's existing SQLite Connection (avoiding the overhead of
+    ///   `reopen_readonly`).
+    pub fn reopen_connection(&self) -> Result<ReopenedTrieStorageConnection<'_, T>, Error> {
+        let data = TrieStorageTransientData {
+            uncommitted_writes: self.data.uncommitted_writes.clone(),
+            cur_block: self.data.cur_block.clone(),
+            cur_block_id: self.data.cur_block_id.clone(),
+
+            read_count: 0,
+            read_backptr_count: 0,
+            read_node_count: 0,
+            read_leaf_count: 0,
+
+            write_count: 0,
+            write_node_count: 0,
+            write_leaf_count: 0,
+
+            trie_ancestor_hash_bytes_cache: None,
+
+            readonly: true,
+            unconfirmed: self.unconfirmed(),
+        };
+        // perf note: should we attempt to clone the cache
+        let cache = TrieCache::default();
+        let blobs = if self.blobs.is_some() {
+            Some(TrieFile::from_db_path(&self.db_path, true)?)
+        } else {
+            None
+        };
+        let bench = TrieBenchmark::new();
+        let hash_calculation_mode = self.hash_calculation_mode;
+        let unconfirmed_block_id = None;
+        Ok(ReopenedTrieStorageConnection {
+            db_path: &self.db_path,
+            db: &self.db,
+            blobs,
+            data,
+            cache,
+            bench,
+            hash_calculation_mode,
+            unconfirmed_block_id,
+            #[cfg(test)]
+            test_genesis_block: self.test_genesis_block.clone(),
+        })
+    }
+
     pub fn transaction(&mut self) -> Result<TrieStorageTransaction<'_, T>, Error> {
         if self.readonly() {
             return Err(Error::ReadOnlyError);