From ebdea194c16873d2daa04247f7934f59dac26587 Mon Sep 17 00:00:00 2001 From: Michael Goldenberg Date: Wed, 9 Jul 2025 11:43:44 -0400 Subject: [PATCH 1/5] refactor(indexeddb): add helper fns for EventCacheStore::load_last_chunk Signed-off-by: Michael Goldenberg --- .../src/event_cache_store/transaction.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/matrix-sdk-indexeddb/src/event_cache_store/transaction.rs b/crates/matrix-sdk-indexeddb/src/event_cache_store/transaction.rs index 00485f4c510..b0e96ee7b9f 100644 --- a/crates/matrix-sdk-indexeddb/src/event_cache_store/transaction.rs +++ b/crates/matrix-sdk-indexeddb/src/event_cache_store/transaction.rs @@ -406,6 +406,18 @@ impl<'a> IndexeddbEventCacheStoreTransaction<'a> { self.get_item_by_key_components::(room_id, chunk_id).await } + /// Query IndexedDB for chunks such that the next chunk matches the given + /// chunk identifier in the given room. If more than one item is found, + /// an error is returned. + pub async fn get_chunk_by_next_chunk_id( + &self, + room_id: &RoomId, + next_chunk_id: &Option, + ) -> Result, IndexeddbEventCacheStoreTransactionError> { + self.get_item_by_key_components::(room_id, next_chunk_id) + .await + } + /// Query IndexedDB for all chunks in the given room pub async fn get_chunks_in_room( &self, @@ -414,6 +426,22 @@ impl<'a> IndexeddbEventCacheStoreTransaction<'a> { self.get_items_in_room::(room_id).await } + /// Query IndexedDB for the number of chunks in the given room. + pub async fn get_chunks_count_in_room( + &self, + room_id: &RoomId, + ) -> Result { + self.get_items_count_in_room::(room_id).await + } + + /// Query IndexedDB for the chunk with the maximum key in the given room. + pub async fn get_max_chunk_by_id( + &self, + room_id: &RoomId, + ) -> Result, IndexeddbEventCacheStoreTransactionError> { + self.get_max_item_by_key::(room_id).await + } + /// Query IndexedDB for given chunk in given room and additionally query /// for events or gap, depending on chunk type, in order to construct the /// full chunk. From 2ffef6b723bf183c37eef312b06affd5e063deac Mon Sep 17 00:00:00 2001 From: Michael Goldenberg Date: Wed, 9 Jul 2025 11:58:02 -0400 Subject: [PATCH 2/5] feat(indexeddb): add IndexedDB-backed impl for EventCacheStore::load_last_chunk Signed-off-by: Michael Goldenberg --- .../src/event_cache_store/error.rs | 14 ++++++- .../src/event_cache_store/mod.rs | 38 ++++++++++++++++--- 2 files changed, 46 insertions(+), 6 deletions(-) diff --git a/crates/matrix-sdk-indexeddb/src/event_cache_store/error.rs b/crates/matrix-sdk-indexeddb/src/event_cache_store/error.rs index 3c468b0a5da..4bc6a847dc6 100644 --- a/crates/matrix-sdk-indexeddb/src/event_cache_store/error.rs +++ b/crates/matrix-sdk-indexeddb/src/event_cache_store/error.rs @@ -33,6 +33,14 @@ impl AsyncErrorDeps for T where T: std::error::Error + SendOutsideWasm + Sync pub enum IndexeddbEventCacheStoreError { #[error("DomException {name} ({code}): {message}")] DomException { name: String, message: String, code: u16 }, + #[error("chunks contain disjoint lists")] + ChunksContainDisjointLists, + #[error("chunks contain cycle")] + ChunksContainCycle, + #[error("unable to load chunk")] + UnableToLoadChunk, + #[error("no max chunk id")] + NoMaxChunkId, #[error("transaction: {0}")] Transaction(#[from] IndexeddbEventCacheStoreTransactionError), #[error("media store: {0}")] @@ -52,7 +60,11 @@ impl From for IndexeddbEventCacheStoreError { impl From for EventCacheStoreError { fn from(value: IndexeddbEventCacheStoreError) -> Self { match value { - IndexeddbEventCacheStoreError::DomException { .. } => { + IndexeddbEventCacheStoreError::DomException { .. } + | IndexeddbEventCacheStoreError::ChunksContainCycle + | IndexeddbEventCacheStoreError::ChunksContainDisjointLists + | IndexeddbEventCacheStoreError::NoMaxChunkId + | IndexeddbEventCacheStoreError::UnableToLoadChunk => { Self::InvalidData { details: value.to_string() } } IndexeddbEventCacheStoreError::Transaction(ref inner) => match inner { diff --git a/crates/matrix-sdk-indexeddb/src/event_cache_store/mod.rs b/crates/matrix-sdk-indexeddb/src/event_cache_store/mod.rs index 42939f2c9e0..28b5800cf4e 100644 --- a/crates/matrix-sdk-indexeddb/src/event_cache_store/mod.rs +++ b/crates/matrix-sdk-indexeddb/src/event_cache_store/mod.rs @@ -36,7 +36,7 @@ use web_sys::IdbTransactionMode; use crate::event_cache_store::{ migrations::current::keys, serializer::IndexeddbEventCacheStoreSerializer, - transaction::IndexeddbEventCacheStoreTransaction, + transaction::{IndexeddbEventCacheStoreTransaction, IndexeddbEventCacheStoreTransactionError}, types::{ChunkType, InBandEvent}, }; @@ -300,10 +300,38 @@ impl_event_cache_store! { (Option>, ChunkIdentifierGenerator), IndexeddbEventCacheStoreError, > { - self.memory_store - .load_last_chunk(linked_chunk_id) - .await - .map_err(IndexeddbEventCacheStoreError::MemoryStore) + let linked_chunk_id = linked_chunk_id.to_owned(); + let room_id = linked_chunk_id.room_id(); + let transaction = self.transaction( + &[keys::LINKED_CHUNKS, keys::EVENTS, keys::GAPS], + IdbTransactionMode::Readonly, + )?; + + if transaction.get_chunks_count_in_room(room_id).await? == 0 { + return Ok((None, ChunkIdentifierGenerator::new_from_scratch())); + } + match transaction.get_chunk_by_next_chunk_id(room_id, &None).await { + Err(IndexeddbEventCacheStoreTransactionError::ItemIsNotUnique) => { + Err(IndexeddbEventCacheStoreError::ChunksContainDisjointLists) + } + Err(e) => Err(e.into()), + Ok(None) => Err(IndexeddbEventCacheStoreError::ChunksContainCycle), + Ok(Some(last_chunk)) => { + let last_chunk_identifier = ChunkIdentifier::new(last_chunk.identifier); + let last_raw_chunk = transaction + .load_chunk_by_id(room_id, &last_chunk_identifier) + .await? + .ok_or(IndexeddbEventCacheStoreError::UnableToLoadChunk)?; + let max_chunk_id = transaction + .get_max_chunk_by_id(room_id) + .await? + .map(|chunk| ChunkIdentifier::new(chunk.identifier)) + .ok_or(IndexeddbEventCacheStoreError::NoMaxChunkId)?; + let generator = + ChunkIdentifierGenerator::new_from_previous_chunk_identifier(max_chunk_id); + Ok((Some(last_raw_chunk), generator)) + } + } } async fn load_previous_chunk( From 111d225088a067a6596686305ecc86a3cf0f008d Mon Sep 17 00:00:00 2001 From: Michael Goldenberg Date: Wed, 9 Jul 2025 12:12:08 -0400 Subject: [PATCH 3/5] test(indexeddb): add IndexedDB-specific integration tests for loading last chunk Signed-off-by: Michael Goldenberg --- .../event_cache_store/integration_tests.rs | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/crates/matrix-sdk-indexeddb/src/event_cache_store/integration_tests.rs b/crates/matrix-sdk-indexeddb/src/event_cache_store/integration_tests.rs index 78a0eef8015..018e83e80a8 100644 --- a/crates/matrix-sdk-indexeddb/src/event_cache_store/integration_tests.rs +++ b/crates/matrix-sdk-indexeddb/src/event_cache_store/integration_tests.rs @@ -435,6 +435,73 @@ pub async fn test_linked_chunk_update_is_a_transaction(store: IndexeddbEventCach assert!(chunks.is_empty()); } +pub async fn test_load_last_chunk(store: IndexeddbEventCacheStore) { + let room_id = &DEFAULT_TEST_ROOM_ID; + let linked_chunk_id = LinkedChunkId::Room(room_id); + let event = |msg: &str| make_test_event(room_id, msg); + + // Case #1: no last chunk. + let (last_chunk, chunk_identifier_generator) = + store.load_last_chunk(linked_chunk_id).await.unwrap(); + assert!(last_chunk.is_none()); + assert_eq!(chunk_identifier_generator.current(), 0); + + // Case #2: only one chunk is present. + let updates = vec![ + Update::NewItemsChunk { previous: None, new: ChunkIdentifier::new(42), next: None }, + Update::PushItems { + at: Position::new(ChunkIdentifier::new(42), 0), + items: vec![event("saucisse de morteau"), event("comté")], + }, + ]; + store.handle_linked_chunk_updates(linked_chunk_id, updates).await.unwrap(); + + let (last_chunk, chunk_identifier_generator) = + store.load_last_chunk(linked_chunk_id).await.unwrap(); + assert_matches!(last_chunk, Some(last_chunk) => { + assert_eq!(last_chunk.identifier, 42); + assert!(last_chunk.previous.is_none()); + assert!(last_chunk.next.is_none()); + assert_matches!(last_chunk.content, ChunkContent::Items(items) => { + assert_eq!(items.len(), 2); + check_test_event(&items[0], "saucisse de morteau"); + check_test_event(&items[1], "comté"); + }); + }); + assert_eq!(chunk_identifier_generator.current(), 42); + + // Case #3: more chunks are present. + let updates = vec![ + Update::NewItemsChunk { + previous: Some(ChunkIdentifier::new(42)), + new: ChunkIdentifier::new(7), + next: None, + }, + Update::PushItems { + at: Position::new(ChunkIdentifier::new(7), 0), + items: vec![event("fondue"), event("gruyère"), event("mont d'or")], + }, + ]; + store.handle_linked_chunk_updates(linked_chunk_id, updates).await.unwrap(); + + let (last_chunk, chunk_identifier_generator) = + store.load_last_chunk(linked_chunk_id).await.unwrap(); + assert_matches!(last_chunk, Some(last_chunk) => { + assert_eq!(last_chunk.identifier, 7); + assert_matches!(last_chunk.previous, Some(previous) => { + assert_eq!(previous, 42); + }); + assert!(last_chunk.next.is_none()); + assert_matches!(last_chunk.content, ChunkContent::Items(items) => { + assert_eq!(items.len(), 3); + check_test_event(&items[0], "fondue"); + check_test_event(&items[1], "gruyère"); + check_test_event(&items[2], "mont d'or"); + }); + }); + assert_eq!(chunk_identifier_generator.current(), 42); +} + /// Macro for generating tests for IndexedDB implementation of /// [`EventCacheStore`] /// @@ -547,6 +614,13 @@ macro_rules! indexeddb_event_cache_store_integration_tests { $crate::event_cache_store::integration_tests::test_linked_chunk_update_is_a_transaction(store) .await } + + #[async_test] + async fn test_load_last_chunk() { + let store = get_event_cache_store().await.expect("Failed to get event cache store"); + $crate::event_cache_store::integration_tests::test_load_last_chunk(store) + .await + } } }; } From 759bc7ff878263b64614843c9f43cd48e372bce7 Mon Sep 17 00:00:00 2001 From: Michael Goldenberg Date: Wed, 9 Jul 2025 12:38:01 -0400 Subject: [PATCH 4/5] refactor(indexeddb): separate transaction and event cache error conversions Signed-off-by: Michael Goldenberg --- .../src/event_cache_store/error.rs | 29 ++++++------------- .../src/event_cache_store/transaction.rs | 23 +++++++++++---- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/crates/matrix-sdk-indexeddb/src/event_cache_store/error.rs b/crates/matrix-sdk-indexeddb/src/event_cache_store/error.rs index 4bc6a847dc6..b8fb67dfdc6 100644 --- a/crates/matrix-sdk-indexeddb/src/event_cache_store/error.rs +++ b/crates/matrix-sdk-indexeddb/src/event_cache_store/error.rs @@ -59,27 +59,16 @@ impl From for IndexeddbEventCacheStoreError { impl From for EventCacheStoreError { fn from(value: IndexeddbEventCacheStoreError) -> Self { + use IndexeddbEventCacheStoreError::*; + match value { - IndexeddbEventCacheStoreError::DomException { .. } - | IndexeddbEventCacheStoreError::ChunksContainCycle - | IndexeddbEventCacheStoreError::ChunksContainDisjointLists - | IndexeddbEventCacheStoreError::NoMaxChunkId - | IndexeddbEventCacheStoreError::UnableToLoadChunk => { - Self::InvalidData { details: value.to_string() } - } - IndexeddbEventCacheStoreError::Transaction(ref inner) => match inner { - IndexeddbEventCacheStoreTransactionError::DomException { .. } => { - Self::InvalidData { details: value.to_string() } - } - IndexeddbEventCacheStoreTransactionError::Serialization(e) => { - Self::Serialization(serde_json::Error::custom(e.to_string())) - } - IndexeddbEventCacheStoreTransactionError::ItemIsNotUnique - | IndexeddbEventCacheStoreTransactionError::ItemNotFound => { - Self::InvalidData { details: value.to_string() } - } - }, - IndexeddbEventCacheStoreError::MemoryStore(inner) => inner, + DomException { .. } + | ChunksContainCycle + | ChunksContainDisjointLists + | NoMaxChunkId + | UnableToLoadChunk => Self::InvalidData { details: value.to_string() }, + Transaction(inner) => inner.into(), + MemoryStore(inner) => inner, } } } diff --git a/crates/matrix-sdk-indexeddb/src/event_cache_store/transaction.rs b/crates/matrix-sdk-indexeddb/src/event_cache_store/transaction.rs index b0e96ee7b9f..45615d0ae0b 100644 --- a/crates/matrix-sdk-indexeddb/src/event_cache_store/transaction.rs +++ b/crates/matrix-sdk-indexeddb/src/event_cache_store/transaction.rs @@ -14,11 +14,14 @@ use indexed_db_futures::{prelude::IdbTransaction, IdbQuerySource}; use matrix_sdk_base::{ - event_cache::{Event as RawEvent, Gap as RawGap}, + event_cache::{store::EventCacheStoreError, Event as RawEvent, Gap as RawGap}, linked_chunk::{ChunkContent, ChunkIdentifier, RawChunk}, }; use ruma::{events::relation::RelationType, OwnedEventId, RoomId}; -use serde::{de::DeserializeOwned, Serialize}; +use serde::{ + de::{DeserializeOwned, Error}, + Serialize, +}; use thiserror::Error; use web_sys::IdbCursorDirection; @@ -55,9 +58,19 @@ impl From for IndexeddbEventCacheStoreTransactionError { impl From for IndexeddbEventCacheStoreTransactionError { fn from(e: serde_wasm_bindgen::Error) -> Self { - Self::Serialization(Box::new(::custom( - e.to_string(), - ))) + Self::Serialization(Box::new(serde_json::Error::custom(e.to_string()))) + } +} + +impl From for EventCacheStoreError { + fn from(value: IndexeddbEventCacheStoreTransactionError) -> Self { + use IndexeddbEventCacheStoreTransactionError::*; + + match value { + DomException { .. } => Self::InvalidData { details: value.to_string() }, + Serialization(e) => Self::Serialization(serde_json::Error::custom(e.to_string())), + ItemIsNotUnique | ItemNotFound => Self::InvalidData { details: value.to_string() }, + } } } From 26eec012571c96e9fb61e130ec7ea300f5d52461 Mon Sep 17 00:00:00 2001 From: Michael Goldenberg Date: Wed, 9 Jul 2025 12:57:19 -0400 Subject: [PATCH 5/5] doc(indexeddb): add explanation of error types in EventCacheStore::load_last_chunk Signed-off-by: Michael Goldenberg --- .../src/event_cache_store/mod.rs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/crates/matrix-sdk-indexeddb/src/event_cache_store/mod.rs b/crates/matrix-sdk-indexeddb/src/event_cache_store/mod.rs index 28b5800cf4e..95490a36ee2 100644 --- a/crates/matrix-sdk-indexeddb/src/event_cache_store/mod.rs +++ b/crates/matrix-sdk-indexeddb/src/event_cache_store/mod.rs @@ -310,12 +310,26 @@ impl_event_cache_store! { if transaction.get_chunks_count_in_room(room_id).await? == 0 { return Ok((None, ChunkIdentifierGenerator::new_from_scratch())); } + // Now that we know we have some chunks in the room, we query IndexedDB + // for the last chunk in the room by getting the chunk which does not + // have a next chunk. match transaction.get_chunk_by_next_chunk_id(room_id, &None).await { Err(IndexeddbEventCacheStoreTransactionError::ItemIsNotUnique) => { + // If there are multiple chunks that do not have a next chunk, that + // means we have more than one last chunk, which means that we have + // more than one list in the room. Err(IndexeddbEventCacheStoreError::ChunksContainDisjointLists) } - Err(e) => Err(e.into()), - Ok(None) => Err(IndexeddbEventCacheStoreError::ChunksContainCycle), + Err(e) => { + // There was some error querying IndexedDB, but it is not necessarily + // a violation of our data constraints. + Err(e.into()) + }, + Ok(None) => { + // If there is no chunk without a next chunk, that means every chunk + // points to another chunk, which means that we have a cycle in our list. + Err(IndexeddbEventCacheStoreError::ChunksContainCycle) + }, Ok(Some(last_chunk)) => { let last_chunk_identifier = ChunkIdentifier::new(last_chunk.identifier); let last_raw_chunk = transaction