Skip to content

Commit 4f29788

Browse files
Centrilgefjonbfops
authored
WASM ABI: add datastore_btree_scan_bsatn & index_id_from_name (#1699)
Signed-off-by: Mazdak Farrokhzad <twingoow@gmail.com> Co-authored-by: Phoebe Goldman <phoebe@clockworklabs.io> Co-authored-by: Zeke Foppa <bfops@users.noreply.github.com>
1 parent 0687062 commit 4f29788

File tree

27 files changed

+844
-84
lines changed

27 files changed

+844
-84
lines changed

Cargo.lock

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,7 @@ tar = "0.4"
221221
tempdir = "0.3.7"
222222
tempfile = "3.8"
223223
termcolor = "1.2.0"
224+
thin-vec = "0.2.13"
224225
thiserror = "1.0.37"
225226
tokio = { version = "1.37", features = ["full"] }
226227
tokio-postgres = { version = "0.7.8", features = ["with-chrono-0_4"] }

crates/bindings-sys/src/lib.rs

Lines changed: 186 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@ use core::mem::MaybeUninit;
88
use core::num::NonZeroU16;
99
use std::ptr;
1010

11-
use spacetimedb_primitives::{errno, errnos, ColId, TableId};
11+
use spacetimedb_primitives::{errno, errnos, ColId, IndexId, TableId};
1212

1313
/// Provides a raw set of sys calls which abstractions can be built atop of.
1414
pub mod raw {
15-
use spacetimedb_primitives::{ColId, TableId};
15+
use spacetimedb_primitives::{ColId, IndexId, TableId};
1616

1717
// this module identifier determines the abi version that modules built with this crate depend
1818
// on. Any non-breaking additions to the abi surface should be put in a new `extern {}` block
@@ -40,6 +40,26 @@ pub mod raw {
4040
/// - `NO_SUCH_TABLE`, when `name` is not the name of a table.
4141
pub fn _table_id_from_name(name: *const u8, name_len: usize, out: *mut TableId) -> u16;
4242

43+
/// Queries the `index_id` associated with the given (index) `name`
44+
/// where `name` is the UTF-8 slice in WASM memory at `name_ptr[..name_len]`.
45+
///
46+
/// The index id is written into the `out` pointer.
47+
///
48+
/// # Traps
49+
///
50+
/// Traps if:
51+
/// - `name_ptr` is NULL or `name` is not in bounds of WASM memory.
52+
/// - `name` is not valid UTF-8.
53+
/// - `out` is NULL or `out[..size_of::<IndexId>()]` is not in bounds of WASM memory.
54+
///
55+
/// # Errors
56+
///
57+
/// Returns an error:
58+
///
59+
/// - `NOT_IN_TRANSACTION`, when called outside of a transaction.
60+
/// - `NO_SUCH_INDEX`, when `name` is not the name of an index.
61+
pub fn _index_id_from_name(name_ptr: *const u8, name_len: usize, out: *mut IndexId) -> u16;
62+
4363
/// Writes the number of rows currently in table identified by `table_id` to `out`.
4464
///
4565
/// # Traps
@@ -72,6 +92,80 @@ pub mod raw {
7292
/// - `NO_SUCH_TABLE`, when `table_id` is not a known ID of a table.
7393
pub fn _datastore_table_scan_bsatn(table_id: TableId, out: *mut RowIter) -> u16;
7494

95+
/// Finds all rows in the index identified by `index_id`,
96+
/// according to the:
97+
/// - `prefix = prefix_ptr[..prefix_len]`,
98+
/// - `rstart = rstart_ptr[..rstart_len]`,
99+
/// - `rend = rend_ptr[..rend_len]`,
100+
/// in WASM memory.
101+
///
102+
/// The index itself has a schema/type.
103+
/// The `prefix` is decoded to the initial `prefix_elems` `AlgebraicType`s
104+
/// whereas `rstart` and `rend` are decoded to the `prefix_elems + 1` `AlgebraicType`
105+
/// where the `AlgebraicValue`s are wrapped in `Bound`.
106+
/// That is, `rstart, rend` are BSATN-encoded `Bound<AlgebraicValue>`s.
107+
///
108+
/// Matching is then defined by equating `prefix`
109+
/// to the initial `prefix_elems` columns of the index
110+
/// and then imposing `rstart` as the starting bound
111+
/// and `rend` as the ending bound on the `prefix_elems + 1` column of the index.
112+
/// Remaining columns of the index are then unbounded.
113+
/// Note that the `prefix` in this case can be empty (`prefix_elems = 0`),
114+
/// in which case this becomes a ranged index scan on a single-col index
115+
/// or even a full table scan if `rstart` and `rend` are both unbounded.
116+
///
117+
/// The relevant table for the index is found implicitly via the `index_id`,
118+
/// which is unique for the module.
119+
///
120+
/// On success, the iterator handle is written to the `out` pointer.
121+
/// This handle can be advanced by [`row_iter_bsatn_advance`].
122+
///
123+
/// # Non-obvious queries
124+
///
125+
/// For an index on columns `[a, b, c]`:
126+
///
127+
/// - `a = x, b = y` is encoded as a prefix `[x, y]`
128+
/// and a range `Range::Unbounded`,
129+
/// or as a prefix `[x]` and a range `rstart = rend = Range::Inclusive(y)`.
130+
/// - `a = x, b = y, c = z` is encoded as a prefix `[x, y]`
131+
/// and a range `rstart = rend = Range::Inclusive(z)`.
132+
/// - A sorted full scan is encoded as an empty prefix
133+
/// and a range `Range::Unbounded`.
134+
///
135+
/// # Traps
136+
///
137+
/// Traps if:
138+
/// - `prefix_elems > 0`
139+
/// and (`prefix_ptr` is NULL or `prefix` is not in bounds of WASM memory).
140+
/// - `rstart` is NULL or `rstart` is not in bounds of WASM memory.
141+
/// - `rend` is NULL or `rend` is not in bounds of WASM memory.
142+
/// - `out` is NULL or `out[..size_of::<RowIter>()]` is not in bounds of WASM memory.
143+
///
144+
/// # Errors
145+
///
146+
/// Returns an error:
147+
///
148+
/// - `NOT_IN_TRANSACTION`, when called outside of a transaction.
149+
/// - `NO_SUCH_INDEX`, when `index_id` is not a known ID of an index.
150+
/// - `WRONG_INDEX_ALGO` if the index is not a btree index.
151+
/// - `BSATN_DECODE_ERROR`, when `prefix` cannot be decoded to
152+
/// a `prefix_elems` number of `AlgebraicValue`
153+
/// typed at the initial `prefix_elems` `AlgebraicType`s of the index's key type.
154+
/// Or when `rstart` or `rend` cannot be decoded to an `Bound<AlgebraicValue>`
155+
/// where the inner `AlgebraicValue`s are
156+
/// typed at the `prefix_elems + 1` `AlgebraicType` of the index's key type.
157+
pub fn _datastore_btree_scan_bsatn(
158+
index_id: IndexId,
159+
prefix_ptr: *const u8,
160+
prefix_len: usize,
161+
prefix_elems: ColId,
162+
rstart_ptr: *const u8, // Bound<AlgebraicValue>
163+
rstart_len: usize,
164+
rend_ptr: *const u8, // Bound<AlgebraicValue>
165+
rend_len: usize,
166+
out: *mut RowIter,
167+
) -> u16;
168+
75169
/// Finds all rows in the table identified by `table_id`,
76170
/// where the row has a column, identified by `col_id`,
77171
/// with data matching the byte string, in WASM memory, pointed to at by `val`.
@@ -597,12 +691,28 @@ unsafe fn call<T: Copy>(f: impl FnOnce(*mut T) -> u16) -> Result<T, Errno> {
597691
///
598692
/// Returns an error:
599693
///
694+
/// - `NOT_IN_TRANSACTION`, when called outside of a transaction.
600695
/// - `NO_SUCH_TABLE`, when `name` is not the name of a table.
601696
#[inline]
602697
pub fn table_id_from_name(name: &str) -> Result<TableId, Errno> {
603698
unsafe { call(|out| raw::_table_id_from_name(name.as_ptr(), name.len(), out)) }
604699
}
605700

701+
/// Queries the `index_id` associated with the given (index) `name`.
702+
///
703+
/// The index id is returned.
704+
///
705+
/// # Errors
706+
///
707+
/// Returns an error:
708+
///
709+
/// - `NOT_IN_TRANSACTION`, when called outside of a transaction.
710+
/// - `NO_SUCH_INDEX`, when `name` is not the name of an index.
711+
#[inline]
712+
pub fn index_id_from_name(name: &str) -> Result<IndexId, Errno> {
713+
unsafe { call(|out| raw::_index_id_from_name(name.as_ptr(), name.len(), out)) }
714+
}
715+
606716
/// Returns the number of rows currently in table identified by `table_id`.
607717
///
608718
/// # Errors
@@ -714,6 +824,80 @@ pub fn datastore_table_scan_bsatn(table_id: TableId) -> Result<RowIter, Errno> {
714824
Ok(RowIter { raw })
715825
}
716826

827+
/// Finds all rows in the index identified by `index_id`,
828+
/// according to the `prefix`, `rstart`, and `rend`.
829+
///
830+
/// The index itself has a schema/type.
831+
/// The `prefix` is decoded to the initial `prefix_elems` `AlgebraicType`s
832+
/// whereas `rstart` and `rend` are decoded to the `prefix_elems + 1` `AlgebraicType`
833+
/// where the `AlgebraicValue`s are wrapped in `Bound`.
834+
/// That is, `rstart, rend` are BSATN-encoded `Bound<AlgebraicValue>`s.
835+
///
836+
/// Matching is then defined by equating `prefix`
837+
/// to the initial `prefix_elems` columns of the index
838+
/// and then imposing `rstart` as the starting bound
839+
/// and `rend` as the ending bound on the `prefix_elems + 1` column of the index.
840+
/// Remaining columns of the index are then unbounded.
841+
/// Note that the `prefix` in this case can be empty (`prefix_elems = 0`),
842+
/// in which case this becomes a ranged index scan on a single-col index
843+
/// or even a full table scan if `rstart` and `rend` are both unbounded.
844+
///
845+
/// The relevant table for the index is found implicitly via the `index_id`,
846+
/// which is unique for the module.
847+
///
848+
/// On success, the iterator handle is written to the `out` pointer.
849+
/// This handle can be advanced by [`row_iter_bsatn_advance`].
850+
///
851+
/// # Non-obvious queries
852+
///
853+
/// For an index on columns `[a, b, c]`:
854+
///
855+
/// - `a = x, b = y` is encoded as a prefix `[x, y]`
856+
/// and a range `Range::Unbounded`,
857+
/// or as a prefix `[x]` and a range `rstart = rend = Range::Inclusive(y)`.
858+
/// - `a = x, b = y, c = z` is encoded as a prefix `[x, y]`
859+
/// and a range `rstart = rend = Range::Inclusive(z)`.
860+
/// - A sorted full scan is encoded as an empty prefix
861+
/// and a range `Range::Unbounded`.
862+
///
863+
/// # Errors
864+
///
865+
/// Returns an error:
866+
///
867+
/// - `NOT_IN_TRANSACTION`, when called outside of a transaction.
868+
/// - `NO_SUCH_INDEX`, when `index_id` is not a known ID of an index.
869+
/// - `WRONG_INDEX_ALGO` if the index is not a btree index.
870+
/// - `BSATN_DECODE_ERROR`, when `prefix` cannot be decoded to
871+
/// a `prefix_elems` number of `AlgebraicValue`
872+
/// typed at the initial `prefix_elems` `AlgebraicType`s of the index's key type.
873+
/// Or when `rstart` or `rend` cannot be decoded to an `Bound<AlgebraicValue>`
874+
/// where the inner `AlgebraicValue`s are
875+
/// typed at the `prefix_elems + 1` `AlgebraicType` of the index's key type.
876+
pub fn datastore_btree_scan_bsatn(
877+
index_id: IndexId,
878+
prefix: &[u8],
879+
prefix_elems: ColId,
880+
rstart: &[u8],
881+
rend: &[u8],
882+
) -> Result<RowIter, Errno> {
883+
let raw = unsafe {
884+
call(|out| {
885+
raw::_datastore_btree_scan_bsatn(
886+
index_id,
887+
prefix.as_ptr(),
888+
prefix.len(),
889+
prefix_elems,
890+
rstart.as_ptr(),
891+
rstart.len(),
892+
rend.as_ptr(),
893+
rend.len(),
894+
out,
895+
)
896+
})?
897+
};
898+
Ok(RowIter { raw })
899+
}
900+
717901
/// Iterate through a table, filtering by an encoded `spacetimedb_lib::filter::Expr`.
718902
///
719903
/// # Errors

crates/core/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ sqlparser.workspace = true
8585
strum.workspace = true
8686
tempfile.workspace = true
8787
thiserror.workspace = true
88+
thin-vec.workspace = true
8889
tokio-util.workspace = true
8990
tokio.workspace = true
9091
tokio-stream = "0.1"

crates/core/src/db/datastore/locking_tx_datastore/committed_state.rs

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ use super::{
22
datastore::Result,
33
sequence::{Sequence, SequencesState},
44
state_view::{Iter, IterByColRange, ScanIterByColRange, StateView},
5-
tx_state::{DeleteTable, TxState},
5+
tx_state::{DeleteTable, IndexIdMap, TxState},
66
};
77
use crate::{
88
db::{
@@ -29,10 +29,9 @@ use spacetimedb_lib::{
2929
address::Address,
3030
db::auth::{StAccess, StTableType},
3131
};
32-
use spacetimedb_primitives::{ColList, TableId};
32+
use spacetimedb_primitives::{ColList, IndexId, TableId};
3333
use spacetimedb_sats::{AlgebraicValue, ProductValue};
3434
use spacetimedb_schema::schema::TableSchema;
35-
3635
use spacetimedb_table::{
3736
blob_store::{BlobStore, HashMapBlobStore},
3837
indexes::{RowPointer, SquashedOffset},
@@ -51,6 +50,8 @@ pub struct CommittedState {
5150
pub(crate) next_tx_offset: u64,
5251
pub(crate) tables: IntMap<TableId, Table>,
5352
pub(crate) blob_store: HashMapBlobStore,
53+
/// Provides fast lookup for index id -> an index.
54+
pub(super) index_id_map: IndexIdMap,
5455
}
5556

5657
impl StateView for CommittedState {
@@ -331,7 +332,9 @@ impl CommittedState {
331332
panic!("Cannot create index for table which doesn't exist in committed state");
332333
};
333334
let index = table.new_index(index_row.index_id, &index_row.columns, index_row.is_unique)?;
334-
table.insert_index(blob_store, index_row.columns, index);
335+
table.insert_index(blob_store, index_row.columns.clone(), index);
336+
self.index_id_map
337+
.insert(index_row.index_id, (index_row.table_id, index_row.columns));
335338
}
336339
Ok(())
337340
}
@@ -443,9 +446,11 @@ impl CommittedState {
443446

444447
// Then, apply inserts. This will re-fill the holes freed by deletions
445448
// before allocating new pages.
446-
447449
self.merge_apply_inserts(&mut tx_data, tx_state.insert_tables, tx_state.blob_store);
448450

451+
// Merge index id fast-lookup map changes.
452+
self.merge_index_map(tx_state.index_id_map, &tx_state.index_id_map_removals);
453+
449454
// If the TX will be logged, record its projected tx offset,
450455
// then increment the counter.
451456
if self.tx_consumes_offset(&tx_data, ctx) {
@@ -537,6 +542,13 @@ impl CommittedState {
537542
}
538543
}
539544

545+
fn merge_index_map(&mut self, index_id_map: IndexIdMap, index_id_map_removals: &[IndexId]) {
546+
for index_id in index_id_map_removals {
547+
self.index_id_map.remove(index_id);
548+
}
549+
self.index_id_map.extend(index_id_map);
550+
}
551+
540552
pub(super) fn get_table(&self, table_id: TableId) -> Option<&Table> {
541553
self.tables.get(&table_id)
542554
}

0 commit comments

Comments
 (0)