From cf2cfde1e48d0852bb0dfbdbc1a5d82ebcc4e3a2 Mon Sep 17 00:00:00 2001 From: Mazdak Farrokhzad Date: Fri, 4 Jul 2025 11:46:11 +0200 Subject: [PATCH] extract spacetimedb-memory-usage & bindings: nix spacetimedb-data-structures dep --- Cargo.lock | 19 +- Cargo.toml | 1 + ...ps__spacetimedb_bindings_dependencies.snap | 25 +- crates/bindings/tests/ui/reducers.stderr | 2 +- crates/bindings/tests/ui/tables.stderr | 14 +- crates/core/Cargo.toml | 1 + crates/core/src/db/datastore/error.rs | 12 +- crates/core/src/error.rs | 4 +- crates/core/src/sql/ast.rs | 4 +- crates/core/src/sql/compiler.rs | 2 +- crates/core/src/sql/execute.rs | 4 +- crates/core/src/sql/type_check.rs | 2 +- .../core/src/subscription/execution_unit.rs | 8 +- crates/core/src/subscription/query.rs | 2 +- crates/core/src/subscription/subscription.rs | 9 +- crates/core/src/vm.rs | 4 +- crates/data-structures/Cargo.toml | 4 +- crates/lib/Cargo.toml | 8 +- crates/lib/src/connection_id.rs | 4 +- crates/lib/src/db/mod.rs | 1 - crates/lib/src/db/raw_def/v8.rs | 57 +--- crates/lib/src/identity.rs | 4 +- crates/lib/src/lib.rs | 1 - crates/memory-usage/Cargo.toml | 19 ++ crates/memory-usage/LICENSE | 0 crates/memory-usage/README.md | 3 + crates/memory-usage/src/lib.rs | 135 ++++++++++ crates/primitives/Cargo.toml | 4 + crates/primitives/src/col_list.rs | 6 +- crates/primitives/src/ids.rs | 3 + crates/sats/Cargo.toml | 7 +- crates/sats/src/layout.rs | 31 ++- crates/sats/src/lib.rs | 5 +- crates/sats/src/memory_usage.rs | 250 ------------------ crates/sats/src/memory_usage_impls.rs | 109 ++++++++ crates/schema/Cargo.toml | 1 + crates/schema/src/def.rs | 1 + .../{lib/src/db => schema/src/def}/error.rs | 4 +- crates/schema/src/def/validate/v8.rs | 71 ++++- crates/schema/src/lib.rs | 1 + crates/{lib => schema}/src/relation.rs | 4 +- crates/schema/src/schema.rs | 7 +- crates/table/Cargo.toml | 5 +- crates/table/src/blob_store.rs | 2 +- crates/vm/src/errors.rs | 2 +- crates/vm/src/eval.rs | 6 +- crates/vm/src/expr.rs | 13 +- crates/vm/src/rel_ops.rs | 2 +- crates/vm/src/relation.rs | 2 +- 49 files changed, 469 insertions(+), 416 deletions(-) create mode 100644 crates/memory-usage/Cargo.toml create mode 100644 crates/memory-usage/LICENSE create mode 100644 crates/memory-usage/README.md create mode 100644 crates/memory-usage/src/lib.rs delete mode 100644 crates/sats/src/memory_usage.rs create mode 100644 crates/sats/src/memory_usage_impls.rs rename crates/{lib/src/db => schema/src/def}/error.rs (97%) rename crates/{lib => schema}/src/relation.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 2bc7629d847..2063360a8f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5529,6 +5529,7 @@ dependencies = [ "spacetimedb-jsonwebtoken", "spacetimedb-jwks", "spacetimedb-lib", + "spacetimedb-memory-usage", "spacetimedb-metrics", "spacetimedb-paths", "spacetimedb-physical-plan", @@ -5575,6 +5576,7 @@ dependencies = [ "nohash-hasher", "serde", "smallvec", + "spacetimedb-memory-usage", "thiserror 1.0.69", ] @@ -5688,13 +5690,23 @@ dependencies = [ "serde", "serde_json", "spacetimedb-bindings-macro", - "spacetimedb-data-structures", + "spacetimedb-memory-usage", "spacetimedb-metrics", "spacetimedb-primitives", "spacetimedb-sats", "thiserror 1.0.69", ] +[[package]] +name = "spacetimedb-memory-usage" +version = "1.2.0" +dependencies = [ + "decorum", + "ethnum", + "hashbrown 0.15.3", + "smallvec", +] + [[package]] name = "spacetimedb-metrics" version = "1.2.0" @@ -5746,6 +5758,7 @@ dependencies = [ "itertools 0.12.1", "nohash-hasher", "proptest", + "spacetimedb-memory-usage", ] [[package]] @@ -5792,7 +5805,7 @@ dependencies = [ "sha3", "smallvec", "spacetimedb-bindings-macro", - "spacetimedb-data-structures", + "spacetimedb-memory-usage", "spacetimedb-metrics", "spacetimedb-primitives", "thiserror 1.0.69", @@ -5803,6 +5816,7 @@ name = "spacetimedb-schema" version = "1.2.0" dependencies = [ "anyhow", + "derive_more", "enum-as-inner", "enum-map", "hashbrown 0.15.3", @@ -5965,6 +5979,7 @@ dependencies = [ "smallvec", "spacetimedb-data-structures", "spacetimedb-lib", + "spacetimedb-memory-usage", "spacetimedb-primitives", "spacetimedb-sats", "spacetimedb-schema", diff --git a/Cargo.toml b/Cargo.toml index 3159af03756..5b66b23bbab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -108,6 +108,7 @@ spacetimedb-durability = { path = "crates/durability", version = "1.2.0" } spacetimedb-execution = { path = "crates/execution", version = "1.2.0" } spacetimedb-expr = { path = "crates/expr", version = "1.2.0" } spacetimedb-lib = { path = "crates/lib", default-features = false, version = "1.2.0" } +spacetimedb-memory-usage = { path = "crates/memory-usage", version = "1.2.0", default-features = false } spacetimedb-metrics = { path = "crates/metrics", version = "1.2.0" } spacetimedb-paths = { path = "crates/paths", version = "1.2.0" } spacetimedb-physical-plan = { path = "crates/physical-plan", version = "1.2.0" } diff --git a/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap b/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap index 8059263f97f..6c97bcc8708 100644 --- a/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap +++ b/crates/bindings/tests/snapshots/deps__spacetimedb_bindings_dependencies.snap @@ -2,7 +2,7 @@ source: crates/bindings/tests/deps.rs expression: "cargo tree -p spacetimedb -f {lib} -e no-dev" --- -total crates: 66 +total crates: 61 spacetimedb ├── bytemuck ├── derive_more @@ -69,22 +69,6 @@ spacetimedb │ ├── hex │ ├── itertools (*) │ ├── spacetimedb_bindings_macro (*) -│ ├── spacetimedb_data_structures -│ │ ├── ahash -│ │ │ ├── cfg_if -│ │ │ ├── once_cell -│ │ │ └── zerocopy -│ │ │ [build-dependencies] -│ │ │ └── version_check -│ │ ├── hashbrown -│ │ │ └── equivalent -│ │ ├── nohash_hasher -│ │ ├── smallvec -│ │ └── thiserror -│ │ └── thiserror_impl -│ │ ├── proc_macro2 (*) -│ │ ├── quote (*) -│ │ └── syn (*) │ ├── spacetimedb_primitives (*) │ ├── spacetimedb_sats │ │ ├── anyhow @@ -117,8 +101,11 @@ spacetimedb │ │ │ └── keccak │ │ ├── smallvec │ │ ├── spacetimedb_bindings_macro (*) -│ │ ├── spacetimedb_data_structures (*) │ │ ├── spacetimedb_primitives (*) -│ │ └── thiserror (*) +│ │ └── thiserror +│ │ └── thiserror_impl +│ │ ├── proc_macro2 (*) +│ │ ├── quote (*) +│ │ └── syn (*) │ └── thiserror (*) └── spacetimedb_primitives (*) diff --git a/crates/bindings/tests/ui/reducers.stderr b/crates/bindings/tests/ui/reducers.stderr index b66ab354668..eee7ef25dc4 100644 --- a/crates/bindings/tests/ui/reducers.stderr +++ b/crates/bindings/tests/ui/reducers.stderr @@ -70,12 +70,12 @@ error[E0277]: the reducer argument `Test` does not implement `SpacetimeType` = help: the following other types implement trait `SpacetimeType`: &T () + AlgebraicType AlgebraicTypeRef Arc ArrayType Box ColId - ColumnAttribute and $N others = note: required for `Test` to implement `ReducerArg` diff --git a/crates/bindings/tests/ui/tables.stderr b/crates/bindings/tests/ui/tables.stderr index aca169f6e98..d003ce5faca 100644 --- a/crates/bindings/tests/ui/tables.stderr +++ b/crates/bindings/tests/ui/tables.stderr @@ -21,12 +21,12 @@ error[E0277]: the column type `Test` does not implement `SpacetimeType` = help: the following other types implement trait `SpacetimeType`: &T () + AlgebraicType AlgebraicTypeRef Alpha Arc ArrayType Box - ColId and $N others = note: required for `Test` to implement `TableColumn` @@ -40,12 +40,12 @@ error[E0277]: the trait bound `Test: SpacetimeType` is not satisfied = help: the following other types implement trait `SpacetimeType`: &T () + AlgebraicType AlgebraicTypeRef Alpha Arc ArrayType Box - ColId and $N others error[E0277]: the trait bound `Test: Deserialize<'de>` is not satisfied @@ -61,11 +61,11 @@ error[E0277]: the trait bound `Test: Deserialize<'de>` is not satisfied &'de [u8] &'de str () + AlgebraicType AlgebraicTypeRef Alpha Arc<[T]> ArrayType - Box and $N others note: required by a bound in `spacetimedb::spacetimedb_lib::de::SeqProductAccess::next_element` --> $WORKSPACE/crates/sats/src/de.rs @@ -83,11 +83,11 @@ error[E0277]: the trait bound `Test: Deserialize<'_>` is not satisfied &'de [u8] &'de str () + AlgebraicType AlgebraicTypeRef Alpha Arc<[T]> ArrayType - Box and $N others note: required by a bound in `get_field_value` --> $WORKSPACE/crates/sats/src/de.rs @@ -104,12 +104,12 @@ error[E0277]: the trait bound `Test: Serialize` is not satisfied = help: the following other types implement trait `Serialize`: &T () + AlgebraicType AlgebraicTypeRef + AlgebraicValue Alpha Arc ArrayType - ArrayValue - Bound and $N others note: required by a bound in `spacetimedb::spacetimedb_lib::ser::SerializeNamedProduct::serialize_element` --> $WORKSPACE/crates/sats/src/ser.rs @@ -135,7 +135,7 @@ error[E0277]: `&'a Alpha` cannot appear as an argument to an index filtering ope &bool ðnum::int::I256 and $N others -note: required by a bound in `UniqueColumn::::ColType, Col>::find` +note: required by a bound in `UniqueColumn::::ColType, Col>::find` --> src/table.rs | | pub fn find(&self, col_val: impl Borrow) -> Option diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index 5599fb11615..1045bff8358 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -21,6 +21,7 @@ spacetimedb-lib = { workspace = true, features = ["serde", "metrics_impls"] } spacetimedb-client-api-messages.workspace = true spacetimedb-commitlog.workspace = true spacetimedb-durability.workspace = true +spacetimedb-memory-usage.workspace = true spacetimedb-metrics.workspace = true spacetimedb-primitives.workspace = true spacetimedb-paths.workspace = true diff --git a/crates/core/src/db/datastore/error.rs b/crates/core/src/db/datastore/error.rs index 536f5097d7a..cd7469679be 100644 --- a/crates/core/src/db/datastore/error.rs +++ b/crates/core/src/db/datastore/error.rs @@ -1,15 +1,11 @@ use super::system_tables::SystemTable; use enum_as_inner::EnumAsInner; -use spacetimedb_lib::buffer::DecodeError; -use spacetimedb_lib::{ - db::{ - error::LibError, - raw_def::{v9::RawSql, RawIndexDefV8}, - }, - AlgebraicType, AlgebraicValue, ProductValue, -}; +use spacetimedb_lib::db::raw_def::{v9::RawSql, RawIndexDefV8}; use spacetimedb_primitives::{ColId, ColList, IndexId, SequenceId, TableId}; +use spacetimedb_sats::buffer::DecodeError; use spacetimedb_sats::{product_value::InvalidFieldError, satn::Satn}; +use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductValue}; +use spacetimedb_schema::def::error::LibError; use spacetimedb_snapshot::SnapshotError; use spacetimedb_table::{ bflatn_to, read_column, diff --git a/crates/core/src/error.rs b/crates/core/src/error.rs index 211a37f6c32..cb853b8bcac 100644 --- a/crates/core/src/error.rs +++ b/crates/core/src/error.rs @@ -16,11 +16,11 @@ use thiserror::Error; use crate::client::ClientActorId; use crate::host::scheduler::ScheduleError; use spacetimedb_lib::buffer::DecodeError; -use spacetimedb_lib::db::error::{LibError, RelationError, SchemaErrors}; -use spacetimedb_lib::relation::FieldName; use spacetimedb_primitives::*; use spacetimedb_sats::hash::Hash; use spacetimedb_sats::product_value::InvalidFieldError; +use spacetimedb_schema::def::error::{LibError, RelationError, SchemaErrors}; +use spacetimedb_schema::relation::FieldName; use spacetimedb_vm::errors::{ErrorKind, ErrorLang, ErrorType, ErrorVm}; use spacetimedb_vm::expr::Crud; diff --git a/crates/core/src/sql/ast.rs b/crates/core/src/sql/ast.rs index b30885cf7f1..57f8c7443d4 100644 --- a/crates/core/src/sql/ast.rs +++ b/crates/core/src/sql/ast.rs @@ -7,11 +7,11 @@ use spacetimedb_data_structures::map::{HashCollectionExt as _, IntMap}; use spacetimedb_expr::check::SchemaView; use spacetimedb_expr::statement::compile_sql_stmt; use spacetimedb_lib::db::auth::StAccess; -use spacetimedb_lib::db::error::RelationError; use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::relation::{ColExpr, FieldName}; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::{AlgebraicType, AlgebraicValue}; +use spacetimedb_schema::def::error::RelationError; +use spacetimedb_schema::relation::{ColExpr, FieldName}; use spacetimedb_schema::schema::{ColumnSchema, TableSchema}; use spacetimedb_vm::errors::ErrorVm; use spacetimedb_vm::expr::{Expr, FieldExpr, FieldOp}; diff --git a/crates/core/src/sql/compiler.rs b/crates/core/src/sql/compiler.rs index b94853cbce7..a5766db35f1 100644 --- a/crates/core/src/sql/compiler.rs +++ b/crates/core/src/sql/compiler.rs @@ -7,8 +7,8 @@ use crate::error::{DBError, PlanError}; use core::ops::Deref; use spacetimedb_data_structures::map::IntMap; use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::relation::{self, ColExpr, DbTable, FieldName, Header}; use spacetimedb_primitives::ColId; +use spacetimedb_schema::relation::{self, ColExpr, DbTable, FieldName, Header}; use spacetimedb_schema::schema::TableSchema; use spacetimedb_vm::expr::{CrudExpr, Expr, FieldExpr, QueryExpr, SourceExpr}; use spacetimedb_vm::operator::OpCmp; diff --git a/crates/core/src/sql/execute.rs b/crates/core/src/sql/execute.rs index 10c08bca898..12c5d15d422 100644 --- a/crates/core/src/sql/execute.rs +++ b/crates/core/src/sql/execute.rs @@ -19,10 +19,10 @@ use anyhow::anyhow; use spacetimedb_expr::statement::Statement; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::metrics::ExecutionMetrics; -use spacetimedb_lib::relation::FieldName; use spacetimedb_lib::Timestamp; use spacetimedb_lib::{AlgebraicType, ProductType, ProductValue}; use spacetimedb_query::{compile_sql_stmt, execute_dml_stmt, execute_select_stmt}; +use spacetimedb_schema::relation::FieldName; use spacetimedb_vm::eval::run_ast; use spacetimedb_vm::expr::{CodeResult, CrudExpr, Expr}; use spacetimedb_vm::relation::MemTable; @@ -320,10 +320,10 @@ pub(crate) mod tests { use spacetimedb_lib::bsatn::ToBsatn; use spacetimedb_lib::db::auth::{StAccess, StTableType}; use spacetimedb_lib::error::{ResultTest, TestError}; - use spacetimedb_lib::relation::Header; use spacetimedb_lib::{AlgebraicValue, Identity}; use spacetimedb_primitives::{col_list, ColId, TableId}; use spacetimedb_sats::{product, AlgebraicType, ArrayValue, ProductType}; + use spacetimedb_schema::relation::Header; use spacetimedb_vm::eval::test_helpers::create_game_data; pub(crate) fn execute_for_testing( diff --git a/crates/core/src/sql/type_check.rs b/crates/core/src/sql/type_check.rs index 6922cb8da6e..e302eeabe95 100644 --- a/crates/core/src/sql/type_check.rs +++ b/crates/core/src/sql/type_check.rs @@ -2,9 +2,9 @@ use crate::error::PlanError; use crate::sql::ast::From; use crate::sql::ast::{Selection, SqlAst}; use spacetimedb_lib::operator::OpQuery; -use spacetimedb_lib::relation::FieldName; use spacetimedb_sats::algebraic_type::fmt::fmt_algebraic_type; use spacetimedb_sats::{AlgebraicType, AlgebraicValue}; +use spacetimedb_schema::relation::FieldName; use spacetimedb_schema::schema::ColumnSchema; use spacetimedb_vm::errors::ErrorType; use spacetimedb_vm::expr::{FieldExpr, FieldOp}; diff --git a/crates/core/src/subscription/execution_unit.rs b/crates/core/src/subscription/execution_unit.rs index bf63150bbbc..188fa1bcb74 100644 --- a/crates/core/src/subscription/execution_unit.rs +++ b/crates/core/src/subscription/execution_unit.rs @@ -11,11 +11,11 @@ use crate::vm::{build_query, TxMode}; use spacetimedb_client_api_messages::websocket::{ Compression, QueryUpdate, RowListLen as _, SingleQueryUpdate, WebsocketFormat, }; -use spacetimedb_lib::db::error::AuthError; -use spacetimedb_lib::relation::DbTable; -use spacetimedb_lib::{Identity, ProductValue}; +use spacetimedb_lib::Identity; use spacetimedb_primitives::TableId; -use spacetimedb_sats::u256; +use spacetimedb_sats::{u256, ProductValue}; +use spacetimedb_schema::def::error::AuthError; +use spacetimedb_schema::relation::DbTable; use spacetimedb_vm::eval::IterRows; use spacetimedb_vm::expr::{AuthAccess, NoInMemUsed, Query, QueryExpr, SourceExpr, SourceId}; use spacetimedb_vm::rel_ops::RelOps; diff --git a/crates/core/src/subscription/query.rs b/crates/core/src/subscription/query.rs index d836a50122a..95b750a99df 100644 --- a/crates/core/src/subscription/query.rs +++ b/crates/core/src/subscription/query.rs @@ -166,10 +166,10 @@ mod tests { use spacetimedb_lib::error::ResultTest; use spacetimedb_lib::identity::AuthCtx; use spacetimedb_lib::metrics::ExecutionMetrics; - use spacetimedb_lib::relation::FieldName; use spacetimedb_lib::Identity; use spacetimedb_primitives::{ColId, TableId}; use spacetimedb_sats::{product, AlgebraicType, ProductType, ProductValue}; + use spacetimedb_schema::relation::FieldName; use spacetimedb_schema::schema::*; use spacetimedb_vm::eval::run_ast; use spacetimedb_vm::eval::test_helpers::{mem_table, mem_table_without_table_name, scalar}; diff --git a/crates/core/src/subscription/subscription.rs b/crates/core/src/subscription/subscription.rs index e2523ae4f0e..f98a7970eb3 100644 --- a/crates/core/src/subscription/subscription.rs +++ b/crates/core/src/subscription/subscription.rs @@ -35,11 +35,12 @@ use itertools::Either; use spacetimedb_client_api_messages::websocket::{Compression, WebsocketFormat}; use spacetimedb_data_structures::map::HashSet; use spacetimedb_lib::db::auth::{StAccess, StTableType}; -use spacetimedb_lib::db::error::AuthError; use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::relation::DbTable; -use spacetimedb_lib::{Identity, ProductValue}; +use spacetimedb_lib::Identity; use spacetimedb_primitives::TableId; +use spacetimedb_sats::ProductValue; +use spacetimedb_schema::def::error::AuthError; +use spacetimedb_schema::relation::DbTable; use spacetimedb_subscription::SubscriptionPlan; use spacetimedb_vm::expr::{self, AuthAccess, IndexJoin, Query, QueryExpr, SourceExpr, SourceProvider, SourceSet}; use spacetimedb_vm::rel_ops::RelOps; @@ -664,9 +665,9 @@ mod tests { use super::*; use crate::db::relational_db::tests_utils::{begin_tx, TestDB}; use crate::sql::compiler::compile_sql; - use spacetimedb_lib::relation::DbTable; use spacetimedb_lib::{error::ResultTest, identity::AuthCtx}; use spacetimedb_sats::{product, AlgebraicType}; + use spacetimedb_schema::relation::DbTable; use spacetimedb_vm::expr::{CrudExpr, IndexJoin, Query, SourceExpr}; #[test] diff --git a/crates/core/src/vm.rs b/crates/core/src/vm.rs index 07b67b20863..79731aeb699 100644 --- a/crates/core/src/vm.rs +++ b/crates/core/src/vm.rs @@ -12,9 +12,9 @@ use core::ops::{Bound, RangeBounds}; use itertools::Itertools; use spacetimedb_data_structures::map::IntMap; use spacetimedb_lib::identity::AuthCtx; -use spacetimedb_lib::relation::{ColExpr, DbTable}; use spacetimedb_primitives::*; use spacetimedb_sats::{AlgebraicValue, ProductValue}; +use spacetimedb_schema::relation::{ColExpr, DbTable}; use spacetimedb_table::static_assert_size; use spacetimedb_table::table::RowRef; use spacetimedb_vm::errors::ErrorVm; @@ -652,9 +652,9 @@ pub(crate) mod tests { use pretty_assertions::assert_eq; use spacetimedb_lib::db::auth::{StAccess, StTableType}; use spacetimedb_lib::error::ResultTest; - use spacetimedb_lib::relation::{FieldName, Header}; use spacetimedb_sats::{product, AlgebraicType, ProductType, ProductValue}; use spacetimedb_schema::def::{BTreeAlgorithm, IndexAlgorithm}; + use spacetimedb_schema::relation::{FieldName, Header}; use spacetimedb_schema::schema::{ColumnSchema, IndexSchema, TableSchema}; use spacetimedb_vm::eval::run_ast; use spacetimedb_vm::eval::test_helpers::{mem_table, mem_table_one_u64, scalar}; diff --git a/crates/data-structures/Cargo.toml b/crates/data-structures/Cargo.toml index d5e52a44e26..c9a82701db2 100644 --- a/crates/data-structures/Cargo.toml +++ b/crates/data-structures/Cargo.toml @@ -6,12 +6,14 @@ license-file = "LICENSE" description = "Assorted data structures used in spacetimedb" [features] +memory-usage = ["dep:spacetimedb-memory-usage"] serde = ["dep:serde", "hashbrown/serde"] [dependencies] +spacetimedb-memory-usage = { workspace = true, optional = true, default-features = false } ahash.workspace = true hashbrown.workspace = true nohash-hasher.workspace = true serde = { workspace = true, optional = true } -thiserror.workspace = true smallvec.workspace = true +thiserror.workspace = true diff --git a/crates/lib/Cargo.toml b/crates/lib/Cargo.toml index 1057d8fb840..dcfe96704b7 100644 --- a/crates/lib/Cargo.toml +++ b/crates/lib/Cargo.toml @@ -23,13 +23,19 @@ proptest = ["dep:proptest", "dep:proptest-derive"] test = ["proptest", "spacetimedb-sats/test"] metrics_impls = ["dep:spacetimedb-metrics", "spacetimedb-sats/metrics_impls"] enum-map = ["dep:enum-map"] +# Gated to avoid including this in `spacetimedb_bindings`. +memory-usage = [ + "dep:spacetimedb-memory-usage", + "spacetimedb-primitives/memory-usage", + "spacetimedb-sats/memory-usage", +] [dependencies] spacetimedb-bindings-macro.workspace = true spacetimedb-sats.workspace = true spacetimedb-primitives.workspace = true +spacetimedb-memory-usage = { workspace = true, optional = true } spacetimedb-metrics = { workspace = true, optional = true } -spacetimedb-data-structures.workspace = true anyhow.workspace = true bitflags.workspace = true diff --git a/crates/lib/src/connection_id.rs b/crates/lib/src/connection_id.rs index 118ca613649..2055e389127 100644 --- a/crates/lib/src/connection_id.rs +++ b/crates/lib/src/connection_id.rs @@ -3,7 +3,6 @@ use core::{fmt, net::Ipv6Addr}; use spacetimedb_bindings_macro::{Deserialize, Serialize}; use spacetimedb_lib::from_hex_pad; use spacetimedb_sats::hex::HexString; -use spacetimedb_sats::memory_usage::MemoryUsage; use spacetimedb_sats::{impl_deserialize, impl_serialize, impl_st, AlgebraicType, AlgebraicValue}; /// A unique identifier for a client connection to a SpacetimeDB database. @@ -33,7 +32,8 @@ pub struct ConnectionId { impl_st!([] ConnectionId, AlgebraicType::connection_id()); -impl MemoryUsage for ConnectionId {} +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for ConnectionId {} #[cfg(feature = "metrics_impls")] impl spacetimedb_metrics::typed_prometheus::AsPrometheusLabel for ConnectionId { diff --git a/crates/lib/src/db/mod.rs b/crates/lib/src/db/mod.rs index 9961772b5ff..6bdbca93d62 100644 --- a/crates/lib/src/db/mod.rs +++ b/crates/lib/src/db/mod.rs @@ -3,5 +3,4 @@ pub mod attr; pub mod auth; pub mod default_element_ordering; -pub mod error; pub mod raw_def; diff --git a/crates/lib/src/db/raw_def/v8.rs b/crates/lib/src/db/raw_def/v8.rs index c306f125a57..dfb6fcda6cb 100644 --- a/crates/lib/src/db/raw_def/v8.rs +++ b/crates/lib/src/db/raw_def/v8.rs @@ -5,7 +5,6 @@ use crate::db::auth::{StAccess, StTableType}; use crate::{AlgebraicType, ProductType, SpacetimeType}; use derive_more::Display; -use spacetimedb_data_structures::map::HashSet; use spacetimedb_primitives::*; // TODO(1.0): move these definitions into this file, @@ -387,7 +386,7 @@ impl RawTableDefV8 { self } - fn gen_constraint_def(&self, kind: Constraints, columns: impl Into) -> RawConstraintDefV8 { + pub fn gen_constraint_def(&self, kind: Constraints, columns: impl Into) -> RawConstraintDefV8 { let columns = columns.into(); RawConstraintDefV8::for_column(&self.table_name, &self.generate_cols_name(&columns), kind, columns) } @@ -454,60 +453,6 @@ impl RawTableDefV8 { ) } - /// Get an iterator deriving [RawIndexDefV8]s from the constraints that require them like `UNIQUE`. - /// - /// It looks into [Self::constraints] for possible duplicates and remove them from the result - pub fn generated_indexes(&self) -> impl Iterator + '_ { - self.constraints - .iter() - // We are only interested in constraints implying an index. - .filter(|x| x.constraints.has_indexed()) - // Create the `IndexDef`. - .map(|x| { - let is_unique = x.constraints.has_unique(); - RawIndexDefV8::for_column(&self.table_name, &x.constraint_name, x.columns.clone(), is_unique) - }) - // Only keep those we don't yet have in the list of indices (checked by name). - .filter(|idx| self.indexes.iter().all(|x| x.index_name != idx.index_name)) - } - - /// Get an iterator deriving [RawSequenceDefV8] from the constraints that require them like `IDENTITY`. - /// - /// It looks into [Self::constraints] for possible duplicates and remove them from the result - pub fn generated_sequences(&self) -> impl Iterator + '_ { - let cols: HashSet<_> = self.sequences.iter().map(|seq| ColList::new(seq.col_pos)).collect(); - - self.constraints - .iter() - // We are only interested in constraints implying a sequence. - .filter(move |x| !cols.contains(&x.columns) && x.constraints.has_autoinc()) - // Create the `SequenceDef`. - .map(|x| RawSequenceDefV8::for_column(&self.table_name, &x.constraint_name, x.columns.head().unwrap())) - // Only keep those we don't yet have in the list of sequences (checked by name). - .filter(|seq| self.sequences.iter().all(|x| x.sequence_name != seq.sequence_name)) - } - - /// Get an iterator deriving [RawConstraintDefV8] from the indexes that require them like `UNIQUE`. - /// - /// It looks into Self::constraints for possible duplicates and remove them from the result - pub fn generated_constraints(&self) -> impl Iterator + '_ { - // Collect the set of all col-lists with a constraint. - let cols: HashSet<_> = self - .constraints - .iter() - .filter(|x| x.constraints.kind() != ConstraintKind::UNSET) - .map(|x| &x.columns) - .collect(); - - // Those indices that are not present in the constraints above - // have constraints generated for them. - // When `idx.is_unique`, a unique constraint is generated rather than an indexed one. - self.indexes - .iter() - .filter(move |idx| !cols.contains(&idx.columns)) - .map(|idx| self.gen_constraint_def(Constraints::from_is_unique(idx.is_unique), idx.columns.clone())) - } - /// Get a column by its position in the table. pub fn get_column(&self, pos: usize) -> Option<&RawColumnDefV8> { self.columns.get(pos) diff --git a/crates/lib/src/identity.rs b/crates/lib/src/identity.rs index 49e996ac5fb..416e75ef9f8 100644 --- a/crates/lib/src/identity.rs +++ b/crates/lib/src/identity.rs @@ -3,7 +3,6 @@ use blake3; use core::mem; use spacetimedb_bindings_macro::{Deserialize, Serialize}; use spacetimedb_sats::hex::HexString; -use spacetimedb_sats::memory_usage::MemoryUsage; use spacetimedb_sats::{impl_st, u256, AlgebraicType, AlgebraicValue}; use std::{fmt, str::FromStr}; @@ -61,7 +60,8 @@ pub struct Identity { impl_st!([] Identity, AlgebraicType::identity()); -impl MemoryUsage for Identity {} +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for Identity {} #[cfg(feature = "metrics_impls")] impl spacetimedb_metrics::typed_prometheus::AsPrometheusLabel for Identity { diff --git a/crates/lib/src/lib.rs b/crates/lib/src/lib.rs index d25e317eabd..886e242cbc2 100644 --- a/crates/lib/src/lib.rs +++ b/crates/lib/src/lib.rs @@ -15,7 +15,6 @@ pub mod identity; pub mod metrics; pub mod operator; pub mod query; -pub mod relation; pub mod scheduler; pub mod st_var; pub mod version; diff --git a/crates/memory-usage/Cargo.toml b/crates/memory-usage/Cargo.toml new file mode 100644 index 00000000000..078ffff1935 --- /dev/null +++ b/crates/memory-usage/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "spacetimedb-memory-usage" +version.workspace = true +edition.workspace = true +license-file = "LICENSE" +description = "Provides the trait `MemoryUsage`" + +[features] +hashbrown = ["dep:hashbrown"] +smallvec = ["dep:smallvec"] +ethnum = ["dep:ethnum"] +decorum = ["dep:decorum"] +default = ["hashbrown", "smallvec", "ethnum", "decorum"] + +[dependencies] +hashbrown = { workspace = true, optional = true } +smallvec = { workspace = true, optional = true } +ethnum = { workspace = true, optional = true } +decorum = { workspace = true, optional = true } diff --git a/crates/memory-usage/LICENSE b/crates/memory-usage/LICENSE new file mode 100644 index 00000000000..e69de29bb2d diff --git a/crates/memory-usage/README.md b/crates/memory-usage/README.md new file mode 100644 index 00000000000..fc5b684dd86 --- /dev/null +++ b/crates/memory-usage/README.md @@ -0,0 +1,3 @@ +> ⚠️ **Internal Crate** ⚠️ +> +> This crate is intended for internal use only. It is **not** stable and may change without notice. diff --git a/crates/memory-usage/src/lib.rs b/crates/memory-usage/src/lib.rs new file mode 100644 index 00000000000..40bbb8fc7ef --- /dev/null +++ b/crates/memory-usage/src/lib.rs @@ -0,0 +1,135 @@ +use core::mem; +use core::sync::atomic::AtomicUsize; + +/// For inspecting how much memory a value is using. +/// +/// This trait specifically measures heap memory. If you want to measure stack memory too, add +/// `mem::size_of_val()` to it. (This only really matters for the outermost type in a hierarchy.) +pub trait MemoryUsage { + /// The **heap** memory usage of this type. The default implementation returns 0. + #[inline(always)] + fn heap_usage(&self) -> usize { + 0 + } +} + +impl MemoryUsage for () {} +impl MemoryUsage for bool {} +impl MemoryUsage for u8 {} +impl MemoryUsage for u16 {} +impl MemoryUsage for u32 {} +impl MemoryUsage for u64 {} +impl MemoryUsage for u128 {} +#[cfg(feature = "ethnum")] +impl MemoryUsage for ethnum::u256 {} +impl MemoryUsage for usize {} +impl MemoryUsage for AtomicUsize {} +impl MemoryUsage for i8 {} +impl MemoryUsage for i16 {} +impl MemoryUsage for i32 {} +impl MemoryUsage for i64 {} +impl MemoryUsage for i128 {} +#[cfg(feature = "ethnum")] +impl MemoryUsage for ethnum::i256 {} +impl MemoryUsage for isize {} +impl MemoryUsage for f32 {} +impl MemoryUsage for f64 {} +#[cfg(feature = "decorum")] +impl MemoryUsage for decorum::Total {} +#[cfg(feature = "decorum")] +impl MemoryUsage for decorum::Total {} + +impl MemoryUsage for &T { + fn heap_usage(&self) -> usize { + (*self).heap_usage() + } +} + +impl MemoryUsage for Box { + fn heap_usage(&self) -> usize { + mem::size_of_val::(self) + T::heap_usage(self) + } +} + +impl MemoryUsage for std::sync::Arc { + fn heap_usage(&self) -> usize { + let refcounts = mem::size_of::() * 2; + refcounts + mem::size_of_val::(self) + T::heap_usage(self) + } +} + +impl MemoryUsage for std::rc::Rc { + fn heap_usage(&self) -> usize { + let refcounts = mem::size_of::() * 2; + refcounts + mem::size_of_val::(self) + T::heap_usage(self) + } +} + +impl MemoryUsage for [T] { + fn heap_usage(&self) -> usize { + self.iter().map(T::heap_usage).sum() + } +} + +impl MemoryUsage for [T; N] { + fn heap_usage(&self) -> usize { + self.iter().map(T::heap_usage).sum() + } +} + +impl MemoryUsage for str {} + +impl MemoryUsage for Option { + fn heap_usage(&self) -> usize { + self.as_ref().map_or(0, T::heap_usage) + } +} + +impl MemoryUsage for (A, B) { + fn heap_usage(&self) -> usize { + self.0.heap_usage() + self.1.heap_usage() + } +} + +impl MemoryUsage for String { + fn heap_usage(&self) -> usize { + self.capacity() + } +} + +impl MemoryUsage for Vec { + fn heap_usage(&self) -> usize { + self.capacity() * mem::size_of::() + self.iter().map(T::heap_usage).sum::() + } +} + +#[cfg(feature = "hashbrown")] +impl MemoryUsage + for hashbrown::HashMap +{ + fn heap_usage(&self) -> usize { + self.allocation_size() + self.iter().map(|(k, v)| k.heap_usage() + v.heap_usage()).sum::() + } +} + +impl MemoryUsage for std::collections::BTreeMap { + fn heap_usage(&self) -> usize { + // NB: this is best-effort, since we don't have a `capacity()` method on `BTreeMap`. + self.len() * mem::size_of::<(K, V)>() + self.iter().map(|(k, v)| k.heap_usage() + v.heap_usage()).sum::() + } +} + +#[cfg(feature = "smallvec")] +impl MemoryUsage for smallvec::SmallVec +where + A::Item: MemoryUsage, +{ + fn heap_usage(&self) -> usize { + self.as_slice().heap_usage() + + if self.spilled() { + self.capacity() * mem::size_of::() + } else { + 0 + } + } +} diff --git a/crates/primitives/Cargo.toml b/crates/primitives/Cargo.toml index 2268fb00235..d8155cd4e9c 100644 --- a/crates/primitives/Cargo.toml +++ b/crates/primitives/Cargo.toml @@ -5,11 +5,15 @@ edition.workspace = true license-file = "LICENSE" description = "Primitives such as TableId and ColumnIndexAttribute" +[features] +memory-usage = ["dep:spacetimedb-memory-usage"] + [dependencies] bitflags.workspace = true either.workspace = true nohash-hasher.workspace = true itertools.workspace = true +spacetimedb-memory-usage = { workspace = true, optional = true, default-features = false } [dev-dependencies] proptest.workspace = true diff --git a/crates/primitives/src/col_list.rs b/crates/primitives/src/col_list.rs index 380f7d3bec2..a702ea51a49 100644 --- a/crates/primitives/src/col_list.rs +++ b/crates/primitives/src/col_list.rs @@ -276,9 +276,11 @@ impl ColList { let addr = unsafe { self.check }; addr & 1 != 0 } +} - #[doc(hidden)] - pub fn heap_size(&self) -> usize { +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for ColList { + fn heap_usage(&self) -> usize { match self.as_inline() { Ok(_) => 0, Err(heap) => heap.capacity() as usize, diff --git a/crates/primitives/src/ids.rs b/crates/primitives/src/ids.rs index e4152618862..fc9a9b69e61 100644 --- a/crates/primitives/src/ids.rs +++ b/crates/primitives/src/ids.rs @@ -49,6 +49,9 @@ macro_rules! system_id { Self(value as _) } } + + #[cfg(feature = "memory-usage")] + impl spacetimedb_memory_usage::MemoryUsage for $name {} }; } // TODO(1.0): convert this into a proper trait. diff --git a/crates/sats/Cargo.toml b/crates/sats/Cargo.toml index 4cc8ad0c9cb..d2dd0a08e87 100644 --- a/crates/sats/Cargo.toml +++ b/crates/sats/Cargo.toml @@ -24,11 +24,16 @@ blake3 = ["dep:blake3"] # which we don't want in `spacetimedb_bindings`. bytestring = ["dep:bytestring"] metrics_impls = ["dep:spacetimedb-metrics"] +# Gated to avoid including this in `spacetimedb_bindings`. +memory-usage = ["dep:spacetimedb-memory-usage", "spacetimedb-primitives/memory-usage"] [dependencies] spacetimedb-bindings-macro.workspace = true -spacetimedb-data-structures.workspace = true spacetimedb-primitives.workspace = true +spacetimedb-memory-usage = { workspace = true, optional = true, default-features = false, features = [ + "ethnum", + "decorum", +] } spacetimedb-metrics = { workspace = true, optional = true } anyhow.workspace = true diff --git a/crates/sats/src/layout.rs b/crates/sats/src/layout.rs index 281a773c815..637c502202f 100644 --- a/crates/sats/src/layout.rs +++ b/crates/sats/src/layout.rs @@ -12,7 +12,6 @@ use crate::{ SumAccess, SumVisitor, ValidNames, VariantAccess as _, VariantVisitor, }, i256, impl_deserialize, impl_serialize, - memory_usage::MemoryUsage, sum_type::{OPTION_NONE_TAG, OPTION_SOME_TAG}, u256, AlgebraicType, AlgebraicValue, ProductType, ProductTypeElement, ProductValue, SumType, SumTypeVariant, SumValue, WithTypespace, @@ -46,7 +45,8 @@ pub const fn align_to(base: usize, required_alignment: usize) -> usize { #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash, Add, Sub)] pub struct Size(pub u16); -impl MemoryUsage for Size {} +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for Size {} // We need to be able to serialize and deserialize `Size` because they appear in the `PageHeader`. impl_serialize!([] Size, (self, ser) => self.0.serialize(ser)); @@ -91,7 +91,8 @@ pub struct Layout { pub fixed: bool, } -impl MemoryUsage for Layout {} +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for Layout {} /// A type which knows what its layout is. /// @@ -145,7 +146,8 @@ pub enum AlgebraicTypeLayout { VarLen(VarLenType), } -impl MemoryUsage for AlgebraicTypeLayout { +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for AlgebraicTypeLayout { fn heap_usage(&self) -> usize { match self { AlgebraicTypeLayout::Sum(x) => x.heap_usage(), @@ -250,7 +252,8 @@ pub struct RowTypeLayout { pub elements: Arc<[ProductTypeElementLayout]>, } -impl MemoryUsage for RowTypeLayout { +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for RowTypeLayout { fn heap_usage(&self) -> usize { let Self { layout, elements } = self; layout.heap_usage() + elements.heap_usage() @@ -334,7 +337,8 @@ impl ProductTypeLayout { } } -impl MemoryUsage for ProductTypeLayout { +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for ProductTypeLayout { fn heap_usage(&self) -> usize { let Self { layout, elements } = self; layout.heap_usage() + elements.heap_usage() @@ -363,7 +367,8 @@ pub struct ProductTypeElementLayout { pub name: Option>, } -impl MemoryUsage for ProductTypeElementLayout { +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for ProductTypeElementLayout { fn heap_usage(&self) -> usize { let Self { offset, ty, name } = self; offset.heap_usage() + ty.heap_usage() + name.heap_usage() @@ -382,7 +387,8 @@ pub struct SumTypeLayout { pub payload_offset: u16, } -impl MemoryUsage for SumTypeLayout { +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for SumTypeLayout { fn heap_usage(&self) -> usize { let Self { layout, @@ -412,7 +418,8 @@ pub struct SumTypeVariantLayout { pub name: Option>, } -impl MemoryUsage for SumTypeVariantLayout { +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for SumTypeVariantLayout { fn heap_usage(&self) -> usize { let Self { ty, name } = self; ty.heap_usage() + name.heap_usage() @@ -440,7 +447,8 @@ pub enum PrimitiveType { F64, } -impl MemoryUsage for PrimitiveType {} +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for PrimitiveType {} impl PrimitiveType { pub fn algebraic_type(&self) -> AlgebraicType { @@ -514,7 +522,8 @@ pub enum VarLenType { Array(Box), } -impl MemoryUsage for VarLenType { +#[cfg(feature = "memory-usage")] +impl spacetimedb_memory_usage::MemoryUsage for VarLenType { fn heap_usage(&self) -> usize { match self { VarLenType::String => 0, diff --git a/crates/sats/src/lib.rs b/crates/sats/src/lib.rs index c07244128ad..8f8243f96e6 100644 --- a/crates/sats/src/lib.rs +++ b/crates/sats/src/lib.rs @@ -11,7 +11,10 @@ pub mod de; pub mod hash; pub mod hex; pub mod layout; -pub mod memory_usage; +#[cfg(feature = "memory-usage")] +mod memory_usage_impls; +#[cfg(feature = "memory-usage")] +pub use spacetimedb_memory_usage as memory_usage; pub mod meta_type; pub mod primitives; pub mod product_type; diff --git a/crates/sats/src/memory_usage.rs b/crates/sats/src/memory_usage.rs deleted file mode 100644 index 32f6377004d..00000000000 --- a/crates/sats/src/memory_usage.rs +++ /dev/null @@ -1,250 +0,0 @@ -use crate::{ - algebraic_value::Packed, i256, u256, AlgebraicType, AlgebraicValue, ArrayType, ArrayValue, ProductType, - ProductTypeElement, ProductValue, SumType, SumTypeVariant, SumValue, F32, F64, -}; -use core::sync::atomic::AtomicUsize; -use std::hash::{BuildHasher, Hash}; -use std::mem; - -/// For inspecting how much memory a value is using. -/// -/// This trait specifically measures heap memory. If you want to measure stack memory too, add -/// `mem::size_of_val()` to it. (This only really matters for the outermost type in a hierarchy.) -pub trait MemoryUsage { - /// The **heap** memory usage of this type. The default implementation returns 0. - #[inline(always)] - fn heap_usage(&self) -> usize { - 0 - } -} - -impl MemoryUsage for () {} -impl MemoryUsage for bool {} -impl MemoryUsage for u8 {} -impl MemoryUsage for u16 {} -impl MemoryUsage for u32 {} -impl MemoryUsage for u64 {} -impl MemoryUsage for u128 {} -impl MemoryUsage for u256 {} -impl MemoryUsage for usize {} -impl MemoryUsage for AtomicUsize {} -impl MemoryUsage for i8 {} -impl MemoryUsage for i16 {} -impl MemoryUsage for i32 {} -impl MemoryUsage for i64 {} -impl MemoryUsage for i128 {} -impl MemoryUsage for i256 {} -impl MemoryUsage for isize {} -impl MemoryUsage for f32 {} -impl MemoryUsage for f64 {} - -impl MemoryUsage for F32 {} -impl MemoryUsage for F64 {} - -impl MemoryUsage for &T { - fn heap_usage(&self) -> usize { - (*self).heap_usage() - } -} - -impl MemoryUsage for Box { - fn heap_usage(&self) -> usize { - mem::size_of_val::(self) + T::heap_usage(self) - } -} - -impl MemoryUsage for std::sync::Arc { - fn heap_usage(&self) -> usize { - let refcounts = mem::size_of::() * 2; - refcounts + mem::size_of_val::(self) + T::heap_usage(self) - } -} - -impl MemoryUsage for std::rc::Rc { - fn heap_usage(&self) -> usize { - let refcounts = mem::size_of::() * 2; - refcounts + mem::size_of_val::(self) + T::heap_usage(self) - } -} - -impl MemoryUsage for [T] { - fn heap_usage(&self) -> usize { - self.iter().map(T::heap_usage).sum() - } -} - -impl MemoryUsage for [T; N] { - fn heap_usage(&self) -> usize { - self.iter().map(T::heap_usage).sum() - } -} - -impl MemoryUsage for str {} - -impl MemoryUsage for Option { - fn heap_usage(&self) -> usize { - self.as_ref().map_or(0, T::heap_usage) - } -} - -impl MemoryUsage for (A, B) { - fn heap_usage(&self) -> usize { - self.0.heap_usage() + self.1.heap_usage() - } -} - -impl MemoryUsage for String { - fn heap_usage(&self) -> usize { - self.capacity() - } -} - -impl MemoryUsage for Vec { - fn heap_usage(&self) -> usize { - self.capacity() * mem::size_of::() + self.iter().map(T::heap_usage).sum::() - } -} - -impl MemoryUsage - for spacetimedb_data_structures::map::HashMap -{ - fn heap_usage(&self) -> usize { - self.allocation_size() + self.iter().map(|(k, v)| k.heap_usage() + v.heap_usage()).sum::() - } -} - -impl MemoryUsage for std::collections::BTreeMap { - fn heap_usage(&self) -> usize { - // NB: this is best-effort, since we don't have a `capacity()` method on `BTreeMap`. - self.len() * mem::size_of::<(K, V)>() + self.iter().map(|(k, v)| k.heap_usage() + v.heap_usage()).sum::() - } -} - -impl MemoryUsage for smallvec::SmallVec -where - A::Item: MemoryUsage, -{ - fn heap_usage(&self) -> usize { - self.as_slice().heap_usage() - + if self.spilled() { - self.capacity() * mem::size_of::() - } else { - 0 - } - } -} - -impl MemoryUsage for spacetimedb_primitives::TableId {} -impl MemoryUsage for spacetimedb_primitives::SequenceId {} -impl MemoryUsage for spacetimedb_primitives::ConstraintId {} -impl MemoryUsage for spacetimedb_primitives::IndexId {} -impl MemoryUsage for spacetimedb_primitives::ColId {} -impl MemoryUsage for spacetimedb_primitives::ColList { - fn heap_usage(&self) -> usize { - self.heap_size() - } -} - -impl MemoryUsage for AlgebraicValue { - fn heap_usage(&self) -> usize { - match self { - AlgebraicValue::Sum(x) => x.heap_usage(), - AlgebraicValue::Product(x) => x.heap_usage(), - AlgebraicValue::Array(x) => x.heap_usage(), - AlgebraicValue::String(x) => x.heap_usage(), - _ => 0, - } - } -} -impl MemoryUsage for SumValue { - fn heap_usage(&self) -> usize { - self.value.heap_usage() - } -} -impl MemoryUsage for ProductValue { - fn heap_usage(&self) -> usize { - self.elements.heap_usage() - } -} -impl MemoryUsage for ArrayValue { - fn heap_usage(&self) -> usize { - match self { - ArrayValue::Sum(v) => v.heap_usage(), - ArrayValue::Product(v) => v.heap_usage(), - ArrayValue::Bool(v) => v.heap_usage(), - ArrayValue::I8(v) => v.heap_usage(), - ArrayValue::U8(v) => v.heap_usage(), - ArrayValue::I16(v) => v.heap_usage(), - ArrayValue::U16(v) => v.heap_usage(), - ArrayValue::I32(v) => v.heap_usage(), - ArrayValue::U32(v) => v.heap_usage(), - ArrayValue::I64(v) => v.heap_usage(), - ArrayValue::U64(v) => v.heap_usage(), - ArrayValue::I128(v) => v.heap_usage(), - ArrayValue::U128(v) => v.heap_usage(), - ArrayValue::I256(v) => v.heap_usage(), - ArrayValue::U256(v) => v.heap_usage(), - ArrayValue::F32(v) => v.heap_usage(), - ArrayValue::F64(v) => v.heap_usage(), - ArrayValue::String(v) => v.heap_usage(), - ArrayValue::Array(v) => v.heap_usage(), - } - } -} -impl MemoryUsage for AlgebraicType { - fn heap_usage(&self) -> usize { - match self { - AlgebraicType::Ref(_) => 0, - AlgebraicType::Sum(x) => x.heap_usage(), - AlgebraicType::Product(x) => x.heap_usage(), - AlgebraicType::Array(x) => x.heap_usage(), - AlgebraicType::String - | AlgebraicType::Bool - | AlgebraicType::I8 - | AlgebraicType::U8 - | AlgebraicType::I16 - | AlgebraicType::U16 - | AlgebraicType::I32 - | AlgebraicType::U32 - | AlgebraicType::I64 - | AlgebraicType::U64 - | AlgebraicType::I128 - | AlgebraicType::U128 - | AlgebraicType::I256 - | AlgebraicType::U256 - | AlgebraicType::F32 - | AlgebraicType::F64 => 0, - } - } -} -impl MemoryUsage for SumType { - fn heap_usage(&self) -> usize { - self.variants.heap_usage() - } -} -impl MemoryUsage for SumTypeVariant { - fn heap_usage(&self) -> usize { - self.name.heap_usage() + self.algebraic_type.heap_usage() - } -} -impl MemoryUsage for ProductType { - fn heap_usage(&self) -> usize { - self.elements.heap_usage() - } -} -impl MemoryUsage for ProductTypeElement { - fn heap_usage(&self) -> usize { - self.name.heap_usage() + self.algebraic_type.heap_usage() - } -} -impl MemoryUsage for ArrayType { - fn heap_usage(&self) -> usize { - self.elem_ty.heap_usage() - } -} - -impl MemoryUsage for Packed { - fn heap_usage(&self) -> usize { - { self.0 }.heap_usage() - } -} diff --git a/crates/sats/src/memory_usage_impls.rs b/crates/sats/src/memory_usage_impls.rs new file mode 100644 index 00000000000..1eca2e8ae59 --- /dev/null +++ b/crates/sats/src/memory_usage_impls.rs @@ -0,0 +1,109 @@ +use crate::{ + algebraic_value::Packed, AlgebraicType, AlgebraicValue, ArrayType, ArrayValue, ProductType, ProductTypeElement, + ProductValue, SumType, SumTypeVariant, SumValue, +}; +use spacetimedb_memory_usage::MemoryUsage; + +impl MemoryUsage for AlgebraicValue { + fn heap_usage(&self) -> usize { + match self { + AlgebraicValue::Sum(x) => x.heap_usage(), + AlgebraicValue::Product(x) => x.heap_usage(), + AlgebraicValue::Array(x) => x.heap_usage(), + AlgebraicValue::String(x) => x.heap_usage(), + _ => 0, + } + } +} +impl MemoryUsage for SumValue { + fn heap_usage(&self) -> usize { + self.value.heap_usage() + } +} +impl MemoryUsage for ProductValue { + fn heap_usage(&self) -> usize { + self.elements.heap_usage() + } +} +impl MemoryUsage for ArrayValue { + fn heap_usage(&self) -> usize { + match self { + ArrayValue::Sum(v) => v.heap_usage(), + ArrayValue::Product(v) => v.heap_usage(), + ArrayValue::Bool(v) => v.heap_usage(), + ArrayValue::I8(v) => v.heap_usage(), + ArrayValue::U8(v) => v.heap_usage(), + ArrayValue::I16(v) => v.heap_usage(), + ArrayValue::U16(v) => v.heap_usage(), + ArrayValue::I32(v) => v.heap_usage(), + ArrayValue::U32(v) => v.heap_usage(), + ArrayValue::I64(v) => v.heap_usage(), + ArrayValue::U64(v) => v.heap_usage(), + ArrayValue::I128(v) => v.heap_usage(), + ArrayValue::U128(v) => v.heap_usage(), + ArrayValue::I256(v) => v.heap_usage(), + ArrayValue::U256(v) => v.heap_usage(), + ArrayValue::F32(v) => v.heap_usage(), + ArrayValue::F64(v) => v.heap_usage(), + ArrayValue::String(v) => v.heap_usage(), + ArrayValue::Array(v) => v.heap_usage(), + } + } +} +impl MemoryUsage for AlgebraicType { + fn heap_usage(&self) -> usize { + match self { + AlgebraicType::Ref(_) => 0, + AlgebraicType::Sum(x) => x.heap_usage(), + AlgebraicType::Product(x) => x.heap_usage(), + AlgebraicType::Array(x) => x.heap_usage(), + AlgebraicType::String + | AlgebraicType::Bool + | AlgebraicType::I8 + | AlgebraicType::U8 + | AlgebraicType::I16 + | AlgebraicType::U16 + | AlgebraicType::I32 + | AlgebraicType::U32 + | AlgebraicType::I64 + | AlgebraicType::U64 + | AlgebraicType::I128 + | AlgebraicType::U128 + | AlgebraicType::I256 + | AlgebraicType::U256 + | AlgebraicType::F32 + | AlgebraicType::F64 => 0, + } + } +} +impl MemoryUsage for SumType { + fn heap_usage(&self) -> usize { + self.variants.heap_usage() + } +} +impl MemoryUsage for SumTypeVariant { + fn heap_usage(&self) -> usize { + self.name.heap_usage() + self.algebraic_type.heap_usage() + } +} +impl MemoryUsage for ProductType { + fn heap_usage(&self) -> usize { + self.elements.heap_usage() + } +} +impl MemoryUsage for ProductTypeElement { + fn heap_usage(&self) -> usize { + self.name.heap_usage() + self.algebraic_type.heap_usage() + } +} +impl MemoryUsage for ArrayType { + fn heap_usage(&self) -> usize { + self.elem_ty.heap_usage() + } +} + +impl MemoryUsage for Packed { + fn heap_usage(&self) -> usize { + { self.0 }.heap_usage() + } +} diff --git a/crates/schema/Cargo.toml b/crates/schema/Cargo.toml index 5dd3449c303..c02756c195b 100644 --- a/crates/schema/Cargo.toml +++ b/crates/schema/Cargo.toml @@ -17,6 +17,7 @@ spacetimedb-data-structures.workspace = true spacetimedb-sql-parser.workspace = true anyhow.workspace = true +derive_more.workspace = true indexmap.workspace = true itertools.workspace = true lazy_static.workspace = true diff --git a/crates/schema/src/def.rs b/crates/schema/src/def.rs index b416581acf5..3c3fad9da6b 100644 --- a/crates/schema/src/def.rs +++ b/crates/schema/src/def.rs @@ -42,6 +42,7 @@ use spacetimedb_sats::AlgebraicType; use spacetimedb_sats::{AlgebraicTypeRef, Typespace}; pub mod deserialize; +pub mod error; pub mod validate; /// A map from `Identifier`s to values of type `T`. diff --git a/crates/lib/src/db/error.rs b/crates/schema/src/def/error.rs similarity index 97% rename from crates/lib/src/db/error.rs rename to crates/schema/src/def/error.rs index 3313be3bceb..83bae995fc1 100644 --- a/crates/lib/src/db/error.rs +++ b/crates/schema/src/def/error.rs @@ -1,10 +1,10 @@ -use crate::db::raw_def::IndexType; use crate::relation::{FieldName, Header}; -use crate::{buffer, AlgebraicType, AlgebraicValue}; use derive_more::Display; +use spacetimedb_lib::db::raw_def::IndexType; use spacetimedb_primitives::{ColId, ColList, TableId}; use spacetimedb_sats::product_value::InvalidFieldError; use spacetimedb_sats::satn::Satn as _; +use spacetimedb_sats::{buffer, AlgebraicType, AlgebraicValue}; use std::fmt; use std::string::FromUtf8Error; use thiserror::Error; diff --git a/crates/schema/src/def/validate/v8.rs b/crates/schema/src/def/validate/v8.rs index 9e184cb6a17..89a853a995e 100644 --- a/crates/schema/src/def/validate/v8.rs +++ b/crates/schema/src/def/validate/v8.rs @@ -1,6 +1,9 @@ //! Backwards-compatibility for the previous version of the schema definition format. //! This will be removed before 1.0. +use crate::def::{validate::Result, ModuleDef}; +use crate::error::{RawColumnName, ValidationError, ValidationErrors}; +use spacetimedb_data_structures::map::HashSet; use spacetimedb_lib::db::raw_def::{v8::*, v9::*}; use spacetimedb_lib::{ // TODO: rename these types globally in a followup PR @@ -11,12 +14,9 @@ use spacetimedb_lib::{ TableDesc as RawTableDescV8, TypeAlias as RawTypeAliasV8, }; -use spacetimedb_primitives::{ColId, ColList}; +use spacetimedb_primitives::{ColId, ColList, ConstraintKind, Constraints}; use spacetimedb_sats::{AlgebraicTypeRef, Typespace, WithTypespace}; -use crate::def::{validate::Result, ModuleDef}; -use crate::error::{RawColumnName, ValidationError, ValidationErrors}; - const INIT_NAME: &str = "__init__"; const IDENTITY_CONNECTED_NAME: &str = "__identity_connected__"; const IDENTITY_DISCONNECTED_NAME: &str = "__identity_disconnected__"; @@ -62,6 +62,63 @@ fn upgrade_module(def: RawModuleDefV8, extra_errors: &mut Vec) } } +/// Get an iterator deriving [RawIndexDefV8]s from the constraints that require them like `UNIQUE`. +/// +/// It looks into [Self::constraints] for possible duplicates and remove them from the result +pub fn generated_indexes(table: &RawTableDefV8) -> impl Iterator + '_ { + table + .constraints + .iter() + // We are only interested in constraints implying an index. + .filter(|x| x.constraints.has_indexed()) + // Create the `IndexDef`. + .map(|x| { + let is_unique = x.constraints.has_unique(); + RawIndexDefV8::for_column(&table.table_name, &x.constraint_name, x.columns.clone(), is_unique) + }) + // Only keep those we don't yet have in the list of indices (checked by name). + .filter(|idx| table.indexes.iter().all(|x| x.index_name != idx.index_name)) +} + +/// Get an iterator deriving [RawSequenceDefV8] from the constraints that require them like `IDENTITY`. +/// +/// It looks into [Self::constraints] for possible duplicates and remove them from the result +pub fn generated_sequences(table: &RawTableDefV8) -> impl Iterator + '_ { + let cols: HashSet<_> = table.sequences.iter().map(|seq| ColList::new(seq.col_pos)).collect(); + + table + .constraints + .iter() + // We are only interested in constraints implying a sequence. + .filter(move |x| !cols.contains(&x.columns) && x.constraints.has_autoinc()) + // Create the `SequenceDef`. + .map(|x| RawSequenceDefV8::for_column(&table.table_name, &x.constraint_name, x.columns.head().unwrap())) + // Only keep those we don't yet have in the list of sequences (checked by name). + .filter(|seq| table.sequences.iter().all(|x| x.sequence_name != seq.sequence_name)) +} + +/// Get an iterator deriving [RawConstraintDefV8] from the indexes that require them like `UNIQUE`. +/// +/// It looks into Self::constraints for possible duplicates and remove them from the result +pub fn generated_constraints(table: &RawTableDefV8) -> impl Iterator + '_ { + // Collect the set of all col-lists with a constraint. + let cols: HashSet<_> = table + .constraints + .iter() + .filter(|x| x.constraints.kind() != ConstraintKind::UNSET) + .map(|x| &x.columns) + .collect(); + + // Those indices that are not present in the constraints above + // have constraints generated for them. + // When `idx.is_unique`, a unique constraint is generated rather than an indexed one. + table + .indexes + .iter() + .filter(move |idx: &&RawIndexDefV8| !cols.contains(&idx.columns)) + .map(|idx| table.gen_constraint_def(Constraints::from_is_unique(idx.is_unique), idx.columns.clone())) +} + /// Upgrade a table, returning a v9 table definition and a stream of v8-only validation errors. fn upgrade_table( table: RawTableDescV8, @@ -70,9 +127,9 @@ fn upgrade_table( ) -> RawTableDefV9 { // First, generate all the various things that are needed. // This is the hairiest part of v8. - let generated_constraints = table.schema.generated_constraints().collect::>(); - let generated_sequences = table.schema.generated_sequences().collect::>(); - let generated_indexes = table.schema.generated_indexes().collect::>(); + let generated_constraints = generated_constraints(&table.schema).collect::>(); + let generated_sequences = generated_sequences(&table.schema).collect::>(); + let generated_indexes = generated_indexes(&table.schema).collect::>(); let RawTableDescV8 { schema: diff --git a/crates/schema/src/lib.rs b/crates/schema/src/lib.rs index 7e849b84ff1..923301fc5a7 100644 --- a/crates/schema/src/lib.rs +++ b/crates/schema/src/lib.rs @@ -6,5 +6,6 @@ pub mod auto_migrate; pub mod def; pub mod error; pub mod identifier; +pub mod relation; pub mod schema; pub mod type_for_generate; diff --git a/crates/lib/src/relation.rs b/crates/schema/src/relation.rs similarity index 99% rename from crates/lib/src/relation.rs rename to crates/schema/src/relation.rs index d11285bf8f0..539f48515d8 100644 --- a/crates/lib/src/relation.rs +++ b/crates/schema/src/relation.rs @@ -1,9 +1,9 @@ -use crate::db::auth::{StAccess, StTableType}; -use crate::db::error::{RelationError, TypeError}; +use crate::def::error::{RelationError, TypeError}; use core::fmt; use core::hash::Hash; use derive_more::From; use spacetimedb_data_structures::map::HashSet; +use spacetimedb_lib::db::auth::{StAccess, StTableType}; use spacetimedb_primitives::{ColId, ColList, ColSet, Constraints, TableId}; use spacetimedb_sats::algebraic_value::AlgebraicValue; use spacetimedb_sats::satn::Satn; diff --git a/crates/schema/src/schema.rs b/crates/schema/src/schema.rs index 541e38098b1..d511cbd3c04 100644 --- a/crates/schema/src/schema.rs +++ b/crates/schema/src/schema.rs @@ -6,17 +6,16 @@ // TODO(1.0): change all the `Box`s in this file to `Identifier`. // This doesn't affect the ABI so can wait until 1.0. +use crate::def::error::{DefType, SchemaError}; +use crate::relation::{combine_constraints, Column, DbTable, FieldName, Header}; use core::mem; use itertools::Itertools; use spacetimedb_lib::db::auth::{StAccess, StTableType}; -use spacetimedb_lib::db::error::{DefType, SchemaError}; use spacetimedb_lib::db::raw_def::v9::RawSql; use spacetimedb_lib::db::raw_def::{generate_cols_name, RawConstraintDefV8}; -use spacetimedb_lib::relation::{combine_constraints, Column, DbTable, FieldName, Header}; -use spacetimedb_lib::{AlgebraicType, ProductType, ProductTypeElement}; use spacetimedb_primitives::*; use spacetimedb_sats::product_value::InvalidFieldError; -use spacetimedb_sats::WithTypespace; +use spacetimedb_sats::{AlgebraicType, ProductType, ProductTypeElement, WithTypespace}; use std::collections::BTreeMap; use std::sync::Arc; diff --git a/crates/table/Cargo.toml b/crates/table/Cargo.toml index 5a4a7cf5341..9fdaedd2d24 100644 --- a/crates/table/Cargo.toml +++ b/crates/table/Cargo.toml @@ -29,10 +29,11 @@ proptest = ["dep:proptest", "dep:proptest-derive", "spacetimedb-sats/proptest"] blake3_pure = ["blake3/pure"] [dependencies] -spacetimedb-data-structures.workspace = true +spacetimedb-data-structures = { workspace = true, features = ["memory-usage"] } +spacetimedb-memory-usage = { workspace = true, features = ["hashbrown", "ethnum", "smallvec"] } spacetimedb-primitives.workspace = true spacetimedb-sats = { workspace = true, features = ["blake3"] } -spacetimedb-lib.workspace = true +spacetimedb-lib = { workspace = true, features = ["memory-usage"] } spacetimedb-schema.workspace = true ahash.workspace = true diff --git a/crates/table/src/blob_store.rs b/crates/table/src/blob_store.rs index f87a8fc614b..42e24366366 100644 --- a/crates/table/src/blob_store.rs +++ b/crates/table/src/blob_store.rs @@ -14,7 +14,7 @@ use blake3::hash; use spacetimedb_data_structures::map::{Entry, HashMap}; use spacetimedb_lib::{de::Deserialize, ser::Serialize}; -use spacetimedb_sats::memory_usage::MemoryUsage; +use spacetimedb_memory_usage::MemoryUsage; /// The content address of a blob-stored object. #[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy, Hash, Debug, Serialize, Deserialize)] diff --git a/crates/vm/src/errors.rs b/crates/vm/src/errors.rs index 963a116021e..d19c0946ff2 100644 --- a/crates/vm/src/errors.rs +++ b/crates/vm/src/errors.rs @@ -1,6 +1,6 @@ -use spacetimedb_lib::db::error::{AuthError, RelationError}; use spacetimedb_lib::operator::OpLogic; use spacetimedb_sats::{AlgebraicType, AlgebraicValue}; +use spacetimedb_schema::def::error::{AuthError, RelationError}; use std::fmt; use thiserror::Error; diff --git a/crates/vm/src/eval.rs b/crates/vm/src/eval.rs index 1dc845c8f31..f24bd662db3 100644 --- a/crates/vm/src/eval.rs +++ b/crates/vm/src/eval.rs @@ -96,9 +96,9 @@ pub mod test_helpers { use crate::relation::MemTable; use core::hash::BuildHasher as _; use spacetimedb_data_structures::map::DefaultHashBuilder; - use spacetimedb_lib::relation::{Column, FieldName, Header}; use spacetimedb_primitives::TableId; use spacetimedb_sats::{product, AlgebraicType, AlgebraicValue, ProductType, ProductValue}; + use spacetimedb_schema::relation::{Column, FieldName, Header}; use std::sync::Arc; pub fn mem_table_without_table_name(mem: &MemTable) -> (&[Column], &[ProductValue]) { @@ -185,11 +185,11 @@ pub mod tests { use crate::expr::{CrudExpr, Query, QueryExpr, SourceExpr, SourceSet}; use crate::iterators::RelIter; use crate::relation::MemTable; - use spacetimedb_lib::db::error::RelationError; use spacetimedb_lib::operator::{OpCmp, OpLogic}; - use spacetimedb_lib::relation::{FieldName, Header}; use spacetimedb_primitives::ColId; use spacetimedb_sats::{product, AlgebraicType, ProductType}; + use spacetimedb_schema::def::error::RelationError; + use spacetimedb_schema::relation::{FieldName, Header}; /// From an original source of `result`s, applies `queries` and returns a final set of results. fn build_query<'a, const N: usize>( diff --git a/crates/vm/src/expr.rs b/crates/vm/src/expr.rs index 499c9bb936c..b360414c11f 100644 --- a/crates/vm/src/expr.rs +++ b/crates/vm/src/expr.rs @@ -8,13 +8,12 @@ use itertools::Itertools; use smallvec::SmallVec; use spacetimedb_data_structures::map::{HashSet, IntMap}; use spacetimedb_lib::db::auth::{StAccess, StTableType}; -use spacetimedb_lib::db::error::{AuthError, RelationError}; -use spacetimedb_lib::relation::{ColExpr, DbTable, FieldName, Header}; -use spacetimedb_lib::{AlgebraicType, Identity}; +use spacetimedb_lib::Identity; use spacetimedb_primitives::*; -use spacetimedb_sats::algebraic_value::AlgebraicValue; use spacetimedb_sats::satn::Satn; -use spacetimedb_sats::ProductValue; +use spacetimedb_sats::{AlgebraicType, AlgebraicValue, ProductValue}; +use spacetimedb_schema::def::error::{AuthError, RelationError}; +use spacetimedb_schema::relation::{ColExpr, DbTable, FieldName, Header}; use spacetimedb_schema::schema::TableSchema; use std::borrow::Cow; use std::cmp::Reverse; @@ -2118,9 +2117,9 @@ impl From for CodeResult { mod tests { use super::*; - use spacetimedb_lib::{db::raw_def::v9::RawModuleDefV9Builder, relation::Column}; + use spacetimedb_lib::db::raw_def::v9::RawModuleDefV9Builder; use spacetimedb_sats::{product, AlgebraicType, ProductType}; - use spacetimedb_schema::{def::ModuleDef, schema::Schema}; + use spacetimedb_schema::{def::ModuleDef, relation::Column, schema::Schema}; use typed_arena::Arena; const ALICE: Identity = Identity::from_byte_array([1; 32]); diff --git a/crates/vm/src/rel_ops.rs b/crates/vm/src/rel_ops.rs index b938b3a9629..47250b71d60 100644 --- a/crates/vm/src/rel_ops.rs +++ b/crates/vm/src/rel_ops.rs @@ -2,8 +2,8 @@ use core::iter; use crate::relation::RelValue; use spacetimedb_data_structures::map::HashMap; -use spacetimedb_lib::relation::ColExpr; use spacetimedb_sats::AlgebraicValue; +use spacetimedb_schema::relation::ColExpr; /// A trait for dealing with fallible iterators for the database. pub trait RelOps<'a> { diff --git a/crates/vm/src/relation.rs b/crates/vm/src/relation.rs index e10e84638ac..1c96413e986 100644 --- a/crates/vm/src/relation.rs +++ b/crates/vm/src/relation.rs @@ -1,10 +1,10 @@ use core::hash::{Hash, Hasher}; use spacetimedb_execution::Row; use spacetimedb_lib::db::auth::StAccess; -use spacetimedb_lib::relation::{ColExpr, ColExprRef, Header}; use spacetimedb_sats::bsatn::{ser::BsatnError, ToBsatn}; use spacetimedb_sats::product_value::ProductValue; use spacetimedb_sats::{impl_serialize, AlgebraicValue}; +use spacetimedb_schema::relation::{ColExpr, ColExprRef, Header}; use spacetimedb_table::read_column::ReadColumn; use spacetimedb_table::table::RowRef; use std::borrow::Cow;