diff --git a/CHANGELOG.md b/CHANGELOG.md index 33bce2626..2f794af95 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,7 @@ - mls-memory-storage: add an in-memory implementation for MLS ([JeffG] at https://github.com/rust-nostr/nostr/pull/839) - mls-sqlite-storage: a sqlite implementation for MLS ([JeffG] at https://github.com/rust-nostr/nostr/pull/842) - mls: add new crate for implementing MLS messaging ([JeffG] at https://github.com/rust-nostr/nostr/pull/843) +- sqldb: add `nostr-sqldb` crate ([Thomas Profelt] at https://github.com/rust-nostr/nostr/pull/835) - pool: add relay monitor ([Yuki Kishimoto] at https://github.com/rust-nostr/nostr/pull/851) - sdk: add `Options::pool` ([Yuki Kishimoto]) - relay-builder: add support for NIP-70 protected events ([Yuki Kishimoto] at https://github.com/rust-nostr/nostr/pull/875) diff --git a/Cargo.lock b/Cargo.lock index 95d418cd0..00c1f4f32 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,7 +74,7 @@ dependencies = [ "cfg-if", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.35", ] [[package]] @@ -86,6 +86,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "amplify" version = "4.6.1" @@ -220,7 +226,7 @@ dependencies = [ "libc", "once_cell", "postage", - "rand", + "rand 0.8.5", "safelog", "serde", "thiserror 2.0.8", @@ -591,6 +597,15 @@ dependencies = [ "serde", ] +[[package]] +name = "btoi" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd6407f73a9b8b6162d8a2ef999fe6afd7cc15902ebf42c5cd296addf17e0ad" +dependencies = [ + "num-traits", +] + [[package]] name = "bumpalo" version = "3.14.0" @@ -642,6 +657,8 @@ version = "1.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c" dependencies = [ + "jobserver", + "libc", "shlex", ] @@ -780,6 +797,15 @@ dependencies = [ "error-code", ] +[[package]] +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + [[package]] name = "coarsetime" version = "0.1.34" @@ -906,6 +932,28 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", + "crossbeam-queue", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -947,7 +995,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "subtle", "zeroize", ] @@ -959,7 +1007,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core", + "rand_core 0.6.4", "typenum", ] @@ -1096,6 +1144,26 @@ version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2" +[[package]] +name = "deadpool" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ed5957ff93768adf7a65ab167a17835c3d2c3c50d084fe305174c112f468e2f" +dependencies = [ + "deadpool-runtime", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +dependencies = [ + "tokio", +] + [[package]] name = "delegate-display" version = "2.1.1" @@ -1287,6 +1355,76 @@ dependencies = [ "zeroize", ] +[[package]] +name = "diesel" +version = "2.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34d3950690ba3a6910126162b47e775e203006d4242a15de912bec6c0a695153" +dependencies = [ + "bitflags 2.9.0", + "byteorder", + "diesel_derives", + "itoa", + "libsqlite3-sys", + "mysqlclient-sys", + "percent-encoding", + "pq-sys", + "serde_json", + "time", + "url", +] + +[[package]] +name = "diesel-async" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51a307ac00f7c23f526a04a77761a0519b9f0eb2838ebf5b905a58580095bdcb" +dependencies = [ + "async-trait", + "deadpool", + "diesel", + "futures-channel", + "futures-util", + "mysql_async", + "mysql_common", + "scoped-futures", + "tokio", + "tokio-postgres", +] + +[[package]] +name = "diesel_derives" +version = "2.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a93958254b70bea63b4187ff73d10180599d9d8d177071b7f91e6da4e0c0ad55" +dependencies = [ + "diesel_table_macro_syntax", + "dsl_auto_type", + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "diesel_migrations" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a73ce704bad4231f001bff3314d91dce4aba0770cee8b233991859abc15c1f6" +dependencies = [ + "diesel", + "migrations_internals", + "migrations_macros", +] + +[[package]] +name = "diesel_table_macro_syntax" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "209c735641a413bc68c4923a9d6ad4bcb3ca306b794edaa7eb0b3228a99ffb25" +dependencies = [ + "syn 2.0.90", +] + [[package]] name = "digest" version = "0.10.7" @@ -1397,6 +1535,20 @@ dependencies = [ "phf", ] +[[package]] +name = "dsl_auto_type" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139ae9aca7527f85f26dd76483eb38533fd84bd571065da1739656ef71c5ff5b" +dependencies = [ + "darling 0.20.10", + "either", + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "dyn-clone" version = "1.0.17" @@ -1436,7 +1588,7 @@ dependencies = [ "curve25519-dalek", "ed25519", "merlin", - "rand_core", + "rand_core 0.6.4", "serde", "sha2", "subtle", @@ -1476,7 +1628,7 @@ dependencies = [ "hkdf", "pem-rfc7468", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sec1", "subtle", "zeroize", @@ -1553,6 +1705,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fallible-iterator" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" + [[package]] name = "fallible-iterator" version = "0.3.0" @@ -1600,7 +1758,7 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ded41244b729663b1e574f1b4fb731469f69f79c17667b5d776b16cda0479449" dependencies = [ - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1667,6 +1825,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -1912,7 +2076,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ "ff", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -1943,6 +2107,17 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + [[package]] name = "hashlink" version = "0.9.1" @@ -2077,7 +2252,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c3f1ae0a26c18d6469a70db1217136056261c4a244b09a755bc60bd4e055b67" dependencies = [ - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -2092,8 +2267,8 @@ dependencies = [ "hpke-rs-crypto", "p256", "p384", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", "sha2", "x25519-dalek", ] @@ -2427,6 +2602,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.77" @@ -2456,6 +2640,15 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "keyed_priority_queue" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee7893dab2e44ae5f9d0173f26ff4aa327c10b01b06a72b52dd9405b628640d" +dependencies = [ + "indexmap 2.5.0", +] + [[package]] name = "keyring" version = "3.6.2" @@ -2598,6 +2791,15 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + [[package]] name = "lru" version = "0.14.0" @@ -2660,6 +2862,16 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "md-5" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" +dependencies = [ + "cfg-if", + "digest", +] + [[package]] name = "memchr" version = "2.7.4" @@ -2683,10 +2895,31 @@ checksum = "58c38e2799fc0978b65dfff8023ec7843e2330bb462f19198840b34b6582397d" dependencies = [ "byteorder", "keccak", - "rand_core", + "rand_core 0.6.4", "zeroize", ] +[[package]] +name = "migrations_internals" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd01039851e82f8799046eabbb354056283fb265c8ec0996af940f4e85a380ff" +dependencies = [ + "serde", + "toml", +] + +[[package]] +name = "migrations_macros" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb161cc72176cb37aa47f1fc520d3ef02263d67d661f44f05d05a079e1237fd" +dependencies = [ + "migrations_internals", + "proc-macro2", + "quote", +] + [[package]] name = "mime" version = "0.3.17" @@ -2731,6 +2964,78 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "mysql_async" +version = "0.34.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0b66e411c31265e879d9814d03721f2daa7ad07337b6308cb4bb0cde7e6fd47" +dependencies = [ + "bytes", + "crossbeam", + "flate2", + "futures-core", + "futures-sink", + "futures-util", + "keyed_priority_queue", + "lru 0.12.5", + "mysql_common", + "pem", + "percent-encoding", + "pin-project", + "rand 0.8.5", + "serde", + "serde_json", + "socket2", + "thiserror 1.0.64", + "tokio", + "tokio-util", + "twox-hash", + "url", +] + +[[package]] +name = "mysql_common" +version = "0.32.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "478b0ff3f7d67b79da2b96f56f334431aef65e15ba4b29dd74a4236e29582bdc" +dependencies = [ + "base64 0.21.7", + "bindgen", + "bitflags 2.9.0", + "btoi", + "byteorder", + "bytes", + "cc", + "cmake", + "crc32fast", + "flate2", + "lazy_static", + "num-bigint", + "num-traits", + "rand 0.8.5", + "regex", + "saturating", + "serde", + "serde_json", + "sha1", + "sha2", + "smallvec", + "subprocess", + "thiserror 1.0.64", + "uuid", + "zstd", +] + +[[package]] +name = "mysqlclient-sys" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f29e21174d84e2622ceb7b0146a9187d36458a3a9ee9a66c9cac22e96493ef9" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "ndk-context" version = "0.1.1" @@ -2844,7 +3149,7 @@ name = "nostr-database" version = "0.42.0" dependencies = [ "flatbuffers", - "lru", + "lru 0.14.0", "nostr", "tokio", ] @@ -2905,7 +3210,7 @@ dependencies = [ name = "nostr-mls-memory-storage" version = "0.42.0" dependencies = [ - "lru", + "lru 0.14.0", "nostr", "nostr-mls-storage", "openmls", @@ -2987,7 +3292,7 @@ dependencies = [ "async-utility", "async-wsocket", "atomic-destructor", - "lru", + "lru 0.14.0", "negentropy", "nostr", "nostr-database", @@ -3014,6 +3319,22 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "nostr-sqldb" +version = "0.41.0" +dependencies = [ + "deadpool", + "diesel", + "diesel-async", + "diesel_migrations", + "nostr", + "nostr-database", + "nostr-relay-builder", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "nostrdb" version = "0.6.1" @@ -3086,7 +3407,7 @@ dependencies = [ "num-integer", "num-iter", "num-traits", - "rand", + "rand 0.8.5", "smallvec", "zeroize", ] @@ -3127,6 +3448,16 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", +] + [[package]] name = "num_enum" version = "0.7.3" @@ -3242,7 +3573,7 @@ dependencies = [ "ed25519-dalek", "openmls_traits", "p256", - "rand", + "rand 0.8.5", "serde", "tls_codec", ] @@ -3275,8 +3606,8 @@ dependencies = [ "openmls_memory_storage", "openmls_traits", "p256", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "serde", "sha2", "thiserror 2.0.8", @@ -3380,7 +3711,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "rand_core", + "rand_core 0.6.4", "sha2", ] @@ -3430,7 +3761,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" dependencies = [ "base64ct", - "rand_core", + "rand_core 0.6.4", "subtle", ] @@ -3450,6 +3781,16 @@ dependencies = [ "hmac", ] +[[package]] +name = "pem" +version = "3.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38af38e8470ac9dee3ce1bae1af9c1671fffc44ddfd8bd1d0a3445bf349a8ef3" +dependencies = [ + "base64 0.22.1", + "serde", +] + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -3482,7 +3823,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ "phf_shared", - "rand", + "rand 0.8.5", ] [[package]] @@ -3610,6 +3951,35 @@ dependencies = [ "thiserror 1.0.64", ] +[[package]] +name = "postgres-protocol" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ff0abab4a9b844b93ef7b81f1efc0a366062aaef2cd702c76256b5dc075c54" +dependencies = [ + "base64 0.22.1", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "hmac", + "md-5", + "memchr", + "rand 0.9.0", + "sha2", + "stringprep", +] + +[[package]] +name = "postgres-types" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613283563cd90e1dfc3518d548caee47e0e725455ed619881f5cf21f36de4b48" +dependencies = [ + "bytes", + "fallible-iterator 0.2.0", + "postgres-protocol", +] + [[package]] name = "powerfmt" version = "0.2.0" @@ -3622,7 +3992,17 @@ version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" dependencies = [ - "zerocopy", + "zerocopy 0.7.35", +] + +[[package]] +name = "pq-sys" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41c852911b98f5981956037b2ca976660612e548986c30af075e753107bc3400" +dependencies = [ + "libc", + "vcpkg", ] [[package]] @@ -3724,7 +4104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fadfaed2cd7f389d0161bb73eeb07b7b78f8691047a6f3e73caaeae55310a4a6" dependencies = [ "bytes", - "rand", + "rand 0.8.5", "ring 0.17.14", "rustc-hash 2.0.0", "rustls", @@ -3769,8 +4149,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3779b94aeb87e8bd4e834cee3650289ee9e0d5677f976ecdb6d219e5f4f6cd94" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", + "zerocopy 0.8.24", ] [[package]] @@ -3780,7 +4171,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -3792,6 +4193,15 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.1", +] + [[package]] name = "rangemap" version = "1.5.1" @@ -4039,7 +4449,7 @@ dependencies = [ "num-traits", "pkcs1", "pkcs8", - "rand_core", + "rand_core 0.6.4", "sha2", "signature", "spki", @@ -4054,7 +4464,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7753b721174eb8ff87a9a0e799e2d7bc3749323e773db92e0984debb00019d6e" dependencies = [ "bitflags 2.9.0", - "fallible-iterator", + "fallible-iterator 0.3.0", "fallible-streaming-iterator", "hashlink", "libsqlite3-sys", @@ -4238,6 +4648,21 @@ dependencies = [ "regex", ] +[[package]] +name = "saturating" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece8e78b2f38ec51c51f5d475df0a7187ba5111b2a28bdc761ee05b075d40a71" + +[[package]] +name = "scoped-futures" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b24aae2d0636530f359e9d5ef0c04669d11c5e756699b27a6a6d845d8329091" +dependencies = [ + "pin-project-lite", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -4276,7 +4701,7 @@ version = "0.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9465315bc9d4566e1724f0fffcbcc446268cb522e60f9a27bcded6b19c108113" dependencies = [ - "rand", + "rand 0.8.5", "secp256k1-sys", "serde", ] @@ -4523,7 +4948,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest", - "rand_core", + "rand_core 0.6.4", ] [[package]] @@ -4638,7 +5063,7 @@ dependencies = [ "p256", "p384", "p521", - "rand_core", + "rand_core 0.6.4", "rsa", "sec1", "sha2", @@ -4655,6 +5080,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stringprep" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4df3d392d81bd458a8a621b8bffbd2302a12ffe288a9d931670948749463b1" +dependencies = [ + "unicode-bidi", + "unicode-normalization", + "unicode-properties", +] + [[package]] name = "strsim" version = "0.10.0" @@ -4689,6 +5125,16 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.6.1" @@ -4941,6 +5387,32 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "tokio-postgres" +version = "0.7.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c95d533c83082bb6490e0189acaa0bbeef9084e60471b696ca6988cd0541fb0" +dependencies = [ + "async-trait", + "byteorder", + "bytes", + "fallible-iterator 0.2.0", + "futures-channel", + "futures-util", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite", + "postgres-protocol", + "postgres-types", + "rand 0.9.0", + "socket2", + "tokio", + "tokio-util", + "whoami", +] + [[package]] name = "tokio-rustls" version = "0.26.0" @@ -5055,8 +5527,8 @@ dependencies = [ "itertools 0.14.0", "libc", "paste", - "rand", - "rand_chacha", + "rand 0.8.5", + "rand_chacha 0.3.1", "serde", "slab", "smallvec", @@ -5095,7 +5567,7 @@ dependencies = [ "derive_more 2.0.1", "educe", "paste", - "rand", + "rand 0.8.5", "smallvec", "thiserror 2.0.8", "tor-basic-utils", @@ -5140,7 +5612,7 @@ dependencies = [ "futures", "oneshot-fused-workaround", "postage", - "rand", + "rand 0.8.5", "safelog", "serde", "thiserror 2.0.8", @@ -5194,7 +5666,7 @@ dependencies = [ "once_cell", "oneshot-fused-workaround", "pin-project", - "rand", + "rand 0.8.5", "retry-error", "safelog", "serde", @@ -5335,7 +5807,7 @@ dependencies = [ "oneshot-fused-workaround", "paste", "postage", - "rand", + "rand 0.8.5", "rusqlite", "safelog", "scopeguard", @@ -5412,7 +5884,7 @@ dependencies = [ "oneshot-fused-workaround", "pin-project", "postage", - "rand", + "rand 0.8.5", "safelog", "serde", "strum", @@ -5448,7 +5920,7 @@ dependencies = [ "itertools 0.14.0", "oneshot-fused-workaround", "postage", - "rand", + "rand 0.8.5", "retry-error", "safelog", "slotmap-careful", @@ -5489,7 +5961,7 @@ dependencies = [ "digest", "itertools 0.14.0", "paste", - "rand", + "rand 0.8.5", "safelog", "signature", "subtle", @@ -5557,8 +6029,8 @@ dependencies = [ "once_cell", "oneshot-fused-workaround", "postage", - "rand", - "rand_core", + "rand 0.8.5", + "rand_core 0.6.4", "retry-error", "safelog", "serde", @@ -5600,7 +6072,7 @@ dependencies = [ "derive_more 2.0.1", "downcast-rs", "paste", - "rand", + "rand 0.8.5", "signature", "ssh-key", "thiserror 2.0.8", @@ -5630,7 +6102,7 @@ dependencies = [ "humantime", "inventory", "itertools 0.14.0", - "rand", + "rand 0.8.5", "serde", "signature", "ssh-key", @@ -5694,7 +6166,7 @@ dependencies = [ "educe", "getrandom 0.2.15", "hex", - "rand_core", + "rand_core 0.6.4", "rsa", "safelog", "serde", @@ -5769,7 +6241,7 @@ dependencies = [ "humantime", "itertools 0.14.0", "num_enum", - "rand", + "rand 0.8.5", "serde", "static_assertions", "strum", @@ -5806,7 +6278,7 @@ dependencies = [ "itertools 0.14.0", "once_cell", "phf", - "rand", + "rand 0.8.5", "serde", "serde_with", "signature", @@ -5884,8 +6356,8 @@ dependencies = [ "hmac", "oneshot-fused-workaround", "pin-project", - "rand", - "rand_core", + "rand 0.8.5", + "rand_core 0.6.4", "safelog", "slotmap-careful", "static_assertions", @@ -5932,7 +6404,7 @@ version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e41754428684bd62892df2c74c2d11128cfbf3f1a8a9aaa1b920fcb90e04961a" dependencies = [ - "rand", + "rand 0.8.5", "serde", "tor-basic-utils", "tor-linkspec", @@ -6132,7 +6604,7 @@ dependencies = [ "http", "httparse", "log", - "rand", + "rand 0.8.5", "rustls", "rustls-pki-types", "sha1", @@ -6140,6 +6612,17 @@ dependencies = [ "utf-8", ] +[[package]] +name = "twox-hash" +version = "1.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +dependencies = [ + "cfg-if", + "rand 0.8.5", + "static_assertions", +] + [[package]] name = "typed-index-collections" version = "3.1.0" @@ -6191,6 +6674,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-properties" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70f2a8b45122e719eb623c01822704c4e0907e7e426a05927e1a1cfff5b75d0" + [[package]] name = "unicode-segmentation" version = "1.12.0" @@ -6368,6 +6857,12 @@ dependencies = [ "wit-bindgen-rt", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasix" version = "0.12.21" @@ -6511,6 +7006,17 @@ dependencies = [ "rustix 0.38.44", ] +[[package]] +name = "whoami" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6994d13118ab492c3c80c1f81928718159254c53c472bf9ce36f8dae4add02a7" +dependencies = [ + "redox_syscall", + "wasite", + "web-sys", +] + [[package]] name = "winapi" version = "0.3.9" @@ -6829,7 +7335,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", - "rand_core", + "rand_core 0.6.4", "serde", "zeroize", ] @@ -6857,7 +7363,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "byteorder", - "zerocopy-derive", + "zerocopy-derive 0.7.35", +] + +[[package]] +name = "zerocopy" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2586fea28e186957ef732a5f8b3be2da217d65c5969d4b1e17f973ebbe876879" +dependencies = [ + "zerocopy-derive 0.8.24", ] [[package]] @@ -6871,6 +7386,17 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "zerocopy-derive" +version = "0.8.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a996a8f63c5c4448cd959ac1bab0aaa3306ccfd060472f85943ee0750f0169be" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "zeroize" version = "1.8.1" @@ -6890,3 +7416,31 @@ dependencies = [ "quote", "syn 2.0.90", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/README.md b/README.md index 7632f1ff5..c27370f1d 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ The project is split up into several crates in the `crates/` directory: * [**nostr-database**](./crates/nostr-database): Database for Nostr apps * [**nostr-lmdb**](./crates/nostr-lmdb): LMDB storage backend * [**nostr-ndb**](./crates/nostr-ndb): [nostrdb](https://github.com/damus-io/nostrdb) storage backend + * [**nostr-sqldb**](./crates/nostr-sqldb): SQL storage backends (PostgreSQL, MySQL and SQLite) * [**nostr-indexeddb**](./crates/nostr-indexeddb): IndexedDB storage backend * [**nostr-mls**](./crates/nostr-mls): A library for implmenting NIP-EE MLS messaging * [**nostr-mls-storage**](./crates/nostr-mls-storage): Storage traits for using MLS messaging diff --git a/contrib/scripts/check-crates.sh b/contrib/scripts/check-crates.sh index ac913ae88..941f75147 100755 --- a/contrib/scripts/check-crates.sh +++ b/contrib/scripts/check-crates.sh @@ -43,6 +43,9 @@ buildargs=( "-p nostr-mls-memory-storage" "-p nostr-mls-sqlite-storage" "-p nostr-mls" + "-p nostr-sqldb --no-default-features --features postgres" # PostgreSQL + "-p nostr-sqldb --no-default-features --features mysql" # MySQL + "-p nostr-sqldb --no-default-features --features sqlite" # SQLite "-p nostr-indexeddb --target wasm32-unknown-unknown" "-p nostr-ndb" "-p nostr-keyring" @@ -59,16 +62,19 @@ buildargs=( ) skip_msrv=( - "-p nostr-lmdb" # MSRV: 1.72.0 - "-p nostr-mls-storage" # MSRV: 1.74.0 - "-p nostr-mls-memory-storage" # MSRV: 1.74.0 - "-p nostr-mls-sqlite-storage" # MSRV: 1.74.0 - "-p nostr-mls" # MSRV: 1.74.0 - "-p nostr-keyring" # MSRV: 1.75.0 - "-p nostr-keyring --features async" # MSRV: 1.75.0 - "-p nostr-sdk --features tor" # MSRV: 1.77.0 - "-p nostr-sdk --all-features" # MSRV: 1.77.0 (since uses lmdb and tor) - "-p nostr-cli" # MSRV: 1.74.0 + "-p nostr-lmdb" # MSRV: 1.72.0 + "-p nostr-mls-storage" # MSRV: 1.74.0 + "-p nostr-mls-memory-storage" # MSRV: 1.74.0 + "-p nostr-mls-sqlite-storage" # MSRV: 1.74.0 + "-p nostr-mls" # MSRV: 1.74.0 + "-p nostr-sqldb --no-default-features --features postgres" # MSRV: 1.82.0 + "-p nostr-sqldb --no-default-features --features mysql" # MSRV: 1.82.0 + "-p nostr-sqldb --no-default-features --features sqlite" # MSRV: 1.82.0 + "-p nostr-keyring" # MSRV: 1.75.0 + "-p nostr-keyring --features async" # MSRV: 1.75.0 + "-p nostr-sdk --features tor" # MSRV: 1.77.0 + "-p nostr-sdk --all-features" # MSRV: 1.77.0 (since uses lmdb and tor) + "-p nostr-cli" # MSRV: 1.74.0 ) for arg in "${buildargs[@]}"; diff --git a/crates/nostr-sqldb/Cargo.toml b/crates/nostr-sqldb/Cargo.toml new file mode 100644 index 000000000..a65464d09 --- /dev/null +++ b/crates/nostr-sqldb/Cargo.toml @@ -0,0 +1,45 @@ +[package] +name = "nostr-sqldb" +version = "0.41.0" +edition = "2021" +description = "SQL storage backend for Nostr apps" +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +readme = "README.md" +rust-version.workspace = true +keywords = ["nostr", "database", "postgres", "mysql", "sqlite"] + +[features] +default = ["postgres"] +postgres = [ + "diesel/postgres", + "diesel-async/postgres", + "diesel_migrations/postgres", +] +mysql = ["diesel/mysql", "diesel-async/mysql", "diesel_migrations/mysql"] +sqlite = [ + "diesel/sqlite", + "diesel-async/sqlite", + "diesel_migrations/sqlite", + "diesel/returning_clauses_for_sqlite_3_35", +] + +[dependencies] +deadpool = { version = "0.12", features = ["managed", "rt_tokio_1"] } +diesel = { version = "2.2", features = ["serde_json"] } +diesel-async = { version = "0.5", features = ["deadpool"] } +diesel_migrations = "2.2" +nostr = { workspace = true, features = ["std"] } +nostr-database = { workspace = true, features = ["flatbuf"] } +tracing.workspace = true + +[dev-dependencies] +nostr-relay-builder.workspace = true +tokio.workspace = true +tracing-subscriber.workspace = true + +[[example]] +name = "postgres-relay" +required-features = ["postgres"] diff --git a/crates/nostr-sqldb/README.md b/crates/nostr-sqldb/README.md new file mode 100644 index 000000000..b2641b6a1 --- /dev/null +++ b/crates/nostr-sqldb/README.md @@ -0,0 +1,25 @@ +# Nostr SQL database backend + +SQL storage backend for nostr apps working with Postgres, SQLite and MySQL. + +## Crate Feature Flags + +The following crate feature flags are available: + +| Feature | Default | Description | +|-------------|:-------:|-------------------------------| +| `postgres` | Yes | Enable support for PostgreSQL | +| `mysql` | No | Enable support for MySQL | +| `sqlite` | No | Enable support for SQLite | + +## State + +**This library is in an ALPHA state**, things that are implemented generally work but the API will change in breaking ways. + +## Donations + +`rust-nostr` is free and open-source. This means we do not earn any revenue by selling it. Instead, we rely on your financial support. If you actively use any of the `rust-nostr` libs/software/services, then please [donate](https://rust-nostr.org/donate). + +## License + +This project is distributed under the MIT software license - see the [LICENSE](../../LICENSE) file for details diff --git a/crates/nostr-sqldb/examples/postgres-relay.rs b/crates/nostr-sqldb/examples/postgres-relay.rs new file mode 100644 index 000000000..72fbfef91 --- /dev/null +++ b/crates/nostr-sqldb/examples/postgres-relay.rs @@ -0,0 +1,31 @@ +// Copyright (c) 2025 Protom +// Distributed under the MIT software license + +use std::time::Duration; + +use nostr_database::prelude::*; +use nostr_relay_builder::prelude::*; +use nostr_sqldb::NostrPostgres; + +// Your database URL +const DB_URL: &str = "postgres://postgres:password@localhost:5432"; + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt::init(); + + // Create a nostr db instance and run pending db migrations if any + let db = NostrPostgres::new(DB_URL).await?; + + // Add db to builder + let builder = RelayBuilder::default().database(db); + + // Create local relay + let relay = LocalRelay::run(builder).await?; + println!("Url: {}", relay.url()); + + // Keep up the program + loop { + tokio::time::sleep(Duration::from_secs(60)).await; + } +} diff --git a/crates/nostr-sqldb/migrations/.keep b/crates/nostr-sqldb/migrations/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/crates/nostr-sqldb/migrations/mysql/.keep b/crates/nostr-sqldb/migrations/mysql/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/down.sql b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/down.sql new file mode 100644 index 000000000..01cb13d3d --- /dev/null +++ b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/down.sql @@ -0,0 +1,3 @@ +-- This file should undo anything in `up.sql` +DROP TABLE IF EXISTS event_tags; +DROP TABLE IF EXISTS events; diff --git a/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql new file mode 100644 index 000000000..ae6d82518 --- /dev/null +++ b/crates/nostr-sqldb/migrations/mysql/2025-04-11-095120_events/up.sql @@ -0,0 +1,26 @@ +-- The actual event data +CREATE TABLE IF NOT EXISTS events ( + id BLOB(32) PRIMARY KEY NOT NULL, + pubkey BLOB(32) NOT NULL, + created_at BIGINT NOT NULL, + kind BIGINT NOT NULL, + payload BLOB NOT NULL, + deleted BOOLEAN NOT NULL +); + +-- Direct indexes +CREATE INDEX event_pubkey ON events (pubkey); +CREATE INDEX event_date ON events (created_at); +CREATE INDEX event_kind ON events (kind); +CREATE INDEX event_deleted ON events (deleted); + +-- The tag index, the primary will give us the index automatically +CREATE TABLE IF NOT EXISTS event_tags ( + tag VARCHAR(64) NOT NULL, + tag_value VARCHAR(512) NOT NULL, + event_id BLOB(32) NOT NULL + REFERENCES events (id) + ON DELETE CASCADE + ON UPDATE CASCADE, + PRIMARY KEY (tag, tag_value, event_id) +); diff --git a/crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/down.sql b/crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/down.sql new file mode 100644 index 000000000..a9f526091 --- /dev/null +++ b/crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/down.sql @@ -0,0 +1,6 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + +DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass); +DROP FUNCTION IF EXISTS diesel_set_updated_at(); diff --git a/crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/up.sql b/crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/up.sql new file mode 100644 index 000000000..d68895b1a --- /dev/null +++ b/crates/nostr-sqldb/migrations/postgres/00000000000000_diesel_initial_setup/up.sql @@ -0,0 +1,36 @@ +-- This file was automatically created by Diesel to setup helper functions +-- and other internal bookkeeping. This file is safe to edit, any future +-- changes will be added to existing projects as new migrations. + + + + +-- Sets up a trigger for the given table to automatically set a column called +-- `updated_at` whenever the row is modified (unless `updated_at` was included +-- in the modified columns) +-- +-- # Example +-- +-- ```sql +-- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW()); +-- +-- SELECT diesel_manage_updated_at('users'); +-- ``` +CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$ +BEGIN + EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s + FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl); +END; +$$ LANGUAGE plpgsql; + +CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$ +BEGIN + IF ( + NEW IS DISTINCT FROM OLD AND + NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at + ) THEN + NEW.updated_at := current_timestamp; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/down.sql b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/down.sql new file mode 100644 index 000000000..ab16f6bdf --- /dev/null +++ b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/down.sql @@ -0,0 +1,3 @@ +-- This file should undo anything in `up.sql` +DROP TABLE event_tags; +DROP TABLE events; diff --git a/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql new file mode 100644 index 000000000..74f300294 --- /dev/null +++ b/crates/nostr-sqldb/migrations/postgres/2025-04-11-095120_events/up.sql @@ -0,0 +1,26 @@ +-- The actual event data +CREATE TABLE events ( + id BYTEA PRIMARY KEY NOT NULL, + pubkey BYTEA NOT NULL, + created_at BIGINT NOT NULL, + kind BIGINT NOT NULL, + payload BYTEA NOT NULL, + deleted BOOLEAN NOT NULL +); + +-- Direct indexes +CREATE INDEX event_pubkey ON events (pubkey); +CREATE INDEX event_date ON events (created_at); +CREATE INDEX event_kind ON events (kind); +CREATE INDEX event_deleted ON events (deleted); + +-- The tag index, the primary will give us the index automatically +CREATE TABLE event_tags ( + tag TEXT NOT NULL, + tag_value TEXT NOT NULL, + event_id BYTEA NOT NULL + REFERENCES events (id) + ON DELETE CASCADE + ON UPDATE CASCADE, + PRIMARY KEY (tag, tag_value, event_id) +); diff --git a/crates/nostr-sqldb/migrations/sqlite/.keep b/crates/nostr-sqldb/migrations/sqlite/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/down.sql b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/down.sql new file mode 100644 index 000000000..ab16f6bdf --- /dev/null +++ b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/down.sql @@ -0,0 +1,3 @@ +-- This file should undo anything in `up.sql` +DROP TABLE event_tags; +DROP TABLE events; diff --git a/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql new file mode 100644 index 000000000..8d1f54314 --- /dev/null +++ b/crates/nostr-sqldb/migrations/sqlite/2025-04-11-095120_events/up.sql @@ -0,0 +1,26 @@ +-- The actual event data +CREATE TABLE events ( + id BLOB PRIMARY KEY NOT NULL, + pubkey BLOB NOT NULL, + created_at BIGINT NOT NULL, + kind BIGINT NOT NULL, + payload BLOB NOT NULL, + deleted BOOLEAN NOT NULL +); + +-- Direct indexes +CREATE INDEX event_pubkey ON events (pubkey); +CREATE INDEX event_date ON events (created_at); +CREATE INDEX event_kind ON events (kind); +CREATE INDEX event_deleted ON events (deleted); + +-- The tag index, the primary will give us the index automatically +CREATE TABLE event_tags ( + tag TEXT NOT NULL, + tag_value TEXT NOT NULL, + event_id BLOB NOT NULL + REFERENCES events (id) + ON DELETE CASCADE + ON UPDATE CASCADE, + PRIMARY KEY (tag, tag_value, event_id) +); diff --git a/crates/nostr-sqldb/mysql.toml b/crates/nostr-sqldb/mysql.toml new file mode 100644 index 000000000..75175050b --- /dev/null +++ b/crates/nostr-sqldb/mysql.toml @@ -0,0 +1,9 @@ +# For documentation on how to configure this file, +# see https://diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema/mysql.rs" +custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] + +[migrations_directory] +dir = "migrations/mysql" diff --git a/crates/nostr-sqldb/postgres.toml b/crates/nostr-sqldb/postgres.toml new file mode 100644 index 000000000..285cb79ad --- /dev/null +++ b/crates/nostr-sqldb/postgres.toml @@ -0,0 +1,9 @@ +# For documentation on how to configure this file, +# see https://diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema/postgres.rs" +custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] + +[migrations_directory] +dir = "migrations/postgres" diff --git a/crates/nostr-sqldb/sqlite.toml b/crates/nostr-sqldb/sqlite.toml new file mode 100644 index 000000000..e1c7f21c0 --- /dev/null +++ b/crates/nostr-sqldb/sqlite.toml @@ -0,0 +1,9 @@ +# For documentation on how to configure this file, +# see https://diesel.rs/guides/configuring-diesel-cli + +[print_schema] +file = "src/schema/sqlite.rs" +custom_type_derives = ["diesel::query_builder::QueryId", "Clone"] + +[migrations_directory] +dir = "migrations/sqlite" diff --git a/crates/nostr-sqldb/src/lib.rs b/crates/nostr-sqldb/src/lib.rs new file mode 100644 index 000000000..7c464c62e --- /dev/null +++ b/crates/nostr-sqldb/src/lib.rs @@ -0,0 +1,23 @@ +#![forbid(unsafe_code)] +#![warn(missing_docs)] +#![warn(rustdoc::bare_urls)] +#![cfg_attr(docsrs, feature(doc_auto_cfg))] +#![doc = include_str!("../README.md")] + +mod migrations; +#[allow(dead_code)] // TODO: to remove when also SQLite and MySQL are implemented +mod model; +#[cfg(feature = "postgres")] +mod postgres; +#[allow(dead_code)] // TODO: to remove when also SQLite and MySQL are implemented +mod query; +mod schema; + +#[cfg(feature = "mysql")] +pub use migrations::mysql::run_migrations; +#[cfg(feature = "postgres")] +pub use migrations::postgres::run_migrations; +#[cfg(feature = "sqlite")] +pub use migrations::sqlite::run_migrations; +#[cfg(feature = "postgres")] +pub use postgres::{postgres_connection_pool, NostrPostgres}; diff --git a/crates/nostr-sqldb/src/migrations/mod.rs b/crates/nostr-sqldb/src/migrations/mod.rs new file mode 100644 index 000000000..13c12d36d --- /dev/null +++ b/crates/nostr-sqldb/src/migrations/mod.rs @@ -0,0 +1,8 @@ +#[cfg(feature = "mysql")] +pub mod mysql; + +#[cfg(feature = "postgres")] +pub mod postgres; + +#[cfg(feature = "sqlite")] +pub mod sqlite; diff --git a/crates/nostr-sqldb/src/migrations/mysql.rs b/crates/nostr-sqldb/src/migrations/mysql.rs new file mode 100644 index 000000000..ae3f171c8 --- /dev/null +++ b/crates/nostr-sqldb/src/migrations/mysql.rs @@ -0,0 +1,19 @@ +use diesel::{Connection, MysqlConnection}; +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; +use nostr_database::DatabaseError; +use tracing::info; + +const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations/mysql"); + +/// programatically run the db migrations +pub fn run_migrations(connection_string: &str) -> Result<(), DatabaseError> { + info!("Running db migrations in mysql database",); + let mut connection = + MysqlConnection::establish(connection_string).map_err(DatabaseError::backend)?; + + let res = connection + .run_pending_migrations(MIGRATIONS) + .map_err(DatabaseError::Backend)?; + info!("Successfully executed mysql db migrations {:?}", res); + Ok(()) +} diff --git a/crates/nostr-sqldb/src/migrations/postgres.rs b/crates/nostr-sqldb/src/migrations/postgres.rs new file mode 100644 index 000000000..f54b32080 --- /dev/null +++ b/crates/nostr-sqldb/src/migrations/postgres.rs @@ -0,0 +1,19 @@ +use diesel::{Connection, PgConnection}; +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; +use nostr_database::DatabaseError; +use tracing::info; + +const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations/postgres"); + +/// programatically run the db migrations +pub fn run_migrations(connection_string: &str) -> Result<(), DatabaseError> { + info!("Running db migrations in postgres database",); + let mut connection = + PgConnection::establish(connection_string).map_err(DatabaseError::backend)?; + + let res = connection + .run_pending_migrations(MIGRATIONS) + .map_err(DatabaseError::Backend)?; + info!("Successfully executed postgres db migrations {:?}", res); + Ok(()) +} diff --git a/crates/nostr-sqldb/src/migrations/sqlite.rs b/crates/nostr-sqldb/src/migrations/sqlite.rs new file mode 100644 index 000000000..34a3e943d --- /dev/null +++ b/crates/nostr-sqldb/src/migrations/sqlite.rs @@ -0,0 +1,19 @@ +use diesel::{Connection, SqliteConnection}; +use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness}; +use nostr_database::DatabaseError; +use tracing::info; + +const MIGRATIONS: EmbeddedMigrations = embed_migrations!("./migrations/sqlite"); + +/// programatically run the db migrations +pub fn run_migrations(connection_string: &str) -> Result<(), DatabaseError> { + info!("Running db migrations in sqlite database",); + let mut connection = + SqliteConnection::establish(connection_string).map_err(DatabaseError::backend)?; + + let res = connection + .run_pending_migrations(MIGRATIONS) + .map_err(DatabaseError::Backend)?; + info!("Successfully executed sqlite db migrations {:?}", res); + Ok(()) +} diff --git a/crates/nostr-sqldb/src/model.rs b/crates/nostr-sqldb/src/model.rs new file mode 100644 index 000000000..c1fec936b --- /dev/null +++ b/crates/nostr-sqldb/src/model.rs @@ -0,0 +1,86 @@ +use std::sync::{Mutex, OnceLock}; + +use diesel::prelude::*; +use nostr::event::Event; +use nostr_database::{DatabaseError, FlatBufferBuilder, FlatBufferEncode}; + +#[cfg(feature = "mysql")] +use crate::schema::mysql::{event_tags, events}; +#[cfg(feature = "postgres")] +use crate::schema::postgres::{event_tags, events}; +#[cfg(feature = "sqlite")] +use crate::schema::sqlite::{event_tags, events}; + +/// DB representation of [`Event`] +#[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] +#[diesel(table_name = events)] +pub struct EventDb { + pub id: Vec, + pub pubkey: Vec, + pub created_at: i64, + pub kind: i64, + pub payload: Vec, + pub deleted: bool, +} + +/// DB representation of [`EventTag`] +#[derive(Queryable, Selectable, Insertable, AsChangeset, Debug, Clone)] +#[diesel(table_name = event_tags)] +pub struct EventTagDb { + pub tag: String, + pub tag_value: String, + pub event_id: Vec, +} + +/// A data container for extracting data from [`Event`] and its tags +#[derive(Debug, Clone)] +pub struct EventDataDb { + pub event: EventDb, + pub tags: Vec, +} + +impl TryFrom<&Event> for EventDataDb { + type Error = DatabaseError; + fn try_from(value: &Event) -> Result { + Ok(Self { + event: EventDb { + id: value.id.as_bytes().to_vec(), + pubkey: value.pubkey.as_bytes().to_vec(), + created_at: value.created_at.as_u64() as i64, + kind: value.kind.as_u16() as i64, + payload: encode_payload(value), + deleted: false, + }, + tags: extract_tags(value), + }) + } +} + +fn encode_payload(value: &Event) -> Vec { + static FB_BUILDER: OnceLock> = OnceLock::new(); + match FB_BUILDER + .get_or_init(|| Mutex::new(FlatBufferBuilder::new())) + .lock() + { + Ok(mut fb_builder) => value.encode(&mut fb_builder).to_vec(), + Err(_) => value.encode(&mut FlatBufferBuilder::new()).to_vec(), + } +} + +fn extract_tags(event: &Event) -> Vec { + event + .tags + .iter() + .filter_map(|tag| { + if let (kind, Some(content)) = (tag.kind(), tag.content()) { + Some(EventTagDb { + tag: kind.to_string(), + tag_value: content.to_string(), + event_id: event.id.as_bytes().to_vec(), + }) + } else { + None + } + }) + .collect() +} diff --git a/crates/nostr-sqldb/src/postgres.rs b/crates/nostr-sqldb/src/postgres.rs new file mode 100644 index 000000000..257ae49d3 --- /dev/null +++ b/crates/nostr-sqldb/src/postgres.rs @@ -0,0 +1,235 @@ +use deadpool::managed::{Object, Pool}; +use diesel::prelude::*; +use diesel::result::{DatabaseErrorKind, Error as DieselError}; +use diesel::QueryResult; +use diesel_async::pooled_connection::AsyncDieselConnectionManager; +use diesel_async::scoped_futures::ScopedFutureExt; +use diesel_async::{AsyncConnection, AsyncPgConnection, RunQueryDsl}; +use nostr::event::*; +use nostr::filter::Filter; +use nostr::types::Timestamp; +use nostr_database::*; +use prelude::BoxedFuture; + +use super::model::{EventDataDb, EventDb}; +use super::schema::postgres::{event_tags, events}; +use crate::query::{build_filter_query, event_by_id, with_limit}; + +/// Shorthand for a database connection pool type +pub type PostgresConnectionPool = Pool>; +pub type PostgresConnection = Object>; + +#[derive(Clone)] +pub struct NostrPostgres { + pool: PostgresConnectionPool, +} + +impl NostrPostgres { + /// Create a new [`NostrPostgres`] instance + pub async fn new(connection_string: C) -> Result + where + C: AsRef, + { + crate::migrations::postgres::run_migrations(connection_string.as_ref())?; + let pool = postgres_connection_pool(connection_string).await?; + Ok(Self { pool }) + } + + pub(crate) async fn get_connection(&self) -> Result { + self.pool.get().await.map_err(DatabaseError::backend) + } + + pub(crate) async fn save( + &self, + event_data: EventDataDb, + ) -> Result { + let mut db = self.get_connection().await?; + let result: QueryResult = db + .transaction(|c| { + async move { + diesel::insert_into(events::table) + .values(&event_data.event) + .execute(c) + .await?; + + diesel::insert_into(event_tags::table) + .values(&event_data.tags) + .execute(c) + .await?; + + Ok(true) + } + .scope_boxed() + }) + .await; + + match result { + Ok(_) => Ok(SaveEventStatus::Success), + Err(e) => match e { + DieselError::DatabaseError(DatabaseErrorKind::UniqueViolation, _) => { + Ok(SaveEventStatus::Rejected(RejectedReason::Duplicate)) + } + e => Err(DatabaseError::backend(e)), + }, + } + } + + pub(crate) async fn event_by_id( + &self, + event_id: &EventId, + ) -> Result, DatabaseError> { + let res = event_by_id(event_id) + .first(&mut self.get_connection().await?) + .await + .optional() + .map_err(DatabaseError::backend)?; + Ok(res) + } +} + +impl NostrEventsDatabase for NostrPostgres { + /// Save [`Event`] into store + /// + /// **This method assumes that [`Event`] was already verified** + fn save_event<'a>( + &'a self, + event: &'a Event, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { self.save(EventDataDb::try_from(event)?).await }) + } + + /// Check event status by ID + /// + /// Check if the event is saved, deleted or not existent. + fn check_id<'a>( + &'a self, + event_id: &'a EventId, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { + let status = match self.event_by_id(event_id).await? { + Some(e) if e.deleted => DatabaseEventStatus::Deleted, + Some(_) => DatabaseEventStatus::Saved, + None => DatabaseEventStatus::NotExistent, + }; + Ok(status) + }) + } + + /// Coordinate feature is not supported yet + fn has_coordinate_been_deleted<'a>( + &'a self, + _coordinate: &'a nostr::nips::nip01::CoordinateBorrow<'a>, + _timestamp: &'a Timestamp, + ) -> BoxedFuture<'a, Result> { + Box::pin(async move { Ok(false) }) + } + + /// Get [`Event`] by [`EventId`] + fn event_by_id<'a>( + &'a self, + event_id: &'a EventId, + ) -> BoxedFuture<'a, Result, DatabaseError>> { + Box::pin(async move { + let event = match self.event_by_id(event_id).await? { + Some(e) if !e.deleted => { + Some(Event::decode(&e.payload).map_err(DatabaseError::backend)?) + } + _ => None, + }; + Ok(event) + }) + } + + /// Count the number of events found with [`Filter`]. + /// + /// Use `Filter::new()` or `Filter::default()` to count all events. + fn count(&self, filter: Filter) -> BoxedFuture> { + Box::pin(async move { + let res: i64 = build_filter_query(filter) + .count() + .get_result(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + Ok(res as usize) + }) + } + + /// Query stored events. + fn query(&self, filter: Filter) -> BoxedFuture> { + let filter = with_limit(filter, 10000); + Box::pin(async move { + let mut events = Events::new(&filter); + let result = build_filter_query(filter.clone()) + .select(EventDb::as_select()) + .load(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + + for item in result.into_iter() { + if let Ok(event) = Event::decode(&item.payload) { + events.insert(event); + } + } + Ok(events) + }) + } + + /// Delete all events that match the [Filter] + fn delete(&self, filter: Filter) -> BoxedFuture> { + let filter = with_limit(filter, 999); + Box::pin(async move { + let filter = build_filter_query(filter); + diesel::update(events::table) + .set(events::deleted.eq(true)) + .filter(events::id.eq_any(filter.select(events::id))) + .execute(&mut self.get_connection().await?) + .await + .map_err(DatabaseError::backend)?; + + Ok(()) + }) + } +} + +impl NostrDatabase for NostrPostgres { + fn backend(&self) -> Backend { + Backend::Custom("Postgres".to_string()) + } +} + +/// Create a new [`NostrPostgres`] instance from an existing connection pool +impl From for NostrPostgres { + fn from(pool: PostgresConnectionPool) -> Self { + Self { pool } + } +} + +/// Create a connection pool for a Postgres database with the given connection string. +pub async fn postgres_connection_pool( + connection_string: C, +) -> Result +where + C: AsRef, +{ + let config = AsyncDieselConnectionManager::::new(connection_string.as_ref()); + let pool: PostgresConnectionPool = Pool::builder(config) + .build() + .map_err(|e| DatabaseError::Backend(Box::new(e)))?; + Ok(pool) +} + +impl std::fmt::Debug for NostrPostgres { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NostrPostgres") + .field("pool", &self.pool.status()) + .finish() + } +} + +/// For now we want to avoid wiping the database +impl NostrDatabaseWipe for NostrPostgres { + #[inline] + fn wipe(&self) -> BoxedFuture> { + Box::pin(async move { Err(DatabaseError::NotSupported) }) + } +} diff --git a/crates/nostr-sqldb/src/query.rs b/crates/nostr-sqldb/src/query.rs new file mode 100644 index 000000000..272536b3d --- /dev/null +++ b/crates/nostr-sqldb/src/query.rs @@ -0,0 +1,126 @@ +use diesel::dsl::{AsSelect, Eq, Filter as DieselFilter, InnerJoin, IntoBoxed, SqlTypeOf}; +use diesel::expression::SqlLiteral; +use diesel::prelude::*; +use diesel::sql_types::Binary; +use nostr::event::*; +use nostr::filter::Filter; +use nostr_database::*; + +use super::model::EventDb; +#[cfg(feature = "mysql")] +use super::schema::mysql::{event_tags, events}; +#[cfg(feature = "postgres")] +use super::schema::postgres::{event_tags, events}; +#[cfg(feature = "sqlite")] +use super::schema::sqlite::{event_tags, events}; + +// filter type of a join query. +type QuerySetJoinTypeDb<'a, DB> = IntoBoxed< + 'a, + DieselFilter< + InnerJoin, + Eq>, + >, + DB, +>; +type SelectEventTypeDb = SqlTypeOf>; +type BoxedEventQueryDb<'a, DB> = events::BoxedQuery<'a, DB, SelectEventTypeDb>; + +#[cfg(feature = "postgres")] +type QuerySetJoinType<'a> = QuerySetJoinTypeDb<'a, diesel::pg::Pg>; +#[cfg(feature = "postgres")] +type BoxedEventQuery<'a> = BoxedEventQueryDb<'a, diesel::pg::Pg>; +#[cfg(feature = "sqlite")] +type QuerySetJoinType<'a> = QuerySetJoinTypeDb<'a, diesel::sqlite::Sqlite>; +#[cfg(feature = "sqlite")] +type BoxedEventQuery<'a> = BoxedEventQueryDb<'a, diesel::sqlite::Sqlite>; +#[cfg(feature = "mysql")] +type QuerySetJoinType<'a> = QuerySetJoinTypeDb<'a, diesel::mysql::Mysql>; +#[cfg(feature = "mysql")] +type BoxedEventQuery<'a> = BoxedEventQueryDb<'a, diesel::mysql::Mysql>; + +pub fn build_filter_query<'a>(filter: Filter) -> QuerySetJoinType<'a> { + let mut query = events::table + .distinct() + .inner_join(event_tags::table) + .filter(events::deleted.eq(false)) + .order_by(events::created_at.desc()) + .into_boxed(); + + if let Some(limit) = filter.limit { + query = query.limit(limit as i64); + } + + if !has_filters(&filter) { + return query; + } + + if let Some(ids) = filter.ids.clone() { + let values = ids + .iter() + .map(|id| id.as_bytes().to_vec()) + .collect::>(); + query = query.filter(events::id.eq_any(values)); + } + + if let Some(authors) = filter.authors.clone() { + let values = authors + .iter() + .map(|a| a.as_bytes().to_vec()) + .collect::>(); + query = query.filter(events::pubkey.eq_any(values)); + } + + if let Some(kinds) = filter.kinds.clone() { + let values = kinds.iter().map(|k| k.as_u16() as i64).collect::>(); + query = query.filter(events::kind.eq_any(values)); + } + + if let Some(since) = filter.since { + query = query.filter(events::created_at.ge(since.as_u64() as i64)); + } + + if let Some(until) = filter.until { + query = query.filter(events::created_at.le(until.as_u64() as i64)); + } + + if !filter.generic_tags.is_empty() { + for (tag, values) in filter.generic_tags.into_iter() { + let values = values.iter().map(|v| v.to_string()).collect::>(); + query = query.filter( + event_tags::tag + .eq(tag.to_string()) + .and(event_tags::tag_value.eq_any(values)), + ); + } + } + + query +} + +/// sets the given default limit on a Nostr filter if not set +pub fn with_limit(filter: Filter, default_limit: usize) -> Filter { + if filter.limit.is_none() { + return filter.limit(default_limit); + } + filter +} + +pub fn event_by_id<'a>(event_id: &EventId) -> BoxedEventQuery<'a> { + let event_id = event_id.as_bytes().to_vec(); + events::table + .select(EventDb::as_select()) + .filter(events::id.eq(event_id)) + .into_boxed() +} + +// determine if the filter has any filters set +fn has_filters(filter: &Filter) -> bool { + filter.ids.is_some() + || filter.authors.is_some() + || filter.kinds.is_some() + || filter.since.is_some() + || filter.until.is_some() + || !filter.generic_tags.is_empty() + || filter.limit.is_some() +} diff --git a/crates/nostr-sqldb/src/schema/mod.rs b/crates/nostr-sqldb/src/schema/mod.rs new file mode 100644 index 000000000..13f8538b0 --- /dev/null +++ b/crates/nostr-sqldb/src/schema/mod.rs @@ -0,0 +1,8 @@ +#[cfg(feature = "postgres")] +pub mod postgres; + +#[cfg(feature = "mysql")] +pub mod mysql; + +#[cfg(feature = "sqlite")] +pub mod sqlite; diff --git a/crates/nostr-sqldb/src/schema/mysql.rs b/crates/nostr-sqldb/src/schema/mysql.rs new file mode 100644 index 000000000..e37f29370 --- /dev/null +++ b/crates/nostr-sqldb/src/schema/mysql.rs @@ -0,0 +1,29 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + event_tags (tag, tag_value, event_id) { + #[max_length = 64] + tag -> Varchar, + #[max_length = 512] + tag_value -> Varchar, + #[max_length = 32] + event_id -> Blob, + } +} + +diesel::table! { + events (id) { + #[max_length = 32] + id -> Blob, + #[max_length = 32] + pubkey -> Blob, + created_at -> Bigint, + kind -> Bigint, + payload -> Blob, + deleted -> Bool, + } +} + +diesel::joinable!(event_tags -> events (event_id)); + +diesel::allow_tables_to_appear_in_same_query!(event_tags, events); diff --git a/crates/nostr-sqldb/src/schema/postgres.rs b/crates/nostr-sqldb/src/schema/postgres.rs new file mode 100644 index 000000000..629ffcf19 --- /dev/null +++ b/crates/nostr-sqldb/src/schema/postgres.rs @@ -0,0 +1,24 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + event_tags (tag, tag_value, event_id) { + tag -> Text, + tag_value -> Text, + event_id -> Bytea, + } +} + +diesel::table! { + events (id) { + id -> Bytea, + pubkey -> Bytea, + created_at -> Int8, + kind -> Int8, + payload -> Bytea, + deleted -> Bool, + } +} + +diesel::joinable!(event_tags -> events (event_id)); + +diesel::allow_tables_to_appear_in_same_query!(event_tags, events); diff --git a/crates/nostr-sqldb/src/schema/sqlite.rs b/crates/nostr-sqldb/src/schema/sqlite.rs new file mode 100644 index 000000000..0a7973a3a --- /dev/null +++ b/crates/nostr-sqldb/src/schema/sqlite.rs @@ -0,0 +1,24 @@ +// @generated automatically by Diesel CLI. + +diesel::table! { + event_tags (tag, tag_value, event_id) { + tag -> Text, + tag_value -> Text, + event_id -> Binary, + } +} + +diesel::table! { + events (id) { + id -> Binary, + pubkey -> Binary, + created_at -> BigInt, + kind -> BigInt, + payload -> Binary, + deleted -> Bool, + } +} + +diesel::joinable!(event_tags -> events (event_id)); + +diesel::allow_tables_to_appear_in_same_query!(event_tags, events);