gossip: init

- Create `nostr-gossip` crate
- Move gossip stuff from `nostr-sdk` to `nostr-gossip`
- Add `gossip` feature to `nostr-sdk`
- Add persistent and in-memory SQLite storage for gossip data
- Add a middleware in the SDK to process events for gossip
- Allows passing additional relays when breaking-down filters

Closes #570

Signed-off-by: Yuki Kishimoto <yukikishimoto@protonmail.com>

Author: yukibtc

Cargo.lock

@@ -28,6 +28,7 @@ negentropy-deprecated = { package = "negentropy", version = "0.3", default-featu
 nostr = { version = "0.39", path = "./crates/nostr", default-features = false }
 nostr-connect = { version = "0.39", path = "./crates/nostr-connect", default-features = false }
 nostr-database = { version = "0.39", path = "./crates/nostr-database", default-features = false }
+nostr-gossip = { version = "0.39", path = "./crates/nostr-gossip", default-features = false }
 nostr-indexeddb = { version = "0.39", path = "./crates/nostr-indexeddb", default-features = false }
 nostr-lmdb = { version = "0.39", path = "./crates/nostr-lmdb", default-features = false }
 nostr-ndb = { version = "0.39", path = "./crates/nostr-ndb", default-features = false }

Cargo.toml

@@ -11,6 +11,7 @@ The project is split up into several crates in the `crates/` directory:
         * [**nostr-lmdb**](./crates/nostr-lmdb): LMDB storage backend
         * [**nostr-ndb**](./crates/nostr-ndb): [nostrdb](https://github.com/damus-io/nostrdb) storage backend
         * [**nostr-indexeddb**](./crates/nostr-indexeddb): IndexedDB storage backend
+    * [**nostr-gossip**](./crates/nostr-gossip): Gossip tracker and storage
     * [**nostr-relay-pool**](./crates/nostr-relay-pool): Nostr Relay Pool
     * [**nostr-sdk**](./crates/nostr-sdk): High level client library
     * [**nwc**](./crates/nwc): Nostr Wallet Connect (NWC) client

README.md

@@ -45,8 +45,10 @@ buildargs=(
     "-p nostr-relay-builder"
     "-p nostr-connect"
     "-p nwc"
+    "-p nostr-gossip"
     "-p nostr-sdk"                                                # No default features
     "-p nostr-sdk --features all-nips"                            # Only NIPs features
+    "-p nostr-sdk --features gossip"                              # Gossip stuff
     "-p nostr-sdk --features tor"                                 # Embedded tor client
     "-p nostr-sdk --all-features"                                 # All features
     "-p nostr-cli"

contrib/scripts/check-crates.sh

@@ -12,6 +12,7 @@ args=(
     "-p nostr-relay-pool"
     "-p nwc"
     "-p nostr-connect"
+    "-p nostr-gossip"
     "-p nostr-sdk"
     "-p nostr-cli"
 )

contrib/scripts/release.sh

@@ -0,0 +1,22 @@
+[package]
+name = "nostr-gossip"
+version = "0.39.0"
+edition = "2021"
+description = "Gossip storage for nostr apps"
+authors.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+readme = "README.md"
+rust-version.workspace = true
+keywords = ["nostr", "gossip"]
+
+[dependencies]
+async-utility.workspace = true
+lru.workspace = true
+nostr = { workspace = true, features = ["std"] }
+rusqlite = { version = "0.32", features = ["bundled"] }
+tracing = { workspace = true, features = ["std"] }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["macros", "rt"] }

crates/nostr-gossip/Cargo.toml

@@ -0,0 +1,13 @@
+# Nostr Gossip
+
+## State
+
+**This library is in an ALPHA state**, things that are implemented generally work but the API will change in breaking ways.
+
+## Donations
+
+`rust-nostr` is free and open-source. This means we do not earn any revenue by selling it. Instead, we rely on your financial support. If you actively use any of the `rust-nostr` libs/software/services, then please [donate](https://rust-nostr.org/donate).
+
+## License
+
+This project is distributed under the MIT software license - see the [LICENSE](../../LICENSE) file for details

crates/nostr-gossip/README.md

@@ -0,0 +1,51 @@
+-- Database settings
+PRAGMA journal_mode = WAL;
+PRAGMA synchronous = NORMAL;
+PRAGMA foreign_keys = ON;
+PRAGMA user_version = 1; -- Schema version
+
+CREATE TABLE IF NOT EXISTS users (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    public_key BLOB NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS relays (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    relay_url TEXT NOT NULL UNIQUE
+);
+
+CREATE TABLE IF NOT EXISTS lists (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    pkid INTEGER NOT NULL, -- Public Key ID
+    kind INTEGER NOT NULL, -- Kind of the list (i.e., 10002 for NIP65, 10050 for NIP17)
+    created_at INTEGER NOT NULL DEFAULT 0, -- Event list created at (`created_at` field of event)
+    last_check INTEGER NOT NULL DEFAULT 0, -- The timestamp of the last check
+    FOREIGN KEY(pkid) REFERENCES users(id) ON DELETE CASCADE ON UPDATE NO ACTION
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS pubkey_list_idx ON lists(pkid,kind);
+
+CREATE TABLE IF NOT EXISTS relays_by_list (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    pkid INTEGER NOT NULL, -- Public Key ID
+    listid INTEGER NOT NULL, -- List ID
+    relayid INTEGER NOT NULL, -- Relay ID
+    metadata TEXT DEFAULT NULL, -- NIP65 metadata: read, write or NULL
+    FOREIGN KEY(pkid) REFERENCES users(id) ON DELETE CASCADE ON UPDATE NO ACTION,
+    FOREIGN KEY(listid) REFERENCES lists(id) ON DELETE CASCADE ON UPDATE NO ACTION,
+    FOREIGN KEY(relayid) REFERENCES relays(id)
+);
+
+CREATE UNIQUE INDEX IF NOT EXISTS pubkey_list_relay_idx ON relays_by_list(pkid,listid,relayid);
+
+-- CREATE TABLE IF NOT EXISTS tracker (
+--     id INTEGER PRIMARY KEY AUTOINCREMENT,
+--     pkid INTEGER NOT NULL, -- Public Key ID
+--     relayid INTEGER NOT NULL, -- Relay ID
+--     last_event INTEGER NOT NULL DEFAULT 0, -- Timestamp of the last event seen for the public key on the relay
+--     score INTEGER NOT NULL DEFAULT 5, -- Score
+--     FOREIGN KEY(pkid) REFERENCES users(id) ON DELETE CASCADE ON UPDATE NO ACTION,
+--     FOREIGN KEY(relayid) REFERENCES relays(id)
+-- );
+--
+-- CREATE UNIQUE INDEX IF NOT EXISTS pubkey_relay_idx ON tracker(pkid,relayid);

crates/nostr-gossip/migrations/001_init.sql

@@ -0,0 +1,13 @@
+// Copyright (c) 2022-2023 Yuki Kishimoto
+// Copyright (c) 2023-2025 Rust Nostr Developers
+// Distributed under the MIT software license
+
+use std::num::NonZeroUsize;
+use std::time::Duration;
+
+/// Max number of relays allowed in NIP17/NIP65 lists
+pub(crate) const MAX_RELAYS_LIST: usize = 5;
+pub(crate) const PUBKEY_METADATA_OUTDATED_AFTER: Duration = Duration::from_secs(60 * 60); // 60 min
+pub(crate) const CHECK_OUTDATED_INTERVAL: Duration = Duration::from_secs(60 * 5); // 5 min
+
+pub(crate) const CACHE_SIZE: Option<NonZeroUsize> = NonZeroUsize::new(10_000);