[store] switch keys from String to (base64 encoded) binary (#28)

Author: roberto-bayardo

Cargo.lock

@@ -13,7 +13,6 @@ tracing = "0.1.41"
 tracing-subscriber = "0.3.19"
 thiserror = "2.0.12"
 axum = "0.8.1"
-serde = "1.0.219"
 base64 = "0.22.1"
 rocksdb = "0.22.0"
 rand = "0.8.5"
@@ -25,7 +24,9 @@ reqwest = "0.12.5"
 tokio-tungstenite = "0.23.1"
 url = "2.5.2"
 futures-util = "0.3.30"
+serde = "1.0.219"
 serde_json = "1.0.120"
+serde_with = { version = "3.14.0" }
 tempfile = "3.10.1"
 portpicker = "0.1.1"
 http = "1.3.1"

Cargo.toml

@@ -38,7 +38,8 @@ paths:
         required: true
         schema:
           type: string
-        description: The key for the store operation.
+          format: byte
+        description: The base64-encoded key for the store operation.
       - $ref: '#/components/parameters/tokenQuery'
     get:
       summary: Get a value
@@ -123,13 +124,15 @@ paths:
           required: false
           schema:
             type: string
-          description: The key to start the query from (inclusive).
+            format: byte
+          description: The key to start the query from (inclusive), base64-encoded.
         - name: end
           in: query
           required: false
           schema:
             type: string
-          description: The key to end the query at (exclusive).
+            format: byte
+          description: The key to end the query at (exclusive), base64-encoded.
         - name: limit
           in: query
           required: false

interface.yaml

@@ -16,6 +16,7 @@ futures-util = { workspace = true }
 http = { workspace = true }
 reqwest = { workspace = true, features = ["json"] }
 serde = { workspace = true, features = ["derive"] }
+serde_with = { workspace = true, features = ["base64"] }
 serde_json = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["full"] }

sdk-rs/Cargo.toml

@@ -4,33 +4,25 @@ use crate::{error::Error, Client as SdkClient};
 use base64::{engine::general_purpose, Engine as _};
 use reqwest::header::{HeaderValue, AUTHORIZATION};
 use serde::{Deserialize, Serialize};
+use serde_with::{base64::Base64, serde_as};
 
 /// The JSON payload for a `get` operation response.
+#[serde_as]
 #[derive(Serialize, Deserialize, Debug)]
 pub struct GetResultPayload {
-    pub value: String,
-}
-
-/// The result of a `get` operation.
-#[derive(Debug)]
-pub struct GetResult {
-    /// The retrieved value.
+    #[serde_as(as = "Base64")]
     pub value: Vec<u8>,
 }
 
-/// An item in the result of a `query` operation. For internal use.
+/// An item in the result of a `query` operation.
+#[serde_as]
 #[derive(Serialize, Deserialize, Debug)]
 pub struct QueryResultItemPayload {
-    pub key: String,
-    pub value: String,
-}
-
-/// An item in the result of a `query` operation.
-#[derive(Debug)]
-pub struct QueryResultItem {
     /// The key of the item.
-    pub key: String,
+    #[serde_as(as = "Base64")]
+    pub key: Vec<u8>,
     /// The value of the item.
+    #[serde_as(as = "Base64")]
     pub value: Vec<u8>,
 }
 
@@ -40,13 +32,6 @@ pub struct QueryResultPayload {
     pub results: Vec<QueryResultItemPayload>,
 }
 
-/// The result of a `query` operation.
-#[derive(Debug)]
-pub struct QueryResult {
-    /// A list of key-value pairs.
-    pub results: Vec<QueryResultItem>,
-}
-
 /// A client for interacting with the key-value store.
 #[derive(Clone)]
 pub struct Client {
@@ -60,8 +45,9 @@ impl Client {
     }
 
     /// Sets a key-value pair in the store.
-    pub async fn set(&self, key: &str, value: Vec<u8>) -> Result<(), Error> {
-        let url = format!("{}/store/{}", self.client.base_url, key);
+    pub async fn set(&self, key: &[u8], value: Vec<u8>) -> Result<(), Error> {
+        let key_b64 = general_purpose::STANDARD.encode(key);
+        let url = format!("{}/store/{}", self.client.base_url, key_b64);
         let mut headers = reqwest::header::HeaderMap::new();
         headers.insert(
             AUTHORIZATION,
@@ -87,8 +73,9 @@ impl Client {
     /// Retrieves a value from the store by its key.
     ///
     /// If the key does not exist, `Ok(None)` is returned.
-    pub async fn get(&self, key: &str) -> Result<Option<GetResult>, Error> {
-        let url = format!("{}/store/{}", self.client.base_url, key);
+    pub async fn get(&self, key: &[u8]) -> Result<Option<GetResultPayload>, Error> {
+        let key_b64 = general_purpose::STANDARD.encode(key);
+        let url = format!("{}/store/{}", self.client.base_url, key_b64);
         let mut headers = reqwest::header::HeaderMap::new();
         headers.insert(
             AUTHORIZATION,
@@ -111,10 +98,7 @@ impl Client {
             return Err(Error::Http(res.status()));
         }
 
-        let payload: GetResultPayload = res.json().await?;
-        let value = general_purpose::STANDARD.decode(payload.value)?;
-
-        Ok(Some(GetResult { value }))
+        Ok(Some(res.json().await?))
     }
 
     /// Queries for a range of key-value pairs.
@@ -126,19 +110,21 @@ impl Client {
     /// * `limit` - The maximum number of results to return. If `None`, all results are returned.
     pub async fn query(
         &self,
-        start: Option<&str>,
-        end: Option<&str>,
+        start: Option<&[u8]>,
+        end: Option<&[u8]>,
         limit: Option<usize>,
-    ) -> Result<QueryResult, Error> {
+    ) -> Result<QueryResultPayload, Error> {
         let mut url = format!("{}/store?", self.client.base_url);
         if let Some(start) = start {
-            url.push_str(&format!("start={}&", start));
+            let start_b64 = general_purpose::STANDARD.encode(start);
+            url.push_str(&format!("start={start_b64}&"));
         }
         if let Some(end) = end {
-            url.push_str(&format!("end={}&", end));
+            let end_b64 = general_purpose::STANDARD.encode(end);
+            url.push_str(&format!("end={end_b64}&"));
         }
         if let Some(limit) = limit {
-            url.push_str(&format!("limit={}", limit));
+            url.push_str(&format!("limit={limit}"));
         }
 
         let mut headers = reqwest::header::HeaderMap::new();
@@ -160,14 +146,7 @@ impl Client {
         }
 
         let payload: QueryResultPayload = res.json().await?;
-        let mut results = Vec::new();
-        for item in payload.results {
-            results.push(QueryResultItem {
-                key: item.key,
-                value: general_purpose::STANDARD.decode(item.value)?,
-            });
-        }
 
-        Ok(QueryResult { results })
+        Ok(payload)
     }
 }

sdk-rs/src/store.rs

@@ -20,7 +20,7 @@ describe('Exoware TS SDK', () => {
     describe('StoreClient', () => {
         it('should set and get a value', async () => {
             const store = client.store();
-            const key = 'test-key';
+            const key = new TextEncoder().encode('test-key');
             const value = Buffer.from('test-value');
 
             await store.set(key, value);
@@ -32,24 +32,25 @@ describe('Exoware TS SDK', () => {
 
         it('should return null for a non-existent key', async () => {
             const store = client.store();
-            const result = await store.get('non-existent-key');
+            const result = await store.get(new TextEncoder().encode('non-existent-key'));
             expect(result).toBeNull();
         });
 
         it('should query for key-value pairs', async () => {
             const store = client.store();
+            const encoder = new TextEncoder();
             const prefix = 'query-test-';
             const pairs = [
-                { key: `${prefix}a`, value: Buffer.from('a') },
-                { key: `${prefix}b`, value: Buffer.from('b') },
-                { key: `${prefix}c`, value: Buffer.from('c') },
+                { key: encoder.encode(`${prefix}a`), value: Buffer.from('a') },
+                { key: encoder.encode(`${prefix}b`), value: Buffer.from('b') },
+                { key: encoder.encode(`${prefix}c`), value: Buffer.from('c') },
             ];
 
             for (const pair of pairs) {
                 await store.set(pair.key, pair.value);
             }
 
-            const result = await store.query(`${prefix}a`, `${prefix}z`);
+            const result = await store.query(encoder.encode(`${prefix}a`), encoder.encode(`${prefix}z`));
             expect(result.results.length).toBe(3);
             expect(result.results.map(r => Buffer.from(r.value))).toEqual(pairs.map(p => p.value));
             expect(result.results.map(r => r.key).sort()).toEqual(pairs.map(p => p.key).sort());
@@ -58,7 +59,7 @@ describe('Exoware TS SDK', () => {
         describe('limits', () => {
             it('should handle key at size limit', async () => {
                 const store = client.store();
-                const key = 'a'.repeat(512);
+                const key = new TextEncoder().encode('a'.repeat(512));
                 const value = Buffer.from('test-value');
 
                 await store.set(key, value);
@@ -70,15 +71,15 @@ describe('Exoware TS SDK', () => {
 
             it('should reject key over size limit', async () => {
                 const store = client.store();
-                const key = 'a'.repeat(513);
+                const key = new TextEncoder().encode('a'.repeat(513));
                 const value = Buffer.from('test-value');
 
                 await expect(store.set(key, value)).rejects.toThrow('HTTP error: 413 Payload Too Large');
             });
 
             it('should handle value at size limit', async () => {
                 const store = client.store();
-                const key = 'value_at_limit';
+                const key = new TextEncoder().encode('value_at_limit');
                 const value = Buffer.alloc(20 * 1024 * 1024);
 
                 await store.set(key, value);
@@ -90,7 +91,7 @@ describe('Exoware TS SDK', () => {
 
             it('should reject value over size limit', async () => {
                 const store = client.store();
-                const key = 'value_over_limit';
+                const key = new TextEncoder().encode('value_over_limit');
                 const value = Buffer.alloc(20 * 1024 * 1024 + 1);
 
                 await expect(store.set(key, value)).rejects.toThrow('HTTP error: 413 Payload Too Large');

sdk-ts/__tests__/sdk.test.ts

@@ -16,7 +16,7 @@ export interface GetResult {
  */
 export interface QueryResultItem {
     /** The key of the item. */
-    key: string;
+    key: Uint8Array;
     /** The value of the item. */
     value: Uint8Array;
 }
@@ -40,8 +40,9 @@ export class StoreClient {
      * @param key The key to set.
      * @param value The value to set.
      */
-    async set(key: string, value: Uint8Array | Buffer): Promise<void> {
-        const url = `${this.client.baseUrl}/store/${key}`;
+    async set(key: Uint8Array, value: Uint8Array | Buffer): Promise<void> {
+        const encodedKey = Base64.fromUint8Array(key);
+        const url = `${this.client.baseUrl}/store/${encodedKey}`;
         try {
             await this.client.httpClient.post(url, value, {
                 headers: { 'Content-Type': 'application/octet-stream' },
@@ -59,8 +60,9 @@ export class StoreClient {
      * @param key The key to retrieve.
      * @returns The value, or `null` if the key does not exist.
      */
-    async get(key: string): Promise<GetResult | null> {
-        const url = `${this.client.baseUrl}/store/${key}`;
+    async get(key: Uint8Array): Promise<GetResult | null> {
+        const encodedKey = Base64.fromUint8Array(key);
+        const url = `${this.client.baseUrl}/store/${encodedKey}`;
         try {
             const response = await this.client.httpClient.get<{ value: string }>(url);
             const value = Base64.toUint8Array(response.data.value);
@@ -82,17 +84,23 @@ export class StoreClient {
      * @param end The key to end the query at (exclusive). If `undefined`, the query continues to the last key.
      * @param limit The maximum number of results to return. If `undefined`, all results are returned.
      */
-    async query(start?: string, end?: string, limit?: number): Promise<QueryResult> {
+    async query(start?: Uint8Array, end?: Uint8Array, limit?: number): Promise<QueryResult> {
         const url = new URL(`${this.client.baseUrl}/store`);
-        if (start) url.searchParams.append('start', start);
-        if (end) url.searchParams.append('end', end);
+        if (start) {
+            const encodedStart = Base64.fromUint8Array(start);
+            url.searchParams.append('start', encodedStart);
+        }
+        if (end) {
+            const encodedEnd = Base64.fromUint8Array(end);
+            url.searchParams.append('end', encodedEnd);
+        }
         if (limit) url.searchParams.append('limit', limit.toString());
 
         try {
             const response = await this.client.httpClient.get<{ results: { key: string, value: string }[] }>(url.toString());
             const results = response.data.results.map(item => ({
-                key: item.key,
-                value: Base64.toUint8Array(item.value),
+                key: Base64.toUint8Array(item.key),
+                value: Base64.toUint8Array(item.value)
             }));
             return { results };
         } catch (error) {

sdk-ts/src/store.ts

@@ -19,6 +19,7 @@ tracing-subscriber = { workspace = true }
 thiserror = { workspace = true }
 axum = { workspace = true, features = ["ws"] }
 serde = { workspace = true, features = ["derive"] }
+serde_with = { workspace = true }
 base64 = { workspace = true }
 rocksdb = { workspace = true }
 rand = { workspace = true }

simulator/Cargo.toml

@@ -39,7 +39,7 @@ async fn main() -> std::process::ExitCode {
     // Initialize the default directory for the persistent store. This will be
     // `$HOME/.exoware_simulator`.
     let home_directory = std::env::var("HOME").expect("$HOME is not configured");
-    let default_directory = PathBuf::from(format!("{}/.exoware_simulator", home_directory));
+    let default_directory = PathBuf::from(format!("{home_directory}/.exoware_simulator"));
     let default_directory: &'static str = default_directory.to_str().unwrap().to_string().leak();
 
     // Define the CLI application and its arguments.

simulator/src/main.rs

@@ -57,7 +57,7 @@ pub async fn run(
     );
 
     // Create a listener for the server on the specified port.
-    let listener = TcpListener::bind(format!("0.0.0.0:{}", port)).await?;
+    let listener = TcpListener::bind(format!("0.0.0.0:{port}")).await?;
     info!(address = %listener.local_addr()?, "server listening");
 
     // Create a router for the server.