Remove old free credential handle (#5864)

* Set cached storage counters to 0 (#5812)

* Set cached storage counters to 0

* u64 to i64 log possible error

* Check addition too

Debug commit

Remove more data from wg storage peer

Put actual ticket type in storage

Simplify add peer

Finish rebase

Pass defguard Peer

Cache less data for consumption

GatewayStorage traits

Wg API trait

Mock test structures

Unit test for peer controller

EcashManager trait

Init test of Authenticator

Remove peer test

* Fix windows different API

* Use make_bincode_serializer like in other places

* Add log_slow_statements to gateway storage

* Use correct LevelFilter

* Fix clippy

* More win fix

* Win clippy

* Use two error variants more

* Use only one Arc<RwLock<T>> instead of many more

* Remove commented test

* Specific trait import

Author: neacsu

Cargo.lock

@@ -234,6 +234,7 @@ digest = "0.10.7"
 dirs = "5.0"
 doc-comment = "0.3"
 dotenvy = "0.15.6"
+dyn-clone = "1.0.19"
 ecdsa = "0.16"
 ed25519-dalek = "2.1"
 encoding_rs = "0.8.35"

Cargo.toml

@@ -28,8 +28,6 @@ pub type HmacSha256 = Hmac<Sha256>;
 pub type Nonce = u64;
 pub type Taken = Option<SystemTime>;
 
-pub const BANDWIDTH_CAP_PER_DAY: u64 = 250 * 1024 * 1024 * 1024; // 250 GB
-
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
 pub struct IpPair {
     pub ipv4: Ipv4Addr,

common/authenticator-requests/src/v5/registration.rs

@@ -11,9 +11,11 @@ rust-version.workspace = true
 readme.workspace = true
 
 [dependencies]
+async-trait = { workspace = true }
 bs58 = { workspace = true }
 cosmwasm-std = { workspace = true }
 cw-utils = { workspace = true }
+dyn-clone = { workspace = true }
 futures = { workspace = true }
 rand = { workspace = true }
 si-scale = { workspace = true }

common/credential-verification/Cargo.toml

@@ -7,25 +7,36 @@ use crate::ClientBandwidth;
 use nym_credentials::ecash::utils::ecash_today;
 use nym_credentials_interface::Bandwidth;
 use nym_gateway_requests::ServerResponse;
-use nym_gateway_storage::GatewayStorage;
+use nym_gateway_storage::traits::BandwidthGatewayStorage;
 use si_scale::helpers::bibytes2;
 use time::OffsetDateTime;
 use tracing::*;
 
 const FREE_TESTNET_BANDWIDTH_VALUE: Bandwidth = Bandwidth::new_unchecked(64 * 1024 * 1024 * 1024); // 64GB
 
-#[derive(Clone)]
 pub struct BandwidthStorageManager {
-    pub(crate) storage: GatewayStorage,
+    pub(crate) storage: Box<dyn BandwidthGatewayStorage + Send + Sync>,
     pub(crate) client_bandwidth: ClientBandwidth,
     pub(crate) client_id: i64,
     pub(crate) bandwidth_cfg: BandwidthFlushingBehaviourConfig,
     pub(crate) only_coconut_credentials: bool,
 }
 
+impl Clone for BandwidthStorageManager {
+    fn clone(&self) -> Self {
+        Self {
+            storage: dyn_clone::clone_box(&*self.storage),
+            client_bandwidth: self.client_bandwidth.clone(),
+            client_id: self.client_id,
+            bandwidth_cfg: self.bandwidth_cfg,
+            only_coconut_credentials: self.only_coconut_credentials,
+        }
+    }
+}
+
 impl BandwidthStorageManager {
     pub fn new(
-        storage: GatewayStorage,
+        storage: Box<dyn BandwidthGatewayStorage + Send + Sync>,
         client_bandwidth: ClientBandwidth,
         client_id: i64,
         bandwidth_cfg: BandwidthFlushingBehaviourConfig,

common/credential-verification/src/bandwidth_storage_manager.rs

@@ -2,12 +2,14 @@
 // SPDX-License-Identifier: GPL-3.0-only
 
 use crate::Error;
+use async_trait::async_trait;
 use credential_sender::CredentialHandler;
 use credential_sender::CredentialHandlerConfig;
 use error::EcashTicketError;
 use futures::channel::mpsc::{self, UnboundedSender};
 use nym_credentials::CredentialSpendingData;
 use nym_credentials_interface::{ClientTicket, CompactEcashError, NymPayInfo, VerificationKeyAuth};
+use nym_gateway_storage::traits::BandwidthGatewayStorage;
 use nym_gateway_storage::GatewayStorage;
 use nym_validator_client::nym_api::EpochId;
 use nym_validator_client::DirectSigningHttpRpcNyxdClient;
@@ -20,6 +22,7 @@ pub mod credential_sender;
 pub mod error;
 mod helpers;
 mod state;
+pub mod traits;
 
 pub const TIME_RANGE_SEC: i64 = 30;
 
@@ -31,44 +34,21 @@ pub struct EcashManager {
     cred_sender: UnboundedSender<ClientTicket>,
 }
 
-impl EcashManager {
-    pub async fn new(
-        credential_handler_cfg: CredentialHandlerConfig,
-        nyxd_client: DirectSigningHttpRpcNyxdClient,
-        pk_bytes: [u8; 32],
-        shutdown: nym_task::TaskClient,
-        storage: GatewayStorage,
-    ) -> Result<Self, Error> {
-        let shared_state = SharedState::new(nyxd_client, storage).await?;
-
-        let (cred_sender, cred_receiver) = mpsc::unbounded();
-
-        let cs =
-            CredentialHandler::new(credential_handler_cfg, cred_receiver, shared_state.clone())
-                .await?;
-        cs.start(shutdown);
-
-        Ok(EcashManager {
-            shared_state,
-            pk_bytes,
-            pay_infos: Default::default(),
-            cred_sender,
-        })
-    }
-
-    pub async fn verification_key(
+#[async_trait]
+impl traits::EcashManager for EcashManager {
+    async fn verification_key(
         &self,
         epoch_id: EpochId,
     ) -> Result<RwLockReadGuard<VerificationKeyAuth>, EcashTicketError> {
         self.shared_state.verification_key(epoch_id).await
     }
 
-    pub fn storage(&self) -> &GatewayStorage {
-        &self.shared_state.storage
+    fn storage(&self) -> Box<dyn BandwidthGatewayStorage + Send + Sync> {
+        dyn_clone::clone_box(&*self.shared_state.storage)
     }
 
     //Check for duplicate pay_info, then check the payment, then insert pay_info if everything succeeded
-    pub async fn check_payment(
+    async fn check_payment(
         &self,
         credential: &CredentialSpendingData,
         aggregated_verification_key: &VerificationKeyAuth,
@@ -88,6 +68,40 @@ impl EcashManager {
             .await
     }
 
+    fn async_verify(&self, ticket: ClientTicket) {
+        // TODO: I guess do something for shutdowns
+        let _ = self
+            .cred_sender
+            .unbounded_send(ticket)
+            .inspect_err(|_| error!("failed to send the client ticket for verification task"));
+    }
+}
+
+impl EcashManager {
+    pub async fn new(
+        credential_handler_cfg: CredentialHandlerConfig,
+        nyxd_client: DirectSigningHttpRpcNyxdClient,
+        pk_bytes: [u8; 32],
+        shutdown: nym_task::TaskClient,
+        storage: GatewayStorage,
+    ) -> Result<Self, Error> {
+        let shared_state = SharedState::new(nyxd_client, Box::new(storage)).await?;
+
+        let (cred_sender, cred_receiver) = mpsc::unbounded();
+
+        let cs =
+            CredentialHandler::new(credential_handler_cfg, cred_receiver, shared_state.clone())
+                .await?;
+        cs.start(shutdown);
+
+        Ok(EcashManager {
+            shared_state,
+            pk_bytes,
+            pay_infos: Default::default(),
+            cred_sender,
+        })
+    }
+
     pub async fn verify_pay_info(&self, pay_info: NymPayInfo) -> Result<usize, EcashTicketError> {
         //Public key check
         if pay_info.pk() != self.pk_bytes {
@@ -152,12 +166,86 @@ impl EcashManager {
         inner.insert(index, pay_info);
         Ok(())
     }
+}
 
-    pub fn async_verify(&self, ticket: ClientTicket) {
-        // TODO: I guess do something for shutdowns
-        let _ = self
-            .cred_sender
-            .unbounded_send(ticket)
-            .inspect_err(|_| error!("failed to send the client ticket for verification task"));
+pub struct MockEcashManager {
+    verfication_key: tokio::sync::RwLock<VerificationKeyAuth>,
+    storage: Box<dyn BandwidthGatewayStorage + Send + Sync>,
+}
+
+impl MockEcashManager {
+    pub fn new(storage: Box<dyn BandwidthGatewayStorage + Send + Sync>) -> Self {
+        Self {
+            verfication_key: tokio::sync::RwLock::new(
+                VerificationKeyAuth::from_bytes(&[
+                    129, 187, 76, 12, 1, 51, 46, 26, 132, 205, 148, 109, 140, 131, 50, 119, 45,
+                    128, 51, 218, 106, 70, 181, 74, 244, 38, 162, 62, 42, 12, 5, 100, 7, 136, 32,
+                    155, 18, 219, 195, 182, 3, 56, 168, 16, 93, 154, 249, 230, 16, 202, 90, 134,
+                    246, 25, 98, 6, 175, 215, 188, 239, 71, 84, 66, 1, 43, 66, 197, 180, 216, 80,
+                    55, 185, 140, 216, 14, 48, 244, 214, 20, 68, 106, 41, 48, 252, 188, 181, 231,
+                    170, 23, 211, 215, 12, 91, 147, 47, 7, 4, 0, 0, 0, 0, 0, 0, 0, 174, 31, 237,
+                    215, 159, 183, 71, 125, 90, 147, 84, 78, 49, 216, 66, 232, 92, 206, 41, 230,
+                    239, 209, 211, 166, 131, 190, 148, 36, 225, 194, 146, 6, 120, 34, 194, 5, 154,
+                    155, 234, 41, 191, 119, 227, 51, 91, 128, 151, 240, 129, 208, 253, 171, 234,
+                    170, 71, 139, 251, 78, 49, 35, 218, 16, 77, 150, 177, 204, 83, 210, 67, 147,
+                    66, 162, 58, 25, 96, 168, 61, 180, 92, 21, 18, 78, 194, 98, 176, 123, 122, 176,
+                    81, 150, 187, 20, 64, 69, 0, 134, 142, 3, 84, 108, 3, 55, 107, 111, 73, 31, 46,
+                    51, 225, 248, 202, 173, 194, 24, 104, 96, 31, 61, 24, 140, 220, 31, 176, 200,
+                    30, 217, 66, 58, 11, 181, 158, 196, 179, 199, 177, 7, 210, 4, 119, 142, 149,
+                    59, 3, 186, 145, 27, 230, 125, 230, 246, 197, 196, 119, 70, 239, 115, 99, 215,
+                    63, 205, 63, 74, 108, 201, 42, 226, 150, 137, 3, 157, 45, 25, 163, 54, 107,
+                    153, 61, 141, 64, 207, 139, 41, 203, 39, 36, 97, 181, 72, 206, 235, 221, 178,
+                    171, 60, 4, 6, 170, 181, 213, 10, 216, 53, 28, 32, 33, 41, 224, 60, 247, 206,
+                    137, 108, 251, 229, 234, 112, 65, 145, 124, 212, 125, 116, 154, 114, 2, 125,
+                    202, 24, 25, 196, 219, 104, 200, 131, 133, 180, 39, 21, 144, 204, 8, 151, 218,
+                    99, 64, 209, 47, 5, 42, 13, 214, 139, 54, 112, 224, 53, 238, 250, 56, 42, 105,
+                    15, 21, 238, 99, 225, 79, 121, 104, 155, 230, 243, 133, 47, 39, 147, 98, 45,
+                    113, 137, 200, 102, 151, 122, 174, 9, 250, 17, 138, 191, 129, 202, 244, 107,
+                    75, 48, 141, 136, 89, 168, 124, 88, 174, 251, 17, 35, 146, 88, 76, 134, 102,
+                    105, 204, 16, 176, 214, 63, 13, 170, 225, 250, 112, 7, 237, 161, 160, 15, 71,
+                    10, 130, 137, 69, 186, 64, 223, 188, 5, 5, 228, 57, 214, 134, 247, 20, 171,
+                    140, 43, 230, 57, 29, 127, 136, 169, 80, 14, 137, 130, 200, 205, 222, 81, 143,
+                    40, 77, 68, 197, 91, 142, 91, 84, 164, 15, 133, 242, 149, 255, 173, 201, 108,
+                    208, 23, 188, 230, 158, 146, 54, 198, 52, 148, 123, 202, 52, 222, 50, 4, 62,
+                    211, 208, 176, 61, 104, 151, 227, 192, 224, 200, 132, 53, 187, 240, 254, 150,
+                    60, 30, 140, 11, 63, 71, 12, 30, 233, 255, 144, 250, 16, 81, 38, 33, 9, 185,
+                    195, 214, 0, 119, 117, 94, 100, 103, 144, 10, 189, 65, 113, 114, 192, 11, 177,
+                    214, 223, 218, 36, 139, 183, 2, 206, 247, 245, 88, 62, 231, 183, 50, 46, 95,
+                    202, 152, 82, 244, 80, 173, 192, 147, 51, 248, 46, 181, 194, 205, 233, 67, 144,
+                    155, 250, 142, 124, 71, 9, 136, 142, 88, 29, 99, 222, 43, 181, 172, 120, 187,
+                    179, 172, 240, 231, 57, 236, 195, 158, 182, 203, 19, 49, 220, 180, 212, 101,
+                    105, 239, 58, 215, 0, 50, 100, 172, 29, 236, 170, 108, 129, 150, 5, 64, 238,
+                    59, 50, 4, 21, 131, 197, 142, 191, 76, 101, 140, 133, 112, 38, 235, 113, 203,
+                    22, 161, 204, 84, 73, 125, 219, 70, 62, 67, 119, 52, 130, 208, 180, 231, 78,
+                    141, 181, 13, 207, 196, 126, 159, 70, 34, 195, 70,
+                ])
+                .unwrap(),
+            ),
+            storage: dyn_clone::clone_box(&*storage),
+        }
     }
 }
+
+#[async_trait]
+impl traits::EcashManager for MockEcashManager {
+    async fn verification_key(
+        &self,
+        _epoch_id: EpochId,
+    ) -> Result<RwLockReadGuard<VerificationKeyAuth>, EcashTicketError> {
+        Ok(self.verfication_key.read().await)
+    }
+
+    fn storage(&self) -> Box<dyn BandwidthGatewayStorage + Send + Sync> {
+        dyn_clone::clone_box(&*self.storage)
+    }
+
+    async fn check_payment(
+        &self,
+        _credential: &CredentialSpendingData,
+        _aggregated_verification_key: &VerificationKeyAuth,
+    ) -> Result<(), EcashTicketError> {
+        Ok(())
+    }
+
+    fn async_verify(&self, _ticket: ClientTicket) {}
+}

common/credential-verification/src/ecash/mod.rs

@@ -6,7 +6,7 @@ use crate::Error;
 use cosmwasm_std::{from_json, CosmosMsg, WasmMsg};
 use nym_credentials_interface::VerificationKeyAuth;
 use nym_ecash_contract_common::msg::ExecuteMsg;
-use nym_gateway_storage::GatewayStorage;
+use nym_gateway_storage::traits::BandwidthGatewayStorage;
 use nym_validator_client::coconut::all_ecash_api_clients;
 use nym_validator_client::nym_api::EpochId;
 use nym_validator_client::nyxd::contract_traits::{
@@ -22,18 +22,28 @@ use tokio::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
 use tracing::{error, trace, warn};
 
 // state shared by different subtasks dealing with credentials
-#[derive(Clone)]
 pub(crate) struct SharedState {
     pub(crate) nyxd_client: Arc<RwLock<DirectSigningHttpRpcNyxdClient>>,
     pub(crate) address: AccountId,
     pub(crate) epoch_data: Arc<RwLock<BTreeMap<EpochId, EpochState>>>,
-    pub(crate) storage: GatewayStorage,
+    pub(crate) storage: Box<dyn BandwidthGatewayStorage + Send + Sync>,
+}
+
+impl Clone for SharedState {
+    fn clone(&self) -> Self {
+        Self {
+            nyxd_client: self.nyxd_client.clone(),
+            address: self.address.clone(),
+            epoch_data: self.epoch_data.clone(),
+            storage: dyn_clone::clone_box(&*self.storage),
+        }
+    }
 }
 
 impl SharedState {
     pub(crate) async fn new(
         nyxd_client: DirectSigningHttpRpcNyxdClient,
-        storage: GatewayStorage,
+        storage: Box<dyn BandwidthGatewayStorage + Send + Sync>,
     ) -> Result<Self, Error> {
         let address = nyxd_client.address();
 

common/credential-verification/src/ecash/state.rs

@@ -0,0 +1,23 @@
+use async_trait::async_trait;
+use nym_credentials::CredentialSpendingData;
+use nym_credentials_interface::{ClientTicket, VerificationKeyAuth};
+use nym_gateway_storage::traits::BandwidthGatewayStorage;
+use nym_validator_client::nym_api::EpochId;
+use tokio::sync::RwLockReadGuard;
+
+use crate::ecash::error::EcashTicketError;
+
+#[async_trait]
+pub trait EcashManager {
+    async fn verification_key(
+        &self,
+        epoch_id: EpochId,
+    ) -> Result<RwLockReadGuard<VerificationKeyAuth>, EcashTicketError>;
+    fn storage(&self) -> Box<dyn BandwidthGatewayStorage + Send + Sync>;
+    async fn check_payment(
+        &self,
+        credential: &CredentialSpendingData,
+        aggregated_verification_key: &VerificationKeyAuth,
+    ) -> Result<(), EcashTicketError>;
+    fn async_verify(&self, ticket: ClientTicket);
+}