From b5738c6a135abfff9609e7e964f4e690c019fe2f Mon Sep 17 00:00:00 2001 From: Ash Kunda <18058966+akundaz@users.noreply.github.com> Date: Mon, 13 Oct 2025 15:21:10 -0400 Subject: [PATCH 1/6] fix make fmt command --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 2830e62..4186d5e 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ build: .PHONY: fmt fmt: - @rustfmt --config-path ./.rustfmt.toml --check ./crates/rproxy/**/* + @cargo +nightly fmt --check .PHONY: help help: From 1af78306effb368b2d92dee0ad0862d9f17d65cc Mon Sep 17 00:00:00 2001 From: Ash Kunda <18058966+akundaz@users.noreply.github.com> Date: Mon, 13 Oct 2025 22:26:26 -0400 Subject: [PATCH 2/6] run `cargo clippy --fix` --- crates/rproxy/src/config/config.rs | 32 ++-- crates/rproxy/src/config/config_authrpc.rs | 8 +- .../src/config/config_circuit_breaker.rs | 4 +- .../rproxy/src/config/config_flashblocks.rs | 4 +- crates/rproxy/src/config/config_rpc.rs | 8 +- crates/rproxy/src/config/config_tls.rs | 32 ++-- crates/rproxy/src/metrics/metrics.rs | 2 +- crates/rproxy/src/proxy/proxy.rs | 2 +- crates/rproxy/src/proxy_http/proxy_http.rs | 94 +++++------ .../proxy_http/proxy_http_inner_authrpc.rs | 4 +- .../src/proxy_http/proxy_http_inner_rpc.rs | 8 +- crates/rproxy/src/proxy_ws/proxy_ws.rs | 152 ++++++++---------- crates/rproxy/src/server/server.rs | 17 +- crates/rproxy/src/utils/utils_compression.rs | 2 +- crates/rproxy/src/utils/utils_loggable.rs | 4 +- 15 files changed, 171 insertions(+), 202 deletions(-) diff --git a/crates/rproxy/src/config/config.rs b/crates/rproxy/src/config/config.rs index e715e73..41bf10c 100644 --- a/crates/rproxy/src/config/config.rs +++ b/crates/rproxy/src/config/config.rs @@ -76,10 +76,10 @@ impl Config { let mut errs: Vec = vec![]; // authrpc proxy - if self.rpc.enabled { - if let Some(_errs) = self.authrpc.validate() { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } + if self.rpc.enabled && + let Some(_errs) = self.authrpc.validate() + { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); } // circuit-breaker @@ -88,10 +88,10 @@ impl Config { } // flashblocks proxy - if self.flashblocks.enabled { - if let Some(_errs) = self.flashblocks.validate() { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } + if self.flashblocks.enabled && + let Some(_errs) = self.flashblocks.validate() + { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); } // logging @@ -105,17 +105,17 @@ impl Config { } // rpc proxy - if self.rpc.enabled { - if let Some(_errs) = self.rpc.validate() { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } + if self.rpc.enabled && + let Some(_errs) = self.rpc.validate() + { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); } // tls - if self.tls.certificate != "" || self.tls.key != "" { - if let Some(_errs) = self.tls.validate() { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } + if (!self.tls.certificate.is_empty() || !self.tls.key.is_empty()) && + let Some(_errs) = self.tls.validate() + { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); } if !self.authrpc.enabled && !self.flashblocks.enabled && !self.rpc.enabled { diff --git a/crates/rproxy/src/config/config_authrpc.rs b/crates/rproxy/src/config/config_authrpc.rs index e50e7c0..a574cf7 100644 --- a/crates/rproxy/src/config/config_authrpc.rs +++ b/crates/rproxy/src/config/config_authrpc.rs @@ -169,7 +169,7 @@ impl ConfigAuthrpc { // backend_url match Url::parse(&self.backend_url) { Ok(url) => { - if let None = url.host() { + if url.host().is_none() { errs.push(ConfigAuthrpcError::BackendUrlMissesHost { url: self.backend_url.clone(), }); @@ -194,9 +194,9 @@ impl ConfigAuthrpc { // mirroring_peer_urls for peer_url in self.mirroring_peer_urls.iter() { - match Url::parse(&peer_url) { + match Url::parse(peer_url) { Ok(url) => { - if let None = url.host() { + if url.host().is_none() { errs.push(ConfigAuthrpcError::PeerUrlMissesHost { url: peer_url.clone() }); } } @@ -235,7 +235,7 @@ impl ConfigAuthrpc { let local_ips = get_all_local_ip_addresses(); self.mirroring_peer_urls.retain(|url| { - let peer_url = Url::parse(&url).expect(ALREADY_VALIDATED); + let peer_url = Url::parse(url).expect(ALREADY_VALIDATED); let peer_host = peer_url.host_str().expect(ALREADY_VALIDATED); if !peer_url.port().eq(&backend_url.port()) { diff --git a/crates/rproxy/src/config/config_circuit_breaker.rs b/crates/rproxy/src/config/config_circuit_breaker.rs index 44127e1..c8d782a 100644 --- a/crates/rproxy/src/config/config_circuit_breaker.rs +++ b/crates/rproxy/src/config/config_circuit_breaker.rs @@ -90,10 +90,10 @@ impl ConfigCircuitBreaker { } // url - if self.url != "" { + if !self.url.is_empty() { match Url::parse(&self.url) { Ok(url) => { - if let None = url.host() { + if url.host().is_none() { errs.push(ConfigCircuitBreakerError::UrlMissesHost { url: self.url.clone(), }); diff --git a/crates/rproxy/src/config/config_flashblocks.rs b/crates/rproxy/src/config/config_flashblocks.rs index 929e94f..c693523 100644 --- a/crates/rproxy/src/config/config_flashblocks.rs +++ b/crates/rproxy/src/config/config_flashblocks.rs @@ -98,13 +98,13 @@ impl ConfigFlashblocks { // backend_url match self.backend_url.parse::() { Ok(uri) => { - if let None = uri.authority() { + if uri.authority().is_none() { errs.push(ConfigFlashblocksError::BackendUrlMissesHost { url: self.backend_url.clone(), }); } - if let None = uri.host() { + if uri.host().is_none() { errs.push(ConfigFlashblocksError::BackendUrlMissesHost { url: self.backend_url.clone(), }); diff --git a/crates/rproxy/src/config/config_rpc.rs b/crates/rproxy/src/config/config_rpc.rs index 3f5d4f6..5b1b578 100644 --- a/crates/rproxy/src/config/config_rpc.rs +++ b/crates/rproxy/src/config/config_rpc.rs @@ -183,7 +183,7 @@ impl ConfigRpc { // backend_url match Url::parse(&self.backend_url) { Ok(url) => { - if let None = url.host() { + if url.host().is_none() { errs.push(ConfigRpcError::BackendUrlMissesHost { url: self.backend_url.clone(), }); @@ -205,9 +205,9 @@ impl ConfigRpc { // mirroring_peer_urls for peer_url in self.mirroring_peer_urls.iter() { - match Url::parse(&peer_url) { + match Url::parse(peer_url) { Ok(url) => { - if let None = url.host() { + if url.host().is_none() { errs.push(ConfigRpcError::PeerUrlMissesHost { url: peer_url.clone() }); } } @@ -246,7 +246,7 @@ impl ConfigRpc { let local_ips = get_all_local_ip_addresses(); self.mirroring_peer_urls.retain(|url| { - let peer_url = Url::parse(&url).expect(ALREADY_VALIDATED); + let peer_url = Url::parse(url).expect(ALREADY_VALIDATED); let peer_host = peer_url.host_str().expect(ALREADY_VALIDATED); if !peer_url.port().eq(&backend_url.port()) { diff --git a/crates/rproxy/src/config/config_tls.rs b/crates/rproxy/src/config/config_tls.rs index fdccd27..ba4fde9 100644 --- a/crates/rproxy/src/config/config_tls.rs +++ b/crates/rproxy/src/config/config_tls.rs @@ -53,11 +53,11 @@ impl ConfigTls { // certificate { - if self.certificate == "" && self.key != "" { + if self.certificate.is_empty() && !self.key.is_empty() { errs.push(ConfigTlsError::MissingCertificate); } - if self.certificate != "" { + if !self.certificate.is_empty() { match File::open(self.certificate.clone()) { Err(err) => { errs.push(ConfigTlsError::InvalidCertificateFile { @@ -104,11 +104,11 @@ impl ConfigTls { // key { - if self.certificate != "" && self.key == "" { + if !self.certificate.is_empty() && self.key.is_empty() { errs.push(ConfigTlsError::MissingKey); } - if self.key != "" { + if !self.key.is_empty() { match File::open(self.key.clone()) { Err(err) => { errs.push(ConfigTlsError::InvalidKeyFile { @@ -155,20 +155,16 @@ impl ConfigTls { // certificate + key { - match (cert, key) { - (Some(cert), Some(key)) => { - if let Err(err) = - ServerConfig::builder().with_no_client_auth().with_single_cert(cert, key) - { - errs.push(ConfigTlsError::InvalidPair { - path_cert: self.certificate.clone(), - path_key: self.key.clone(), - err: err.to_string(), - }); - } + if let (Some(cert), Some(key)) = (cert, key) { + if let Err(err) = + ServerConfig::builder().with_no_client_auth().with_single_cert(cert, key) + { + errs.push(ConfigTlsError::InvalidPair { + path_cert: self.certificate.clone(), + path_key: self.key.clone(), + err: err.to_string(), + }); } - - (_, _) => {} } } @@ -179,7 +175,7 @@ impl ConfigTls { } pub(crate) fn enabled(&self) -> bool { - self.certificate != "" && self.key != "" + !self.certificate.is_empty() && !self.key.is_empty() } pub(crate) fn key(&self) -> &PrivateKeyDer<'static> { diff --git a/crates/rproxy/src/metrics/metrics.rs b/crates/rproxy/src/metrics/metrics.rs index 223f28b..01501cd 100644 --- a/crates/rproxy/src/metrics/metrics.rs +++ b/crates/rproxy/src/metrics/metrics.rs @@ -269,7 +269,7 @@ impl Metrics { self: Arc, canceller: tokio_util::sync::CancellationToken, ) -> Result<(), Box> { - let listen_address = self.config.listen_address().clone(); + let listen_address = self.config.listen_address(); let listener = match self.listen() { Ok(listener) => listener, diff --git a/crates/rproxy/src/proxy/proxy.rs b/crates/rproxy/src/proxy/proxy.rs index b4283cb..e54d90f 100644 --- a/crates/rproxy/src/proxy/proxy.rs +++ b/crates/rproxy/src/proxy/proxy.rs @@ -122,7 +122,7 @@ impl Drop for ProxyConnectionGuard { fn drop(&mut self) { let val = self.client_connections_count.fetch_sub(1, Ordering::Relaxed) - 1; - let metric_labels = LabelsProxy { proxy: &self.proxy_name }; + let metric_labels = LabelsProxy { proxy: self.proxy_name }; self.metrics.client_connections_active_count.get_or_create(&metric_labels).set(val); self.metrics.client_connections_closed_count.get_or_create(&metric_labels).inc(); diff --git a/crates/rproxy/src/proxy_http/proxy_http.rs b/crates/rproxy/src/proxy_http/proxy_http.rs index 87871b0..f29bf35 100644 --- a/crates/rproxy/src/proxy_http/proxy_http.rs +++ b/crates/rproxy/src/proxy_http/proxy_http.rs @@ -88,7 +88,7 @@ where let backend = ProxyHttpBackendEndpoint::new( inner.clone(), - id.clone(), + id, shared.metrics.clone(), config.backend_url(), connections_limit, @@ -102,7 +102,7 @@ where .map(|peer_url| { ProxyHttpBackendEndpoint::new( shared.inner(), - id.clone(), + id, shared.metrics.clone(), peer_url.to_owned(), config.backend_max_concurrent_requests(), @@ -114,7 +114,7 @@ where ); let postprocessor = ProxyHttpPostprocessor:: { - worker_id: id.clone(), + worker_id: id, inner: inner.clone(), metrics: shared.metrics.clone(), mirroring_peers: peers.clone(), @@ -132,7 +132,7 @@ where canceller: tokio_util::sync::CancellationToken, resetter: broadcast::Sender<()>, ) -> Result<(), Box> { - let listen_address = config.listen_address().clone(); + let listen_address = config.listen_address(); let listener = match Self::listen(&config) { Ok(listener) => listener, @@ -306,8 +306,8 @@ where let info = ProxyHttpRequestInfo::new(&cli_req, cli_req.conn_data::()); - let id = info.id.clone(); - let connection_id = info.connection_id.clone(); + let id = info.id; + let connection_id = info.connection_id; let bck_req = this.backend.new_backend_request(&info); let bck_req_body = ProxyHttpRequestBody::new(this.clone(), info, cli_req_body, timestamp); @@ -350,8 +350,8 @@ where } fn postprocess_client_request(&self, req: ProxiedHttpRequest) { - let id = req.info.id.clone(); - let connection_id = req.info.connection_id.clone(); + let id = req.info.id; + let connection_id = req.info.connection_id; if let Err(_) = self.requests.insert_sync(id, req) { error!( @@ -589,7 +589,7 @@ where } .to_owned(); - if method != "" { + if !method.is_empty() { // single-shot request let params = match match message.get_mut("params") { @@ -629,7 +629,7 @@ where } "engine_newPayloadV4" => { - if params.len() < 1 { + if params.is_empty() { return; } @@ -654,7 +654,7 @@ where } "eth_sendBundle" => { - if params.len() < 1 { + if params.is_empty() { return; } @@ -698,15 +698,14 @@ where None => return, }; - if let Some(execution_payload) = result.get_mut("executionPayload") { - if let Some(transactions) = execution_payload.get_mut("transactions") { - if let Some(transactions) = transactions.as_array_mut() { - // engine_getPayloadV4 + if let Some(execution_payload) = result.get_mut("executionPayload") && + let Some(transactions) = execution_payload.get_mut("transactions") && + let Some(transactions) = transactions.as_array_mut() + { + // engine_getPayloadV4 - for transaction in transactions { - raw_transaction_to_hash(transaction); - } - } + for transaction in transactions { + raw_transaction_to_hash(transaction); } } } @@ -890,7 +889,7 @@ where fn handle(&mut self, msg: ProxiedHttpCombo, ctx: &mut Self::Context) -> Self::Result { let inner = self.inner.clone(); let metrics = self.metrics.clone(); - let worker_id = self.worker_id.clone(); + let worker_id = self.worker_id; let mirroring_peers = self.mirroring_peers.clone(); let mut mirroring_peer_round_robin_index = self.mirroring_peer_round_robin_index.load(Ordering::Relaxed); @@ -1000,7 +999,7 @@ where let start = UtcDateTime::now(); let inner = self.inner.clone(); - let worker_id = self.worker_id.clone(); + let worker_id = self.worker_id; let metrics = self.metrics.clone(); let mrr_req = self.new_backend_request(&cli_req.info); @@ -1100,11 +1099,11 @@ impl ProxyHttpRequestInfo { // append remote ip to x-forwarded-for if let Some(peer_addr) = req.connection_info().peer_addr() { let mut forwarded_for = String::new(); - if let Some(ff) = req.headers().get(header::X_FORWARDED_FOR) { - if let Ok(ff) = ff.to_str() { - forwarded_for.push_str(ff); - forwarded_for.push_str(", "); - } + if let Some(ff) = req.headers().get(header::X_FORWARDED_FOR) && + let Ok(ff) = ff.to_str() + { + forwarded_for.push_str(ff); + forwarded_for.push_str(", "); } forwarded_for.push_str(peer_addr); if let Ok(forwarded_for) = HeaderValue::from_str(&forwarded_for) { @@ -1113,21 +1112,19 @@ impl ProxyHttpRequestInfo { } // set x-forwarded-proto if it's not already set - if req.connection_info().scheme() != "" { - if None == req.headers().get(header::X_FORWARDED_PROTO) { - if let Ok(forwarded_proto) = HeaderValue::from_str(req.connection_info().scheme()) { - headers.insert(header::X_FORWARDED_PROTO, forwarded_proto); - } - } + if req.connection_info().scheme() != "" && + req.headers().get(header::X_FORWARDED_PROTO).is_none() && + let Ok(forwarded_proto) = HeaderValue::from_str(req.connection_info().scheme()) + { + headers.insert(header::X_FORWARDED_PROTO, forwarded_proto); } // set x-forwarded-host if it's not already set - if req.connection_info().scheme() != "" { - if None == req.headers().get(header::X_FORWARDED_HOST) { - if let Ok(forwarded_host) = HeaderValue::from_str(req.connection_info().scheme()) { - headers.insert(header::X_FORWARDED_HOST, forwarded_host); - } - } + if req.connection_info().scheme() != "" && + req.headers().get(header::X_FORWARDED_HOST).is_none() && + let Ok(forwarded_host) = HeaderValue::from_str(req.connection_info().scheme()) + { + headers.insert(header::X_FORWARDED_HOST, forwarded_host); } // remote address from the guard has port, and connection info has ip @@ -1165,12 +1162,12 @@ impl ProxyHttpRequestInfo { #[inline] pub(crate) fn id(&self) -> Uuid { - self.id.clone() + self.id } #[inline] pub(crate) fn connection_id(&self) -> Uuid { - self.connection_id.clone() + self.connection_id } #[inline] @@ -1209,7 +1206,7 @@ impl ProxyHttpResponseInfo { #[inline] pub(crate) fn id(&self) -> Uuid { - self.id.clone() + self.id } fn content_encoding(&self) -> String { @@ -1306,12 +1303,7 @@ where if let Some(info) = mem::take(this.info) { let proxy = this.proxy.clone(); - let req = ProxiedHttpRequest::new( - info, - mem::take(this.body), - this.start.clone(), - end, - ); + let req = ProxiedHttpRequest::new(info, mem::take(this.body), *this.start, end); proxy.postprocess_client_request(req); } @@ -1408,12 +1400,8 @@ where if let Some(info) = mem::take(this.info) { let proxy = this.proxy.clone(); - let res = ProxiedHttpResponse::new( - info, - mem::take(this.body), - this.start.clone(), - end, - ); + let res = + ProxiedHttpResponse::new(info, mem::take(this.body), *this.start, end); proxy.postprocess_backend_response(res); } diff --git a/crates/rproxy/src/proxy_http/proxy_http_inner_authrpc.rs b/crates/rproxy/src/proxy_http/proxy_http_inner_authrpc.rs index 831d086..e16cc5a 100644 --- a/crates/rproxy/src/proxy_http/proxy_http_inner_authrpc.rs +++ b/crates/rproxy/src/proxy_http/proxy_http_inner_authrpc.rs @@ -46,7 +46,7 @@ impl ProxyHttpInner for ProxyHttpInnerAuthrpc { { return false; } - return true; + true } match jrpc_req { @@ -58,7 +58,7 @@ impl ProxyHttpInner for ProxyHttpInnerAuthrpc { return true; } } - return false; + false } } } diff --git a/crates/rproxy/src/proxy_http/proxy_http_inner_rpc.rs b/crates/rproxy/src/proxy_http/proxy_http_inner_rpc.rs index f5da0c8..9be9051 100644 --- a/crates/rproxy/src/proxy_http/proxy_http_inner_rpc.rs +++ b/crates/rproxy/src/proxy_http/proxy_http_inner_rpc.rs @@ -47,7 +47,7 @@ impl ProxyHttpInner for ProxyHttpInnerRpc { return false; } - return mirror_errored_requests || jrpc_res.error.is_none() + mirror_errored_requests || jrpc_res.error.is_none() } match jrpc_req { @@ -67,11 +67,11 @@ impl ProxyHttpInner for ProxyHttpInnerRpc { } }; - return should_mirror( + should_mirror( jrpc_req_single, &jrpc_res_single, self.config.mirror_errored_requests, - ); + ) } JrpcRequestMetaMaybeBatch::Batch(jrpc_req_batch) => { @@ -106,7 +106,7 @@ impl ProxyHttpInner for ProxyHttpInnerRpc { return true; } } - return false; + false } } } diff --git a/crates/rproxy/src/proxy_ws/proxy_ws.rs b/crates/rproxy/src/proxy_ws/proxy_ws.rs index 72fdfb8..e10c47a 100644 --- a/crates/rproxy/src/proxy_ws/proxy_ws.rs +++ b/crates/rproxy/src/proxy_ws/proxy_ws.rs @@ -46,13 +46,13 @@ use crate::{ const WS_PING_INTERVAL_SECONDS: u64 = 1; -const WS_CLI_ERROR: &'static str = "client error"; -const WS_BCK_ERROR: &'static str = "backend error"; -const WS_BCK_TIMEOUT: &'static str = "backend error"; -const WS_CLOSE_OK: &'static str = ""; +const WS_CLI_ERROR: &str = "client error"; +const WS_BCK_ERROR: &str = "backend error"; +const WS_BCK_TIMEOUT: &str = "backend error"; +const WS_CLOSE_OK: &str = ""; -const WS_LABEL_BACKEND: &'static str = "backend"; -const WS_LABEL_CLIENT: &'static str = "client"; +const WS_LABEL_BACKEND: &str = "backend"; +const WS_LABEL_CLIENT: &str = "client"; // ProxyWs ------------------------------------------------------------- @@ -90,7 +90,7 @@ where let config = shared.config(); - let backend = ProxyWsBackendEndpoint::new(id.clone(), config.backend_url()); + let backend = ProxyWsBackendEndpoint::new(id, config.backend_url()); let postprocessor = ProxyWsPostprocessor:: { inner: shared.inner.clone(), @@ -125,7 +125,7 @@ where canceller: tokio_util::sync::CancellationToken, resetter: broadcast::Sender<()>, ) -> Result<(), Box> { - let listen_address = config.listen_address().clone(); + let listen_address = config.listen_address(); let listener = match Self::listen(&config) { Ok(listener) => listener, @@ -582,7 +582,7 @@ where start: timestamp, end: UtcDateTime::now(), }); - return Ok(()); + Ok(()) } // text @@ -619,7 +619,7 @@ where start: timestamp, end: UtcDateTime::now(), }); - return Ok(()); + Ok(()) } // ping @@ -634,31 +634,28 @@ where ); return Err(WS_CLI_ERROR); } - return Ok(()); + Ok(()) } // pong actix_ws::Message::Pong(bytes) => { - if let Some(pong) = ProxyWsPing::from_bytes(bytes) { - if let Some((_, ping)) = this.pings.remove_sync(&pong.id) { - if pong == ping { - this.ping_balance_cli.dec(); - this.shared - .metrics - .ws_latency_client - .get_or_create(&LabelsProxyWs { - proxy: P::name(), - destination: WS_LABEL_BACKEND, - }) - .record( - (1000000.0 * - (timestamp - pong.timestamp).as_seconds_f64() / - 2.0) - as i64, - ); - return Ok(()); - } - } + if let Some(pong) = ProxyWsPing::from_bytes(bytes) && + let Some((_, ping)) = this.pings.remove_sync(&pong.id) && + pong == ping + { + this.ping_balance_cli.dec(); + this.shared + .metrics + .ws_latency_client + .get_or_create(&LabelsProxyWs { + proxy: P::name(), + destination: WS_LABEL_BACKEND, + }) + .record( + (1000000.0 * (timestamp - pong.timestamp).as_seconds_f64() / + 2.0) as i64, + ); + return Ok(()); } warn!( proxy = P::name(), @@ -666,7 +663,7 @@ where worker_id = %this.id, "Unexpected websocket pong received from client", ); - return Ok(()); + Ok(()) } // close @@ -691,12 +688,10 @@ where ); return Err(WS_BCK_ERROR); } - return Err(WS_CLOSE_OK); + Err(WS_CLOSE_OK) } - _ => { - return Ok(()); - } + _ => Ok(()), } } @@ -708,7 +703,7 @@ where error = ?err, "Client websocket stream error" ); - return Err(WS_CLI_ERROR); + Err(WS_CLI_ERROR) } None => { @@ -718,7 +713,7 @@ where worker_id = %this.id, "Client had closed websocket stream" ); - return Err(WS_CLOSE_OK); + Err(WS_CLOSE_OK) } } } @@ -760,7 +755,7 @@ where start: timestamp, end: UtcDateTime::now(), }); - return Ok(()); + Ok(()) } // text @@ -789,7 +784,7 @@ where start: timestamp, end: UtcDateTime::now(), }); - return Ok(()); + Ok(()) } // ping @@ -804,31 +799,28 @@ where ); return Err(WS_BCK_ERROR); } - return Ok(()); + Ok(()) } // pong tungstenite::Message::Pong(bytes) => { - if let Some(pong) = ProxyWsPing::from_bytes(bytes) { - if let Some((_, ping)) = this.pings.remove_sync(&pong.id) { - if pong == ping { - this.ping_balance_bck.dec(); - this.shared - .metrics - .ws_latency_backend - .get_or_create(&LabelsProxyWs { - proxy: P::name(), - destination: WS_LABEL_BACKEND, - }) - .record( - (1000000.0 * - (timestamp - pong.timestamp).as_seconds_f64() / - 2.0) - as i64, - ); - return Ok(()); - } - } + if let Some(pong) = ProxyWsPing::from_bytes(bytes) && + let Some((_, ping)) = this.pings.remove_sync(&pong.id) && + pong == ping + { + this.ping_balance_bck.dec(); + this.shared + .metrics + .ws_latency_backend + .get_or_create(&LabelsProxyWs { + proxy: P::name(), + destination: WS_LABEL_BACKEND, + }) + .record( + (1000000.0 * (timestamp - pong.timestamp).as_seconds_f64() / + 2.0) as i64, + ); + return Ok(()); } warn!( proxy = P::name(), @@ -836,7 +828,7 @@ where worker_id = %this.id, "Unexpected websocket pong received from backend", ); - return Ok(()); + Ok(()) } // close @@ -858,12 +850,10 @@ where ); return Err(WS_CLI_ERROR); } - return Err(WS_CLOSE_OK); + Err(WS_CLOSE_OK) } - _ => { - return Ok(()); - } + _ => Ok(()), } } @@ -875,7 +865,7 @@ where error = ?err, "Backend websocket stream error" ); - return Err(WS_BCK_ERROR); + Err(WS_BCK_ERROR) } None => { @@ -885,7 +875,7 @@ where worker_id = %this.id, "Backend had closed websocket stream" ); - return Err(WS_CLOSE_OK); + Err(WS_CLOSE_OK) } } } @@ -909,7 +899,7 @@ where let json_msg = if config.log_backend_messages() { Loggable(&Self::maybe_sanitise( config.log_sanitise(), - serde_json::from_slice(&msg).unwrap_or_default(), + serde_json::from_slice(msg).unwrap_or_default(), )) } else { Loggable(&serde_json::Value::Null) @@ -931,7 +921,7 @@ where let json_msg = if config.log_backend_messages() { Loggable(&Self::maybe_sanitise( config.log_sanitise(), - serde_json::from_str(&msg).unwrap_or_default(), + serde_json::from_str(msg).unwrap_or_default(), )) } else { Loggable(&serde_json::Value::Null) @@ -953,7 +943,7 @@ where let json_msg = if config.log_client_messages() { Loggable(&Self::maybe_sanitise( config.log_sanitise(), - serde_json::from_slice(&msg).unwrap_or_default(), + serde_json::from_slice(msg).unwrap_or_default(), )) } else { Loggable(&serde_json::Value::Null) @@ -975,7 +965,7 @@ where let json_msg = if config.log_client_messages() { Loggable(&Self::maybe_sanitise( config.log_sanitise(), - serde_json::from_str(&msg).unwrap_or_default(), + serde_json::from_str(msg).unwrap_or_default(), )) } else { Loggable(&serde_json::Value::Null) @@ -1000,15 +990,13 @@ where return message; } - if let Some(object) = message.as_object_mut() { - if let Some(diff) = object.get_mut("diff") { - if let Some(transactions) = diff.get_mut("transactions") { - if let Some(transactions) = transactions.as_array_mut() { - for transaction in transactions { - raw_transaction_to_hash(transaction); - } - } - } + if let Some(object) = message.as_object_mut() && + let Some(diff) = object.get_mut("diff") && + let Some(transactions) = diff.get_mut("transactions") && + let Some(transactions) = transactions.as_array_mut() + { + for transaction in transactions { + raw_transaction_to_hash(transaction); } } @@ -1193,7 +1181,7 @@ where fn handle(&mut self, msg: ProxyWsMessage, ctx: &mut Self::Context) -> Self::Result { let inner = self.inner.clone(); let metrics = self.metrics.clone(); - let worker_id = self.worker_id.clone(); + let worker_id = self.worker_id; ctx.spawn( async move { diff --git a/crates/rproxy/src/server/server.rs b/crates/rproxy/src/server/server.rs index 12bf1a9..7315cf7 100644 --- a/crates/rproxy/src/server/server.rs +++ b/crates/rproxy/src/server/server.rs @@ -46,7 +46,7 @@ impl Server { } // spawn circuit-breaker - if config.circuit_breaker.url != "" { + if !config.circuit_breaker.url.is_empty() { let canceller = canceller.clone(); let resetter = resetter.clone(); @@ -86,7 +86,7 @@ impl Server { let resetter = resetter.clone(); services.push(tokio::spawn(async move { - let res = ProxyHttp::::run( + ProxyHttp::::run( config, tls, metrics, @@ -101,8 +101,7 @@ impl Server { "Failed to start http-proxy, terminating...", ); canceller.cancel(); - }); - res + }) })); } @@ -115,7 +114,7 @@ impl Server { let resetter = resetter.clone(); services.push(tokio::spawn(async move { - let res = ProxyHttp::::run( + ProxyHttp::::run( config, tls, metrics, @@ -130,8 +129,7 @@ impl Server { "Failed to start http-proxy, terminating...", ); canceller.cancel(); - }); - res + }) })); } @@ -144,7 +142,7 @@ impl Server { let resetter = resetter.clone(); services.push(tokio::spawn(async move { - let res = ProxyWs::::run( + ProxyWs::::run( config, tls, metrics, @@ -159,8 +157,7 @@ impl Server { "Failed to start websocket-proxy, terminating...", ); canceller.cancel(); - }); - res + }) })); } diff --git a/crates/rproxy/src/utils/utils_compression.rs b/crates/rproxy/src/utils/utils_compression.rs index 3ea8be6..56cdc49 100644 --- a/crates/rproxy/src/utils/utils_compression.rs +++ b/crates/rproxy/src/utils/utils_compression.rs @@ -48,5 +48,5 @@ pub fn decompress(body: Bytes, size: usize, content_encoding: String) -> (Bytes, _ => {} } - return (body.clone(), size); + (body.clone(), size) } diff --git a/crates/rproxy/src/utils/utils_loggable.rs b/crates/rproxy/src/utils/utils_loggable.rs index 65c0931..f41ba4f 100644 --- a/crates/rproxy/src/utils/utils_loggable.rs +++ b/crates/rproxy/src/utils/utils_loggable.rs @@ -53,7 +53,7 @@ impl valuable::Listable for Loggable<'_> { if let serde_json::Value::Array(arr) = &self.0 { return (arr.len(), Some(arr.len())); } - return (0, Some(0)); + (0, Some(0)) } } @@ -62,6 +62,6 @@ impl valuable::Mappable for Loggable<'_> { if let serde_json::Value::Object(obj) = &self.0 { return (obj.len(), Some(obj.len())); } - return (0, Some(0)); + (0, Some(0)) } } From e701382a10663b61d1f5755577eb0fbf49ec920d Mon Sep 17 00:00:00 2001 From: Ash Kunda <18058966+akundaz@users.noreply.github.com> Date: Mon, 13 Oct 2025 15:52:54 -0400 Subject: [PATCH 3/6] manual clippy fixes --- crates/rproxy/src/config/config_logging.rs | 4 +-- crates/rproxy/src/config/config_tls.rs | 17 ++++++----- crates/rproxy/src/proxy_http/proxy_http.rs | 28 +++++++++---------- .../proxy_http/proxy_http_inner_authrpc.rs | 3 +- crates/rproxy/src/proxy_ws/proxy_ws.rs | 2 +- 5 files changed, 25 insertions(+), 29 deletions(-) diff --git a/crates/rproxy/src/config/config_logging.rs b/crates/rproxy/src/config/config_logging.rs index a46a265..055e6aa 100644 --- a/crates/rproxy/src/config/config_logging.rs +++ b/crates/rproxy/src/config/config_logging.rs @@ -45,7 +45,7 @@ impl ConfigLogging { pub(crate) fn setup_logging(&self) { match self.format { - ConfigLogFormat::JSON => { + ConfigLogFormat::Json => { tracing_subscriber::registry() .with(EnvFilter::from(self.level.clone())) .with(fmt::layer().json().flatten_event(true)) @@ -66,7 +66,7 @@ impl ConfigLogging { #[derive(Clone, Debug, clap::ValueEnum)] pub(crate) enum ConfigLogFormat { - JSON, + Json, Text, } diff --git a/crates/rproxy/src/config/config_tls.rs b/crates/rproxy/src/config/config_tls.rs index ba4fde9..4aadf0c 100644 --- a/crates/rproxy/src/config/config_tls.rs +++ b/crates/rproxy/src/config/config_tls.rs @@ -155,16 +155,15 @@ impl ConfigTls { // certificate + key { - if let (Some(cert), Some(key)) = (cert, key) { - if let Err(err) = + if let (Some(cert), Some(key)) = (cert, key) && + let Err(err) = ServerConfig::builder().with_no_client_auth().with_single_cert(cert, key) - { - errs.push(ConfigTlsError::InvalidPair { - path_cert: self.certificate.clone(), - path_key: self.key.clone(), - err: err.to_string(), - }); - } + { + errs.push(ConfigTlsError::InvalidPair { + path_cert: self.certificate.clone(), + path_key: self.key.clone(), + err: err.to_string(), + }); } } diff --git a/crates/rproxy/src/proxy_http/proxy_http.rs b/crates/rproxy/src/proxy_http/proxy_http.rs index f29bf35..072bd25 100644 --- a/crates/rproxy/src/proxy_http/proxy_http.rs +++ b/crates/rproxy/src/proxy_http/proxy_http.rs @@ -215,7 +215,7 @@ where let handler = server.handle(); let mut resetter = resetter.subscribe(); tokio::spawn(async move { - if let Ok(_) = resetter.recv().await { + if resetter.recv().await.is_ok() { info!(proxy = P::name(), "Reset signal received, stopping http-proxy..."); handler.stop(true).await; } @@ -353,7 +353,7 @@ where let id = req.info.id; let connection_id = req.info.connection_id; - if let Err(_) = self.requests.insert_sync(id, req) { + if self.requests.insert_sync(id, req).is_err() { error!( proxy = P::name(), request_id = %id, @@ -580,13 +580,11 @@ where None => return, }; - let method = match match message.get_key_value("method") { + let method = (match message.get_key_value("method") { Some((_, method)) => method.as_str(), None => None, - } { - Some(method) => method, - None => "", - } + }) + .unwrap_or_default() .to_owned(); if !method.is_empty() { @@ -1230,7 +1228,7 @@ where info: Option, start: UtcDateTime, - body: Box>, + body: Vec, #[pin] stream: S, @@ -1252,7 +1250,7 @@ where info: Some(info), stream: body, start: timestamp, - body: Box::new(Vec::new()), // TODO: preallocate reasonable size + body: Vec::new(), // TODO: preallocate reasonable size } } } @@ -1326,7 +1324,7 @@ where info: Option, start: UtcDateTime, - body: Box>, + body: Vec, #[pin] stream: S, @@ -1349,7 +1347,7 @@ where proxy, stream: body, start: timestamp, - body: Box::new(Vec::new()), // TODO: preallocate reasonable size + body: Vec::new(), // TODO: preallocate reasonable size info: Some(ProxyHttpResponseInfo::new(id, status, headers)), } } @@ -1429,14 +1427,14 @@ pub(crate) struct ProxiedHttpRequest { impl ProxiedHttpRequest { pub(crate) fn new( info: ProxyHttpRequestInfo, - body: Box>, + body: Vec, start: UtcDateTime, end: UtcDateTime, ) -> Self { let size = body.len(); Self { info, - body: Bytes::from(*body), + body: Bytes::from(body), size, decompressed_body: Bytes::new(), decompressed_size: 0, @@ -1478,14 +1476,14 @@ pub(crate) struct ProxiedHttpResponse { impl ProxiedHttpResponse { pub(crate) fn new( info: ProxyHttpResponseInfo, - body: Box>, + body: Vec, start: UtcDateTime, end: UtcDateTime, ) -> Self { let size = body.len(); Self { info, - body: Bytes::from(*body), + body: Bytes::from(body), size, decompressed_body: Bytes::new(), decompressed_size: 0, diff --git a/crates/rproxy/src/proxy_http/proxy_http_inner_authrpc.rs b/crates/rproxy/src/proxy_http/proxy_http_inner_authrpc.rs index e16cc5a..8cf9c8c 100644 --- a/crates/rproxy/src/proxy_http/proxy_http_inner_authrpc.rs +++ b/crates/rproxy/src/proxy_http/proxy_http_inner_authrpc.rs @@ -39,8 +39,7 @@ impl ProxyHttpInner for ProxyHttpInnerAuthrpc { fn should_mirror(jrpc_req: &JrpcRequestMeta) -> bool { let method = jrpc_req.method(); - if true && - !method.starts_with("engine_forkchoiceUpdated") && + if !method.starts_with("engine_forkchoiceUpdated") && !method.starts_with("engine_newPayload") && !method.starts_with("miner_setMaxDASize") { diff --git a/crates/rproxy/src/proxy_ws/proxy_ws.rs b/crates/rproxy/src/proxy_ws/proxy_ws.rs index e10c47a..dc95825 100644 --- a/crates/rproxy/src/proxy_ws/proxy_ws.rs +++ b/crates/rproxy/src/proxy_ws/proxy_ws.rs @@ -190,7 +190,7 @@ where let handler = proxy.handle(); let mut resetter = resetter.subscribe(); tokio::spawn(async move { - if let Ok(_) = resetter.recv().await { + if resetter.recv().await.is_ok() { info!(proxy = P::name(), "Reset signal received, stopping websocket-proxy..."); handler.stop(true).await; } From 2c15522e26b5770df6a4b65b6aa30fc8ca5444b3 Mon Sep 17 00:00:00 2001 From: Ash Kunda <18058966+akundaz@users.noreply.github.com> Date: Mon, 13 Oct 2025 22:42:21 -0400 Subject: [PATCH 4/6] resolve module_inception clippy issue --- .../src/circuit_breaker/circuit_breaker.rs | 202 --- crates/rproxy/src/circuit_breaker/mod.rs | 204 ++- crates/rproxy/src/config/config.rs | 206 --- crates/rproxy/src/config/mod.rs | 191 ++- crates/rproxy/src/jrpc/jrpc.rs | 100 -- crates/rproxy/src/jrpc/mod.rs | 102 +- crates/rproxy/src/metrics/metrics.rs | 369 ---- crates/rproxy/src/metrics/mod.rs | 361 +++- crates/rproxy/src/proxy/mod.rs | 140 +- crates/rproxy/src/proxy/proxy.rs | 138 -- crates/rproxy/src/proxy_http/mod.rs | 1528 ++++++++++++++++- crates/rproxy/src/proxy_http/proxy_http.rs | 1523 ---------------- crates/rproxy/src/proxy_ws/mod.rs | 1303 +++++++++++++- crates/rproxy/src/server/mod.rs | 237 ++- crates/rproxy/src/server/server.rs | 235 --- 15 files changed, 4043 insertions(+), 2796 deletions(-) delete mode 100644 crates/rproxy/src/circuit_breaker/circuit_breaker.rs delete mode 100644 crates/rproxy/src/config/config.rs delete mode 100644 crates/rproxy/src/jrpc/jrpc.rs delete mode 100644 crates/rproxy/src/metrics/metrics.rs delete mode 100644 crates/rproxy/src/proxy/proxy.rs delete mode 100644 crates/rproxy/src/proxy_http/proxy_http.rs delete mode 100644 crates/rproxy/src/server/server.rs diff --git a/crates/rproxy/src/circuit_breaker/circuit_breaker.rs b/crates/rproxy/src/circuit_breaker/circuit_breaker.rs deleted file mode 100644 index 8e734ae..0000000 --- a/crates/rproxy/src/circuit_breaker/circuit_breaker.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::{sync::Arc, time::Duration}; - -use awc::{ - Client, - Connector, - http::{self, Method, header}, -}; -use parking_lot::Mutex; -use tokio::sync::broadcast; -use tracing::{debug, error, warn}; - -use crate::config::ConfigCircuitBreaker; - -// CircuitBreakerInner ------------------------------------------------- - -struct CircuitBreakerInner { - curr_status: Status, - last_status: Status, - - streak_length: usize, -} - -// CircuitBreaker ------------------------------------------------------ - -pub(crate) struct CircuitBreaker { - config: ConfigCircuitBreaker, - inner: Arc>, - client: Client, -} - -impl CircuitBreaker { - pub(crate) fn new(config: ConfigCircuitBreaker) -> Self { - let client = Self::client(&config); - - Self { - config, - client, - inner: Arc::new(Mutex::new(CircuitBreakerInner { - curr_status: Status::Healthy, - last_status: Status::Healthy, - - streak_length: 0, - })), - } - } - - #[inline] - pub(crate) fn name() -> &'static str { - "circuit-breaker" - } - - #[inline] - fn timeout(config: &ConfigCircuitBreaker) -> Duration { - std::cmp::min(Duration::from_secs(5), config.poll_interval * 3 / 4) - } - - #[inline] - fn max_threshold(config: &ConfigCircuitBreaker) -> usize { - std::cmp::max(config.threshold_healthy, config.threshold_unhealthy) + 1 - } - - #[inline] - fn client(config: &ConfigCircuitBreaker) -> Client { - let host = config - .url() - .host() - .unwrap() // safety: verified on start - .to_string(); - let timeout = Self::timeout(config); - - Client::builder() - .add_default_header((header::HOST, host)) - .connector(Connector::new().timeout(timeout).handshake_timeout(timeout)) - .timeout(timeout) - .finish() - } - - pub(crate) async fn run( - self, - canceller: tokio_util::sync::CancellationToken, - resetter: broadcast::Sender<()>, - ) { - let canceller = canceller.clone(); - let resetter = resetter.clone(); - - let mut ticker = tokio::time::interval(self.config.poll_interval); - ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - - // spawning locally b/c actix client is thread-local by design - if let Err(err) = tokio::task::spawn_local(async move { - loop { - let resetter = resetter.clone(); - - tokio::select! { - _ = ticker.tick() => { - self.poll(resetter).await; - } - - _ = canceller.cancelled() => { - break - } - } - } - }) - .await - { - warn!( - service = Self::name(), - error = ?err, - "Failure while running circuit-breaker", - ); - } - } - - async fn poll(&self, resetter: broadcast::Sender<()>) { - let req = self - .client - .request(Method::GET, self.config.url.clone()) - .timeout(Self::timeout(&self.config)); - - let status = match req.send().await { - Ok(res) => match res.status() { - http::StatusCode::OK => Status::Healthy, - _ => { - debug!( - service = Self::name(), - status = %res.status(), - "Unexpected backend status", - ); - Status::Unhealthy - } - }, - - Err(err) => { - debug!( - service = Self::name(), - error = ?err, - "Failed to poll health-status of the backend", - ); - Status::Unhealthy - } - }; - - let mut this = self.inner.lock(); - - if this.last_status == status { - if this.streak_length < Self::max_threshold(&self.config) { - this.streak_length += 1; // prevent overflow on long healthy runs - } - } else { - this.streak_length = 1 - } - this.last_status = status; - - match (this.curr_status.clone(), this.last_status.clone()) { - (Status::Healthy, Status::Healthy) => {} - - (Status::Healthy, Status::Unhealthy) => { - if this.streak_length < self.config.threshold_unhealthy { - return; - } - this.curr_status = Status::Unhealthy; - - warn!(service = Self::name(), "Backend became unhealthy, resetting..."); - - if let Err(err) = resetter.send(()) { - error!( - from = Self::name(), - error = ?err, - "Failed to broadcast reset signal", - ); - } - } - - (Status::Unhealthy, Status::Unhealthy) => { - warn!(service = Self::name(), "Backend is still unhealthy, resetting..."); - - if let Err(err) = resetter.send(()) { - error!( - from = Self::name(), - error = ?err, - "Failed to broadcast reset signal", - ); - } - } - - (Status::Unhealthy, Status::Healthy) => { - if this.streak_length == self.config.threshold_healthy { - this.curr_status = Status::Healthy; - } - } - } - } -} - -// Status -------------------------------------------------------------- - -#[derive(Clone, PartialEq)] -enum Status { - Healthy, - Unhealthy, -} diff --git a/crates/rproxy/src/circuit_breaker/mod.rs b/crates/rproxy/src/circuit_breaker/mod.rs index 9a13f67..8e734ae 100644 --- a/crates/rproxy/src/circuit_breaker/mod.rs +++ b/crates/rproxy/src/circuit_breaker/mod.rs @@ -1,2 +1,202 @@ -mod circuit_breaker; -pub(crate) use circuit_breaker::CircuitBreaker; +use std::{sync::Arc, time::Duration}; + +use awc::{ + Client, + Connector, + http::{self, Method, header}, +}; +use parking_lot::Mutex; +use tokio::sync::broadcast; +use tracing::{debug, error, warn}; + +use crate::config::ConfigCircuitBreaker; + +// CircuitBreakerInner ------------------------------------------------- + +struct CircuitBreakerInner { + curr_status: Status, + last_status: Status, + + streak_length: usize, +} + +// CircuitBreaker ------------------------------------------------------ + +pub(crate) struct CircuitBreaker { + config: ConfigCircuitBreaker, + inner: Arc>, + client: Client, +} + +impl CircuitBreaker { + pub(crate) fn new(config: ConfigCircuitBreaker) -> Self { + let client = Self::client(&config); + + Self { + config, + client, + inner: Arc::new(Mutex::new(CircuitBreakerInner { + curr_status: Status::Healthy, + last_status: Status::Healthy, + + streak_length: 0, + })), + } + } + + #[inline] + pub(crate) fn name() -> &'static str { + "circuit-breaker" + } + + #[inline] + fn timeout(config: &ConfigCircuitBreaker) -> Duration { + std::cmp::min(Duration::from_secs(5), config.poll_interval * 3 / 4) + } + + #[inline] + fn max_threshold(config: &ConfigCircuitBreaker) -> usize { + std::cmp::max(config.threshold_healthy, config.threshold_unhealthy) + 1 + } + + #[inline] + fn client(config: &ConfigCircuitBreaker) -> Client { + let host = config + .url() + .host() + .unwrap() // safety: verified on start + .to_string(); + let timeout = Self::timeout(config); + + Client::builder() + .add_default_header((header::HOST, host)) + .connector(Connector::new().timeout(timeout).handshake_timeout(timeout)) + .timeout(timeout) + .finish() + } + + pub(crate) async fn run( + self, + canceller: tokio_util::sync::CancellationToken, + resetter: broadcast::Sender<()>, + ) { + let canceller = canceller.clone(); + let resetter = resetter.clone(); + + let mut ticker = tokio::time::interval(self.config.poll_interval); + ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + + // spawning locally b/c actix client is thread-local by design + if let Err(err) = tokio::task::spawn_local(async move { + loop { + let resetter = resetter.clone(); + + tokio::select! { + _ = ticker.tick() => { + self.poll(resetter).await; + } + + _ = canceller.cancelled() => { + break + } + } + } + }) + .await + { + warn!( + service = Self::name(), + error = ?err, + "Failure while running circuit-breaker", + ); + } + } + + async fn poll(&self, resetter: broadcast::Sender<()>) { + let req = self + .client + .request(Method::GET, self.config.url.clone()) + .timeout(Self::timeout(&self.config)); + + let status = match req.send().await { + Ok(res) => match res.status() { + http::StatusCode::OK => Status::Healthy, + _ => { + debug!( + service = Self::name(), + status = %res.status(), + "Unexpected backend status", + ); + Status::Unhealthy + } + }, + + Err(err) => { + debug!( + service = Self::name(), + error = ?err, + "Failed to poll health-status of the backend", + ); + Status::Unhealthy + } + }; + + let mut this = self.inner.lock(); + + if this.last_status == status { + if this.streak_length < Self::max_threshold(&self.config) { + this.streak_length += 1; // prevent overflow on long healthy runs + } + } else { + this.streak_length = 1 + } + this.last_status = status; + + match (this.curr_status.clone(), this.last_status.clone()) { + (Status::Healthy, Status::Healthy) => {} + + (Status::Healthy, Status::Unhealthy) => { + if this.streak_length < self.config.threshold_unhealthy { + return; + } + this.curr_status = Status::Unhealthy; + + warn!(service = Self::name(), "Backend became unhealthy, resetting..."); + + if let Err(err) = resetter.send(()) { + error!( + from = Self::name(), + error = ?err, + "Failed to broadcast reset signal", + ); + } + } + + (Status::Unhealthy, Status::Unhealthy) => { + warn!(service = Self::name(), "Backend is still unhealthy, resetting..."); + + if let Err(err) = resetter.send(()) { + error!( + from = Self::name(), + error = ?err, + "Failed to broadcast reset signal", + ); + } + } + + (Status::Unhealthy, Status::Healthy) => { + if this.streak_length == self.config.threshold_healthy { + this.curr_status = Status::Healthy; + } + } + } + } +} + +// Status -------------------------------------------------------------- + +#[derive(Clone, PartialEq)] +enum Status { + Healthy, + Unhealthy, +} diff --git a/crates/rproxy/src/config/config.rs b/crates/rproxy/src/config/config.rs deleted file mode 100644 index 41bf10c..0000000 --- a/crates/rproxy/src/config/config.rs +++ /dev/null @@ -1,206 +0,0 @@ -use std::{process, sync::LazyLock}; - -use clap::Parser; -use thiserror::Error; - -use crate::config::{ - ConfigAuthrpc, - ConfigAuthrpcError, - ConfigCircuitBreaker, - ConfigCircuitBreakerError, - ConfigFlashblocks, - ConfigFlashblocksError, - ConfigLogError, - ConfigLogging, - ConfigMetrics, - ConfigMetricsError, - ConfigRpc, - ConfigRpcError, - ConfigTls, - ConfigTlsError, -}; - -pub(crate) const ALREADY_VALIDATED: &str = "parameter must have been validated already"; - -pub(crate) static PARALLELISM: LazyLock = - LazyLock::new(|| std::thread::available_parallelism().map_or(2, std::num::NonZero::get)); - -pub(crate) static PARALLELISM_STRING: LazyLock = LazyLock::new(|| PARALLELISM.to_string()); - -// Config -------------------------------------------------------------- - -#[derive(Clone, Parser)] -#[command(about, author, long_about = None, term_width = 90, version)] -pub struct Config { - #[command(flatten)] - pub(crate) authrpc: ConfigAuthrpc, - - #[command(flatten)] - pub(crate) circuit_breaker: ConfigCircuitBreaker, - - #[command(flatten)] - pub(crate) flashblocks: ConfigFlashblocks, - - #[command(flatten)] - pub(crate) logging: ConfigLogging, - - #[command(flatten)] - pub(crate) metrics: ConfigMetrics, - - #[command(flatten)] - pub(crate) rpc: ConfigRpc, - - #[command(flatten)] - pub(crate) tls: ConfigTls, -} - -impl Config { - pub fn setup() -> Self { - let mut res = Config::parse(); - - if let Some(errs) = res.clone().validate() { - for err in errs.iter() { - eprintln!("fatal: {}", err); - } - process::exit(1); - }; - - res.logging.setup_logging(); - - res.preprocess(); - - res - } - - pub(crate) fn validate(self) -> Option> { - let mut errs: Vec = vec![]; - - // authrpc proxy - if self.rpc.enabled && - let Some(_errs) = self.authrpc.validate() - { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } - - // circuit-breaker - if let Some(_errs) = self.circuit_breaker.validate() { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } - - // flashblocks proxy - if self.flashblocks.enabled && - let Some(_errs) = self.flashblocks.validate() - { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } - - // logging - if let Some(_errs) = self.logging.validate() { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } - - // metrics - if let Some(_errs) = self.metrics.validate() { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } - - // rpc proxy - if self.rpc.enabled && - let Some(_errs) = self.rpc.validate() - { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } - - // tls - if (!self.tls.certificate.is_empty() || !self.tls.key.is_empty()) && - let Some(_errs) = self.tls.validate() - { - errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); - } - - if !self.authrpc.enabled && !self.flashblocks.enabled && !self.rpc.enabled { - errs.push(ConfigError::NoEnabledProxies); - } - - match errs.len() { - 0 => None, - _ => Some(errs), - } - } - - pub(crate) fn preprocess(&mut self) { - self.authrpc.preprocess(); - self.rpc.preprocess(); - } -} - -// ConfigError --------------------------------------------------------- - -#[derive(Debug, Error)] -pub(crate) enum ConfigError { - #[error("invalid authrpc proxy configuration: {0}")] - ConfigAuthrpcInvalid(ConfigAuthrpcError), - - #[error("invalid circuit-breaker configuration: {0}")] - ConfigCircuitBreakerInvalid(ConfigCircuitBreakerError), - - #[error("invalid flashblocks proxy configuration: {0}")] - ConfigFlashblocksInvalid(ConfigFlashblocksError), - - #[error("invalid logging configuration: {0}")] - ConfigLoggingInvalid(ConfigLogError), - - #[error("invalid metrics configuration: {0}")] - ConfigMetricsInvalid(ConfigMetricsError), - - #[error("invalid rpc proxy configuration: {0}")] - ConfigRpcInvalid(ConfigRpcError), - - #[error("invalid tls configuration: {0}")] - ConfigTlsInvalid(ConfigTlsError), - - #[error("no enabled proxies")] - NoEnabledProxies, -} - -impl From for ConfigError { - fn from(err: ConfigAuthrpcError) -> Self { - Self::ConfigAuthrpcInvalid(err) - } -} - -impl From for ConfigError { - fn from(err: ConfigCircuitBreakerError) -> Self { - Self::ConfigCircuitBreakerInvalid(err) - } -} - -impl From for ConfigError { - fn from(err: ConfigFlashblocksError) -> Self { - Self::ConfigFlashblocksInvalid(err) - } -} - -impl From for ConfigError { - fn from(err: ConfigLogError) -> Self { - Self::ConfigLoggingInvalid(err) - } -} - -impl From for ConfigError { - fn from(err: ConfigMetricsError) -> Self { - Self::ConfigMetricsInvalid(err) - } -} - -impl From for ConfigError { - fn from(err: ConfigRpcError) -> Self { - Self::ConfigRpcInvalid(err) - } -} - -impl From for ConfigError { - fn from(err: ConfigTlsError) -> Self { - Self::ConfigTlsInvalid(err) - } -} diff --git a/crates/rproxy/src/config/mod.rs b/crates/rproxy/src/config/mod.rs index 4202988..8c67aac 100644 --- a/crates/rproxy/src/config/mod.rs +++ b/crates/rproxy/src/config/mod.rs @@ -23,8 +23,193 @@ mod config_rpc; pub(crate) use config_rpc::*; mod config_tls; +use std::{process, sync::LazyLock}; + +use clap::Parser; pub(crate) use config_tls::*; +use thiserror::Error; + +pub(crate) const ALREADY_VALIDATED: &str = "parameter must have been validated already"; + +pub(crate) static PARALLELISM: LazyLock = + LazyLock::new(|| std::thread::available_parallelism().map_or(2, std::num::NonZero::get)); + +pub(crate) static PARALLELISM_STRING: LazyLock = LazyLock::new(|| PARALLELISM.to_string()); + +// Config -------------------------------------------------------------- + +#[derive(Clone, Parser)] +#[command(about, author, long_about = None, term_width = 90, version)] +pub struct Config { + #[command(flatten)] + pub(crate) authrpc: ConfigAuthrpc, + + #[command(flatten)] + pub(crate) circuit_breaker: ConfigCircuitBreaker, + + #[command(flatten)] + pub(crate) flashblocks: ConfigFlashblocks, + + #[command(flatten)] + pub(crate) logging: ConfigLogging, + + #[command(flatten)] + pub(crate) metrics: ConfigMetrics, + + #[command(flatten)] + pub(crate) rpc: ConfigRpc, + + #[command(flatten)] + pub(crate) tls: ConfigTls, +} + +impl Config { + pub fn setup() -> Self { + let mut res = Config::parse(); + + if let Some(errs) = res.clone().validate() { + for err in errs.iter() { + eprintln!("fatal: {}", err); + } + process::exit(1); + }; + + res.logging.setup_logging(); + + res.preprocess(); + + res + } + + pub(crate) fn validate(self) -> Option> { + let mut errs: Vec = vec![]; + + // authrpc proxy + if self.rpc.enabled && + let Some(_errs) = self.authrpc.validate() + { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); + } + + // circuit-breaker + if let Some(_errs) = self.circuit_breaker.validate() { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); + } + + // flashblocks proxy + if self.flashblocks.enabled && + let Some(_errs) = self.flashblocks.validate() + { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); + } + + // logging + if let Some(_errs) = self.logging.validate() { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); + } + + // metrics + if let Some(_errs) = self.metrics.validate() { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); + } + + // rpc proxy + if self.rpc.enabled && + let Some(_errs) = self.rpc.validate() + { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); + } + + // tls + if (!self.tls.certificate.is_empty() || !self.tls.key.is_empty()) && + let Some(_errs) = self.tls.validate() + { + errs.append(&mut _errs.into_iter().map(|err| err.into()).collect()); + } + + if !self.authrpc.enabled && !self.flashblocks.enabled && !self.rpc.enabled { + errs.push(ConfigError::NoEnabledProxies); + } + + match errs.len() { + 0 => None, + _ => Some(errs), + } + } + + pub(crate) fn preprocess(&mut self) { + self.authrpc.preprocess(); + self.rpc.preprocess(); + } +} + +// ConfigError --------------------------------------------------------- + +#[derive(Debug, Error)] +pub(crate) enum ConfigError { + #[error("invalid authrpc proxy configuration: {0}")] + ConfigAuthrpcInvalid(ConfigAuthrpcError), + + #[error("invalid circuit-breaker configuration: {0}")] + ConfigCircuitBreakerInvalid(ConfigCircuitBreakerError), + + #[error("invalid flashblocks proxy configuration: {0}")] + ConfigFlashblocksInvalid(ConfigFlashblocksError), + + #[error("invalid logging configuration: {0}")] + ConfigLoggingInvalid(ConfigLogError), + + #[error("invalid metrics configuration: {0}")] + ConfigMetricsInvalid(ConfigMetricsError), + + #[error("invalid rpc proxy configuration: {0}")] + ConfigRpcInvalid(ConfigRpcError), + + #[error("invalid tls configuration: {0}")] + ConfigTlsInvalid(ConfigTlsError), + + #[error("no enabled proxies")] + NoEnabledProxies, +} + +impl From for ConfigError { + fn from(err: ConfigAuthrpcError) -> Self { + Self::ConfigAuthrpcInvalid(err) + } +} + +impl From for ConfigError { + fn from(err: ConfigCircuitBreakerError) -> Self { + Self::ConfigCircuitBreakerInvalid(err) + } +} + +impl From for ConfigError { + fn from(err: ConfigFlashblocksError) -> Self { + Self::ConfigFlashblocksInvalid(err) + } +} + +impl From for ConfigError { + fn from(err: ConfigLogError) -> Self { + Self::ConfigLoggingInvalid(err) + } +} + +impl From for ConfigError { + fn from(err: ConfigMetricsError) -> Self { + Self::ConfigMetricsInvalid(err) + } +} + +impl From for ConfigError { + fn from(err: ConfigRpcError) -> Self { + Self::ConfigRpcInvalid(err) + } +} -mod config; -pub use config::Config; -pub(crate) use config::*; +impl From for ConfigError { + fn from(err: ConfigTlsError) -> Self { + Self::ConfigTlsInvalid(err) + } +} diff --git a/crates/rproxy/src/jrpc/jrpc.rs b/crates/rproxy/src/jrpc/jrpc.rs deleted file mode 100644 index b2f7d61..0000000 --- a/crates/rproxy/src/jrpc/jrpc.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::borrow::Cow; - -use serde::Deserialize; - -// JrpcError ----------------------------------------------------------- - -#[derive(Clone, Deserialize)] -pub(crate) struct JrpcError { - // pub(crate) code: i64, - // pub(crate) message: String, -} - -// JrpcRequestMeta ----------------------------------------------------- - -const JRPC_METHOD_FCUV1_WITH_PAYLOAD: Cow<'static, str> = - Cow::Borrowed("engine_forkchoiceUpdatedV1_withPayload"); -const JRPC_METHOD_FCUV2_WITH_PAYLOAD: Cow<'static, str> = - Cow::Borrowed("engine_forkchoiceUpdatedV2_withPayload"); -const JRPC_METHOD_FCUV3_WITH_PAYLOAD: Cow<'static, str> = - Cow::Borrowed("engine_forkchoiceUpdatedV3_withPayload"); - -pub(crate) struct JrpcRequestMeta { - method: Cow<'static, str>, - method_enriched: Cow<'static, str>, -} - -impl JrpcRequestMeta { - #[inline] - pub(crate) fn method(&self) -> Cow<'static, str> { - self.method.clone() - } - - #[inline] - pub(crate) fn method_enriched(&self) -> Cow<'static, str> { - self.method_enriched.clone() - } -} - -impl<'a> Deserialize<'a> for JrpcRequestMeta { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'a>, - { - #[derive(Deserialize)] - struct JrpcRequestMetaWire { - method: Cow<'static, str>, - params: Vec, - } - - let wire = JrpcRequestMetaWire::deserialize(deserializer)?; - - let mut params_count = 0; - for param in wire.params.iter() { - if !param.is_null() { - params_count += 1; - } - } - - if params_count < 2 { - return Ok(Self { method: wire.method.clone(), method_enriched: wire.method.clone() }); - } - - let method_enriched = match wire.method.as_ref() { - "engine_forkchoiceUpdatedV1" => JRPC_METHOD_FCUV1_WITH_PAYLOAD.clone(), - "engine_forkchoiceUpdatedV2" => JRPC_METHOD_FCUV2_WITH_PAYLOAD.clone(), - "engine_forkchoiceUpdatedV3" => JRPC_METHOD_FCUV3_WITH_PAYLOAD.clone(), - - _ => wire.method.clone(), - }; - - Ok(Self { method: wire.method, method_enriched }) - } -} - -// JrpcRequestMetaMaybeBatch ------------------------------------------- - -const JRPC_METHOD_BATCH: Cow<'static, str> = Cow::Borrowed("batch"); - -#[derive(Deserialize)] -#[serde(untagged)] -pub(crate) enum JrpcRequestMetaMaybeBatch { - Single(JrpcRequestMeta), - Batch(Vec), -} - -impl JrpcRequestMetaMaybeBatch { - pub(crate) fn method_enriched(&self) -> Cow<'static, str> { - match self { - Self::Single(jrpc) => jrpc.method_enriched.clone(), - Self::Batch(_) => JRPC_METHOD_BATCH.clone(), - } - } -} - -// JrpcResponseMeta ---------------------------------------------------- - -#[derive(Clone, Deserialize)] -pub(crate) struct JrpcResponseMeta { - pub(crate) error: Option, -} diff --git a/crates/rproxy/src/jrpc/mod.rs b/crates/rproxy/src/jrpc/mod.rs index 5742f20..b2f7d61 100644 --- a/crates/rproxy/src/jrpc/mod.rs +++ b/crates/rproxy/src/jrpc/mod.rs @@ -1,2 +1,100 @@ -mod jrpc; -pub(crate) use jrpc::*; +use std::borrow::Cow; + +use serde::Deserialize; + +// JrpcError ----------------------------------------------------------- + +#[derive(Clone, Deserialize)] +pub(crate) struct JrpcError { + // pub(crate) code: i64, + // pub(crate) message: String, +} + +// JrpcRequestMeta ----------------------------------------------------- + +const JRPC_METHOD_FCUV1_WITH_PAYLOAD: Cow<'static, str> = + Cow::Borrowed("engine_forkchoiceUpdatedV1_withPayload"); +const JRPC_METHOD_FCUV2_WITH_PAYLOAD: Cow<'static, str> = + Cow::Borrowed("engine_forkchoiceUpdatedV2_withPayload"); +const JRPC_METHOD_FCUV3_WITH_PAYLOAD: Cow<'static, str> = + Cow::Borrowed("engine_forkchoiceUpdatedV3_withPayload"); + +pub(crate) struct JrpcRequestMeta { + method: Cow<'static, str>, + method_enriched: Cow<'static, str>, +} + +impl JrpcRequestMeta { + #[inline] + pub(crate) fn method(&self) -> Cow<'static, str> { + self.method.clone() + } + + #[inline] + pub(crate) fn method_enriched(&self) -> Cow<'static, str> { + self.method_enriched.clone() + } +} + +impl<'a> Deserialize<'a> for JrpcRequestMeta { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'a>, + { + #[derive(Deserialize)] + struct JrpcRequestMetaWire { + method: Cow<'static, str>, + params: Vec, + } + + let wire = JrpcRequestMetaWire::deserialize(deserializer)?; + + let mut params_count = 0; + for param in wire.params.iter() { + if !param.is_null() { + params_count += 1; + } + } + + if params_count < 2 { + return Ok(Self { method: wire.method.clone(), method_enriched: wire.method.clone() }); + } + + let method_enriched = match wire.method.as_ref() { + "engine_forkchoiceUpdatedV1" => JRPC_METHOD_FCUV1_WITH_PAYLOAD.clone(), + "engine_forkchoiceUpdatedV2" => JRPC_METHOD_FCUV2_WITH_PAYLOAD.clone(), + "engine_forkchoiceUpdatedV3" => JRPC_METHOD_FCUV3_WITH_PAYLOAD.clone(), + + _ => wire.method.clone(), + }; + + Ok(Self { method: wire.method, method_enriched }) + } +} + +// JrpcRequestMetaMaybeBatch ------------------------------------------- + +const JRPC_METHOD_BATCH: Cow<'static, str> = Cow::Borrowed("batch"); + +#[derive(Deserialize)] +#[serde(untagged)] +pub(crate) enum JrpcRequestMetaMaybeBatch { + Single(JrpcRequestMeta), + Batch(Vec), +} + +impl JrpcRequestMetaMaybeBatch { + pub(crate) fn method_enriched(&self) -> Cow<'static, str> { + match self { + Self::Single(jrpc) => jrpc.method_enriched.clone(), + Self::Batch(_) => JRPC_METHOD_BATCH.clone(), + } + } +} + +// JrpcResponseMeta ---------------------------------------------------- + +#[derive(Clone, Deserialize)] +pub(crate) struct JrpcResponseMeta { + pub(crate) error: Option, +} diff --git a/crates/rproxy/src/metrics/metrics.rs b/crates/rproxy/src/metrics/metrics.rs deleted file mode 100644 index 01501cd..0000000 --- a/crates/rproxy/src/metrics/metrics.rs +++ /dev/null @@ -1,369 +0,0 @@ -use std::{net::TcpListener, sync::Arc, time::Duration}; - -use actix_web::{ - App, - HttpRequest, - HttpResponse, - HttpServer, - middleware::{NormalizePath, TrailingSlash}, - web, -}; -use awc::http::Method; -use prometheus_client::{ - metrics::{counter::Counter, family::Family, gauge::Gauge}, - registry::{Registry, Unit}, -}; -use socket2::{SockAddr, Socket, TcpKeepalive}; -use tracing::{error, info}; - -use crate::{ - config::ConfigMetrics, - metrics::{ - Candlestick, - LabelsProxy, - LabelsProxyClientInfo, - LabelsProxyHttpJrpc, - LabelsProxyWs, - }, -}; - -// Metrics ------------------------------------------------------------- - -pub(crate) struct Metrics { - config: ConfigMetrics, - registry: Registry, - - pub(crate) client_connections_active_count: Family, - pub(crate) client_connections_established_count: Family, - pub(crate) client_connections_closed_count: Family, - pub(crate) client_info: Family, - - pub(crate) http_latency_backend: Family, - pub(crate) http_latency_delta: Family, - pub(crate) http_latency_total: Family, - - pub(crate) http_mirror_success_count: Family, - pub(crate) http_mirror_failure_count: Family, - - pub(crate) http_proxy_success_count: Family, - pub(crate) http_proxy_failure_count: Family, - - pub(crate) http_request_size: Family, - pub(crate) http_response_size: Family, - - pub(crate) http_request_decompressed_size: Family, - pub(crate) http_response_decompressed_size: Family, - - pub(crate) tls_certificate_valid_not_before: Gauge, - pub(crate) tls_certificate_valid_not_after: Gauge, - - pub(crate) ws_latency_backend: Family, - pub(crate) ws_latency_client: Family, - pub(crate) ws_latency_proxy: Family, - - pub(crate) ws_message_size: Family, - - pub(crate) ws_proxy_success_count: Family, - pub(crate) ws_proxy_failure_count: Family, -} - -impl Metrics { - pub(crate) fn new(config: ConfigMetrics) -> Self { - let mut this = Metrics { - config, - registry: Registry::with_prefix("rproxy"), - - client_connections_active_count: Family::default(), - client_connections_established_count: Family::default(), - client_connections_closed_count: Family::default(), - - client_info: Family::default(), - - http_latency_backend: Family::default(), - http_latency_delta: Family::default(), - http_latency_total: Family::default(), - - http_mirror_success_count: Family::default(), - http_mirror_failure_count: Family::default(), - - http_proxy_success_count: Family::default(), - http_proxy_failure_count: Family::default(), - - http_request_size: Family::default(), - http_response_size: Family::default(), - - http_request_decompressed_size: Family::default(), - http_response_decompressed_size: Family::default(), - - tls_certificate_valid_not_before: Gauge::default(), - tls_certificate_valid_not_after: Gauge::default(), - - ws_latency_backend: Family::default(), - ws_latency_client: Family::default(), - ws_latency_proxy: Family::default(), - - ws_message_size: Family::default(), - - ws_proxy_success_count: Family::default(), - ws_proxy_failure_count: Family::default(), - }; - - this.registry.register( - "client_connections_active_count", - "count of active client connections", - this.client_connections_active_count.clone(), - ); - - this.registry.register( - "client_connections_established_count", - "count of client connections established", - this.client_connections_established_count.clone(), - ); - - this.registry.register( - "client_info", - "general information about the client", - this.client_info.clone(), - ); - - this.registry.register( - "client_connections_closed_count", - "count of client connections closed", - this.client_connections_closed_count.clone(), - ); - - this.registry.register_with_unit( - "http_latency_backend", - "latency of backend http responses (interval b/w end of client's request and begin of backend's response)", - Unit::Other(String::from("nanoseconds")), - this.http_latency_backend.clone(), - ); - - this.registry.register_with_unit( - "http_latency_delta", - "latency delta (http_latency_total - http_latency_backend)", - Unit::Other(String::from("nanoseconds")), - this.http_latency_delta.clone(), - ); - - this.registry.register_with_unit( - "http_latency_total", - "overall latency of http requests (interval b/w begin of client's request and end of forwarded response)", - Unit::Other(String::from("nanoseconds")), - this.http_latency_total.clone(), - ); - - this.registry.register( - "http_mirror_success_count", - "count of successfully mirrored http requests/responses", - this.http_mirror_success_count.clone(), - ); - - this.registry.register( - "http_mirror_failure_count", - "count of failures to mirror http request/response", - this.http_mirror_failure_count.clone(), - ); - - this.registry.register( - "http_proxy_success_count", - "count of successfully proxied http requests/responses", - this.http_proxy_success_count.clone(), - ); - - this.registry.register( - "http_proxy_failure_count", - "count of failures to proxy http request/response", - this.http_proxy_failure_count.clone(), - ); - - this.registry.register_with_unit( - "http_request_size", - "sizes of incoming http requests", - Unit::Bytes, - this.http_request_size.clone(), - ); - - this.registry.register_with_unit( - "http_response_size", - "sizes of proxied http responses", - Unit::Bytes, - this.http_response_size.clone(), - ); - - this.registry.register_with_unit( - "http_request_decompressed_size", - "decompressed sizes of incoming http requests", - Unit::Bytes, - this.http_request_decompressed_size.clone(), - ); - - this.registry.register_with_unit( - "http_response_decompressed_size", - "decompressed sizes of proxied http responses", - Unit::Bytes, - this.http_response_decompressed_size.clone(), - ); - - this.registry.register( - "tls_certificate_valid_not_before", - "tls certificate's not-valid-before timestamp", - this.tls_certificate_valid_not_before.clone(), - ); - - this.registry.register( - "tls_certificate_valid_not_after", - "tls certificate's not-valid-after timestamp", - this.tls_certificate_valid_not_after.clone(), - ); - - this.registry.register_with_unit( - "ws_latency_backend", - "round-trip-time of websocket pings to backend divided by 2", - Unit::Other(String::from("nanoseconds")), - this.ws_latency_backend.clone(), - ); - - this.registry.register_with_unit( - "ws_latency_client", - "round-trip-time of websocket pings to backend divided by 2", - Unit::Other(String::from("nanoseconds")), - this.ws_latency_client.clone(), - ); - - this.registry.register_with_unit( - "ws_latency_proxy", - "time to process the websocket message by the proxy", - Unit::Other(String::from("nanoseconds")), - this.ws_latency_proxy.clone(), - ); - - this.registry.register_with_unit( - "ws_message_size", - "sizes of proxied websocket messages", - Unit::Bytes, - this.ws_message_size.clone(), - ); - - this.registry.register( - "ws_proxy_success_count", - "count of successfully proxied websocket messages", - this.ws_proxy_success_count.clone(), - ); - - this.registry.register( - "ws_proxy_failure_count", - "count of failures to proxy websocket message", - this.ws_proxy_failure_count.clone(), - ); - - this - } - - #[inline] - pub(crate) fn name() -> &'static str { - "metrics" - } - - pub(crate) async fn run( - self: Arc, - canceller: tokio_util::sync::CancellationToken, - ) -> Result<(), Box> { - let listen_address = self.config.listen_address(); - - let listener = match self.listen() { - Ok(listener) => listener, - Err(err) => { - error!( - service = Self::name(), - addr = %&self.config.listen_address(), - error = ?err, - "Failed to initialise a socket" - ); - return Err(Box::new(err)); - } - }; - - let server = match HttpServer::new(move || { - App::new() - .app_data(web::Data::new(self.clone())) - .wrap(NormalizePath::new(TrailingSlash::Trim)) - .default_service(web::route().to(Self::receive)) - }) - .workers(1) - .shutdown_signal(canceller.cancelled_owned()) - .listen(listener) - { - Ok(metrics) => metrics, - Err(err) => { - error!(service = Self::name(), error = ?err, "Failed to initialise http service"); - return Err(Box::new(err)); - } - }; - - info!( - service = Self::name(), - listen_address = %listen_address, - "Starting http service...", - ); - - if let Err(err) = server.run().await { - error!(service = Self::name(), error = ?err, "Failure while running http service") - } - - info!(service = Self::name(), "Stopped http service"); - - Ok(()) - } - - fn listen(&self) -> std::io::Result { - let socket = Socket::new( - socket2::Domain::for_address(self.config.listen_address()), - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - - // must use non-blocking with tokio - socket.set_nonblocking(true)?; - - // allow time to flush buffers on close - socket.set_linger(Some(Duration::from_secs(1)))?; - - // allow binding to the socket whlie there are still TIME_WAIT conns - socket.set_reuse_address(true)?; - - socket.set_tcp_keepalive( - &TcpKeepalive::new() - .with_time(Duration::from_secs(15)) - .with_interval(Duration::from_secs(15)) - .with_retries(4), - )?; - - socket.bind(&SockAddr::from(self.config.listen_address()))?; - - socket.listen(16)?; - - Ok(socket.into()) - } - - async fn receive( - req: HttpRequest, - _: web::Payload, - this: web::Data>, - ) -> Result { - if req.method() != Method::GET { - return Ok(HttpResponse::BadRequest().finish()); - } - - let mut body = String::new(); - - if let Err(err) = prometheus_client::encoding::text::encode(&mut body, &this.registry) { - error!(service = Self::name(), error = ?err, "Failed to encode metrics"); - return Ok(HttpResponse::InternalServerError().finish()); - } - - Ok(HttpResponse::Ok() - .content_type("application/openmetrics-text; version=1.0.0; charset=utf-8") - .body(body)) - } -} diff --git a/crates/rproxy/src/metrics/mod.rs b/crates/rproxy/src/metrics/mod.rs index 3cf2ed5..e4f30c3 100644 --- a/crates/rproxy/src/metrics/mod.rs +++ b/crates/rproxy/src/metrics/mod.rs @@ -2,7 +2,364 @@ mod metrics_candlestick; pub(crate) use metrics_candlestick::Candlestick; mod metrics_labels; +use std::{net::TcpListener, sync::Arc, time::Duration}; + +use actix_web::{ + App, + HttpRequest, + HttpResponse, + HttpServer, + middleware::{NormalizePath, TrailingSlash}, + web, +}; +use awc::http::Method; pub(crate) use metrics_labels::*; +use prometheus_client::{ + metrics::{counter::Counter, family::Family, gauge::Gauge}, + registry::{Registry, Unit}, +}; +use socket2::{SockAddr, Socket, TcpKeepalive}; +use tracing::{error, info}; + +use crate::config::ConfigMetrics; + +// Metrics ------------------------------------------------------------- + +pub(crate) struct Metrics { + config: ConfigMetrics, + registry: Registry, + + pub(crate) client_connections_active_count: Family, + pub(crate) client_connections_established_count: Family, + pub(crate) client_connections_closed_count: Family, + pub(crate) client_info: Family, + + pub(crate) http_latency_backend: Family, + pub(crate) http_latency_delta: Family, + pub(crate) http_latency_total: Family, + + pub(crate) http_mirror_success_count: Family, + pub(crate) http_mirror_failure_count: Family, + + pub(crate) http_proxy_success_count: Family, + pub(crate) http_proxy_failure_count: Family, + + pub(crate) http_request_size: Family, + pub(crate) http_response_size: Family, + + pub(crate) http_request_decompressed_size: Family, + pub(crate) http_response_decompressed_size: Family, + + pub(crate) tls_certificate_valid_not_before: Gauge, + pub(crate) tls_certificate_valid_not_after: Gauge, + + pub(crate) ws_latency_backend: Family, + pub(crate) ws_latency_client: Family, + pub(crate) ws_latency_proxy: Family, + + pub(crate) ws_message_size: Family, + + pub(crate) ws_proxy_success_count: Family, + pub(crate) ws_proxy_failure_count: Family, +} + +impl Metrics { + pub(crate) fn new(config: ConfigMetrics) -> Self { + let mut this = Metrics { + config, + registry: Registry::with_prefix("rproxy"), + + client_connections_active_count: Family::default(), + client_connections_established_count: Family::default(), + client_connections_closed_count: Family::default(), + + client_info: Family::default(), + + http_latency_backend: Family::default(), + http_latency_delta: Family::default(), + http_latency_total: Family::default(), + + http_mirror_success_count: Family::default(), + http_mirror_failure_count: Family::default(), + + http_proxy_success_count: Family::default(), + http_proxy_failure_count: Family::default(), + + http_request_size: Family::default(), + http_response_size: Family::default(), + + http_request_decompressed_size: Family::default(), + http_response_decompressed_size: Family::default(), + + tls_certificate_valid_not_before: Gauge::default(), + tls_certificate_valid_not_after: Gauge::default(), + + ws_latency_backend: Family::default(), + ws_latency_client: Family::default(), + ws_latency_proxy: Family::default(), + + ws_message_size: Family::default(), + + ws_proxy_success_count: Family::default(), + ws_proxy_failure_count: Family::default(), + }; + + this.registry.register( + "client_connections_active_count", + "count of active client connections", + this.client_connections_active_count.clone(), + ); + + this.registry.register( + "client_connections_established_count", + "count of client connections established", + this.client_connections_established_count.clone(), + ); + + this.registry.register( + "client_info", + "general information about the client", + this.client_info.clone(), + ); + + this.registry.register( + "client_connections_closed_count", + "count of client connections closed", + this.client_connections_closed_count.clone(), + ); + + this.registry.register_with_unit( + "http_latency_backend", + "latency of backend http responses (interval b/w end of client's request and begin of backend's response)", + Unit::Other(String::from("nanoseconds")), + this.http_latency_backend.clone(), + ); + + this.registry.register_with_unit( + "http_latency_delta", + "latency delta (http_latency_total - http_latency_backend)", + Unit::Other(String::from("nanoseconds")), + this.http_latency_delta.clone(), + ); + + this.registry.register_with_unit( + "http_latency_total", + "overall latency of http requests (interval b/w begin of client's request and end of forwarded response)", + Unit::Other(String::from("nanoseconds")), + this.http_latency_total.clone(), + ); + + this.registry.register( + "http_mirror_success_count", + "count of successfully mirrored http requests/responses", + this.http_mirror_success_count.clone(), + ); + + this.registry.register( + "http_mirror_failure_count", + "count of failures to mirror http request/response", + this.http_mirror_failure_count.clone(), + ); + + this.registry.register( + "http_proxy_success_count", + "count of successfully proxied http requests/responses", + this.http_proxy_success_count.clone(), + ); + + this.registry.register( + "http_proxy_failure_count", + "count of failures to proxy http request/response", + this.http_proxy_failure_count.clone(), + ); + + this.registry.register_with_unit( + "http_request_size", + "sizes of incoming http requests", + Unit::Bytes, + this.http_request_size.clone(), + ); + + this.registry.register_with_unit( + "http_response_size", + "sizes of proxied http responses", + Unit::Bytes, + this.http_response_size.clone(), + ); + + this.registry.register_with_unit( + "http_request_decompressed_size", + "decompressed sizes of incoming http requests", + Unit::Bytes, + this.http_request_decompressed_size.clone(), + ); + + this.registry.register_with_unit( + "http_response_decompressed_size", + "decompressed sizes of proxied http responses", + Unit::Bytes, + this.http_response_decompressed_size.clone(), + ); + + this.registry.register( + "tls_certificate_valid_not_before", + "tls certificate's not-valid-before timestamp", + this.tls_certificate_valid_not_before.clone(), + ); + + this.registry.register( + "tls_certificate_valid_not_after", + "tls certificate's not-valid-after timestamp", + this.tls_certificate_valid_not_after.clone(), + ); + + this.registry.register_with_unit( + "ws_latency_backend", + "round-trip-time of websocket pings to backend divided by 2", + Unit::Other(String::from("nanoseconds")), + this.ws_latency_backend.clone(), + ); + + this.registry.register_with_unit( + "ws_latency_client", + "round-trip-time of websocket pings to backend divided by 2", + Unit::Other(String::from("nanoseconds")), + this.ws_latency_client.clone(), + ); + + this.registry.register_with_unit( + "ws_latency_proxy", + "time to process the websocket message by the proxy", + Unit::Other(String::from("nanoseconds")), + this.ws_latency_proxy.clone(), + ); + + this.registry.register_with_unit( + "ws_message_size", + "sizes of proxied websocket messages", + Unit::Bytes, + this.ws_message_size.clone(), + ); + + this.registry.register( + "ws_proxy_success_count", + "count of successfully proxied websocket messages", + this.ws_proxy_success_count.clone(), + ); + + this.registry.register( + "ws_proxy_failure_count", + "count of failures to proxy websocket message", + this.ws_proxy_failure_count.clone(), + ); + + this + } + + #[inline] + pub(crate) fn name() -> &'static str { + "metrics" + } + + pub(crate) async fn run( + self: Arc, + canceller: tokio_util::sync::CancellationToken, + ) -> Result<(), Box> { + let listen_address = self.config.listen_address(); + + let listener = match self.listen() { + Ok(listener) => listener, + Err(err) => { + error!( + service = Self::name(), + addr = %&self.config.listen_address(), + error = ?err, + "Failed to initialise a socket" + ); + return Err(Box::new(err)); + } + }; + + let server = match HttpServer::new(move || { + App::new() + .app_data(web::Data::new(self.clone())) + .wrap(NormalizePath::new(TrailingSlash::Trim)) + .default_service(web::route().to(Self::receive)) + }) + .workers(1) + .shutdown_signal(canceller.cancelled_owned()) + .listen(listener) + { + Ok(metrics) => metrics, + Err(err) => { + error!(service = Self::name(), error = ?err, "Failed to initialise http service"); + return Err(Box::new(err)); + } + }; + + info!( + service = Self::name(), + listen_address = %listen_address, + "Starting http service...", + ); + + if let Err(err) = server.run().await { + error!(service = Self::name(), error = ?err, "Failure while running http service") + } + + info!(service = Self::name(), "Stopped http service"); + + Ok(()) + } + + fn listen(&self) -> std::io::Result { + let socket = Socket::new( + socket2::Domain::for_address(self.config.listen_address()), + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + + // must use non-blocking with tokio + socket.set_nonblocking(true)?; + + // allow time to flush buffers on close + socket.set_linger(Some(Duration::from_secs(1)))?; + + // allow binding to the socket whlie there are still TIME_WAIT conns + socket.set_reuse_address(true)?; + + socket.set_tcp_keepalive( + &TcpKeepalive::new() + .with_time(Duration::from_secs(15)) + .with_interval(Duration::from_secs(15)) + .with_retries(4), + )?; + + socket.bind(&SockAddr::from(self.config.listen_address()))?; + + socket.listen(16)?; + + Ok(socket.into()) + } + + async fn receive( + req: HttpRequest, + _: web::Payload, + this: web::Data>, + ) -> Result { + if req.method() != Method::GET { + return Ok(HttpResponse::BadRequest().finish()); + } + + let mut body = String::new(); + + if let Err(err) = prometheus_client::encoding::text::encode(&mut body, &this.registry) { + error!(service = Self::name(), error = ?err, "Failed to encode metrics"); + return Ok(HttpResponse::InternalServerError().finish()); + } -mod metrics; -pub(crate) use metrics::Metrics; + Ok(HttpResponse::Ok() + .content_type("application/openmetrics-text; version=1.0.0; charset=utf-8") + .body(body)) + } +} diff --git a/crates/rproxy/src/proxy/mod.rs b/crates/rproxy/src/proxy/mod.rs index c677ef9..e54d90f 100644 --- a/crates/rproxy/src/proxy/mod.rs +++ b/crates/rproxy/src/proxy/mod.rs @@ -1,2 +1,138 @@ -mod proxy; -pub(crate) use proxy::{Proxy, ProxyConnectionGuard, ProxyInner}; +use std::{ + any::Any, + sync::{ + Arc, + atomic::{AtomicI64, Ordering}, + }, +}; + +use actix_web::dev::Extensions; +use tracing::{debug, warn}; +use uuid::Uuid; + +use crate::metrics::{LabelsProxy, Metrics}; + +// Proxy --------------------------------------------------------------- + +pub(crate) trait Proxy

+where + P: ProxyInner, +{ + fn on_connect( + metrics: Arc, + client_connections_count: Arc, + ) -> impl Fn(&dyn Any, &mut Extensions) { + move |connection, extensions| { + { + let val = client_connections_count.fetch_add(1, Ordering::Relaxed) + 1; + let metric_labels = LabelsProxy { proxy: P::name() }; + + metrics.client_connections_active_count.get_or_create(&metric_labels).set(val); + metrics.client_connections_established_count.get_or_create(&metric_labels).inc(); + } + + let stream: Option<&actix_web::rt::net::TcpStream> = if let Some(stream) = connection.downcast_ref::>() { + let (stream, _) = stream.get_ref(); + Some(stream) + } else if let Some(stream) = connection.downcast_ref::() { + Some(stream) + } else { + warn!("Unexpected stream type"); + None + }; + + if let Some(stream) = stream { + let id = Uuid::now_v7(); + + let remote_addr = match stream.peer_addr() { + Ok(local_addr) => Some(local_addr.to_string()), + Err(err) => { + warn!(proxy = P::name(), error = ?err, "Failed to get remote address"); + None + } + }; + let local_addr = match stream.local_addr() { + Ok(local_addr) => Some(local_addr.to_string()), + Err(err) => { + warn!(proxy = P::name(), error = ?err, "Failed to get remote address"); + None + } + }; + + debug!( + proxy = P::name(), + connection_id = %id, + remote_addr = remote_addr.as_ref().map_or("unknown", |v| v.as_str()), + local_addr = local_addr.as_ref().map_or("unknown", |v| v.as_str()), + "Client connection open" + ); + + extensions.insert(ProxyConnectionGuard::new( + id, + P::name(), + remote_addr, + local_addr, + &metrics, + client_connections_count.clone(), + )); + } + } + } +} + +// ProxyInner ---------------------------------------------------------- + +pub(crate) trait ProxyInner: 'static { + fn name() -> &'static str; +} + +// ProxyConnectionGuard ------------------------------------------------ + +pub struct ProxyConnectionGuard { + pub id: Uuid, + pub remote_addr: Option, + pub local_addr: Option, + + proxy_name: &'static str, + metrics: Arc, + client_connections_count: Arc, +} + +impl ProxyConnectionGuard { + fn new( + id: Uuid, + proxy_name: &'static str, + remote_addr: Option, + local_addr: Option, + metrics: &Arc, + client_connections_count: Arc, + ) -> Self { + Self { + id, + remote_addr, + local_addr, + proxy_name, + metrics: metrics.clone(), + client_connections_count, + } + } +} + +impl Drop for ProxyConnectionGuard { + fn drop(&mut self) { + let val = self.client_connections_count.fetch_sub(1, Ordering::Relaxed) - 1; + + let metric_labels = LabelsProxy { proxy: self.proxy_name }; + + self.metrics.client_connections_active_count.get_or_create(&metric_labels).set(val); + self.metrics.client_connections_closed_count.get_or_create(&metric_labels).inc(); + + debug!( + proxy = self.proxy_name, + connection_id = %self.id, + remote_addr = self.remote_addr.as_ref().map_or("unknown", |v| v.as_str()), + local_addr = self.local_addr.as_ref().map_or("unknown", |v| v.as_str()), + "Client connection closed" + ); + } +} diff --git a/crates/rproxy/src/proxy/proxy.rs b/crates/rproxy/src/proxy/proxy.rs deleted file mode 100644 index e54d90f..0000000 --- a/crates/rproxy/src/proxy/proxy.rs +++ /dev/null @@ -1,138 +0,0 @@ -use std::{ - any::Any, - sync::{ - Arc, - atomic::{AtomicI64, Ordering}, - }, -}; - -use actix_web::dev::Extensions; -use tracing::{debug, warn}; -use uuid::Uuid; - -use crate::metrics::{LabelsProxy, Metrics}; - -// Proxy --------------------------------------------------------------- - -pub(crate) trait Proxy

-where - P: ProxyInner, -{ - fn on_connect( - metrics: Arc, - client_connections_count: Arc, - ) -> impl Fn(&dyn Any, &mut Extensions) { - move |connection, extensions| { - { - let val = client_connections_count.fetch_add(1, Ordering::Relaxed) + 1; - let metric_labels = LabelsProxy { proxy: P::name() }; - - metrics.client_connections_active_count.get_or_create(&metric_labels).set(val); - metrics.client_connections_established_count.get_or_create(&metric_labels).inc(); - } - - let stream: Option<&actix_web::rt::net::TcpStream> = if let Some(stream) = connection.downcast_ref::>() { - let (stream, _) = stream.get_ref(); - Some(stream) - } else if let Some(stream) = connection.downcast_ref::() { - Some(stream) - } else { - warn!("Unexpected stream type"); - None - }; - - if let Some(stream) = stream { - let id = Uuid::now_v7(); - - let remote_addr = match stream.peer_addr() { - Ok(local_addr) => Some(local_addr.to_string()), - Err(err) => { - warn!(proxy = P::name(), error = ?err, "Failed to get remote address"); - None - } - }; - let local_addr = match stream.local_addr() { - Ok(local_addr) => Some(local_addr.to_string()), - Err(err) => { - warn!(proxy = P::name(), error = ?err, "Failed to get remote address"); - None - } - }; - - debug!( - proxy = P::name(), - connection_id = %id, - remote_addr = remote_addr.as_ref().map_or("unknown", |v| v.as_str()), - local_addr = local_addr.as_ref().map_or("unknown", |v| v.as_str()), - "Client connection open" - ); - - extensions.insert(ProxyConnectionGuard::new( - id, - P::name(), - remote_addr, - local_addr, - &metrics, - client_connections_count.clone(), - )); - } - } - } -} - -// ProxyInner ---------------------------------------------------------- - -pub(crate) trait ProxyInner: 'static { - fn name() -> &'static str; -} - -// ProxyConnectionGuard ------------------------------------------------ - -pub struct ProxyConnectionGuard { - pub id: Uuid, - pub remote_addr: Option, - pub local_addr: Option, - - proxy_name: &'static str, - metrics: Arc, - client_connections_count: Arc, -} - -impl ProxyConnectionGuard { - fn new( - id: Uuid, - proxy_name: &'static str, - remote_addr: Option, - local_addr: Option, - metrics: &Arc, - client_connections_count: Arc, - ) -> Self { - Self { - id, - remote_addr, - local_addr, - proxy_name, - metrics: metrics.clone(), - client_connections_count, - } - } -} - -impl Drop for ProxyConnectionGuard { - fn drop(&mut self) { - let val = self.client_connections_count.fetch_sub(1, Ordering::Relaxed) - 1; - - let metric_labels = LabelsProxy { proxy: self.proxy_name }; - - self.metrics.client_connections_active_count.get_or_create(&metric_labels).set(val); - self.metrics.client_connections_closed_count.get_or_create(&metric_labels).inc(); - - debug!( - proxy = self.proxy_name, - connection_id = %self.id, - remote_addr = self.remote_addr.as_ref().map_or("unknown", |v| v.as_str()), - local_addr = self.local_addr.as_ref().map_or("unknown", |v| v.as_str()), - "Client connection closed" - ); - } -} diff --git a/crates/rproxy/src/proxy_http/mod.rs b/crates/rproxy/src/proxy_http/mod.rs index 93d77b6..ee8249e 100644 --- a/crates/rproxy/src/proxy_http/mod.rs +++ b/crates/rproxy/src/proxy_http/mod.rs @@ -4,13 +4,1527 @@ pub(crate) use proxy_http_inner_authrpc::ProxyHttpInnerAuthrpc; mod proxy_http_inner_rpc; pub(crate) use proxy_http_inner_rpc::ProxyHttpInnerRpc; -mod proxy_http; -pub(crate) use proxy_http::{ - ProxiedHttpRequest, - ProxiedHttpResponse, - ProxyHttp, - ProxyHttpRequestInfo, +mod proxy_http_inner; +use std::{ + borrow::Cow, + fmt::Debug, + marker::PhantomData, + mem, + pin::Pin, + str::FromStr, + sync::{ + Arc, + atomic::{AtomicI64, AtomicUsize, Ordering}, + }, + task::{Context, Poll}, }; -mod proxy_http_inner; +use actix::{Actor, AsyncContext, WrapFuture}; +use actix_web::{ + self, + App, + HttpRequest, + HttpResponse, + HttpResponseBuilder, + HttpServer, + body::BodySize, + http::{StatusCode, header}, + middleware::{NormalizePath, TrailingSlash}, + web, +}; +use awc::{ + Client, + ClientRequest, + ClientResponse, + Connector, + body::MessageBody, + error::HeaderValue, + http::{Method, header::HeaderMap}, +}; +use bytes::Bytes; +use futures::TryStreamExt; +use futures_core::Stream; +use pin_project::pin_project; pub(crate) use proxy_http_inner::ProxyHttpInner; +use scc::HashMap; +use time::{UtcDateTime, format_description::well_known::Iso8601}; +use tokio::sync::broadcast; +use tracing::{debug, error, info, warn}; +use url::Url; +use uuid::Uuid; +use x509_parser::asn1_rs::ToStatic; + +use crate::{ + config::{ConfigProxyHttp, ConfigProxyHttpMirroringStrategy, ConfigTls, PARALLELISM}, + jrpc::JrpcRequestMetaMaybeBatch, + metrics::{LabelsProxy, LabelsProxyClientInfo, LabelsProxyHttpJrpc, Metrics}, + proxy::{Proxy, ProxyConnectionGuard}, + utils::{Loggable, decompress, is_hop_by_hop_header, raw_transaction_to_hash}, +}; + +const TCP_KEEPALIVE_ATTEMPTS: u32 = 8; + +// ProxyHttp ----------------------------------------------------------- + +pub(crate) struct ProxyHttp +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + id: Uuid, + + shared: ProxyHttpSharedState, + + backend: ProxyHttpBackendEndpoint, + requests: HashMap, + postprocessor: actix::Addr>, +} + +impl ProxyHttp +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + fn new(shared: ProxyHttpSharedState, connections_limit: usize) -> Self { + let id = Uuid::now_v7(); + + debug!(proxy = P::name(), worker_id = %id, "Creating http-proxy worker..."); + + let config = shared.config(); + let inner = shared.inner(); + + let backend = ProxyHttpBackendEndpoint::new( + inner.clone(), + id, + shared.metrics.clone(), + config.backend_url(), + connections_limit, + config.backend_timeout(), + ); + + let peers: Arc>>> = Arc::new( + config + .mirroring_peer_urls() + .iter() + .map(|peer_url| { + ProxyHttpBackendEndpoint::new( + shared.inner(), + id, + shared.metrics.clone(), + peer_url.to_owned(), + config.backend_max_concurrent_requests(), + config.backend_timeout(), + ) + .start() + }) + .collect(), + ); + + let postprocessor = ProxyHttpPostprocessor:: { + worker_id: id, + inner: inner.clone(), + metrics: shared.metrics.clone(), + mirroring_peers: peers.clone(), + mirroring_peer_round_robin_index: AtomicUsize::new(0), + } + .start(); + + Self { id, shared, backend, requests: HashMap::default(), postprocessor } + } + + pub(crate) async fn run( + config: C, + tls: ConfigTls, + metrics: Arc, + canceller: tokio_util::sync::CancellationToken, + resetter: broadcast::Sender<()>, + ) -> Result<(), Box> { + let listen_address = config.listen_address(); + + let listener = match Self::listen(&config) { + Ok(listener) => listener, + Err(err) => { + error!( + proxy = P::name(), + addr = %config.listen_address(), + error = ?err, + "Failed to initialise a socket" + ); + return Err(Box::new(err)); + } + }; + + let workers_count = + std::cmp::min(PARALLELISM.to_static(), config.backend_max_concurrent_requests()); + let max_concurrent_requests_per_worker = + config.backend_max_concurrent_requests() / workers_count; + if workers_count * max_concurrent_requests_per_worker < + config.backend_max_concurrent_requests() + { + warn!( + "Max backend concurrent requests must be a round of available parallelism ({}), therefore it's clamped at {} (instead of {})", + PARALLELISM.to_static(), + workers_count * max_concurrent_requests_per_worker, + config.backend_max_concurrent_requests() + ); + } + + let shared = ProxyHttpSharedState::::new(config, &metrics); + let client_connections_count = shared.client_connections_count.clone(); + + info!( + proxy = P::name(), + listen_address = %listen_address, + workers_count = workers_count, + max_concurrent_requests_per_worker = max_concurrent_requests_per_worker, + "Starting http-proxy..." + ); + + let proxy = HttpServer::new(move || { + let this = + web::Data::new(Self::new(shared.clone(), max_concurrent_requests_per_worker)); + + App::new() + .app_data(this) + .wrap(NormalizePath::new(TrailingSlash::Trim)) + .default_service(web::route().to(Self::receive)) + }) + .on_connect(Self::on_connect(metrics, client_connections_count)) + .shutdown_signal(canceller.cancelled_owned()) + .workers(workers_count); + + let server = match if tls.enabled() { + let cert = tls.certificate().clone(); + let key = tls.key().clone_key(); + + proxy.listen_rustls_0_23( + listener, + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(cert, key) + .unwrap(), // safety: verified on start + ) + } else { + proxy.listen(listener) + } { + Ok(server) => server, + Err(err) => { + error!( + proxy = P::name(), + error = ?err, + "Failed to initialise http-proxy", + ); + return Err(Box::new(err)); + } + } + .run(); + + let handler = server.handle(); + let mut resetter = resetter.subscribe(); + tokio::spawn(async move { + if resetter.recv().await.is_ok() { + info!(proxy = P::name(), "Reset signal received, stopping http-proxy..."); + handler.stop(true).await; + } + }); + + if let Err(err) = server.await { + error!(proxy = P::name(), error = ?err, "Failure while running http-proxy") + } + + info!(proxy = P::name(), "Stopped http-proxy"); + + Ok(()) + } + + fn listen(config: &C) -> std::io::Result { + let socket = socket2::Socket::new( + socket2::Domain::for_address(config.listen_address()), + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + + // must use non-blocking with tokio + socket.set_nonblocking(true)?; + + // allow time to flush buffers on close + socket.set_linger(Some(config.backend_timeout()))?; + + // allow binding while there are still residual connections in TIME_WAIT + socket.set_reuse_address(true)?; + + if !config.idle_connection_timeout().is_zero() { + socket.set_tcp_keepalive( + &socket2::TcpKeepalive::new() + .with_time( + config.idle_connection_timeout().div_f64(f64::from(TCP_KEEPALIVE_ATTEMPTS)), + ) + .with_interval( + config.idle_connection_timeout().div_f64(f64::from(TCP_KEEPALIVE_ATTEMPTS)), + ) + .with_retries(TCP_KEEPALIVE_ATTEMPTS - 1), + )?; + } + + socket.bind(&socket2::SockAddr::from(config.listen_address()))?; + + socket.listen(1024)?; + + Ok(socket.into()) + } + + fn to_client_response(bck_res: &ClientResponse) -> HttpResponseBuilder { + let mut cli_res = HttpResponse::build(bck_res.status()); + + for (name, header) in bck_res.headers().iter() { + if is_hop_by_hop_header(name) { + continue; + } + if let Ok(hname) = header::HeaderName::from_str(name.as_str()) { + cli_res.append_header((hname, header.clone())); + } + } + + cli_res + } + + /// receive accepts client's (frontend) request and proxies it to + /// backend + async fn receive( + cli_req: HttpRequest, + cli_req_body: web::Payload, + this: web::Data, + ) -> Result { + let timestamp = UtcDateTime::now(); + + if let Some(user_agent) = cli_req.headers().get(header::USER_AGENT) && + !user_agent.is_empty() && + let Ok(user_agent) = user_agent.to_str() + { + this.shared + .metrics + .client_info + .get_or_create(&LabelsProxyClientInfo { + proxy: P::name(), + user_agent: user_agent.to_string(), + }) + .inc(); + } + + let info = ProxyHttpRequestInfo::new(&cli_req, cli_req.conn_data::()); + + let id = info.id; + let connection_id = info.connection_id; + + let bck_req = this.backend.new_backend_request(&info); + let bck_req_body = ProxyHttpRequestBody::new(this.clone(), info, cli_req_body, timestamp); + + let bck_res = match bck_req.send_stream(bck_req_body).await { + Ok(res) => res, + Err(err) => { + warn!( + proxy = P::name(), + request_id = %id, + connection_id = %connection_id, + worker_id = %this.id, + backend_url = %this.backend.url, + error = ?err, + "Failed to proxy a request", + ); + this.shared + .metrics + .http_proxy_failure_count + .get_or_create(&LabelsProxy { proxy: P::name() }) + .inc(); + return Ok(HttpResponse::BadGateway().body(format!("Backend error: {:?}", err))); + } + }; + + let timestamp = UtcDateTime::now(); + let status = bck_res.status(); + let mut cli_res = Self::to_client_response(&bck_res); + + let bck_body = ProxyHttpResponseBody::new( + this, + id, + status, + bck_res.headers().clone(), + bck_res.into_stream(), + timestamp, + ); + + Ok(cli_res.streaming(bck_body)) + } + + fn postprocess_client_request(&self, req: ProxiedHttpRequest) { + let id = req.info.id; + let connection_id = req.info.connection_id; + + if self.requests.insert_sync(id, req).is_err() { + error!( + proxy = P::name(), + request_id = %id, + connection_id = %connection_id, + worker_id = %self.id, + "Duplicate request id", + ); + }; + } + + fn postprocess_backend_response(&self, bck_res: ProxiedHttpResponse) { + let cli_req = match self.requests.remove_sync(&bck_res.info.id) { + Some((_, req)) => req, + None => { + error!( + proxy = P::name(), + request_id = %bck_res.info.id, + worker_id = %self.id, + "Proxied http response for unmatching request", + ); + return; + } + }; + + // hand over to postprocessor asynchronously so that we can return the + // response to the client as early as possible + self.postprocessor.do_send(ProxiedHttpCombo { req: cli_req, res: bck_res }); + } + + fn finalise_proxying( + mut cli_req: ProxiedHttpRequest, + mut bck_res: ProxiedHttpResponse, + inner: Arc

, + worker_id: Uuid, + metrics: Arc, + mirroring_peers: Arc>>>, + mut mirroring_peer_round_robin_index: usize, + ) { + if cli_req.decompressed_size < cli_req.size { + (cli_req.decompressed_body, cli_req.decompressed_size) = + decompress(cli_req.body.clone(), cli_req.size, cli_req.info.content_encoding()); + } + + if bck_res.decompressed_size < bck_res.size { + (bck_res.decompressed_body, bck_res.decompressed_size) = + decompress(bck_res.body.clone(), bck_res.size, bck_res.info.content_encoding()); + } + + match serde_json::from_slice::(&cli_req.decompressed_body) { + Ok(jrpc) => { + if inner.should_mirror(&jrpc, &cli_req, &bck_res) { + let mirrors_count = match inner.config().mirroring_strategy() { + ConfigProxyHttpMirroringStrategy::FanOut => mirroring_peers.len(), + ConfigProxyHttpMirroringStrategy::RoundRobin => 1, + ConfigProxyHttpMirroringStrategy::RoundRobinPairs => 2, + }; + + for _ in 0..mirrors_count { + let mirroring_peer = &mirroring_peers[mirroring_peer_round_robin_index]; + mirroring_peer_round_robin_index += 1; + if mirroring_peer_round_robin_index >= mirroring_peers.len() { + mirroring_peer_round_robin_index = 0; + } + + let mut req = cli_req.clone(); + req.info.jrpc_method_enriched = jrpc.method_enriched(); + mirroring_peer.do_send(req.clone()); + } + } + + Self::maybe_log_proxied_request_and_response( + &jrpc, + &cli_req, + &bck_res, + inner.clone(), + worker_id, + ); + + Self::emit_metrics_on_proxy_success(&jrpc, &cli_req, &bck_res, metrics.clone()); + } + + Err(err) => { + warn!( + proxy = P::name(), + request_id = %cli_req.info.id, + connection_id = %cli_req.info.connection_id, + worker_id = %worker_id, + error = ?err, + "Failed to parse json-rpc request", + ); + } + } + } + + fn postprocess_mirrored_response( + mut cli_req: ProxiedHttpRequest, + mut mrr_res: ProxiedHttpResponse, + inner: Arc

, + metrics: Arc, + worker_id: Uuid, + ) { + if cli_req.decompressed_size < cli_req.size { + (cli_req.decompressed_body, cli_req.decompressed_size) = + decompress(cli_req.body.clone(), cli_req.size, cli_req.info.content_encoding()); + } + + if mrr_res.decompressed_size < mrr_res.size { + (mrr_res.decompressed_body, mrr_res.decompressed_size) = + decompress(mrr_res.body.clone(), mrr_res.size, mrr_res.info.content_encoding()); + } + + Self::maybe_log_mirrored_request(&cli_req, &mrr_res, worker_id, inner.config()); + + metrics + .http_mirror_success_count + .get_or_create(&LabelsProxyHttpJrpc { + proxy: P::name(), + jrpc_method: cli_req.info.jrpc_method_enriched, + }) + .inc(); + } + + fn maybe_log_proxied_request_and_response( + jrpc: &JrpcRequestMetaMaybeBatch, + req: &ProxiedHttpRequest, + res: &ProxiedHttpResponse, + inner: Arc

, + worker_id: Uuid, + ) { + let config = inner.config(); + + let json_req = if config.log_proxied_requests() { + Loggable(&Self::maybe_sanitise( + config.log_sanitise(), + serde_json::from_slice(&req.decompressed_body).unwrap_or_default(), + )) + } else { + Loggable(&serde_json::Value::Null) + }; + + let json_res = if config.log_proxied_responses() { + Loggable(&Self::maybe_sanitise( + config.log_sanitise(), + serde_json::from_slice(&res.decompressed_body).unwrap_or_default(), + )) + } else { + Loggable(&serde_json::Value::Null) + }; + + info!( + proxy = P::name(), + request_id = %req.info.id, + connection_id = %req.info.connection_id, + worker_id = %worker_id, + jrpc_method = %jrpc.method_enriched(), + http_status = res.status(), + remote_addr = req.info().remote_addr, + ts_request_received = req.start().format(&Iso8601::DEFAULT).unwrap_or_default(), + latency_backend = (res.start() - req.end()).as_seconds_f64(), + latency_total = (res.end() - req.start()).as_seconds_f64(), + json_request = tracing::field::valuable(&json_req), + json_response = tracing::field::valuable(&json_res), + "Proxied request" + ); + } + + fn maybe_log_mirrored_request( + req: &ProxiedHttpRequest, + res: &ProxiedHttpResponse, + worker_id: Uuid, + config: &C, + ) { + let json_req = if config.log_mirrored_requests() { + Loggable(&Self::maybe_sanitise( + config.log_sanitise(), + serde_json::from_slice(&req.decompressed_body).unwrap_or_default(), + )) + } else { + Loggable(&serde_json::Value::Null) + }; + + let json_res = if config.log_mirrored_responses() { + Loggable(&Self::maybe_sanitise( + config.log_sanitise(), + serde_json::from_slice(&res.decompressed_body).unwrap_or_default(), + )) + } else { + Loggable(&serde_json::Value::Null) + }; + + info!( + proxy = P::name(), + request_id = %req.info.id, + connection_id = %req.info.connection_id, + worker_id = %worker_id, + jrpc_method = %req.info.jrpc_method_enriched, + http_status = res.status(), + remote_addr = req.info().remote_addr, + ts_request_received = req.start().format(&Iso8601::DEFAULT).unwrap_or_default(), + latency_backend = (res.start() - req.end()).as_seconds_f64(), + latency_total = (res.end() - req.start()).as_seconds_f64(), + json_request = tracing::field::valuable(&json_req), + json_response = tracing::field::valuable(&json_res), + "Mirrored request" + ); + } + + fn maybe_sanitise(do_sanitise: bool, mut message: serde_json::Value) -> serde_json::Value { + if do_sanitise { + sanitise(&mut message); + } + return message; + + fn sanitise(message: &mut serde_json::Value) { + if let Some(batch) = message.as_array_mut() { + for item in batch { + sanitise(item); + } + return; + } + + let message = match message.as_object_mut() { + Some(message) => message, + None => return, + }; + + let method = (match message.get_key_value("method") { + Some((_, method)) => method.as_str(), + None => None, + }) + .unwrap_or_default() + .to_owned(); + + if !method.is_empty() { + // single-shot request + + let params = match match message.get_mut("params") { + Some(params) => params, + None => return, + } + .as_array_mut() + { + Some(params) => params, + None => return, + }; + + match method.as_str() { + "engine_forkchoiceUpdatedV3" => { + if params.len() < 2 { + return; + } + + let execution_payload = match params[1].as_object_mut() { + Some(execution_payload) => execution_payload, + None => return, + }; + + let transactions = match match execution_payload.get_mut("transactions") { + Some(transactions) => transactions, + None => return, + } + .as_array_mut() + { + Some(transactions) => transactions, + None => return, + }; + + for transaction in transactions { + raw_transaction_to_hash(transaction); + } + } + + "engine_newPayloadV4" => { + if params.is_empty() { + return; + } + + let execution_payload = match params[0].as_object_mut() { + Some(execution_payload) => execution_payload, + None => return, + }; + + let transactions = match match execution_payload.get_mut("transactions") { + Some(transactions) => transactions, + None => return, + } + .as_array_mut() + { + Some(transactions) => transactions, + None => return, + }; + + for transaction in transactions { + raw_transaction_to_hash(transaction); + } + } + + "eth_sendBundle" => { + if params.is_empty() { + return; + } + + let execution_payload = match params[0].as_object_mut() { + Some(execution_payload) => execution_payload, + None => return, + }; + + let transactions = match match execution_payload.get_mut("txs") { + Some(transactions) => transactions, + None => return, + } + .as_array_mut() + { + Some(transactions) => transactions, + None => return, + }; + + for transaction in transactions { + raw_transaction_to_hash(transaction); + } + } + + "eth_sendRawTransaction" => { + for transaction in params { + raw_transaction_to_hash(transaction); + } + } + + _ => { + return; + } + } + } + + let result = match match message.get_mut("result") { + Some(result) => result.as_object_mut(), + None => return, + } { + Some(result) => result, + None => return, + }; + + if let Some(execution_payload) = result.get_mut("executionPayload") && + let Some(transactions) = execution_payload.get_mut("transactions") && + let Some(transactions) = transactions.as_array_mut() + { + // engine_getPayloadV4 + + for transaction in transactions { + raw_transaction_to_hash(transaction); + } + } + } + } + + fn emit_metrics_on_proxy_success( + jrpc: &JrpcRequestMetaMaybeBatch, + req: &ProxiedHttpRequest, + res: &ProxiedHttpResponse, + metrics: Arc, + ) { + let metric_labels_jrpc = match jrpc { + JrpcRequestMetaMaybeBatch::Single(jrpc) => { + LabelsProxyHttpJrpc { jrpc_method: jrpc.method_enriched(), proxy: P::name() } + } + + JrpcRequestMetaMaybeBatch::Batch(_) => { + LabelsProxyHttpJrpc { jrpc_method: Cow::Borrowed("batch"), proxy: P::name() } + } + }; + + let latency_backend = 1000000.0 * (res.start() - req.end()).as_seconds_f64(); + let latency_total = 1000000.0 * (res.end() - req.start()).as_seconds_f64(); + + // latency_backend + metrics + .http_latency_backend + .get_or_create(&metric_labels_jrpc) + .record(latency_backend.round() as i64); + + // latency_delta + metrics + .http_latency_delta + .get_or_create(&metric_labels_jrpc) + .record((latency_total - latency_backend).round() as i64); + + // latency_total + metrics + .http_latency_total + .get_or_create(&metric_labels_jrpc) + .record(latency_total.round() as i64); + + // proxy_success_count + match jrpc { + JrpcRequestMetaMaybeBatch::Single(_) => { + metrics.http_proxy_success_count.get_or_create(&metric_labels_jrpc).inc(); + } + + JrpcRequestMetaMaybeBatch::Batch(batch) => { + for jrpc in batch.iter() { + let metric_labels_jrpc = LabelsProxyHttpJrpc { + jrpc_method: jrpc.method_enriched(), + proxy: P::name(), + }; + metrics.http_proxy_success_count.get_or_create(&metric_labels_jrpc).inc(); + } + } + } + + // proxied_request_size + metrics.http_request_size.get_or_create_owned(&metric_labels_jrpc).record(req.size as i64); + + // proxied_response_size + metrics.http_response_size.get_or_create_owned(&metric_labels_jrpc).record(res.size as i64); + + // proxied_request_decompressed_size + metrics + .http_request_decompressed_size + .get_or_create_owned(&metric_labels_jrpc) + .record(req.decompressed_size as i64); + + // proxied_response_decompressed_size + metrics + .http_response_decompressed_size + .get_or_create_owned(&metric_labels_jrpc) + .record(res.decompressed_size as i64); + } +} + +impl Proxy

for ProxyHttp +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ +} + +impl Drop for ProxyHttp +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + fn drop(&mut self) { + debug!( + proxy = P::name(), + worker_id = %self.id, + "Destroying http-proxy worker...", + ); + } +} + +// ProxyHttpSharedState ------------------------------------------------ + +#[derive(Clone)] +struct ProxyHttpSharedState +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + inner: Arc

, + metrics: Arc, + + client_connections_count: Arc, + + _config: PhantomData, +} + +impl ProxyHttpSharedState +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + fn new(config: C, metrics: &Arc) -> Self { + Self { + inner: Arc::new(P::new(config)), + metrics: metrics.clone(), + client_connections_count: Arc::new(AtomicI64::new(0)), + _config: PhantomData, + } + } + + #[inline] + fn config(&self) -> &C { + self.inner.config() + } + + #[inline] + fn inner(&self) -> Arc

{ + self.inner.clone() + } +} + +// ProxyHttpPostprocessor ---------------------------------------------- + +struct ProxyHttpPostprocessor +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + inner: Arc

, + worker_id: Uuid, + metrics: Arc, + + /// mirroring_peers is the vector of endpoints for mirroring peers. + mirroring_peers: Arc>>>, + + /// mirroring_peer_round_robin_index is used for round-robin mirroring + /// strategy. it holds the index of the mirroring peers that will be + /// used for the next round of mirroring. + mirroring_peer_round_robin_index: AtomicUsize, +} + +impl actix::Actor for ProxyHttpPostprocessor +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(1024); + } +} + +impl actix::Handler for ProxyHttpPostprocessor +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + type Result = (); + + fn handle(&mut self, msg: ProxiedHttpCombo, ctx: &mut Self::Context) -> Self::Result { + let inner = self.inner.clone(); + let metrics = self.metrics.clone(); + let worker_id = self.worker_id; + let mirroring_peers = self.mirroring_peers.clone(); + let mut mirroring_peer_round_robin_index = + self.mirroring_peer_round_robin_index.load(Ordering::Relaxed); + + ctx.spawn( + async move { + ProxyHttp::::finalise_proxying( + msg.req, + msg.res, + inner, + worker_id, + metrics, + mirroring_peers, + mirroring_peer_round_robin_index, + ); + } + .into_actor(self), + ); + + mirroring_peer_round_robin_index += 1; + if mirroring_peer_round_robin_index >= self.mirroring_peers.len() { + mirroring_peer_round_robin_index = 0; + } + self.mirroring_peer_round_robin_index + .store(mirroring_peer_round_robin_index, Ordering::Relaxed); + } +} + +// ProxyHttpBackendEndpoint -------------------------------------------- + +pub(crate) struct ProxyHttpBackendEndpoint +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + inner: Arc

, + worker_id: Uuid, + metrics: Arc, + + client: Client, + url: Url, + + _config: PhantomData, +} + +impl ProxyHttpBackendEndpoint +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + fn new( + inner: Arc

, + worker_id: Uuid, + metrics: Arc, + url: Url, + connections_limit: usize, + timeout: std::time::Duration, + ) -> Self { + let host = url + .host() + .unwrap() // safety: verified on start + .to_string(); + + let client = Client::builder() + .add_default_header((header::HOST, host)) + .connector(Connector::new().conn_keep_alive(2 * timeout).limit(connections_limit)) + .timeout(timeout) + .finish(); + + Self { inner, worker_id, metrics, client, url, _config: PhantomData } + } + + fn new_backend_request(&self, info: &ProxyHttpRequestInfo) -> ClientRequest { + let mut url = self.url.clone(); + url.set_path(&info.path); + + let mut req = self.client.request(info.method.clone(), url.as_str()).no_decompress(); + + for (header, value) in info.headers.iter() { + req = req.insert_header((header.clone(), value.clone())); + } + + req + } +} + +impl actix::Actor for ProxyHttpBackendEndpoint +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(1024); + } +} + +impl actix::Handler for ProxyHttpBackendEndpoint +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + type Result = (); + + fn handle(&mut self, cli_req: ProxiedHttpRequest, ctx: &mut Self::Context) -> Self::Result { + let start = UtcDateTime::now(); + + let inner = self.inner.clone(); + let worker_id = self.worker_id; + let metrics = self.metrics.clone(); + + let mrr_req = self.new_backend_request(&cli_req.info); + let mrr_req_body = cli_req.body.clone(); + + ctx.spawn( + async move { + match mrr_req.send_body(mrr_req_body).await { + Ok(mut bck_res) => { + let end = UtcDateTime::now(); + + match bck_res.body().await { + Ok(mrr_res_body) => { + let size = match mrr_res_body.size() { + BodySize::Sized(size) => size, // Body is always sized + BodySize::None => 0, + BodySize::Stream => 0, + }; + let info = ProxyHttpResponseInfo::new( + cli_req.info.id, + bck_res.status(), + bck_res.headers().clone(), + ); + let mrr_res = ProxiedHttpResponse { + info, + body: mrr_res_body, + size: size as usize, + decompressed_body: Bytes::new(), + decompressed_size: 0, + start, + end, + }; + ProxyHttp::::postprocess_mirrored_response( + cli_req, mrr_res, inner, metrics, worker_id, + ); + } + Err(err) => { + warn!( + proxy = P::name(), + request_id = %cli_req.info.id, + connection_id = %cli_req.info.connection_id, + error = ?err, + "Failed to mirror a request", + ); + metrics + .http_mirror_failure_count + .get_or_create(&LabelsProxy { proxy: P::name() }) + .inc(); + } + }; + } + + Err(err) => { + warn!( + proxy = P::name(), + request_id = %cli_req.info.id, + connection_id = %cli_req.info.connection_id, + error = ?err, + "Failed to mirror a request", + ); + metrics + .http_mirror_failure_count + .get_or_create(&LabelsProxy { proxy: P::name() }) + .inc(); + } + } + } + .into_actor(self), + ); + } +} + +// ProxyHttpRequestInfo ------------------------------------------------ + +#[derive(Clone)] +pub(crate) struct ProxyHttpRequestInfo { + id: Uuid, + connection_id: Uuid, + remote_addr: Option, + method: Method, + path: String, + path_and_query: String, + headers: HeaderMap, + jrpc_method_enriched: Cow<'static, str>, +} + +impl ProxyHttpRequestInfo { + pub(crate) fn new(req: &HttpRequest, guard: Option<&ProxyConnectionGuard>) -> Self { + // copy over only non hop-by-hop headers + let mut headers = HeaderMap::new(); + for (header, value) in req.headers().iter() { + if !is_hop_by_hop_header(header) { + headers.insert(header.clone(), value.clone()); + } + } + + // append remote ip to x-forwarded-for + if let Some(peer_addr) = req.connection_info().peer_addr() { + let mut forwarded_for = String::new(); + if let Some(ff) = req.headers().get(header::X_FORWARDED_FOR) && + let Ok(ff) = ff.to_str() + { + forwarded_for.push_str(ff); + forwarded_for.push_str(", "); + } + forwarded_for.push_str(peer_addr); + if let Ok(forwarded_for) = HeaderValue::from_str(&forwarded_for) { + headers.insert(header::X_FORWARDED_FOR, forwarded_for); + } + } + + // set x-forwarded-proto if it's not already set + if req.connection_info().scheme() != "" && + req.headers().get(header::X_FORWARDED_PROTO).is_none() && + let Ok(forwarded_proto) = HeaderValue::from_str(req.connection_info().scheme()) + { + headers.insert(header::X_FORWARDED_PROTO, forwarded_proto); + } + + // set x-forwarded-host if it's not already set + if req.connection_info().scheme() != "" && + req.headers().get(header::X_FORWARDED_HOST).is_none() && + let Ok(forwarded_host) = HeaderValue::from_str(req.connection_info().scheme()) + { + headers.insert(header::X_FORWARDED_HOST, forwarded_host); + } + + // remote address from the guard has port, and connection info has ip + // address only => we prefer the guard + let remote_addr = match guard { + Some(guard) => match guard.remote_addr.clone() { + Some(remote_addr) => Some(remote_addr), + None => req.connection_info().peer_addr().map(String::from), + }, + None => req.connection_info().peer_addr().map(String::from), + }; + + let path = match req.path() { + "" => "/", + val => val, + } + .to_string(); + + let path_and_query = match req.query_string() { + "" => path.clone(), + val => format!("{}?{}", path, val), + }; + + Self { + id: Uuid::now_v7(), + connection_id: Uuid::now_v7(), + remote_addr, + method: req.method().clone(), + path, + path_and_query, + headers, + jrpc_method_enriched: Cow::Borrowed(""), + } + } + + #[inline] + pub(crate) fn id(&self) -> Uuid { + self.id + } + + #[inline] + pub(crate) fn connection_id(&self) -> Uuid { + self.connection_id + } + + #[inline] + fn content_encoding(&self) -> String { + self.headers + .get(header::CONTENT_ENCODING) + .map(|h| h.to_str().unwrap_or_default()) + .map(|h| h.to_string()) + .unwrap_or_default() + } + + #[inline] + pub fn path_and_query(&self) -> &str { + &self.path_and_query + } + + #[inline] + pub fn remote_addr(&self) -> &Option { + &self.remote_addr + } +} + +// ProxyHttpResponseInfo ----------------------------------------------- + +#[derive(Clone)] +pub(crate) struct ProxyHttpResponseInfo { + id: Uuid, + status: StatusCode, + headers: HeaderMap, // TODO: perhaps we don't need all headers, just select ones +} + +impl ProxyHttpResponseInfo { + pub(crate) fn new(id: Uuid, status: StatusCode, headers: HeaderMap) -> Self { + Self { id, status, headers } + } + + #[inline] + pub(crate) fn id(&self) -> Uuid { + self.id + } + + fn content_encoding(&self) -> String { + self.headers + .get(header::CONTENT_ENCODING) + .map(|h| h.to_str().unwrap_or_default()) + .map(|h| h.to_string()) + .unwrap_or_default() + } +} + +// ProxyHttpRequestBody ------------------------------------------------ + +#[pin_project] +struct ProxyHttpRequestBody +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + proxy: web::Data>, + + info: Option, + start: UtcDateTime, + body: Vec, + + #[pin] + stream: S, +} + +impl ProxyHttpRequestBody +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + fn new( + worker: web::Data>, + info: ProxyHttpRequestInfo, + body: S, + timestamp: UtcDateTime, + ) -> Self { + Self { + proxy: worker, + info: Some(info), + stream: body, + start: timestamp, + body: Vec::new(), // TODO: preallocate reasonable size + } + } +} + +impl Stream for ProxyHttpRequestBody +where + S: Stream>, + E: Debug, + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); + + match this.stream.poll_next(cx) { + Poll::Pending => Poll::Pending, + + Poll::Ready(Some(Ok(chunk))) => { + this.body.extend_from_slice(&chunk); + Poll::Ready(Some(Ok(chunk))) + } + + Poll::Ready(Some(Err(err))) => { + if let Some(info) = mem::take(this.info) { + warn!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + error = ?err, + "Proxy http request stream error", + ); + } else { + warn!( + proxy = P::name(), + error = ?err, + request_id = "unknown", + "Proxy http request stream error", + ); + } + Poll::Ready(Some(Err(err))) + } + + Poll::Ready(None) => { + let end = UtcDateTime::now(); + + if let Some(info) = mem::take(this.info) { + let proxy = this.proxy.clone(); + + let req = ProxiedHttpRequest::new(info, mem::take(this.body), *this.start, end); + + proxy.postprocess_client_request(req); + } + + Poll::Ready(None) + } + } + } +} + +// ProxyHttpResponseBody ----------------------------------------------- + +#[pin_project] +struct ProxyHttpResponseBody +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + proxy: web::Data>, + + info: Option, + start: UtcDateTime, + body: Vec, + + #[pin] + stream: S, +} + +impl ProxyHttpResponseBody +where + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + fn new( + proxy: web::Data>, + id: Uuid, + status: StatusCode, + headers: HeaderMap, + body: S, + timestamp: UtcDateTime, + ) -> Self { + Self { + proxy, + stream: body, + start: timestamp, + body: Vec::new(), // TODO: preallocate reasonable size + info: Some(ProxyHttpResponseInfo::new(id, status, headers)), + } + } +} + +impl Stream for ProxyHttpResponseBody +where + S: Stream>, + E: Debug, + C: ConfigProxyHttp, + P: ProxyHttpInner, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); + + match this.stream.poll_next(cx) { + Poll::Pending => Poll::Pending, + + Poll::Ready(Some(Ok(chunk))) => { + this.body.extend_from_slice(&chunk); + Poll::Ready(Some(Ok(chunk))) + } + + Poll::Ready(Some(Err(err))) => { + if let Some(info) = mem::take(this.info) { + warn!( + proxy = P::name(), + request_id = %info.id(), + error = ?err, + "Proxy http response stream error", + ); + } else { + warn!( + proxy = P::name(), + error = ?err, + request_id = "unknown", + "Proxy http response stream error", + ); + } + Poll::Ready(Some(Err(err))) + } + + Poll::Ready(None) => { + let end = UtcDateTime::now(); + + if let Some(info) = mem::take(this.info) { + let proxy = this.proxy.clone(); + + let res = + ProxiedHttpResponse::new(info, mem::take(this.body), *this.start, end); + + proxy.postprocess_backend_response(res); + } + + Poll::Ready(None) + } + } + } +} + +// ProxiedHttpRequest -------------------------------------------------- + +#[derive(Clone, actix::Message)] +#[rtype(result = "()")] +pub(crate) struct ProxiedHttpRequest { + info: ProxyHttpRequestInfo, + body: Bytes, + size: usize, + decompressed_body: Bytes, + decompressed_size: usize, + start: UtcDateTime, + end: UtcDateTime, +} + +impl ProxiedHttpRequest { + pub(crate) fn new( + info: ProxyHttpRequestInfo, + body: Vec, + start: UtcDateTime, + end: UtcDateTime, + ) -> Self { + let size = body.len(); + Self { + info, + body: Bytes::from(body), + size, + decompressed_body: Bytes::new(), + decompressed_size: 0, + start, + end, + } + } + + #[inline] + pub(crate) fn info(&self) -> &ProxyHttpRequestInfo { + &self.info + } + + #[inline] + pub(crate) fn start(&self) -> UtcDateTime { + self.start + } + + #[inline] + pub(crate) fn end(&self) -> UtcDateTime { + self.end + } +} + +// ProxiedHttpResponse ------------------------------------------------- + +#[derive(Clone, actix::Message)] +#[rtype(result = "()")] +pub(crate) struct ProxiedHttpResponse { + info: ProxyHttpResponseInfo, + body: Bytes, + size: usize, + decompressed_body: Bytes, + decompressed_size: usize, + start: UtcDateTime, + end: UtcDateTime, +} + +impl ProxiedHttpResponse { + pub(crate) fn new( + info: ProxyHttpResponseInfo, + body: Vec, + start: UtcDateTime, + end: UtcDateTime, + ) -> Self { + let size = body.len(); + Self { + info, + body: Bytes::from(body), + size, + decompressed_body: Bytes::new(), + decompressed_size: 0, + start, + end, + } + } + + #[inline] + pub(crate) fn status(&self) -> &str { + self.info.status.as_str() + } + + #[inline] + pub(crate) fn decompressed_body(&self) -> Bytes { + self.decompressed_body.clone() + } + + #[inline] + pub(crate) fn start(&self) -> UtcDateTime { + self.start + } + + #[inline] + pub(crate) fn end(&self) -> UtcDateTime { + self.end + } +} + +// ProxiedHttpCombo ---------------------------------------------------- + +#[derive(Clone, actix::Message)] +#[rtype(result = "()")] +struct ProxiedHttpCombo { + req: ProxiedHttpRequest, + res: ProxiedHttpResponse, +} diff --git a/crates/rproxy/src/proxy_http/proxy_http.rs b/crates/rproxy/src/proxy_http/proxy_http.rs deleted file mode 100644 index 072bd25..0000000 --- a/crates/rproxy/src/proxy_http/proxy_http.rs +++ /dev/null @@ -1,1523 +0,0 @@ -use std::{ - borrow::Cow, - fmt::Debug, - marker::PhantomData, - mem, - pin::Pin, - str::FromStr, - sync::{ - Arc, - atomic::{AtomicI64, AtomicUsize, Ordering}, - }, - task::{Context, Poll}, -}; - -use actix::{Actor, AsyncContext, WrapFuture}; -use actix_web::{ - self, - App, - HttpRequest, - HttpResponse, - HttpResponseBuilder, - HttpServer, - body::BodySize, - http::{StatusCode, header}, - middleware::{NormalizePath, TrailingSlash}, - web, -}; -use awc::{ - Client, - ClientRequest, - ClientResponse, - Connector, - body::MessageBody, - error::HeaderValue, - http::{Method, header::HeaderMap}, -}; -use bytes::Bytes; -use futures::TryStreamExt; -use futures_core::Stream; -use pin_project::pin_project; -use scc::HashMap; -use time::{UtcDateTime, format_description::well_known::Iso8601}; -use tokio::sync::broadcast; -use tracing::{debug, error, info, warn}; -use url::Url; -use uuid::Uuid; -use x509_parser::asn1_rs::ToStatic; - -use crate::{ - config::{ConfigProxyHttp, ConfigProxyHttpMirroringStrategy, ConfigTls, PARALLELISM}, - jrpc::JrpcRequestMetaMaybeBatch, - metrics::{LabelsProxy, LabelsProxyClientInfo, LabelsProxyHttpJrpc, Metrics}, - proxy::{Proxy, ProxyConnectionGuard}, - proxy_http::ProxyHttpInner, - utils::{Loggable, decompress, is_hop_by_hop_header, raw_transaction_to_hash}, -}; - -const TCP_KEEPALIVE_ATTEMPTS: u32 = 8; - -// ProxyHttp ----------------------------------------------------------- - -pub(crate) struct ProxyHttp -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - id: Uuid, - - shared: ProxyHttpSharedState, - - backend: ProxyHttpBackendEndpoint, - requests: HashMap, - postprocessor: actix::Addr>, -} - -impl ProxyHttp -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - fn new(shared: ProxyHttpSharedState, connections_limit: usize) -> Self { - let id = Uuid::now_v7(); - - debug!(proxy = P::name(), worker_id = %id, "Creating http-proxy worker..."); - - let config = shared.config(); - let inner = shared.inner(); - - let backend = ProxyHttpBackendEndpoint::new( - inner.clone(), - id, - shared.metrics.clone(), - config.backend_url(), - connections_limit, - config.backend_timeout(), - ); - - let peers: Arc>>> = Arc::new( - config - .mirroring_peer_urls() - .iter() - .map(|peer_url| { - ProxyHttpBackendEndpoint::new( - shared.inner(), - id, - shared.metrics.clone(), - peer_url.to_owned(), - config.backend_max_concurrent_requests(), - config.backend_timeout(), - ) - .start() - }) - .collect(), - ); - - let postprocessor = ProxyHttpPostprocessor:: { - worker_id: id, - inner: inner.clone(), - metrics: shared.metrics.clone(), - mirroring_peers: peers.clone(), - mirroring_peer_round_robin_index: AtomicUsize::new(0), - } - .start(); - - Self { id, shared, backend, requests: HashMap::default(), postprocessor } - } - - pub(crate) async fn run( - config: C, - tls: ConfigTls, - metrics: Arc, - canceller: tokio_util::sync::CancellationToken, - resetter: broadcast::Sender<()>, - ) -> Result<(), Box> { - let listen_address = config.listen_address(); - - let listener = match Self::listen(&config) { - Ok(listener) => listener, - Err(err) => { - error!( - proxy = P::name(), - addr = %config.listen_address(), - error = ?err, - "Failed to initialise a socket" - ); - return Err(Box::new(err)); - } - }; - - let workers_count = - std::cmp::min(PARALLELISM.to_static(), config.backend_max_concurrent_requests()); - let max_concurrent_requests_per_worker = - config.backend_max_concurrent_requests() / workers_count; - if workers_count * max_concurrent_requests_per_worker < - config.backend_max_concurrent_requests() - { - warn!( - "Max backend concurrent requests must be a round of available parallelism ({}), therefore it's clamped at {} (instead of {})", - PARALLELISM.to_static(), - workers_count * max_concurrent_requests_per_worker, - config.backend_max_concurrent_requests() - ); - } - - let shared = ProxyHttpSharedState::::new(config, &metrics); - let client_connections_count = shared.client_connections_count.clone(); - - info!( - proxy = P::name(), - listen_address = %listen_address, - workers_count = workers_count, - max_concurrent_requests_per_worker = max_concurrent_requests_per_worker, - "Starting http-proxy..." - ); - - let proxy = HttpServer::new(move || { - let this = - web::Data::new(Self::new(shared.clone(), max_concurrent_requests_per_worker)); - - App::new() - .app_data(this) - .wrap(NormalizePath::new(TrailingSlash::Trim)) - .default_service(web::route().to(Self::receive)) - }) - .on_connect(Self::on_connect(metrics, client_connections_count)) - .shutdown_signal(canceller.cancelled_owned()) - .workers(workers_count); - - let server = match if tls.enabled() { - let cert = tls.certificate().clone(); - let key = tls.key().clone_key(); - - proxy.listen_rustls_0_23( - listener, - rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(cert, key) - .unwrap(), // safety: verified on start - ) - } else { - proxy.listen(listener) - } { - Ok(server) => server, - Err(err) => { - error!( - proxy = P::name(), - error = ?err, - "Failed to initialise http-proxy", - ); - return Err(Box::new(err)); - } - } - .run(); - - let handler = server.handle(); - let mut resetter = resetter.subscribe(); - tokio::spawn(async move { - if resetter.recv().await.is_ok() { - info!(proxy = P::name(), "Reset signal received, stopping http-proxy..."); - handler.stop(true).await; - } - }); - - if let Err(err) = server.await { - error!(proxy = P::name(), error = ?err, "Failure while running http-proxy") - } - - info!(proxy = P::name(), "Stopped http-proxy"); - - Ok(()) - } - - fn listen(config: &C) -> std::io::Result { - let socket = socket2::Socket::new( - socket2::Domain::for_address(config.listen_address()), - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - - // must use non-blocking with tokio - socket.set_nonblocking(true)?; - - // allow time to flush buffers on close - socket.set_linger(Some(config.backend_timeout()))?; - - // allow binding while there are still residual connections in TIME_WAIT - socket.set_reuse_address(true)?; - - if !config.idle_connection_timeout().is_zero() { - socket.set_tcp_keepalive( - &socket2::TcpKeepalive::new() - .with_time( - config.idle_connection_timeout().div_f64(f64::from(TCP_KEEPALIVE_ATTEMPTS)), - ) - .with_interval( - config.idle_connection_timeout().div_f64(f64::from(TCP_KEEPALIVE_ATTEMPTS)), - ) - .with_retries(TCP_KEEPALIVE_ATTEMPTS - 1), - )?; - } - - socket.bind(&socket2::SockAddr::from(config.listen_address()))?; - - socket.listen(1024)?; - - Ok(socket.into()) - } - - fn to_client_response(bck_res: &ClientResponse) -> HttpResponseBuilder { - let mut cli_res = HttpResponse::build(bck_res.status()); - - for (name, header) in bck_res.headers().iter() { - if is_hop_by_hop_header(name) { - continue; - } - if let Ok(hname) = header::HeaderName::from_str(name.as_str()) { - cli_res.append_header((hname, header.clone())); - } - } - - cli_res - } - - /// receive accepts client's (frontend) request and proxies it to - /// backend - async fn receive( - cli_req: HttpRequest, - cli_req_body: web::Payload, - this: web::Data, - ) -> Result { - let timestamp = UtcDateTime::now(); - - if let Some(user_agent) = cli_req.headers().get(header::USER_AGENT) && - !user_agent.is_empty() && - let Ok(user_agent) = user_agent.to_str() - { - this.shared - .metrics - .client_info - .get_or_create(&LabelsProxyClientInfo { - proxy: P::name(), - user_agent: user_agent.to_string(), - }) - .inc(); - } - - let info = ProxyHttpRequestInfo::new(&cli_req, cli_req.conn_data::()); - - let id = info.id; - let connection_id = info.connection_id; - - let bck_req = this.backend.new_backend_request(&info); - let bck_req_body = ProxyHttpRequestBody::new(this.clone(), info, cli_req_body, timestamp); - - let bck_res = match bck_req.send_stream(bck_req_body).await { - Ok(res) => res, - Err(err) => { - warn!( - proxy = P::name(), - request_id = %id, - connection_id = %connection_id, - worker_id = %this.id, - backend_url = %this.backend.url, - error = ?err, - "Failed to proxy a request", - ); - this.shared - .metrics - .http_proxy_failure_count - .get_or_create(&LabelsProxy { proxy: P::name() }) - .inc(); - return Ok(HttpResponse::BadGateway().body(format!("Backend error: {:?}", err))); - } - }; - - let timestamp = UtcDateTime::now(); - let status = bck_res.status(); - let mut cli_res = Self::to_client_response(&bck_res); - - let bck_body = ProxyHttpResponseBody::new( - this, - id, - status, - bck_res.headers().clone(), - bck_res.into_stream(), - timestamp, - ); - - Ok(cli_res.streaming(bck_body)) - } - - fn postprocess_client_request(&self, req: ProxiedHttpRequest) { - let id = req.info.id; - let connection_id = req.info.connection_id; - - if self.requests.insert_sync(id, req).is_err() { - error!( - proxy = P::name(), - request_id = %id, - connection_id = %connection_id, - worker_id = %self.id, - "Duplicate request id", - ); - }; - } - - fn postprocess_backend_response(&self, bck_res: ProxiedHttpResponse) { - let cli_req = match self.requests.remove_sync(&bck_res.info.id) { - Some((_, req)) => req, - None => { - error!( - proxy = P::name(), - request_id = %bck_res.info.id, - worker_id = %self.id, - "Proxied http response for unmatching request", - ); - return; - } - }; - - // hand over to postprocessor asynchronously so that we can return the - // response to the client as early as possible - self.postprocessor.do_send(ProxiedHttpCombo { req: cli_req, res: bck_res }); - } - - fn finalise_proxying( - mut cli_req: ProxiedHttpRequest, - mut bck_res: ProxiedHttpResponse, - inner: Arc

, - worker_id: Uuid, - metrics: Arc, - mirroring_peers: Arc>>>, - mut mirroring_peer_round_robin_index: usize, - ) { - if cli_req.decompressed_size < cli_req.size { - (cli_req.decompressed_body, cli_req.decompressed_size) = - decompress(cli_req.body.clone(), cli_req.size, cli_req.info.content_encoding()); - } - - if bck_res.decompressed_size < bck_res.size { - (bck_res.decompressed_body, bck_res.decompressed_size) = - decompress(bck_res.body.clone(), bck_res.size, bck_res.info.content_encoding()); - } - - match serde_json::from_slice::(&cli_req.decompressed_body) { - Ok(jrpc) => { - if inner.should_mirror(&jrpc, &cli_req, &bck_res) { - let mirrors_count = match inner.config().mirroring_strategy() { - ConfigProxyHttpMirroringStrategy::FanOut => mirroring_peers.len(), - ConfigProxyHttpMirroringStrategy::RoundRobin => 1, - ConfigProxyHttpMirroringStrategy::RoundRobinPairs => 2, - }; - - for _ in 0..mirrors_count { - let mirroring_peer = &mirroring_peers[mirroring_peer_round_robin_index]; - mirroring_peer_round_robin_index += 1; - if mirroring_peer_round_robin_index >= mirroring_peers.len() { - mirroring_peer_round_robin_index = 0; - } - - let mut req = cli_req.clone(); - req.info.jrpc_method_enriched = jrpc.method_enriched(); - mirroring_peer.do_send(req.clone()); - } - } - - Self::maybe_log_proxied_request_and_response( - &jrpc, - &cli_req, - &bck_res, - inner.clone(), - worker_id, - ); - - Self::emit_metrics_on_proxy_success(&jrpc, &cli_req, &bck_res, metrics.clone()); - } - - Err(err) => { - warn!( - proxy = P::name(), - request_id = %cli_req.info.id, - connection_id = %cli_req.info.connection_id, - worker_id = %worker_id, - error = ?err, - "Failed to parse json-rpc request", - ); - } - } - } - - fn postprocess_mirrored_response( - mut cli_req: ProxiedHttpRequest, - mut mrr_res: ProxiedHttpResponse, - inner: Arc

, - metrics: Arc, - worker_id: Uuid, - ) { - if cli_req.decompressed_size < cli_req.size { - (cli_req.decompressed_body, cli_req.decompressed_size) = - decompress(cli_req.body.clone(), cli_req.size, cli_req.info.content_encoding()); - } - - if mrr_res.decompressed_size < mrr_res.size { - (mrr_res.decompressed_body, mrr_res.decompressed_size) = - decompress(mrr_res.body.clone(), mrr_res.size, mrr_res.info.content_encoding()); - } - - Self::maybe_log_mirrored_request(&cli_req, &mrr_res, worker_id, inner.config()); - - metrics - .http_mirror_success_count - .get_or_create(&LabelsProxyHttpJrpc { - proxy: P::name(), - jrpc_method: cli_req.info.jrpc_method_enriched, - }) - .inc(); - } - - fn maybe_log_proxied_request_and_response( - jrpc: &JrpcRequestMetaMaybeBatch, - req: &ProxiedHttpRequest, - res: &ProxiedHttpResponse, - inner: Arc

, - worker_id: Uuid, - ) { - let config = inner.config(); - - let json_req = if config.log_proxied_requests() { - Loggable(&Self::maybe_sanitise( - config.log_sanitise(), - serde_json::from_slice(&req.decompressed_body).unwrap_or_default(), - )) - } else { - Loggable(&serde_json::Value::Null) - }; - - let json_res = if config.log_proxied_responses() { - Loggable(&Self::maybe_sanitise( - config.log_sanitise(), - serde_json::from_slice(&res.decompressed_body).unwrap_or_default(), - )) - } else { - Loggable(&serde_json::Value::Null) - }; - - info!( - proxy = P::name(), - request_id = %req.info.id, - connection_id = %req.info.connection_id, - worker_id = %worker_id, - jrpc_method = %jrpc.method_enriched(), - http_status = res.status(), - remote_addr = req.info().remote_addr, - ts_request_received = req.start().format(&Iso8601::DEFAULT).unwrap_or_default(), - latency_backend = (res.start() - req.end()).as_seconds_f64(), - latency_total = (res.end() - req.start()).as_seconds_f64(), - json_request = tracing::field::valuable(&json_req), - json_response = tracing::field::valuable(&json_res), - "Proxied request" - ); - } - - fn maybe_log_mirrored_request( - req: &ProxiedHttpRequest, - res: &ProxiedHttpResponse, - worker_id: Uuid, - config: &C, - ) { - let json_req = if config.log_mirrored_requests() { - Loggable(&Self::maybe_sanitise( - config.log_sanitise(), - serde_json::from_slice(&req.decompressed_body).unwrap_or_default(), - )) - } else { - Loggable(&serde_json::Value::Null) - }; - - let json_res = if config.log_mirrored_responses() { - Loggable(&Self::maybe_sanitise( - config.log_sanitise(), - serde_json::from_slice(&res.decompressed_body).unwrap_or_default(), - )) - } else { - Loggable(&serde_json::Value::Null) - }; - - info!( - proxy = P::name(), - request_id = %req.info.id, - connection_id = %req.info.connection_id, - worker_id = %worker_id, - jrpc_method = %req.info.jrpc_method_enriched, - http_status = res.status(), - remote_addr = req.info().remote_addr, - ts_request_received = req.start().format(&Iso8601::DEFAULT).unwrap_or_default(), - latency_backend = (res.start() - req.end()).as_seconds_f64(), - latency_total = (res.end() - req.start()).as_seconds_f64(), - json_request = tracing::field::valuable(&json_req), - json_response = tracing::field::valuable(&json_res), - "Mirrored request" - ); - } - - fn maybe_sanitise(do_sanitise: bool, mut message: serde_json::Value) -> serde_json::Value { - if do_sanitise { - sanitise(&mut message); - } - return message; - - fn sanitise(message: &mut serde_json::Value) { - if let Some(batch) = message.as_array_mut() { - for item in batch { - sanitise(item); - } - return; - } - - let message = match message.as_object_mut() { - Some(message) => message, - None => return, - }; - - let method = (match message.get_key_value("method") { - Some((_, method)) => method.as_str(), - None => None, - }) - .unwrap_or_default() - .to_owned(); - - if !method.is_empty() { - // single-shot request - - let params = match match message.get_mut("params") { - Some(params) => params, - None => return, - } - .as_array_mut() - { - Some(params) => params, - None => return, - }; - - match method.as_str() { - "engine_forkchoiceUpdatedV3" => { - if params.len() < 2 { - return; - } - - let execution_payload = match params[1].as_object_mut() { - Some(execution_payload) => execution_payload, - None => return, - }; - - let transactions = match match execution_payload.get_mut("transactions") { - Some(transactions) => transactions, - None => return, - } - .as_array_mut() - { - Some(transactions) => transactions, - None => return, - }; - - for transaction in transactions { - raw_transaction_to_hash(transaction); - } - } - - "engine_newPayloadV4" => { - if params.is_empty() { - return; - } - - let execution_payload = match params[0].as_object_mut() { - Some(execution_payload) => execution_payload, - None => return, - }; - - let transactions = match match execution_payload.get_mut("transactions") { - Some(transactions) => transactions, - None => return, - } - .as_array_mut() - { - Some(transactions) => transactions, - None => return, - }; - - for transaction in transactions { - raw_transaction_to_hash(transaction); - } - } - - "eth_sendBundle" => { - if params.is_empty() { - return; - } - - let execution_payload = match params[0].as_object_mut() { - Some(execution_payload) => execution_payload, - None => return, - }; - - let transactions = match match execution_payload.get_mut("txs") { - Some(transactions) => transactions, - None => return, - } - .as_array_mut() - { - Some(transactions) => transactions, - None => return, - }; - - for transaction in transactions { - raw_transaction_to_hash(transaction); - } - } - - "eth_sendRawTransaction" => { - for transaction in params { - raw_transaction_to_hash(transaction); - } - } - - _ => { - return; - } - } - } - - let result = match match message.get_mut("result") { - Some(result) => result.as_object_mut(), - None => return, - } { - Some(result) => result, - None => return, - }; - - if let Some(execution_payload) = result.get_mut("executionPayload") && - let Some(transactions) = execution_payload.get_mut("transactions") && - let Some(transactions) = transactions.as_array_mut() - { - // engine_getPayloadV4 - - for transaction in transactions { - raw_transaction_to_hash(transaction); - } - } - } - } - - fn emit_metrics_on_proxy_success( - jrpc: &JrpcRequestMetaMaybeBatch, - req: &ProxiedHttpRequest, - res: &ProxiedHttpResponse, - metrics: Arc, - ) { - let metric_labels_jrpc = match jrpc { - JrpcRequestMetaMaybeBatch::Single(jrpc) => { - LabelsProxyHttpJrpc { jrpc_method: jrpc.method_enriched(), proxy: P::name() } - } - - JrpcRequestMetaMaybeBatch::Batch(_) => { - LabelsProxyHttpJrpc { jrpc_method: Cow::Borrowed("batch"), proxy: P::name() } - } - }; - - let latency_backend = 1000000.0 * (res.start() - req.end()).as_seconds_f64(); - let latency_total = 1000000.0 * (res.end() - req.start()).as_seconds_f64(); - - // latency_backend - metrics - .http_latency_backend - .get_or_create(&metric_labels_jrpc) - .record(latency_backend.round() as i64); - - // latency_delta - metrics - .http_latency_delta - .get_or_create(&metric_labels_jrpc) - .record((latency_total - latency_backend).round() as i64); - - // latency_total - metrics - .http_latency_total - .get_or_create(&metric_labels_jrpc) - .record(latency_total.round() as i64); - - // proxy_success_count - match jrpc { - JrpcRequestMetaMaybeBatch::Single(_) => { - metrics.http_proxy_success_count.get_or_create(&metric_labels_jrpc).inc(); - } - - JrpcRequestMetaMaybeBatch::Batch(batch) => { - for jrpc in batch.iter() { - let metric_labels_jrpc = LabelsProxyHttpJrpc { - jrpc_method: jrpc.method_enriched(), - proxy: P::name(), - }; - metrics.http_proxy_success_count.get_or_create(&metric_labels_jrpc).inc(); - } - } - } - - // proxied_request_size - metrics.http_request_size.get_or_create_owned(&metric_labels_jrpc).record(req.size as i64); - - // proxied_response_size - metrics.http_response_size.get_or_create_owned(&metric_labels_jrpc).record(res.size as i64); - - // proxied_request_decompressed_size - metrics - .http_request_decompressed_size - .get_or_create_owned(&metric_labels_jrpc) - .record(req.decompressed_size as i64); - - // proxied_response_decompressed_size - metrics - .http_response_decompressed_size - .get_or_create_owned(&metric_labels_jrpc) - .record(res.decompressed_size as i64); - } -} - -impl Proxy

for ProxyHttp -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ -} - -impl Drop for ProxyHttp -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - fn drop(&mut self) { - debug!( - proxy = P::name(), - worker_id = %self.id, - "Destroying http-proxy worker...", - ); - } -} - -// ProxyHttpSharedState ------------------------------------------------ - -#[derive(Clone)] -struct ProxyHttpSharedState -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - inner: Arc

, - metrics: Arc, - - client_connections_count: Arc, - - _config: PhantomData, -} - -impl ProxyHttpSharedState -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - fn new(config: C, metrics: &Arc) -> Self { - Self { - inner: Arc::new(P::new(config)), - metrics: metrics.clone(), - client_connections_count: Arc::new(AtomicI64::new(0)), - _config: PhantomData, - } - } - - #[inline] - fn config(&self) -> &C { - self.inner.config() - } - - #[inline] - fn inner(&self) -> Arc

{ - self.inner.clone() - } -} - -// ProxyHttpPostprocessor ---------------------------------------------- - -struct ProxyHttpPostprocessor -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - inner: Arc

, - worker_id: Uuid, - metrics: Arc, - - /// mirroring_peers is the vector of endpoints for mirroring peers. - mirroring_peers: Arc>>>, - - /// mirroring_peer_round_robin_index is used for round-robin mirroring - /// strategy. it holds the index of the mirroring peers that will be - /// used for the next round of mirroring. - mirroring_peer_round_robin_index: AtomicUsize, -} - -impl actix::Actor for ProxyHttpPostprocessor -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(1024); - } -} - -impl actix::Handler for ProxyHttpPostprocessor -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - type Result = (); - - fn handle(&mut self, msg: ProxiedHttpCombo, ctx: &mut Self::Context) -> Self::Result { - let inner = self.inner.clone(); - let metrics = self.metrics.clone(); - let worker_id = self.worker_id; - let mirroring_peers = self.mirroring_peers.clone(); - let mut mirroring_peer_round_robin_index = - self.mirroring_peer_round_robin_index.load(Ordering::Relaxed); - - ctx.spawn( - async move { - ProxyHttp::::finalise_proxying( - msg.req, - msg.res, - inner, - worker_id, - metrics, - mirroring_peers, - mirroring_peer_round_robin_index, - ); - } - .into_actor(self), - ); - - mirroring_peer_round_robin_index += 1; - if mirroring_peer_round_robin_index >= self.mirroring_peers.len() { - mirroring_peer_round_robin_index = 0; - } - self.mirroring_peer_round_robin_index - .store(mirroring_peer_round_robin_index, Ordering::Relaxed); - } -} - -// ProxyHttpBackendEndpoint -------------------------------------------- - -pub(crate) struct ProxyHttpBackendEndpoint -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - inner: Arc

, - worker_id: Uuid, - metrics: Arc, - - client: Client, - url: Url, - - _config: PhantomData, -} - -impl ProxyHttpBackendEndpoint -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - fn new( - inner: Arc

, - worker_id: Uuid, - metrics: Arc, - url: Url, - connections_limit: usize, - timeout: std::time::Duration, - ) -> Self { - let host = url - .host() - .unwrap() // safety: verified on start - .to_string(); - - let client = Client::builder() - .add_default_header((header::HOST, host)) - .connector(Connector::new().conn_keep_alive(2 * timeout).limit(connections_limit)) - .timeout(timeout) - .finish(); - - Self { inner, worker_id, metrics, client, url, _config: PhantomData } - } - - fn new_backend_request(&self, info: &ProxyHttpRequestInfo) -> ClientRequest { - let mut url = self.url.clone(); - url.set_path(&info.path); - - let mut req = self.client.request(info.method.clone(), url.as_str()).no_decompress(); - - for (header, value) in info.headers.iter() { - req = req.insert_header((header.clone(), value.clone())); - } - - req - } -} - -impl actix::Actor for ProxyHttpBackendEndpoint -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(1024); - } -} - -impl actix::Handler for ProxyHttpBackendEndpoint -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - type Result = (); - - fn handle(&mut self, cli_req: ProxiedHttpRequest, ctx: &mut Self::Context) -> Self::Result { - let start = UtcDateTime::now(); - - let inner = self.inner.clone(); - let worker_id = self.worker_id; - let metrics = self.metrics.clone(); - - let mrr_req = self.new_backend_request(&cli_req.info); - let mrr_req_body = cli_req.body.clone(); - - ctx.spawn( - async move { - match mrr_req.send_body(mrr_req_body).await { - Ok(mut bck_res) => { - let end = UtcDateTime::now(); - - match bck_res.body().await { - Ok(mrr_res_body) => { - let size = match mrr_res_body.size() { - BodySize::Sized(size) => size, // Body is always sized - BodySize::None => 0, - BodySize::Stream => 0, - }; - let info = ProxyHttpResponseInfo::new( - cli_req.info.id, - bck_res.status(), - bck_res.headers().clone(), - ); - let mrr_res = ProxiedHttpResponse { - info, - body: mrr_res_body, - size: size as usize, - decompressed_body: Bytes::new(), - decompressed_size: 0, - start, - end, - }; - ProxyHttp::::postprocess_mirrored_response( - cli_req, mrr_res, inner, metrics, worker_id, - ); - } - Err(err) => { - warn!( - proxy = P::name(), - request_id = %cli_req.info.id, - connection_id = %cli_req.info.connection_id, - error = ?err, - "Failed to mirror a request", - ); - metrics - .http_mirror_failure_count - .get_or_create(&LabelsProxy { proxy: P::name() }) - .inc(); - } - }; - } - - Err(err) => { - warn!( - proxy = P::name(), - request_id = %cli_req.info.id, - connection_id = %cli_req.info.connection_id, - error = ?err, - "Failed to mirror a request", - ); - metrics - .http_mirror_failure_count - .get_or_create(&LabelsProxy { proxy: P::name() }) - .inc(); - } - } - } - .into_actor(self), - ); - } -} - -// ProxyHttpRequestInfo ------------------------------------------------ - -#[derive(Clone)] -pub(crate) struct ProxyHttpRequestInfo { - id: Uuid, - connection_id: Uuid, - remote_addr: Option, - method: Method, - path: String, - path_and_query: String, - headers: HeaderMap, - jrpc_method_enriched: Cow<'static, str>, -} - -impl ProxyHttpRequestInfo { - pub(crate) fn new(req: &HttpRequest, guard: Option<&ProxyConnectionGuard>) -> Self { - // copy over only non hop-by-hop headers - let mut headers = HeaderMap::new(); - for (header, value) in req.headers().iter() { - if !is_hop_by_hop_header(header) { - headers.insert(header.clone(), value.clone()); - } - } - - // append remote ip to x-forwarded-for - if let Some(peer_addr) = req.connection_info().peer_addr() { - let mut forwarded_for = String::new(); - if let Some(ff) = req.headers().get(header::X_FORWARDED_FOR) && - let Ok(ff) = ff.to_str() - { - forwarded_for.push_str(ff); - forwarded_for.push_str(", "); - } - forwarded_for.push_str(peer_addr); - if let Ok(forwarded_for) = HeaderValue::from_str(&forwarded_for) { - headers.insert(header::X_FORWARDED_FOR, forwarded_for); - } - } - - // set x-forwarded-proto if it's not already set - if req.connection_info().scheme() != "" && - req.headers().get(header::X_FORWARDED_PROTO).is_none() && - let Ok(forwarded_proto) = HeaderValue::from_str(req.connection_info().scheme()) - { - headers.insert(header::X_FORWARDED_PROTO, forwarded_proto); - } - - // set x-forwarded-host if it's not already set - if req.connection_info().scheme() != "" && - req.headers().get(header::X_FORWARDED_HOST).is_none() && - let Ok(forwarded_host) = HeaderValue::from_str(req.connection_info().scheme()) - { - headers.insert(header::X_FORWARDED_HOST, forwarded_host); - } - - // remote address from the guard has port, and connection info has ip - // address only => we prefer the guard - let remote_addr = match guard { - Some(guard) => match guard.remote_addr.clone() { - Some(remote_addr) => Some(remote_addr), - None => req.connection_info().peer_addr().map(String::from), - }, - None => req.connection_info().peer_addr().map(String::from), - }; - - let path = match req.path() { - "" => "/", - val => val, - } - .to_string(); - - let path_and_query = match req.query_string() { - "" => path.clone(), - val => format!("{}?{}", path, val), - }; - - Self { - id: Uuid::now_v7(), - connection_id: Uuid::now_v7(), - remote_addr, - method: req.method().clone(), - path, - path_and_query, - headers, - jrpc_method_enriched: Cow::Borrowed(""), - } - } - - #[inline] - pub(crate) fn id(&self) -> Uuid { - self.id - } - - #[inline] - pub(crate) fn connection_id(&self) -> Uuid { - self.connection_id - } - - #[inline] - fn content_encoding(&self) -> String { - self.headers - .get(header::CONTENT_ENCODING) - .map(|h| h.to_str().unwrap_or_default()) - .map(|h| h.to_string()) - .unwrap_or_default() - } - - #[inline] - pub fn path_and_query(&self) -> &str { - &self.path_and_query - } - - #[inline] - pub fn remote_addr(&self) -> &Option { - &self.remote_addr - } -} - -// ProxyHttpResponseInfo ----------------------------------------------- - -#[derive(Clone)] -pub(crate) struct ProxyHttpResponseInfo { - id: Uuid, - status: StatusCode, - headers: HeaderMap, // TODO: perhaps we don't need all headers, just select ones -} - -impl ProxyHttpResponseInfo { - pub(crate) fn new(id: Uuid, status: StatusCode, headers: HeaderMap) -> Self { - Self { id, status, headers } - } - - #[inline] - pub(crate) fn id(&self) -> Uuid { - self.id - } - - fn content_encoding(&self) -> String { - self.headers - .get(header::CONTENT_ENCODING) - .map(|h| h.to_str().unwrap_or_default()) - .map(|h| h.to_string()) - .unwrap_or_default() - } -} - -// ProxyHttpRequestBody ------------------------------------------------ - -#[pin_project] -struct ProxyHttpRequestBody -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - proxy: web::Data>, - - info: Option, - start: UtcDateTime, - body: Vec, - - #[pin] - stream: S, -} - -impl ProxyHttpRequestBody -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - fn new( - worker: web::Data>, - info: ProxyHttpRequestInfo, - body: S, - timestamp: UtcDateTime, - ) -> Self { - Self { - proxy: worker, - info: Some(info), - stream: body, - start: timestamp, - body: Vec::new(), // TODO: preallocate reasonable size - } - } -} - -impl Stream for ProxyHttpRequestBody -where - S: Stream>, - E: Debug, - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.project(); - - match this.stream.poll_next(cx) { - Poll::Pending => Poll::Pending, - - Poll::Ready(Some(Ok(chunk))) => { - this.body.extend_from_slice(&chunk); - Poll::Ready(Some(Ok(chunk))) - } - - Poll::Ready(Some(Err(err))) => { - if let Some(info) = mem::take(this.info) { - warn!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - error = ?err, - "Proxy http request stream error", - ); - } else { - warn!( - proxy = P::name(), - error = ?err, - request_id = "unknown", - "Proxy http request stream error", - ); - } - Poll::Ready(Some(Err(err))) - } - - Poll::Ready(None) => { - let end = UtcDateTime::now(); - - if let Some(info) = mem::take(this.info) { - let proxy = this.proxy.clone(); - - let req = ProxiedHttpRequest::new(info, mem::take(this.body), *this.start, end); - - proxy.postprocess_client_request(req); - } - - Poll::Ready(None) - } - } - } -} - -// ProxyHttpResponseBody ----------------------------------------------- - -#[pin_project] -struct ProxyHttpResponseBody -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - proxy: web::Data>, - - info: Option, - start: UtcDateTime, - body: Vec, - - #[pin] - stream: S, -} - -impl ProxyHttpResponseBody -where - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - fn new( - proxy: web::Data>, - id: Uuid, - status: StatusCode, - headers: HeaderMap, - body: S, - timestamp: UtcDateTime, - ) -> Self { - Self { - proxy, - stream: body, - start: timestamp, - body: Vec::new(), // TODO: preallocate reasonable size - info: Some(ProxyHttpResponseInfo::new(id, status, headers)), - } - } -} - -impl Stream for ProxyHttpResponseBody -where - S: Stream>, - E: Debug, - C: ConfigProxyHttp, - P: ProxyHttpInner, -{ - type Item = Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.project(); - - match this.stream.poll_next(cx) { - Poll::Pending => Poll::Pending, - - Poll::Ready(Some(Ok(chunk))) => { - this.body.extend_from_slice(&chunk); - Poll::Ready(Some(Ok(chunk))) - } - - Poll::Ready(Some(Err(err))) => { - if let Some(info) = mem::take(this.info) { - warn!( - proxy = P::name(), - request_id = %info.id(), - error = ?err, - "Proxy http response stream error", - ); - } else { - warn!( - proxy = P::name(), - error = ?err, - request_id = "unknown", - "Proxy http response stream error", - ); - } - Poll::Ready(Some(Err(err))) - } - - Poll::Ready(None) => { - let end = UtcDateTime::now(); - - if let Some(info) = mem::take(this.info) { - let proxy = this.proxy.clone(); - - let res = - ProxiedHttpResponse::new(info, mem::take(this.body), *this.start, end); - - proxy.postprocess_backend_response(res); - } - - Poll::Ready(None) - } - } - } -} - -// ProxiedHttpRequest -------------------------------------------------- - -#[derive(Clone, actix::Message)] -#[rtype(result = "()")] -pub(crate) struct ProxiedHttpRequest { - info: ProxyHttpRequestInfo, - body: Bytes, - size: usize, - decompressed_body: Bytes, - decompressed_size: usize, - start: UtcDateTime, - end: UtcDateTime, -} - -impl ProxiedHttpRequest { - pub(crate) fn new( - info: ProxyHttpRequestInfo, - body: Vec, - start: UtcDateTime, - end: UtcDateTime, - ) -> Self { - let size = body.len(); - Self { - info, - body: Bytes::from(body), - size, - decompressed_body: Bytes::new(), - decompressed_size: 0, - start, - end, - } - } - - #[inline] - pub(crate) fn info(&self) -> &ProxyHttpRequestInfo { - &self.info - } - - #[inline] - pub(crate) fn start(&self) -> UtcDateTime { - self.start - } - - #[inline] - pub(crate) fn end(&self) -> UtcDateTime { - self.end - } -} - -// ProxiedHttpResponse ------------------------------------------------- - -#[derive(Clone, actix::Message)] -#[rtype(result = "()")] -pub(crate) struct ProxiedHttpResponse { - info: ProxyHttpResponseInfo, - body: Bytes, - size: usize, - decompressed_body: Bytes, - decompressed_size: usize, - start: UtcDateTime, - end: UtcDateTime, -} - -impl ProxiedHttpResponse { - pub(crate) fn new( - info: ProxyHttpResponseInfo, - body: Vec, - start: UtcDateTime, - end: UtcDateTime, - ) -> Self { - let size = body.len(); - Self { - info, - body: Bytes::from(body), - size, - decompressed_body: Bytes::new(), - decompressed_size: 0, - start, - end, - } - } - - #[inline] - pub(crate) fn status(&self) -> &str { - self.info.status.as_str() - } - - #[inline] - pub(crate) fn decompressed_body(&self) -> Bytes { - self.decompressed_body.clone() - } - - #[inline] - pub(crate) fn start(&self) -> UtcDateTime { - self.start - } - - #[inline] - pub(crate) fn end(&self) -> UtcDateTime { - self.end - } -} - -// ProxiedHttpCombo ---------------------------------------------------- - -#[derive(Clone, actix::Message)] -#[rtype(result = "()")] -struct ProxiedHttpCombo { - req: ProxiedHttpRequest, - res: ProxiedHttpResponse, -} diff --git a/crates/rproxy/src/proxy_ws/mod.rs b/crates/rproxy/src/proxy_ws/mod.rs index 770cf87..8115521 100644 --- a/crates/rproxy/src/proxy_ws/mod.rs +++ b/crates/rproxy/src/proxy_ws/mod.rs @@ -1,8 +1,1305 @@ -mod proxy_ws; - mod proxy_ws_flashblocks; mod proxy_ws_inner; -pub(crate) use proxy_ws::ProxyWs; +use std::{ + io::Write, + marker::PhantomData, + str::FromStr, + sync::{ + Arc, + atomic::{AtomicI64, Ordering}, + }, + time::Duration, +}; + +use actix::{Actor, AsyncContext, WrapFuture}; +use actix_web::{ + App, + HttpRequest, + HttpResponse, + HttpServer, + middleware::{NormalizePath, TrailingSlash}, + web, +}; +use actix_ws::{MessageStream, Session}; +use bytes::{Buf, BufMut, Bytes, BytesMut}; +use futures::{ + SinkExt, + StreamExt, + stream::{SplitSink, SplitStream}, +}; +use prometheus_client::metrics::gauge::Atomic; pub(crate) use proxy_ws_flashblocks::ProxyWsInnerFlashblocks; pub(crate) use proxy_ws_inner::ProxyWsInner; +use scc::HashMap; +use time::{UtcDateTime, format_description::well_known::Iso8601}; +use tokio::{net::TcpStream, sync::broadcast}; +use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; +use tracing::{error, info, trace, warn}; +use tungstenite::Utf8Bytes; +use uuid::Uuid; +use x509_parser::asn1_rs::ToStatic; + +use crate::{ + config::{ConfigProxyWs, ConfigTls, PARALLELISM}, + metrics::{LabelsProxyWs, Metrics}, + proxy::{Proxy, ProxyConnectionGuard}, + proxy_http::ProxyHttpRequestInfo, + utils::{Loggable, raw_transaction_to_hash}, +}; + +const WS_PING_INTERVAL_SECONDS: u64 = 1; + +const WS_CLI_ERROR: &str = "client error"; +const WS_BCK_ERROR: &str = "backend error"; +const WS_BCK_TIMEOUT: &str = "backend error"; +const WS_CLOSE_OK: &str = ""; + +const WS_LABEL_BACKEND: &str = "backend"; +const WS_LABEL_CLIENT: &str = "client"; + +// ProxyWs ------------------------------------------------------------- + +pub(crate) struct ProxyWs +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + id: Uuid, + + shared: ProxyWsSharedState, + postprocessor: actix::Addr>, + canceller: tokio_util::sync::CancellationToken, + + backend: ProxyWsBackendEndpoint, + + pings: HashMap, + ping_balance_cli: AtomicI64, + ping_balance_bck: AtomicI64, + + _config: PhantomData, + _proxy: PhantomData

, +} + +impl ProxyWs +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + fn new( + shared: ProxyWsSharedState, + canceller: tokio_util::sync::CancellationToken, + ) -> Self { + let id = Uuid::now_v7(); + + let config = shared.config(); + + let backend = ProxyWsBackendEndpoint::new(id, config.backend_url()); + + let postprocessor = ProxyWsPostprocessor:: { + inner: shared.inner.clone(), + metrics: shared.metrics.clone(), + worker_id: id, + _config: PhantomData, + } + .start(); + + Self { + id, + shared, + postprocessor, + canceller, + backend, + pings: HashMap::new(), + ping_balance_bck: AtomicI64::new(0), + ping_balance_cli: AtomicI64::new(0), + _config: PhantomData, + _proxy: PhantomData, + } + } + + fn config(&self) -> &C { + self.shared.config() + } + + pub(crate) async fn run( + config: C, + tls: ConfigTls, + metrics: Arc, + canceller: tokio_util::sync::CancellationToken, + resetter: broadcast::Sender<()>, + ) -> Result<(), Box> { + let listen_address = config.listen_address(); + + let listener = match Self::listen(&config) { + Ok(listener) => listener, + Err(err) => { + error!( + proxy = P::name(), + addr = %config.listen_address(), + error = ?err, + "Failed to initialise a socket" + ); + return Err(Box::new(err)); + } + }; + + let workers_count = PARALLELISM.to_static(); + + let shared = ProxyWsSharedState::::new(config, &metrics); + let client_connections_count = shared.client_connections_count.clone(); + let worker_canceller = canceller.clone(); + + info!( + proxy = P::name(), + listen_address = %listen_address, + workers_count = workers_count, + "Starting websocket-proxy...", + ); + + let server = HttpServer::new(move || { + let this = web::Data::new(Self::new(shared.clone(), worker_canceller.clone())); + + App::new() + .app_data(this) + .wrap(NormalizePath::new(TrailingSlash::Trim)) + .default_service(web::route().to(Self::receive)) + }) + .on_connect(Self::on_connect(metrics, client_connections_count)) + .shutdown_signal(canceller.cancelled_owned()) + .workers(workers_count); + + let proxy = match if tls.enabled() { + let cert = tls.certificate().clone(); + let key = tls.key().clone_key(); + + server.listen_rustls_0_23( + listener, + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(cert, key) + .unwrap(), + ) + } else { + server.listen(listener) + } { + Ok(server) => server, + Err(err) => { + error!(proxy = P::name(), error = ?err, "Failed to initialise websocket-proxy"); + return Err(Box::new(err)); + } + } + .run(); + + let handler = proxy.handle(); + let mut resetter = resetter.subscribe(); + tokio::spawn(async move { + if resetter.recv().await.is_ok() { + info!(proxy = P::name(), "Reset signal received, stopping websocket-proxy..."); + handler.stop(true).await; + } + }); + + if let Err(err) = proxy.await { + error!(proxy = P::name(), error = ?err, "Failure while running websocket-proxy") + } + + info!(proxy = P::name(), "Stopped websocket-proxy"); + + Ok(()) + } + + fn listen(config: &C) -> std::io::Result { + let socket = socket2::Socket::new( + socket2::Domain::for_address(config.listen_address()), + socket2::Type::STREAM, + Some(socket2::Protocol::TCP), + )?; + + // must use non-blocking with tokio + socket.set_nonblocking(true)?; + + // allow time to flush buffers on close + socket.set_linger(Some(config.backend_timeout()))?; + + // allow binding to the socket whlie there are still TIME_WAIT conns + socket.set_reuse_address(true)?; + + socket.bind(&socket2::SockAddr::from(config.listen_address()))?; + + socket.listen(1024)?; + + Ok(socket.into()) + } + + async fn receive( + cli_req: HttpRequest, + cli_req_body: web::Payload, + this: web::Data, + ) -> Result { + let info = ProxyHttpRequestInfo::new(&cli_req, cli_req.conn_data::()); + + let (res, cli_tx, cli_rx) = match actix_ws::handle(&cli_req, cli_req_body) { + Ok(res) => res, + Err(err) => { + error!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to upgrade to websocket", + ); + return Err(err); + } + }; + + actix_web::rt::spawn(Self::handshake(this, cli_tx, cli_rx, info)); + + Ok(res) + } + + async fn handshake( + this: web::Data, + cli_tx: Session, + cli_rx: MessageStream, + info: ProxyHttpRequestInfo, + ) { + let bck_uri = this.backend.new_backend_uri(&info); + trace!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + worker_id = %this.id, + backend_uri = %bck_uri, + "Starting websocket handshake...", + ); + + let (bck_stream, _) = match tokio::time::timeout( + this.config().backend_timeout(), + tokio_tungstenite::connect_async(bck_uri), + ) + .await + { + Ok(Ok(res)) => res, + + Ok(Err(err)) => { + error!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to establish backend websocket session" + ); + + if let Err(err) = cli_tx + .close(Some(actix_ws::CloseReason { + code: awc::ws::CloseCode::Error, + description: Some(String::from(WS_BCK_ERROR)), + })) + .await + { + error!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to close client websocket session" + ); + }; + return; + } + + Err(_) => { + error!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "Timed out to establish backend websocket session" + ); + + if let Err(err) = cli_tx + .close(Some(actix_ws::CloseReason { + code: awc::ws::CloseCode::Again, + description: Some(String::from(WS_BCK_TIMEOUT)), + })) + .await + { + error!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to close client websocket session" + ); + } + return; + } + }; + + let (bck_tx, bck_rx) = bck_stream.split(); + + Self::pump(this, info, cli_tx, cli_rx, bck_tx, bck_rx).await; + } + + async fn pump( + this: web::Data, + info: ProxyHttpRequestInfo, + mut cli_tx: Session, + mut cli_rx: MessageStream, + mut bck_tx: SplitSink>, tungstenite::Message>, + mut bck_rx: SplitStream>>, + ) { + info!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "Starting websocket pump..." + ); + + let info = Arc::new(info); + + let mut heartbeat = tokio::time::interval(Duration::from_secs(WS_PING_INTERVAL_SECONDS)); + + let mut pumping: Result<(), &str> = Ok(()); + + while pumping.is_ok() && !this.canceller.is_cancelled() { + tokio::select! { + _ = this.canceller.cancelled() => { + break; + } + + // ping both sides + _ = heartbeat.tick() => { + pumping = Self::heartbeat(&this, info.clone(), &mut cli_tx, &mut bck_tx).await; + } + + // client => backend + cli_msg = cli_rx.next() => { + pumping = Self::pump_cli_to_bck( + &this, + info.clone(), + UtcDateTime::now(), + cli_msg, + &mut bck_tx, + &mut cli_tx + ).await; + } + + // backend => client + bck_msg = bck_rx.next() => { + pumping = Self::pump_bck_to_cli( + &this, + info.clone(), + UtcDateTime::now(), + bck_msg, + &mut cli_tx, + &mut bck_tx + ).await; + } + } + } + + if let Err(msg) = pumping && + msg != WS_CLOSE_OK + { + if let Err(err) = cli_tx + .close(Some(actix_ws::CloseReason { + code: awc::ws::CloseCode::Error, + description: Some(String::from(msg)), + })) + .await + { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to close client websocket session" + ); + } + + if let Err(err) = bck_tx + .send(tungstenite::Message::Close(Some(tungstenite::protocol::CloseFrame { + code: tungstenite::protocol::frame::coding::CloseCode::Error, + reason: msg.into(), + }))) + .await + { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to close backend websocket session" + ); + } + } else { + if let Err(err) = cli_tx + .close(Some(actix_ws::CloseReason { + code: awc::ws::CloseCode::Normal, + description: None, + })) + .await + { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to close client websocket session" + ); + } + + if let Err(err) = bck_tx + .send(tungstenite::Message::Close(Some(tungstenite::protocol::CloseFrame { + code: tungstenite::protocol::frame::coding::CloseCode::Normal, + reason: Utf8Bytes::default(), + }))) + .await + { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to close backend websocket session" + ); + } + } + + info!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "Stopped websocket pump" + ); + } + + async fn heartbeat( + this: &web::Data, + info: Arc, + cli_tx: &mut Session, + bck_tx: &mut SplitSink>, tungstenite::Message>, + ) -> Result<(), &'static str> { + let ping_threshold = + (1 + this.config().backend_timeout().as_secs() / WS_PING_INTERVAL_SECONDS) as i64; + + { + // ping -> client + + if this.ping_balance_cli.load(Ordering::Relaxed) > ping_threshold { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "More than {} websocket pings sent to client didn't return, terminating the pump...", ping_threshold, + ); + return Err(WS_CLI_ERROR); + } + + let cli_ping = ProxyWsPing::new(info.connection_id()); + if let Err(err) = cli_tx.ping(&cli_ping.to_slice()).await { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to send ping websocket message to client" + ); + return Err(WS_CLI_ERROR); + } + let _ = this.pings.insert_sync(cli_ping.id, cli_ping); + this.ping_balance_cli.inc(); + } + + { + // ping -> backend + + if this.ping_balance_bck.load(Ordering::Relaxed) > ping_threshold { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "More than {} websocket pings sent to backend didn't return, terminating the pump...", ping_threshold, + ); + return Err(WS_BCK_ERROR); + } + + let bck_ping = ProxyWsPing::new(info.connection_id()); + if let Err(err) = bck_tx.send(tungstenite::Message::Ping(bck_ping.to_bytes())).await { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to send ping websocket message to backend" + ); + return Err(WS_BCK_ERROR); + } + let _ = this.pings.insert_sync(bck_ping.id, bck_ping); + this.ping_balance_bck.inc(); + } + Ok(()) + } + + async fn pump_cli_to_bck( + this: &web::Data, + info: Arc, + timestamp: UtcDateTime, + cli_msg: Option>, + bck_tx: &mut SplitSink>, tungstenite::Message>, + cli_tx: &mut Session, + ) -> Result<(), &'static str> { + match cli_msg { + Some(Ok(msg)) => { + match msg { + // binary + actix_ws::Message::Binary(bytes) => { + if let Err(err) = + bck_tx.send(tungstenite::Message::Binary(bytes.clone())).await + { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to proxy binary websocket message to backend" + ); + this.shared + .metrics + .ws_proxy_failure_count + .get_or_create(&LabelsProxyWs { + proxy: P::name(), + destination: WS_LABEL_BACKEND, + }) + .inc(); + return Err(WS_BCK_ERROR); + } + this.postprocessor.do_send(ProxyWsMessage::ClientToBackendBinary { + msg: bytes, + info, + start: timestamp, + end: UtcDateTime::now(), + }); + Ok(()) + } + + // text + actix_ws::Message::Text(text) => { + if let Err(err) = bck_tx + .send(tungstenite::Message::Text(unsafe { + // safety: it's from client's ws message => must be valid utf-8 + tungstenite::protocol::frame::Utf8Bytes::from_bytes_unchecked( + text.clone().into_bytes(), + ) + })) + .await + { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to proxy text websocket message to backend" + ); + this.shared + .metrics + .ws_proxy_failure_count + .get_or_create(&LabelsProxyWs { + proxy: P::name(), + destination: WS_LABEL_BACKEND, + }) + .inc(); + return Err(WS_BCK_ERROR); + } + this.postprocessor.do_send(ProxyWsMessage::ClientToBackendText { + msg: text, + info, + start: timestamp, + end: UtcDateTime::now(), + }); + Ok(()) + } + + // ping + actix_ws::Message::Ping(bytes) => { + if let Err(err) = cli_tx.pong(&bytes).await { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to return pong message to client" + ); + return Err(WS_CLI_ERROR); + } + Ok(()) + } + + // pong + actix_ws::Message::Pong(bytes) => { + if let Some(pong) = ProxyWsPing::from_bytes(bytes) && + let Some((_, ping)) = this.pings.remove_sync(&pong.id) && + pong == ping + { + this.ping_balance_cli.dec(); + this.shared + .metrics + .ws_latency_client + .get_or_create(&LabelsProxyWs { + proxy: P::name(), + destination: WS_LABEL_BACKEND, + }) + .record( + (1000000.0 * (timestamp - pong.timestamp).as_seconds_f64() / + 2.0) as i64, + ); + return Ok(()); + } + warn!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "Unexpected websocket pong received from client", + ); + Ok(()) + } + + // close + actix_ws::Message::Close(reason) => { + if let Err(err) = bck_tx + .send(tungstenite::Message::Close(reason.map(|r| { + tungstenite::protocol::CloseFrame { + code: tungstenite::protocol::frame::coding::CloseCode::from( + u16::from(r.code), + ), + reason: r.description.unwrap_or_default().into(), + } + }))) + .await + { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to proxy close websocket message to backend" + ); + return Err(WS_BCK_ERROR); + } + Err(WS_CLOSE_OK) + } + + _ => Ok(()), + } + } + + Some(Err(err)) => { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Client websocket stream error" + ); + Err(WS_CLI_ERROR) + } + + None => { + info!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "Client had closed websocket stream" + ); + Err(WS_CLOSE_OK) + } + } + } + + async fn pump_bck_to_cli( + this: &web::Data, + info: Arc, + timestamp: UtcDateTime, + bck_msg: Option>, + cli_tx: &mut Session, + bck_tx: &mut SplitSink>, tungstenite::Message>, + ) -> Result<(), &'static str> { + match bck_msg { + Some(Ok(msg)) => { + match msg { + // binary + tungstenite::Message::Binary(bytes) => { + if let Err(err) = cli_tx.binary(bytes.clone()).await { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to proxy binary websocket message to client" + ); + this.shared + .metrics + .ws_proxy_failure_count + .get_or_create(&LabelsProxyWs { + proxy: P::name(), + destination: WS_LABEL_CLIENT, + }) + .inc(); + return Err(WS_CLI_ERROR); + } + this.postprocessor.do_send(ProxyWsMessage::BackendToClientBinary { + msg: bytes, + info, + start: timestamp, + end: UtcDateTime::now(), + }); + Ok(()) + } + + // text + tungstenite::Message::Text(text) => { + if let Err(err) = cli_tx.text(text.clone().as_str()).await { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to proxy text websocket message to client" + ); + this.shared + .metrics + .ws_proxy_failure_count + .get_or_create(&LabelsProxyWs { + proxy: P::name(), + destination: WS_LABEL_CLIENT, + }) + .inc(); + return Err(WS_CLI_ERROR); + } + this.postprocessor.do_send(ProxyWsMessage::BackendToClientText { + msg: text, + info, + start: timestamp, + end: UtcDateTime::now(), + }); + Ok(()) + } + + // ping + tungstenite::Message::Ping(bytes) => { + if let Err(err) = bck_tx.send(tungstenite::Message::Pong(bytes)).await { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to return pong message to backend" + ); + return Err(WS_BCK_ERROR); + } + Ok(()) + } + + // pong + tungstenite::Message::Pong(bytes) => { + if let Some(pong) = ProxyWsPing::from_bytes(bytes) && + let Some((_, ping)) = this.pings.remove_sync(&pong.id) && + pong == ping + { + this.ping_balance_bck.dec(); + this.shared + .metrics + .ws_latency_backend + .get_or_create(&LabelsProxyWs { + proxy: P::name(), + destination: WS_LABEL_BACKEND, + }) + .record( + (1000000.0 * (timestamp - pong.timestamp).as_seconds_f64() / + 2.0) as i64, + ); + return Ok(()); + } + warn!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "Unexpected websocket pong received from backend", + ); + Ok(()) + } + + // close + tungstenite::Message::Close(reason) => { + if let Err(err) = cli_tx + .clone() // .close() consumes it + .close(reason.map(|reason| actix_ws::CloseReason { + code: u16::from(reason.code).into(), + description: reason.reason.to_string().into(), + })) + .await + { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Failed to proxy close websocket message to client" + ); + return Err(WS_CLI_ERROR); + } + Err(WS_CLOSE_OK) + } + + _ => Ok(()), + } + } + + Some(Err(err)) => { + error!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + error = ?err, + "Backend websocket stream error" + ); + Err(WS_BCK_ERROR) + } + + None => { + info!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %this.id, + "Backend had closed websocket stream" + ); + Err(WS_CLOSE_OK) + } + } + } + + fn finalise_proxying( + msg: ProxyWsMessage, + inner: Arc

, + metrics: Arc, + worker_id: Uuid, + ) { + Self::maybe_log_proxied_message(&msg, inner.clone(), worker_id); + + Self::emit_metrics_on_proxy_success(&msg, metrics.clone()); + } + + fn maybe_log_proxied_message(msg: &ProxyWsMessage, inner: Arc

, worker_id: Uuid) { + let config = inner.config(); + + match msg { + ProxyWsMessage::BackendToClientBinary { msg, info, start, end } => { + let json_msg = if config.log_backend_messages() { + Loggable(&Self::maybe_sanitise( + config.log_sanitise(), + serde_json::from_slice(msg).unwrap_or_default(), + )) + } else { + Loggable(&serde_json::Value::Null) + }; + + info!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %worker_id, + remote_addr = info.remote_addr(), + ts_message_received = start.format(&Iso8601::DEFAULT).unwrap_or_default(), + latency_proxying = (*end - *start).as_seconds_f64(), + message = tracing::field::valuable(&json_msg), + "Proxied binary message to client", + ); + } + + ProxyWsMessage::BackendToClientText { msg, info, start, end } => { + let json_msg = if config.log_backend_messages() { + Loggable(&Self::maybe_sanitise( + config.log_sanitise(), + serde_json::from_str(msg).unwrap_or_default(), + )) + } else { + Loggable(&serde_json::Value::Null) + }; + + info!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %worker_id, + remote_addr = info.remote_addr(), + ts_message_received = start.format(&Iso8601::DEFAULT).unwrap_or_default(), + latency_proxying = (*end - *start).as_seconds_f64(), + message = tracing::field::valuable(&json_msg), + "Proxied text message to client", + ); + } + + ProxyWsMessage::ClientToBackendBinary { msg, info, start, end } => { + let json_msg = if config.log_client_messages() { + Loggable(&Self::maybe_sanitise( + config.log_sanitise(), + serde_json::from_slice(msg).unwrap_or_default(), + )) + } else { + Loggable(&serde_json::Value::Null) + }; + + info!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %worker_id, + remote_addr = info.remote_addr(), + ts_message_received = start.format(&Iso8601::DEFAULT).unwrap_or_default(), + latency_proxying = (*end - *start).as_seconds_f64(), + message = tracing::field::valuable(&json_msg), + "Proxied binary message to backend", + ); + } + + ProxyWsMessage::ClientToBackendText { msg, info, start, end } => { + let json_msg = if config.log_client_messages() { + Loggable(&Self::maybe_sanitise( + config.log_sanitise(), + serde_json::from_str(msg).unwrap_or_default(), + )) + } else { + Loggable(&serde_json::Value::Null) + }; + + info!( + proxy = P::name(), + connection_id = %info.connection_id(), + worker_id = %worker_id, + remote_addr = info.remote_addr(), + ts_message_received = start.format(&Iso8601::DEFAULT).unwrap_or_default(), + latency_proxying = (*end - *start).as_seconds_f64(), + message = tracing::field::valuable(&json_msg), + "Proxied text message to backend", + ); + } + } + } + + fn maybe_sanitise(do_sanitise: bool, mut message: serde_json::Value) -> serde_json::Value { + if !do_sanitise { + return message; + } + + if let Some(object) = message.as_object_mut() && + let Some(diff) = object.get_mut("diff") && + let Some(transactions) = diff.get_mut("transactions") && + let Some(transactions) = transactions.as_array_mut() + { + for transaction in transactions { + raw_transaction_to_hash(transaction); + } + } + + message + } + + fn emit_metrics_on_proxy_success(msg: &ProxyWsMessage, metrics: Arc) { + match msg { + ProxyWsMessage::BackendToClientBinary { msg, info: _, start, end } => { + let labels = LabelsProxyWs { proxy: P::name(), destination: WS_LABEL_CLIENT }; + metrics + .ws_latency_proxy + .get_or_create(&labels) + .record((1000000.0 * (*end - *start).as_seconds_f64()) as i64); + metrics.ws_proxy_success_count.get_or_create_owned(&labels).inc(); + metrics.ws_message_size.get_or_create_owned(&labels).record(msg.len() as i64); + } + + ProxyWsMessage::BackendToClientText { msg, info: _, start, end } => { + let labels = LabelsProxyWs { proxy: P::name(), destination: WS_LABEL_CLIENT }; + metrics + .ws_latency_proxy + .get_or_create(&labels) + .record((1000000.0 * (*end - *start).as_seconds_f64()) as i64); + metrics.ws_proxy_success_count.get_or_create_owned(&labels).inc(); + metrics.ws_message_size.get_or_create_owned(&labels).record(msg.len() as i64); + } + + ProxyWsMessage::ClientToBackendBinary { msg, info: _, start, end } => { + let labels = LabelsProxyWs { proxy: P::name(), destination: WS_LABEL_BACKEND }; + metrics + .ws_latency_proxy + .get_or_create(&labels) + .record((1000000.0 * (*end - *start).as_seconds_f64()) as i64); + metrics.ws_proxy_success_count.get_or_create_owned(&labels).inc(); + metrics.ws_message_size.get_or_create_owned(&labels).record(msg.len() as i64); + } + + ProxyWsMessage::ClientToBackendText { msg, info: _, start, end } => { + let labels = LabelsProxyWs { proxy: P::name(), destination: WS_LABEL_BACKEND }; + metrics + .ws_latency_proxy + .get_or_create(&labels) + .record((1000000.0 * (*end - *start).as_seconds_f64()) as i64); + metrics.ws_proxy_success_count.get_or_create_owned(&labels).inc(); + metrics.ws_message_size.get_or_create_owned(&labels).record(msg.len() as i64); + } + } + } +} + +impl Proxy

for ProxyWs +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ +} + +// ProxyWsSharedState -------------------------------------------------- + +#[derive(Clone)] +struct ProxyWsSharedState +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + inner: Arc

, + metrics: Arc, + + client_connections_count: Arc, + + _config: PhantomData, +} + +impl ProxyWsSharedState +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + fn new(config: C, metrics: &Arc) -> Self { + Self { + inner: Arc::new(P::new(config)), + metrics: metrics.clone(), + client_connections_count: Arc::new(AtomicI64::new(0)), + _config: PhantomData, + } + } + + #[inline] + fn config(&self) -> &C { + self.inner.config() + } +} + +// ProxyWsBackendEndpoint ---------------------------------------------- + +struct ProxyWsBackendEndpoint +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + worker_id: Uuid, + + url: tungstenite::http::Uri, + + _config: PhantomData, + _inner: PhantomData

, +} + +impl ProxyWsBackendEndpoint +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + fn new(worker_id: Uuid, url: tungstenite::http::Uri) -> Self { + Self { worker_id, url, _config: PhantomData, _inner: PhantomData } + } + + fn new_backend_uri(&self, info: &ProxyHttpRequestInfo) -> tungstenite::http::Uri { + let mut parts = self.url.clone().into_parts(); + let pq = tungstenite::http::uri::PathAndQuery::from_str(info.path_and_query()) + .inspect_err(|err| { + error!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + worker_id = %self.worker_id, + error = ?err, + "Failed to re-parse client request's path and query", + ); + }) + .ok(); + parts.path_and_query = pq; + + tungstenite::http::Uri::from_parts(parts) + .inspect_err(|err| { + error!( + proxy = P::name(), + request_id = %info.id(), + connection_id = %info.connection_id(), + worker_id = %self.worker_id, + error = ?err, "Failed to construct backend URI, defaulting to the base one", + ); + }) + .unwrap_or(self.url.clone()) + } +} + +// ProxyWsPostprocessor + +struct ProxyWsPostprocessor +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + inner: Arc

, + worker_id: Uuid, + metrics: Arc, + + _config: PhantomData, +} + +impl actix::Actor for ProxyWsPostprocessor +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(1024); + } +} + +impl actix::Handler for ProxyWsPostprocessor +where + C: ConfigProxyWs, + P: ProxyWsInner, +{ + type Result = (); + + fn handle(&mut self, msg: ProxyWsMessage, ctx: &mut Self::Context) -> Self::Result { + let inner = self.inner.clone(); + let metrics = self.metrics.clone(); + let worker_id = self.worker_id; + + ctx.spawn( + async move { + ProxyWs::::finalise_proxying(msg, inner, metrics, worker_id); + } + .into_actor(self), + ); + } +} + +// ProxyWsMessage ------------------------------------------------------ + +#[derive(Clone, actix::Message)] +#[rtype(result = "()")] +enum ProxyWsMessage { + BackendToClientBinary { + msg: bytes::Bytes, + info: Arc, + start: UtcDateTime, + end: UtcDateTime, + }, + + BackendToClientText { + msg: tungstenite::protocol::frame::Utf8Bytes, + info: Arc, + start: UtcDateTime, + end: UtcDateTime, + }, + + ClientToBackendBinary { + msg: bytes::Bytes, + info: Arc, + start: UtcDateTime, + end: UtcDateTime, + }, + + ClientToBackendText { + msg: bytestring::ByteString, + info: Arc, + start: UtcDateTime, + end: UtcDateTime, + }, +} + +// ProxyWsPing --------------------------------------------------------- + +#[derive(PartialEq, Eq)] +struct ProxyWsPing { + id: Uuid, + connection_id: Uuid, + timestamp: UtcDateTime, +} + +impl ProxyWsPing { + fn new(connection_id: Uuid) -> Self { + Self { id: Uuid::now_v7(), connection_id, timestamp: UtcDateTime::now() } + } + + fn to_bytes(&self) -> Bytes { + let mut bytes = BytesMut::with_capacity(48); + bytes.put_u128(self.id.as_u128()); + bytes.put_u128(self.connection_id.as_u128()); + bytes.put_i128(self.timestamp.unix_timestamp_nanos()); + bytes.freeze() + } + + fn from_bytes(mut bytes: Bytes) -> Option { + if bytes.len() != 48 { + return None; + } + + let id = Uuid::from_u128(bytes.get_u128()); + let connection_id = Uuid::from_u128(bytes.get_u128()); + let timestamp = match UtcDateTime::from_unix_timestamp_nanos(bytes.get_i128()) { + Ok(timestamp) => timestamp, + Err(_) => return None, + }; + + Some(Self { id, connection_id, timestamp }) + } + + fn to_slice(&self) -> [u8; 48] { + let res: [u8; 48] = [0; 48]; + let mut cur = std::io::Cursor::new(res); + + let _ = cur.write(self.id.as_bytes()); + let _ = cur.write(self.connection_id.as_bytes()); + let _ = cur.write(&self.timestamp.unix_timestamp_nanos().to_be_bytes()); + + cur.into_inner() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn proxy_ws_ping_encode_decode() { + let ping = ProxyWsPing::new(Uuid::now_v7()); + + { + let pong = ProxyWsPing::from_bytes(ping.to_bytes()); + assert!(pong.is_some(), "must be some"); + let pong = pong.unwrap(); // safety: just verified + assert!(pong == ping, "must be the same"); + } + + { + let slice = ping.to_slice(); + let pong = ProxyWsPing::from_bytes(Bytes::copy_from_slice(&slice)); + assert!(pong.is_some(), "must be some"); + let pong = pong.unwrap(); // safety: just verified + assert!(pong == ping, "must be the same"); + } + } +} diff --git a/crates/rproxy/src/server/mod.rs b/crates/rproxy/src/server/mod.rs index dc1344f..7315cf7 100644 --- a/crates/rproxy/src/server/mod.rs +++ b/crates/rproxy/src/server/mod.rs @@ -1,2 +1,235 @@ -mod server; -pub use server::Server; +use std::{error::Error, sync::Arc}; + +use tokio::{ + signal::unix::{SignalKind, signal}, + sync::broadcast, + task::JoinHandle, +}; +use tokio_util::sync::CancellationToken; +use tracing::{error, info}; + +use crate::{ + circuit_breaker::CircuitBreaker, + config::{Config, ConfigAuthrpc, ConfigFlashblocks, ConfigRpc}, + metrics::Metrics, + proxy::ProxyInner, + proxy_http::{ProxyHttp, ProxyHttpInnerAuthrpc, ProxyHttpInnerRpc}, + proxy_ws::{ProxyWs, ProxyWsInnerFlashblocks}, + utils::tls_certificate_validity_timestamps, +}; + +// Proxy --------------------------------------------------------------- + +pub struct Server {} + +impl Server { + pub async fn run(config: Config) -> Result<(), Box> { + let canceller = Server::wait_for_shutdown_signal(); + let resetter = Server::wait_for_reset_signal(canceller.clone()); + + // spawn metrics service + let metrics = Arc::new(Metrics::new(config.metrics.clone())); + { + let canceller = canceller.clone(); + let metrics = metrics.clone(); + + tokio::spawn(async move { + metrics.run(canceller).await.inspect_err(|err| { + error!( + service = Metrics::name(), + error = ?err, + "Failed to start metrics service", + ); + std::process::exit(-1); + }) + }); + } + + // spawn circuit-breaker + if !config.circuit_breaker.url.is_empty() { + let canceller = canceller.clone(); + let resetter = resetter.clone(); + + let _ = std::thread::spawn(move || { + let rt = match tokio::runtime::Builder::new_current_thread().enable_all().build() { + Ok(rt) => rt, + Err(err) => { + error!(error = ?err, "Failed to initialise a single-threaded runtime for circuit-breaker"); + std::process::exit(-1); + } + }; + + let circuit_breaker = CircuitBreaker::new(config.circuit_breaker.clone()); + + tokio::task::LocalSet::new() + .block_on(&rt, async move { circuit_breaker.run(canceller, resetter).await }) + }); + } + + while !canceller.is_cancelled() { + if config.tls.enabled() { + let metrics = metrics.clone(); + let (not_before, not_after) = + tls_certificate_validity_timestamps(config.tls.certificate()); + metrics.tls_certificate_valid_not_before.set(not_before); + metrics.tls_certificate_valid_not_after.set(not_after); + } + + let mut services: Vec>>> = Vec::new(); + + // spawn authrpc proxy + if config.authrpc.enabled { + let tls = config.tls.clone(); + let config = config.authrpc.clone(); + let metrics = metrics.clone(); + let canceller = canceller.clone(); + let resetter = resetter.clone(); + + services.push(tokio::spawn(async move { + ProxyHttp::::run( + config, + tls, + metrics, + canceller.clone(), + resetter, + ) + .await + .inspect_err(|err| { + error!( + proxy = ProxyHttpInnerRpc::name(), + error = ?err, + "Failed to start http-proxy, terminating...", + ); + canceller.cancel(); + }) + })); + } + + // spawn rpc proxy + if config.rpc.enabled { + let tls = config.tls.clone(); + let config = config.rpc.clone(); + let metrics = metrics.clone(); + let canceller = canceller.clone(); + let resetter = resetter.clone(); + + services.push(tokio::spawn(async move { + ProxyHttp::::run( + config, + tls, + metrics, + canceller.clone(), + resetter, + ) + .await + .inspect_err(|err| { + error!( + proxy = ProxyHttpInnerRpc::name(), + error = ?err, + "Failed to start http-proxy, terminating...", + ); + canceller.cancel(); + }) + })); + } + + // spawn flashblocks proxy + if config.flashblocks.enabled { + let tls = config.tls.clone(); + let config = config.flashblocks.clone(); + let metrics = metrics.clone(); + let canceller = canceller.clone(); + let resetter = resetter.clone(); + + services.push(tokio::spawn(async move { + ProxyWs::::run( + config, + tls, + metrics, + canceller.clone(), + resetter, + ) + .await + .inspect_err(|err| { + error!( + proxy = ProxyHttpInnerRpc::name(), + error = ?err, + "Failed to start websocket-proxy, terminating...", + ); + canceller.cancel(); + }) + })); + } + + futures::future::join_all(services).await; + } + + Ok(()) + } + + fn wait_for_shutdown_signal() -> CancellationToken { + let canceller = tokio_util::sync::CancellationToken::new(); + + { + let canceller = canceller.clone(); + + tokio::spawn(async move { + let sigint = async { + signal(SignalKind::interrupt()) + .expect("failed to install sigint handler") + .recv() + .await; + }; + + let sigterm = async { + signal(SignalKind::terminate()) + .expect("failed to install sigterm handler") + .recv() + .await; + }; + + tokio::select! { + _ = sigint => {}, + _ = sigterm => {}, + } + + info!("Shutdown signal received, stopping..."); + + canceller.cancel(); + }); + } + + canceller + } + + fn wait_for_reset_signal(canceller: CancellationToken) -> broadcast::Sender<()> { + let (resetter, _) = broadcast::channel::<()>(2); + + { + let resetter = resetter.clone(); + + tokio::spawn(async move { + let mut hangup = + signal(SignalKind::hangup()).expect("failed to install sighup handler"); + + loop { + tokio::select! { + _ = hangup.recv() => { + info!("Hangup signal received, resetting..."); + + if let Err(err) = resetter.send(()) { + error!(from = "sighup", error = ?err, "Failed to broadcast reset signal"); + } + } + + _ = canceller.cancelled() => { + return + }, + } + } + }); + } + + resetter + } +} diff --git a/crates/rproxy/src/server/server.rs b/crates/rproxy/src/server/server.rs deleted file mode 100644 index 7315cf7..0000000 --- a/crates/rproxy/src/server/server.rs +++ /dev/null @@ -1,235 +0,0 @@ -use std::{error::Error, sync::Arc}; - -use tokio::{ - signal::unix::{SignalKind, signal}, - sync::broadcast, - task::JoinHandle, -}; -use tokio_util::sync::CancellationToken; -use tracing::{error, info}; - -use crate::{ - circuit_breaker::CircuitBreaker, - config::{Config, ConfigAuthrpc, ConfigFlashblocks, ConfigRpc}, - metrics::Metrics, - proxy::ProxyInner, - proxy_http::{ProxyHttp, ProxyHttpInnerAuthrpc, ProxyHttpInnerRpc}, - proxy_ws::{ProxyWs, ProxyWsInnerFlashblocks}, - utils::tls_certificate_validity_timestamps, -}; - -// Proxy --------------------------------------------------------------- - -pub struct Server {} - -impl Server { - pub async fn run(config: Config) -> Result<(), Box> { - let canceller = Server::wait_for_shutdown_signal(); - let resetter = Server::wait_for_reset_signal(canceller.clone()); - - // spawn metrics service - let metrics = Arc::new(Metrics::new(config.metrics.clone())); - { - let canceller = canceller.clone(); - let metrics = metrics.clone(); - - tokio::spawn(async move { - metrics.run(canceller).await.inspect_err(|err| { - error!( - service = Metrics::name(), - error = ?err, - "Failed to start metrics service", - ); - std::process::exit(-1); - }) - }); - } - - // spawn circuit-breaker - if !config.circuit_breaker.url.is_empty() { - let canceller = canceller.clone(); - let resetter = resetter.clone(); - - let _ = std::thread::spawn(move || { - let rt = match tokio::runtime::Builder::new_current_thread().enable_all().build() { - Ok(rt) => rt, - Err(err) => { - error!(error = ?err, "Failed to initialise a single-threaded runtime for circuit-breaker"); - std::process::exit(-1); - } - }; - - let circuit_breaker = CircuitBreaker::new(config.circuit_breaker.clone()); - - tokio::task::LocalSet::new() - .block_on(&rt, async move { circuit_breaker.run(canceller, resetter).await }) - }); - } - - while !canceller.is_cancelled() { - if config.tls.enabled() { - let metrics = metrics.clone(); - let (not_before, not_after) = - tls_certificate_validity_timestamps(config.tls.certificate()); - metrics.tls_certificate_valid_not_before.set(not_before); - metrics.tls_certificate_valid_not_after.set(not_after); - } - - let mut services: Vec>>> = Vec::new(); - - // spawn authrpc proxy - if config.authrpc.enabled { - let tls = config.tls.clone(); - let config = config.authrpc.clone(); - let metrics = metrics.clone(); - let canceller = canceller.clone(); - let resetter = resetter.clone(); - - services.push(tokio::spawn(async move { - ProxyHttp::::run( - config, - tls, - metrics, - canceller.clone(), - resetter, - ) - .await - .inspect_err(|err| { - error!( - proxy = ProxyHttpInnerRpc::name(), - error = ?err, - "Failed to start http-proxy, terminating...", - ); - canceller.cancel(); - }) - })); - } - - // spawn rpc proxy - if config.rpc.enabled { - let tls = config.tls.clone(); - let config = config.rpc.clone(); - let metrics = metrics.clone(); - let canceller = canceller.clone(); - let resetter = resetter.clone(); - - services.push(tokio::spawn(async move { - ProxyHttp::::run( - config, - tls, - metrics, - canceller.clone(), - resetter, - ) - .await - .inspect_err(|err| { - error!( - proxy = ProxyHttpInnerRpc::name(), - error = ?err, - "Failed to start http-proxy, terminating...", - ); - canceller.cancel(); - }) - })); - } - - // spawn flashblocks proxy - if config.flashblocks.enabled { - let tls = config.tls.clone(); - let config = config.flashblocks.clone(); - let metrics = metrics.clone(); - let canceller = canceller.clone(); - let resetter = resetter.clone(); - - services.push(tokio::spawn(async move { - ProxyWs::::run( - config, - tls, - metrics, - canceller.clone(), - resetter, - ) - .await - .inspect_err(|err| { - error!( - proxy = ProxyHttpInnerRpc::name(), - error = ?err, - "Failed to start websocket-proxy, terminating...", - ); - canceller.cancel(); - }) - })); - } - - futures::future::join_all(services).await; - } - - Ok(()) - } - - fn wait_for_shutdown_signal() -> CancellationToken { - let canceller = tokio_util::sync::CancellationToken::new(); - - { - let canceller = canceller.clone(); - - tokio::spawn(async move { - let sigint = async { - signal(SignalKind::interrupt()) - .expect("failed to install sigint handler") - .recv() - .await; - }; - - let sigterm = async { - signal(SignalKind::terminate()) - .expect("failed to install sigterm handler") - .recv() - .await; - }; - - tokio::select! { - _ = sigint => {}, - _ = sigterm => {}, - } - - info!("Shutdown signal received, stopping..."); - - canceller.cancel(); - }); - } - - canceller - } - - fn wait_for_reset_signal(canceller: CancellationToken) -> broadcast::Sender<()> { - let (resetter, _) = broadcast::channel::<()>(2); - - { - let resetter = resetter.clone(); - - tokio::spawn(async move { - let mut hangup = - signal(SignalKind::hangup()).expect("failed to install sighup handler"); - - loop { - tokio::select! { - _ = hangup.recv() => { - info!("Hangup signal received, resetting..."); - - if let Err(err) = resetter.send(()) { - error!(from = "sighup", error = ?err, "Failed to broadcast reset signal"); - } - } - - _ = canceller.cancelled() => { - return - }, - } - } - }); - } - - resetter - } -} From 7b25cb659409e57d547063e00ce989da48fcc648 Mon Sep 17 00:00:00 2001 From: Ash Kunda <18058966+akundaz@users.noreply.github.com> Date: Mon, 13 Oct 2025 22:44:56 -0400 Subject: [PATCH 5/6] add lint make rule --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 4186d5e..ed54621 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,10 @@ build: fmt: @cargo +nightly fmt --check +.PHONY: lint +lint: + @cargo +nightly clippy --all-features -- -D warnings + .PHONY: help help: @cargo run -- --help From af980666394c05e5461a710dfc62b84eac30565a Mon Sep 17 00:00:00 2001 From: Ash Kunda <18058966+akundaz@users.noreply.github.com> Date: Tue, 14 Oct 2025 10:23:19 -0400 Subject: [PATCH 6/6] use modern module organization --- .../mod.rs => circuit_breaker.rs} | 0 .../rproxy/src/{config/mod.rs => config.rs} | 0 crates/rproxy/src/{jrpc/mod.rs => jrpc.rs} | 0 .../rproxy/src/{metrics/mod.rs => metrics.rs} | 0 crates/rproxy/src/{proxy/mod.rs => proxy.rs} | 0 .../src/{proxy_http/mod.rs => proxy_http.rs} | 0 .../src/{proxy_ws/mod.rs => proxy_ws.rs} | 0 crates/rproxy/src/proxy_ws/proxy_ws.rs | 1301 ----------------- .../rproxy/src/{server/mod.rs => server.rs} | 0 crates/rproxy/src/{utils/mod.rs => utils.rs} | 0 10 files changed, 1301 deletions(-) rename crates/rproxy/src/{circuit_breaker/mod.rs => circuit_breaker.rs} (100%) rename crates/rproxy/src/{config/mod.rs => config.rs} (100%) rename crates/rproxy/src/{jrpc/mod.rs => jrpc.rs} (100%) rename crates/rproxy/src/{metrics/mod.rs => metrics.rs} (100%) rename crates/rproxy/src/{proxy/mod.rs => proxy.rs} (100%) rename crates/rproxy/src/{proxy_http/mod.rs => proxy_http.rs} (100%) rename crates/rproxy/src/{proxy_ws/mod.rs => proxy_ws.rs} (100%) delete mode 100644 crates/rproxy/src/proxy_ws/proxy_ws.rs rename crates/rproxy/src/{server/mod.rs => server.rs} (100%) rename crates/rproxy/src/{utils/mod.rs => utils.rs} (100%) diff --git a/crates/rproxy/src/circuit_breaker/mod.rs b/crates/rproxy/src/circuit_breaker.rs similarity index 100% rename from crates/rproxy/src/circuit_breaker/mod.rs rename to crates/rproxy/src/circuit_breaker.rs diff --git a/crates/rproxy/src/config/mod.rs b/crates/rproxy/src/config.rs similarity index 100% rename from crates/rproxy/src/config/mod.rs rename to crates/rproxy/src/config.rs diff --git a/crates/rproxy/src/jrpc/mod.rs b/crates/rproxy/src/jrpc.rs similarity index 100% rename from crates/rproxy/src/jrpc/mod.rs rename to crates/rproxy/src/jrpc.rs diff --git a/crates/rproxy/src/metrics/mod.rs b/crates/rproxy/src/metrics.rs similarity index 100% rename from crates/rproxy/src/metrics/mod.rs rename to crates/rproxy/src/metrics.rs diff --git a/crates/rproxy/src/proxy/mod.rs b/crates/rproxy/src/proxy.rs similarity index 100% rename from crates/rproxy/src/proxy/mod.rs rename to crates/rproxy/src/proxy.rs diff --git a/crates/rproxy/src/proxy_http/mod.rs b/crates/rproxy/src/proxy_http.rs similarity index 100% rename from crates/rproxy/src/proxy_http/mod.rs rename to crates/rproxy/src/proxy_http.rs diff --git a/crates/rproxy/src/proxy_ws/mod.rs b/crates/rproxy/src/proxy_ws.rs similarity index 100% rename from crates/rproxy/src/proxy_ws/mod.rs rename to crates/rproxy/src/proxy_ws.rs diff --git a/crates/rproxy/src/proxy_ws/proxy_ws.rs b/crates/rproxy/src/proxy_ws/proxy_ws.rs deleted file mode 100644 index dc95825..0000000 --- a/crates/rproxy/src/proxy_ws/proxy_ws.rs +++ /dev/null @@ -1,1301 +0,0 @@ -use std::{ - io::Write, - marker::PhantomData, - str::FromStr, - sync::{ - Arc, - atomic::{AtomicI64, Ordering}, - }, - time::Duration, -}; - -use actix::{Actor, AsyncContext, WrapFuture}; -use actix_web::{ - App, - HttpRequest, - HttpResponse, - HttpServer, - middleware::{NormalizePath, TrailingSlash}, - web, -}; -use actix_ws::{MessageStream, Session}; -use bytes::{Buf, BufMut, Bytes, BytesMut}; -use futures::{ - SinkExt, - StreamExt, - stream::{SplitSink, SplitStream}, -}; -use prometheus_client::metrics::gauge::Atomic; -use scc::HashMap; -use time::{UtcDateTime, format_description::well_known::Iso8601}; -use tokio::{net::TcpStream, sync::broadcast}; -use tokio_tungstenite::{MaybeTlsStream, WebSocketStream}; -use tracing::{error, info, trace, warn}; -use tungstenite::Utf8Bytes; -use uuid::Uuid; -use x509_parser::asn1_rs::ToStatic; - -use crate::{ - config::{ConfigProxyWs, ConfigTls, PARALLELISM}, - metrics::{LabelsProxyWs, Metrics}, - proxy::{Proxy, ProxyConnectionGuard}, - proxy_http::ProxyHttpRequestInfo, - proxy_ws::ProxyWsInner, - utils::{Loggable, raw_transaction_to_hash}, -}; - -const WS_PING_INTERVAL_SECONDS: u64 = 1; - -const WS_CLI_ERROR: &str = "client error"; -const WS_BCK_ERROR: &str = "backend error"; -const WS_BCK_TIMEOUT: &str = "backend error"; -const WS_CLOSE_OK: &str = ""; - -const WS_LABEL_BACKEND: &str = "backend"; -const WS_LABEL_CLIENT: &str = "client"; - -// ProxyWs ------------------------------------------------------------- - -pub(crate) struct ProxyWs -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - id: Uuid, - - shared: ProxyWsSharedState, - postprocessor: actix::Addr>, - canceller: tokio_util::sync::CancellationToken, - - backend: ProxyWsBackendEndpoint, - - pings: HashMap, - ping_balance_cli: AtomicI64, - ping_balance_bck: AtomicI64, - - _config: PhantomData, - _proxy: PhantomData

, -} - -impl ProxyWs -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - fn new( - shared: ProxyWsSharedState, - canceller: tokio_util::sync::CancellationToken, - ) -> Self { - let id = Uuid::now_v7(); - - let config = shared.config(); - - let backend = ProxyWsBackendEndpoint::new(id, config.backend_url()); - - let postprocessor = ProxyWsPostprocessor:: { - inner: shared.inner.clone(), - metrics: shared.metrics.clone(), - worker_id: id, - _config: PhantomData, - } - .start(); - - Self { - id, - shared, - postprocessor, - canceller, - backend, - pings: HashMap::new(), - ping_balance_bck: AtomicI64::new(0), - ping_balance_cli: AtomicI64::new(0), - _config: PhantomData, - _proxy: PhantomData, - } - } - - fn config(&self) -> &C { - self.shared.config() - } - - pub(crate) async fn run( - config: C, - tls: ConfigTls, - metrics: Arc, - canceller: tokio_util::sync::CancellationToken, - resetter: broadcast::Sender<()>, - ) -> Result<(), Box> { - let listen_address = config.listen_address(); - - let listener = match Self::listen(&config) { - Ok(listener) => listener, - Err(err) => { - error!( - proxy = P::name(), - addr = %config.listen_address(), - error = ?err, - "Failed to initialise a socket" - ); - return Err(Box::new(err)); - } - }; - - let workers_count = PARALLELISM.to_static(); - - let shared = ProxyWsSharedState::::new(config, &metrics); - let client_connections_count = shared.client_connections_count.clone(); - let worker_canceller = canceller.clone(); - - info!( - proxy = P::name(), - listen_address = %listen_address, - workers_count = workers_count, - "Starting websocket-proxy...", - ); - - let server = HttpServer::new(move || { - let this = web::Data::new(Self::new(shared.clone(), worker_canceller.clone())); - - App::new() - .app_data(this) - .wrap(NormalizePath::new(TrailingSlash::Trim)) - .default_service(web::route().to(Self::receive)) - }) - .on_connect(Self::on_connect(metrics, client_connections_count)) - .shutdown_signal(canceller.cancelled_owned()) - .workers(workers_count); - - let proxy = match if tls.enabled() { - let cert = tls.certificate().clone(); - let key = tls.key().clone_key(); - - server.listen_rustls_0_23( - listener, - rustls::ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(cert, key) - .unwrap(), - ) - } else { - server.listen(listener) - } { - Ok(server) => server, - Err(err) => { - error!(proxy = P::name(), error = ?err, "Failed to initialise websocket-proxy"); - return Err(Box::new(err)); - } - } - .run(); - - let handler = proxy.handle(); - let mut resetter = resetter.subscribe(); - tokio::spawn(async move { - if resetter.recv().await.is_ok() { - info!(proxy = P::name(), "Reset signal received, stopping websocket-proxy..."); - handler.stop(true).await; - } - }); - - if let Err(err) = proxy.await { - error!(proxy = P::name(), error = ?err, "Failure while running websocket-proxy") - } - - info!(proxy = P::name(), "Stopped websocket-proxy"); - - Ok(()) - } - - fn listen(config: &C) -> std::io::Result { - let socket = socket2::Socket::new( - socket2::Domain::for_address(config.listen_address()), - socket2::Type::STREAM, - Some(socket2::Protocol::TCP), - )?; - - // must use non-blocking with tokio - socket.set_nonblocking(true)?; - - // allow time to flush buffers on close - socket.set_linger(Some(config.backend_timeout()))?; - - // allow binding to the socket whlie there are still TIME_WAIT conns - socket.set_reuse_address(true)?; - - socket.bind(&socket2::SockAddr::from(config.listen_address()))?; - - socket.listen(1024)?; - - Ok(socket.into()) - } - - async fn receive( - cli_req: HttpRequest, - cli_req_body: web::Payload, - this: web::Data, - ) -> Result { - let info = ProxyHttpRequestInfo::new(&cli_req, cli_req.conn_data::()); - - let (res, cli_tx, cli_rx) = match actix_ws::handle(&cli_req, cli_req_body) { - Ok(res) => res, - Err(err) => { - error!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to upgrade to websocket", - ); - return Err(err); - } - }; - - actix_web::rt::spawn(Self::handshake(this, cli_tx, cli_rx, info)); - - Ok(res) - } - - async fn handshake( - this: web::Data, - cli_tx: Session, - cli_rx: MessageStream, - info: ProxyHttpRequestInfo, - ) { - let bck_uri = this.backend.new_backend_uri(&info); - trace!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - worker_id = %this.id, - backend_uri = %bck_uri, - "Starting websocket handshake...", - ); - - let (bck_stream, _) = match tokio::time::timeout( - this.config().backend_timeout(), - tokio_tungstenite::connect_async(bck_uri), - ) - .await - { - Ok(Ok(res)) => res, - - Ok(Err(err)) => { - error!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to establish backend websocket session" - ); - - if let Err(err) = cli_tx - .close(Some(actix_ws::CloseReason { - code: awc::ws::CloseCode::Error, - description: Some(String::from(WS_BCK_ERROR)), - })) - .await - { - error!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to close client websocket session" - ); - }; - return; - } - - Err(_) => { - error!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "Timed out to establish backend websocket session" - ); - - if let Err(err) = cli_tx - .close(Some(actix_ws::CloseReason { - code: awc::ws::CloseCode::Again, - description: Some(String::from(WS_BCK_TIMEOUT)), - })) - .await - { - error!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to close client websocket session" - ); - } - return; - } - }; - - let (bck_tx, bck_rx) = bck_stream.split(); - - Self::pump(this, info, cli_tx, cli_rx, bck_tx, bck_rx).await; - } - - async fn pump( - this: web::Data, - info: ProxyHttpRequestInfo, - mut cli_tx: Session, - mut cli_rx: MessageStream, - mut bck_tx: SplitSink>, tungstenite::Message>, - mut bck_rx: SplitStream>>, - ) { - info!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "Starting websocket pump..." - ); - - let info = Arc::new(info); - - let mut heartbeat = tokio::time::interval(Duration::from_secs(WS_PING_INTERVAL_SECONDS)); - - let mut pumping: Result<(), &str> = Ok(()); - - while pumping.is_ok() && !this.canceller.is_cancelled() { - tokio::select! { - _ = this.canceller.cancelled() => { - break; - } - - // ping both sides - _ = heartbeat.tick() => { - pumping = Self::heartbeat(&this, info.clone(), &mut cli_tx, &mut bck_tx).await; - } - - // client => backend - cli_msg = cli_rx.next() => { - pumping = Self::pump_cli_to_bck( - &this, - info.clone(), - UtcDateTime::now(), - cli_msg, - &mut bck_tx, - &mut cli_tx - ).await; - } - - // backend => client - bck_msg = bck_rx.next() => { - pumping = Self::pump_bck_to_cli( - &this, - info.clone(), - UtcDateTime::now(), - bck_msg, - &mut cli_tx, - &mut bck_tx - ).await; - } - } - } - - if let Err(msg) = pumping && - msg != WS_CLOSE_OK - { - if let Err(err) = cli_tx - .close(Some(actix_ws::CloseReason { - code: awc::ws::CloseCode::Error, - description: Some(String::from(msg)), - })) - .await - { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to close client websocket session" - ); - } - - if let Err(err) = bck_tx - .send(tungstenite::Message::Close(Some(tungstenite::protocol::CloseFrame { - code: tungstenite::protocol::frame::coding::CloseCode::Error, - reason: msg.into(), - }))) - .await - { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to close backend websocket session" - ); - } - } else { - if let Err(err) = cli_tx - .close(Some(actix_ws::CloseReason { - code: awc::ws::CloseCode::Normal, - description: None, - })) - .await - { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to close client websocket session" - ); - } - - if let Err(err) = bck_tx - .send(tungstenite::Message::Close(Some(tungstenite::protocol::CloseFrame { - code: tungstenite::protocol::frame::coding::CloseCode::Normal, - reason: Utf8Bytes::default(), - }))) - .await - { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to close backend websocket session" - ); - } - } - - info!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "Stopped websocket pump" - ); - } - - async fn heartbeat( - this: &web::Data, - info: Arc, - cli_tx: &mut Session, - bck_tx: &mut SplitSink>, tungstenite::Message>, - ) -> Result<(), &'static str> { - let ping_threshold = - (1 + this.config().backend_timeout().as_secs() / WS_PING_INTERVAL_SECONDS) as i64; - - { - // ping -> client - - if this.ping_balance_cli.load(Ordering::Relaxed) > ping_threshold { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "More than {} websocket pings sent to client didn't return, terminating the pump...", ping_threshold, - ); - return Err(WS_CLI_ERROR); - } - - let cli_ping = ProxyWsPing::new(info.connection_id()); - if let Err(err) = cli_tx.ping(&cli_ping.to_slice()).await { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to send ping websocket message to client" - ); - return Err(WS_CLI_ERROR); - } - let _ = this.pings.insert_sync(cli_ping.id, cli_ping); - this.ping_balance_cli.inc(); - } - - { - // ping -> backend - - if this.ping_balance_bck.load(Ordering::Relaxed) > ping_threshold { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "More than {} websocket pings sent to backend didn't return, terminating the pump...", ping_threshold, - ); - return Err(WS_BCK_ERROR); - } - - let bck_ping = ProxyWsPing::new(info.connection_id()); - if let Err(err) = bck_tx.send(tungstenite::Message::Ping(bck_ping.to_bytes())).await { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to send ping websocket message to backend" - ); - return Err(WS_BCK_ERROR); - } - let _ = this.pings.insert_sync(bck_ping.id, bck_ping); - this.ping_balance_bck.inc(); - } - Ok(()) - } - - async fn pump_cli_to_bck( - this: &web::Data, - info: Arc, - timestamp: UtcDateTime, - cli_msg: Option>, - bck_tx: &mut SplitSink>, tungstenite::Message>, - cli_tx: &mut Session, - ) -> Result<(), &'static str> { - match cli_msg { - Some(Ok(msg)) => { - match msg { - // binary - actix_ws::Message::Binary(bytes) => { - if let Err(err) = - bck_tx.send(tungstenite::Message::Binary(bytes.clone())).await - { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to proxy binary websocket message to backend" - ); - this.shared - .metrics - .ws_proxy_failure_count - .get_or_create(&LabelsProxyWs { - proxy: P::name(), - destination: WS_LABEL_BACKEND, - }) - .inc(); - return Err(WS_BCK_ERROR); - } - this.postprocessor.do_send(ProxyWsMessage::ClientToBackendBinary { - msg: bytes, - info, - start: timestamp, - end: UtcDateTime::now(), - }); - Ok(()) - } - - // text - actix_ws::Message::Text(text) => { - if let Err(err) = bck_tx - .send(tungstenite::Message::Text(unsafe { - // safety: it's from client's ws message => must be valid utf-8 - tungstenite::protocol::frame::Utf8Bytes::from_bytes_unchecked( - text.clone().into_bytes(), - ) - })) - .await - { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to proxy text websocket message to backend" - ); - this.shared - .metrics - .ws_proxy_failure_count - .get_or_create(&LabelsProxyWs { - proxy: P::name(), - destination: WS_LABEL_BACKEND, - }) - .inc(); - return Err(WS_BCK_ERROR); - } - this.postprocessor.do_send(ProxyWsMessage::ClientToBackendText { - msg: text, - info, - start: timestamp, - end: UtcDateTime::now(), - }); - Ok(()) - } - - // ping - actix_ws::Message::Ping(bytes) => { - if let Err(err) = cli_tx.pong(&bytes).await { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to return pong message to client" - ); - return Err(WS_CLI_ERROR); - } - Ok(()) - } - - // pong - actix_ws::Message::Pong(bytes) => { - if let Some(pong) = ProxyWsPing::from_bytes(bytes) && - let Some((_, ping)) = this.pings.remove_sync(&pong.id) && - pong == ping - { - this.ping_balance_cli.dec(); - this.shared - .metrics - .ws_latency_client - .get_or_create(&LabelsProxyWs { - proxy: P::name(), - destination: WS_LABEL_BACKEND, - }) - .record( - (1000000.0 * (timestamp - pong.timestamp).as_seconds_f64() / - 2.0) as i64, - ); - return Ok(()); - } - warn!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "Unexpected websocket pong received from client", - ); - Ok(()) - } - - // close - actix_ws::Message::Close(reason) => { - if let Err(err) = bck_tx - .send(tungstenite::Message::Close(reason.map(|r| { - tungstenite::protocol::CloseFrame { - code: tungstenite::protocol::frame::coding::CloseCode::from( - u16::from(r.code), - ), - reason: r.description.unwrap_or_default().into(), - } - }))) - .await - { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to proxy close websocket message to backend" - ); - return Err(WS_BCK_ERROR); - } - Err(WS_CLOSE_OK) - } - - _ => Ok(()), - } - } - - Some(Err(err)) => { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Client websocket stream error" - ); - Err(WS_CLI_ERROR) - } - - None => { - info!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "Client had closed websocket stream" - ); - Err(WS_CLOSE_OK) - } - } - } - - async fn pump_bck_to_cli( - this: &web::Data, - info: Arc, - timestamp: UtcDateTime, - bck_msg: Option>, - cli_tx: &mut Session, - bck_tx: &mut SplitSink>, tungstenite::Message>, - ) -> Result<(), &'static str> { - match bck_msg { - Some(Ok(msg)) => { - match msg { - // binary - tungstenite::Message::Binary(bytes) => { - if let Err(err) = cli_tx.binary(bytes.clone()).await { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to proxy binary websocket message to client" - ); - this.shared - .metrics - .ws_proxy_failure_count - .get_or_create(&LabelsProxyWs { - proxy: P::name(), - destination: WS_LABEL_CLIENT, - }) - .inc(); - return Err(WS_CLI_ERROR); - } - this.postprocessor.do_send(ProxyWsMessage::BackendToClientBinary { - msg: bytes, - info, - start: timestamp, - end: UtcDateTime::now(), - }); - Ok(()) - } - - // text - tungstenite::Message::Text(text) => { - if let Err(err) = cli_tx.text(text.clone().as_str()).await { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to proxy text websocket message to client" - ); - this.shared - .metrics - .ws_proxy_failure_count - .get_or_create(&LabelsProxyWs { - proxy: P::name(), - destination: WS_LABEL_CLIENT, - }) - .inc(); - return Err(WS_CLI_ERROR); - } - this.postprocessor.do_send(ProxyWsMessage::BackendToClientText { - msg: text, - info, - start: timestamp, - end: UtcDateTime::now(), - }); - Ok(()) - } - - // ping - tungstenite::Message::Ping(bytes) => { - if let Err(err) = bck_tx.send(tungstenite::Message::Pong(bytes)).await { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to return pong message to backend" - ); - return Err(WS_BCK_ERROR); - } - Ok(()) - } - - // pong - tungstenite::Message::Pong(bytes) => { - if let Some(pong) = ProxyWsPing::from_bytes(bytes) && - let Some((_, ping)) = this.pings.remove_sync(&pong.id) && - pong == ping - { - this.ping_balance_bck.dec(); - this.shared - .metrics - .ws_latency_backend - .get_or_create(&LabelsProxyWs { - proxy: P::name(), - destination: WS_LABEL_BACKEND, - }) - .record( - (1000000.0 * (timestamp - pong.timestamp).as_seconds_f64() / - 2.0) as i64, - ); - return Ok(()); - } - warn!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "Unexpected websocket pong received from backend", - ); - Ok(()) - } - - // close - tungstenite::Message::Close(reason) => { - if let Err(err) = cli_tx - .clone() // .close() consumes it - .close(reason.map(|reason| actix_ws::CloseReason { - code: u16::from(reason.code).into(), - description: reason.reason.to_string().into(), - })) - .await - { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Failed to proxy close websocket message to client" - ); - return Err(WS_CLI_ERROR); - } - Err(WS_CLOSE_OK) - } - - _ => Ok(()), - } - } - - Some(Err(err)) => { - error!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - error = ?err, - "Backend websocket stream error" - ); - Err(WS_BCK_ERROR) - } - - None => { - info!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %this.id, - "Backend had closed websocket stream" - ); - Err(WS_CLOSE_OK) - } - } - } - - fn finalise_proxying( - msg: ProxyWsMessage, - inner: Arc

, - metrics: Arc, - worker_id: Uuid, - ) { - Self::maybe_log_proxied_message(&msg, inner.clone(), worker_id); - - Self::emit_metrics_on_proxy_success(&msg, metrics.clone()); - } - - fn maybe_log_proxied_message(msg: &ProxyWsMessage, inner: Arc

, worker_id: Uuid) { - let config = inner.config(); - - match msg { - ProxyWsMessage::BackendToClientBinary { msg, info, start, end } => { - let json_msg = if config.log_backend_messages() { - Loggable(&Self::maybe_sanitise( - config.log_sanitise(), - serde_json::from_slice(msg).unwrap_or_default(), - )) - } else { - Loggable(&serde_json::Value::Null) - }; - - info!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %worker_id, - remote_addr = info.remote_addr(), - ts_message_received = start.format(&Iso8601::DEFAULT).unwrap_or_default(), - latency_proxying = (*end - *start).as_seconds_f64(), - message = tracing::field::valuable(&json_msg), - "Proxied binary message to client", - ); - } - - ProxyWsMessage::BackendToClientText { msg, info, start, end } => { - let json_msg = if config.log_backend_messages() { - Loggable(&Self::maybe_sanitise( - config.log_sanitise(), - serde_json::from_str(msg).unwrap_or_default(), - )) - } else { - Loggable(&serde_json::Value::Null) - }; - - info!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %worker_id, - remote_addr = info.remote_addr(), - ts_message_received = start.format(&Iso8601::DEFAULT).unwrap_or_default(), - latency_proxying = (*end - *start).as_seconds_f64(), - message = tracing::field::valuable(&json_msg), - "Proxied text message to client", - ); - } - - ProxyWsMessage::ClientToBackendBinary { msg, info, start, end } => { - let json_msg = if config.log_client_messages() { - Loggable(&Self::maybe_sanitise( - config.log_sanitise(), - serde_json::from_slice(msg).unwrap_or_default(), - )) - } else { - Loggable(&serde_json::Value::Null) - }; - - info!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %worker_id, - remote_addr = info.remote_addr(), - ts_message_received = start.format(&Iso8601::DEFAULT).unwrap_or_default(), - latency_proxying = (*end - *start).as_seconds_f64(), - message = tracing::field::valuable(&json_msg), - "Proxied binary message to backend", - ); - } - - ProxyWsMessage::ClientToBackendText { msg, info, start, end } => { - let json_msg = if config.log_client_messages() { - Loggable(&Self::maybe_sanitise( - config.log_sanitise(), - serde_json::from_str(msg).unwrap_or_default(), - )) - } else { - Loggable(&serde_json::Value::Null) - }; - - info!( - proxy = P::name(), - connection_id = %info.connection_id(), - worker_id = %worker_id, - remote_addr = info.remote_addr(), - ts_message_received = start.format(&Iso8601::DEFAULT).unwrap_or_default(), - latency_proxying = (*end - *start).as_seconds_f64(), - message = tracing::field::valuable(&json_msg), - "Proxied text message to backend", - ); - } - } - } - - fn maybe_sanitise(do_sanitise: bool, mut message: serde_json::Value) -> serde_json::Value { - if !do_sanitise { - return message; - } - - if let Some(object) = message.as_object_mut() && - let Some(diff) = object.get_mut("diff") && - let Some(transactions) = diff.get_mut("transactions") && - let Some(transactions) = transactions.as_array_mut() - { - for transaction in transactions { - raw_transaction_to_hash(transaction); - } - } - - message - } - - fn emit_metrics_on_proxy_success(msg: &ProxyWsMessage, metrics: Arc) { - match msg { - ProxyWsMessage::BackendToClientBinary { msg, info: _, start, end } => { - let labels = LabelsProxyWs { proxy: P::name(), destination: WS_LABEL_CLIENT }; - metrics - .ws_latency_proxy - .get_or_create(&labels) - .record((1000000.0 * (*end - *start).as_seconds_f64()) as i64); - metrics.ws_proxy_success_count.get_or_create_owned(&labels).inc(); - metrics.ws_message_size.get_or_create_owned(&labels).record(msg.len() as i64); - } - - ProxyWsMessage::BackendToClientText { msg, info: _, start, end } => { - let labels = LabelsProxyWs { proxy: P::name(), destination: WS_LABEL_CLIENT }; - metrics - .ws_latency_proxy - .get_or_create(&labels) - .record((1000000.0 * (*end - *start).as_seconds_f64()) as i64); - metrics.ws_proxy_success_count.get_or_create_owned(&labels).inc(); - metrics.ws_message_size.get_or_create_owned(&labels).record(msg.len() as i64); - } - - ProxyWsMessage::ClientToBackendBinary { msg, info: _, start, end } => { - let labels = LabelsProxyWs { proxy: P::name(), destination: WS_LABEL_BACKEND }; - metrics - .ws_latency_proxy - .get_or_create(&labels) - .record((1000000.0 * (*end - *start).as_seconds_f64()) as i64); - metrics.ws_proxy_success_count.get_or_create_owned(&labels).inc(); - metrics.ws_message_size.get_or_create_owned(&labels).record(msg.len() as i64); - } - - ProxyWsMessage::ClientToBackendText { msg, info: _, start, end } => { - let labels = LabelsProxyWs { proxy: P::name(), destination: WS_LABEL_BACKEND }; - metrics - .ws_latency_proxy - .get_or_create(&labels) - .record((1000000.0 * (*end - *start).as_seconds_f64()) as i64); - metrics.ws_proxy_success_count.get_or_create_owned(&labels).inc(); - metrics.ws_message_size.get_or_create_owned(&labels).record(msg.len() as i64); - } - } - } -} - -impl Proxy

for ProxyWs -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ -} - -// ProxyWsSharedState -------------------------------------------------- - -#[derive(Clone)] -struct ProxyWsSharedState -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - inner: Arc

, - metrics: Arc, - - client_connections_count: Arc, - - _config: PhantomData, -} - -impl ProxyWsSharedState -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - fn new(config: C, metrics: &Arc) -> Self { - Self { - inner: Arc::new(P::new(config)), - metrics: metrics.clone(), - client_connections_count: Arc::new(AtomicI64::new(0)), - _config: PhantomData, - } - } - - #[inline] - fn config(&self) -> &C { - self.inner.config() - } -} - -// ProxyWsBackendEndpoint ---------------------------------------------- - -struct ProxyWsBackendEndpoint -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - worker_id: Uuid, - - url: tungstenite::http::Uri, - - _config: PhantomData, - _inner: PhantomData

, -} - -impl ProxyWsBackendEndpoint -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - fn new(worker_id: Uuid, url: tungstenite::http::Uri) -> Self { - Self { worker_id, url, _config: PhantomData, _inner: PhantomData } - } - - fn new_backend_uri(&self, info: &ProxyHttpRequestInfo) -> tungstenite::http::Uri { - let mut parts = self.url.clone().into_parts(); - let pq = tungstenite::http::uri::PathAndQuery::from_str(info.path_and_query()) - .inspect_err(|err| { - error!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - worker_id = %self.worker_id, - error = ?err, - "Failed to re-parse client request's path and query", - ); - }) - .ok(); - parts.path_and_query = pq; - - tungstenite::http::Uri::from_parts(parts) - .inspect_err(|err| { - error!( - proxy = P::name(), - request_id = %info.id(), - connection_id = %info.connection_id(), - worker_id = %self.worker_id, - error = ?err, "Failed to construct backend URI, defaulting to the base one", - ); - }) - .unwrap_or(self.url.clone()) - } -} - -// ProxyWsPostprocessor - -struct ProxyWsPostprocessor -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - inner: Arc

, - worker_id: Uuid, - metrics: Arc, - - _config: PhantomData, -} - -impl actix::Actor for ProxyWsPostprocessor -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(1024); - } -} - -impl actix::Handler for ProxyWsPostprocessor -where - C: ConfigProxyWs, - P: ProxyWsInner, -{ - type Result = (); - - fn handle(&mut self, msg: ProxyWsMessage, ctx: &mut Self::Context) -> Self::Result { - let inner = self.inner.clone(); - let metrics = self.metrics.clone(); - let worker_id = self.worker_id; - - ctx.spawn( - async move { - ProxyWs::::finalise_proxying(msg, inner, metrics, worker_id); - } - .into_actor(self), - ); - } -} - -// ProxyWsMessage ------------------------------------------------------ - -#[derive(Clone, actix::Message)] -#[rtype(result = "()")] -enum ProxyWsMessage { - BackendToClientBinary { - msg: bytes::Bytes, - info: Arc, - start: UtcDateTime, - end: UtcDateTime, - }, - - BackendToClientText { - msg: tungstenite::protocol::frame::Utf8Bytes, - info: Arc, - start: UtcDateTime, - end: UtcDateTime, - }, - - ClientToBackendBinary { - msg: bytes::Bytes, - info: Arc, - start: UtcDateTime, - end: UtcDateTime, - }, - - ClientToBackendText { - msg: bytestring::ByteString, - info: Arc, - start: UtcDateTime, - end: UtcDateTime, - }, -} - -// ProxyWsPing --------------------------------------------------------- - -#[derive(PartialEq, Eq)] -struct ProxyWsPing { - id: Uuid, - connection_id: Uuid, - timestamp: UtcDateTime, -} - -impl ProxyWsPing { - fn new(connection_id: Uuid) -> Self { - Self { id: Uuid::now_v7(), connection_id, timestamp: UtcDateTime::now() } - } - - fn to_bytes(&self) -> Bytes { - let mut bytes = BytesMut::with_capacity(48); - bytes.put_u128(self.id.as_u128()); - bytes.put_u128(self.connection_id.as_u128()); - bytes.put_i128(self.timestamp.unix_timestamp_nanos()); - bytes.freeze() - } - - fn from_bytes(mut bytes: Bytes) -> Option { - if bytes.len() != 48 { - return None; - } - - let id = Uuid::from_u128(bytes.get_u128()); - let connection_id = Uuid::from_u128(bytes.get_u128()); - let timestamp = match UtcDateTime::from_unix_timestamp_nanos(bytes.get_i128()) { - Ok(timestamp) => timestamp, - Err(_) => return None, - }; - - Some(Self { id, connection_id, timestamp }) - } - - fn to_slice(&self) -> [u8; 48] { - let res: [u8; 48] = [0; 48]; - let mut cur = std::io::Cursor::new(res); - - let _ = cur.write(self.id.as_bytes()); - let _ = cur.write(self.connection_id.as_bytes()); - let _ = cur.write(&self.timestamp.unix_timestamp_nanos().to_be_bytes()); - - cur.into_inner() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn proxy_ws_ping_encode_decode() { - let ping = ProxyWsPing::new(Uuid::now_v7()); - - { - let pong = ProxyWsPing::from_bytes(ping.to_bytes()); - assert!(pong.is_some(), "must be some"); - let pong = pong.unwrap(); // safety: just verified - assert!(pong == ping, "must be the same"); - } - - { - let slice = ping.to_slice(); - let pong = ProxyWsPing::from_bytes(Bytes::copy_from_slice(&slice)); - assert!(pong.is_some(), "must be some"); - let pong = pong.unwrap(); // safety: just verified - assert!(pong == ping, "must be the same"); - } - } -} diff --git a/crates/rproxy/src/server/mod.rs b/crates/rproxy/src/server.rs similarity index 100% rename from crates/rproxy/src/server/mod.rs rename to crates/rproxy/src/server.rs diff --git a/crates/rproxy/src/utils/mod.rs b/crates/rproxy/src/utils.rs similarity index 100% rename from crates/rproxy/src/utils/mod.rs rename to crates/rproxy/src/utils.rs