feat(policy): Allow outbound hostname metrics (#3770)

Outbound hostname metrics were recently disabled. This conditionally re-enables those through a `LINKERD2_PROXY_OUTBOUND_METRICS_HOSTNAME_LABELS` env var, wired through the policy/routing config with the option of individual policies and routes to set this separately from the global config.

Signed-off-by: Scott Fleener <scott@buoyant.io>

Author: sfleen

linkerd/app/outbound/src/lib.rs

@@ -6,11 +6,11 @@
 #![allow(opaque_hidden_inferred_bound)]
 #![forbid(unsafe_code)]
 
-use linkerd_app_core::http_tracing::SpanSink;
 use linkerd_app_core::{
     config::{ProxyConfig, QueueConfig},
     drain,
     exp_backoff::ExponentialBackoff,
+    http_tracing::SpanSink,
     identity, io,
     metrics::prom,
     profiles,
@@ -143,6 +143,7 @@ impl Outbound<()> {
         client: C,
         backoff: ExponentialBackoff,
         limits: ReceiveLimits,
+        export_hostname_labels: bool,
     ) -> impl policy::GetPolicy
     where
         C: tonic::client::GrpcService<tonic::body::BoxBody, Error = Error>,
@@ -151,12 +152,18 @@ impl Outbound<()> {
         C::ResponseBody: Send + 'static,
         C::Future: Send,
     {
-        policy::Api::new(workload, limits, Duration::from_secs(10), client)
-            .into_watch(backoff)
-            .map_result(|response| match response {
-                Err(e) => Err(e.into()),
-                Ok(rsp) => Ok(rsp.into_inner()),
-            })
+        policy::Api::new(
+            workload,
+            limits,
+            Duration::from_secs(10),
+            export_hostname_labels,
+            client,
+        )
+        .into_watch(backoff)
+        .map_result(|res| match res {
+            Err(e) => Err(e.into()),
+            Ok(rsp) => Ok(rsp.into_inner()),
+        })
     }
 
     #[cfg(any(test, feature = "test-util"))]

linkerd/app/outbound/src/policy/api.rs

@@ -8,7 +8,7 @@ use linkerd_app_core::{
     svc::Service,
     Addr, Error, Recover, Result,
 };
-use linkerd_proxy_client_policy::ClientPolicy;
+use linkerd_proxy_client_policy::{ClientPolicy, ClientPolicyOverrides};
 use linkerd_tonic_stream::{LimitReceiveFuture, ReceiveLimits};
 use linkerd_tonic_watch::StreamWatch;
 use std::sync::Arc;
@@ -19,6 +19,7 @@ pub(crate) struct Api<S> {
     workload: Arc<str>,
     limits: ReceiveLimits,
     default_detect_timeout: time::Duration,
+    export_hostname_labels: bool,
     client: Client<S>,
 }
 
@@ -39,12 +40,14 @@ where
         workload: Arc<str>,
         limits: ReceiveLimits,
         default_detect_timeout: time::Duration,
+        export_hostname_labels: bool,
         client: S,
     ) -> Self {
         Self {
             workload,
             limits,
             default_detect_timeout,
+            export_hostname_labels,
             client: Client::new(client),
         }
     }
@@ -86,6 +89,9 @@ where
         };
 
         let detect_timeout = self.default_detect_timeout;
+        let overrides = ClientPolicyOverrides {
+            export_hostname_labels: self.export_hostname_labels,
+        };
         let limits = self.limits;
         let mut client = self.client.clone();
         Box::pin(async move {
@@ -96,7 +102,7 @@ where
                     // If the server returned an invalid client policy, we
                     // default to using an invalid policy that causes all
                     // requests to report an internal error.
-                    let policy = ClientPolicy::try_from(up).unwrap_or_else(|error| {
+                    let policy = ClientPolicy::try_from(overrides, up).unwrap_or_else(|error| {
                         tracing::warn!(%error, "Client policy misconfigured");
                         INVALID_POLICY
                             .get_or_init(|| ClientPolicy::invalid(detect_timeout))

linkerd/app/src/env.rs

@@ -146,6 +146,9 @@ pub const ENV_OUTBOUND_MAX_IN_FLIGHT: &str = "LINKERD2_PROXY_OUTBOUND_MAX_IN_FLI
 const ENV_OUTBOUND_DISABLE_INFORMATIONAL_HEADERS: &str =
     "LINKERD2_PROXY_OUTBOUND_DISABLE_INFORMATIONAL_HEADERS";
 
+const ENV_OUTBOUND_METRICS_HOSTNAME_LABELS: &str =
+    "LINKERD2_PROXY_OUTBOUND_METRICS_HOSTNAME_LABELS";
+
 const ENV_TRACE_ATTRIBUTES_PATH: &str = "LINKERD2_PROXY_TRACE_ATTRIBUTES_PATH";
 const ENV_TRACE_PROTOCOL: &str = "LINKERD2_PROXY_TRACE_PROTOCOL";
 const ENV_TRACE_SERVICE_NAME: &str = "LINKERD2_PROXY_TRACE_SERVICE_NAME";
@@ -791,11 +794,14 @@ pub fn parse_config<S: Strings>(strings: &S) -> Result<super::Config, EnvError>
                 },
             }
         };
+        let export_hostname_labels =
+            parse(strings, ENV_OUTBOUND_METRICS_HOSTNAME_LABELS, parse_bool)?.unwrap_or(false);
 
         policy::Config {
             control,
             workload,
             limits,
+            export_hostname_labels,
         }
     };
 

linkerd/app/src/lib.rs

@@ -174,6 +174,7 @@ impl Config {
         }?;
 
         debug!("Building Policy client");
+        let export_hostname_labels = policy.export_hostname_labels;
         let policies = {
             let control_metrics =
                 ControlMetrics::register(registry.sub_registry_with_prefix("control_policy"));
@@ -243,6 +244,7 @@ impl Config {
             policies.client.clone(),
             policies.backoff,
             policies.limits,
+            export_hostname_labels,
         );
 
         let dst_addr = dst.addr.clone();

linkerd/app/src/policy.rs

@@ -15,6 +15,7 @@ pub struct Config {
     pub control: control::Config,
     pub workload: String,
     pub limits: ReceiveLimits,
+    pub export_hostname_labels: bool,
 }
 
 /// Handles to policy service clients.

linkerd/proxy/client-policy/src/grpc.rs

@@ -105,7 +105,7 @@ pub mod proto {
         proto::{
             BackendSet, InvalidBackend, InvalidDistribution, InvalidFailureAccrual, InvalidMeta,
         },
-        Meta, RouteBackend, RouteDistribution,
+        ClientPolicyOverrides, Meta, RouteBackend, RouteDistribution,
     };
     use linkerd2_proxy_api::outbound::{self, grpc_route};
     use linkerd_http_route::{
@@ -184,22 +184,22 @@ pub mod proto {
         Redirect(#[from] InvalidRequestRedirect),
     }
 
-    impl TryFrom<outbound::proxy_protocol::Grpc> for Grpc {
-        type Error = InvalidGrpcRoute;
-        fn try_from(proto: outbound::proxy_protocol::Grpc) -> Result<Self, Self::Error> {
+    impl Grpc {
+        pub fn try_from(
+            overrides: ClientPolicyOverrides,
+            proto: outbound::proxy_protocol::Grpc,
+        ) -> Result<Self, InvalidGrpcRoute> {
             let routes = proto
                 .routes
                 .into_iter()
-                .map(try_route)
+                .map(|p| try_route(overrides, p))
                 .collect::<Result<Arc<[_]>, _>>()?;
             Ok(Self {
                 routes,
                 failure_accrual: proto.failure_accrual.try_into()?,
             })
         }
-    }
 
-    impl Grpc {
         pub fn fill_backends(&self, set: &mut BackendSet) {
             for Route { ref rules, .. } in &*self.routes {
                 for Rule { ref policy, .. } in rules {
@@ -209,7 +209,10 @@ pub mod proto {
         }
     }
 
-    fn try_route(proto: outbound::GrpcRoute) -> Result<Route, InvalidGrpcRoute> {
+    fn try_route(
+        overrides: ClientPolicyOverrides,
+        proto: outbound::GrpcRoute,
+    ) -> Result<Route, InvalidGrpcRoute> {
         let outbound::GrpcRoute {
             hosts,
             rules,
@@ -227,14 +230,15 @@ pub mod proto {
 
         let rules = rules
             .into_iter()
-            .map(|rule| try_rule(&meta, rule))
+            .map(|rule| try_rule(&meta, overrides, rule))
             .collect::<Result<Vec<_>, _>>()?;
 
         Ok(Route { hosts, rules })
     }
 
     fn try_rule(
         meta: &Arc<Meta>,
+        overrides: ClientPolicyOverrides,
         proto: outbound::grpc_route::Rule,
     ) -> Result<Rule, InvalidGrpcRoute> {
         #[allow(deprecated)]
@@ -262,13 +266,8 @@ pub mod proto {
             .ok_or(InvalidGrpcRoute::Missing("distribution"))?
             .try_into()?;
 
-        let export_hostname_labels = false;
-        let mut params = RouteParams::try_from_proto(
-            timeouts,
-            retry,
-            allow_l5d_request_headers,
-            export_hostname_labels,
-        )?;
+        let mut params =
+            RouteParams::try_from_proto(timeouts, retry, allow_l5d_request_headers, overrides)?;
         let legacy = request_timeout.map(TryInto::try_into).transpose()?;
         params.timeouts.request = params.timeouts.request.or(legacy);
 
@@ -288,7 +287,7 @@ pub mod proto {
             timeouts: Option<linkerd2_proxy_api::http_route::Timeouts>,
             retry: Option<grpc_route::Retry>,
             allow_l5d_request_headers: bool,
-            export_hostname_labels: bool,
+            overrides: ClientPolicyOverrides,
         ) -> Result<Self, InvalidGrpcRoute> {
             Ok(Self {
                 retry: retry.map(Retry::try_from).transpose()?,
@@ -297,7 +296,7 @@ pub mod proto {
                     .transpose()?
                     .unwrap_or_default(),
                 allow_l5d_request_headers,
-                export_hostname_labels,
+                export_hostname_labels: overrides.export_hostname_labels,
             })
         }
     }

linkerd/proxy/client-policy/src/http.rs

@@ -125,7 +125,7 @@ pub mod proto {
         proto::{
             BackendSet, InvalidBackend, InvalidDistribution, InvalidFailureAccrual, InvalidMeta,
         },
-        Meta, RouteBackend, RouteDistribution,
+        ClientPolicyOverrides, Meta, RouteBackend, RouteDistribution,
     };
     use linkerd2_proxy_api::outbound::{self, http_route};
     use linkerd_http_route::http::{
@@ -217,13 +217,15 @@ pub mod proto {
         }
     }
 
-    impl TryFrom<outbound::proxy_protocol::Http1> for Http1 {
-        type Error = InvalidHttpRoute;
-        fn try_from(proto: outbound::proxy_protocol::Http1) -> Result<Self, Self::Error> {
+    impl Http1 {
+        pub fn try_from(
+            overrides: ClientPolicyOverrides,
+            proto: outbound::proxy_protocol::Http1,
+        ) -> Result<Self, InvalidHttpRoute> {
             let routes = proto
                 .routes
                 .into_iter()
-                .map(try_route)
+                .map(|p| try_route(overrides, p))
                 .collect::<Result<Arc<[_]>, _>>()?;
             Ok(Self {
                 routes,
@@ -232,13 +234,15 @@ pub mod proto {
         }
     }
 
-    impl TryFrom<outbound::proxy_protocol::Http2> for Http2 {
-        type Error = InvalidHttpRoute;
-        fn try_from(proto: outbound::proxy_protocol::Http2) -> Result<Self, Self::Error> {
+    impl Http2 {
+        pub fn try_from(
+            overrides: ClientPolicyOverrides,
+            proto: outbound::proxy_protocol::Http2,
+        ) -> Result<Self, InvalidHttpRoute> {
             let routes = proto
                 .routes
                 .into_iter()
-                .map(try_route)
+                .map(|p| try_route(overrides, p))
                 .collect::<Result<Arc<[_]>, _>>()?;
             Ok(Self {
                 routes,
@@ -247,7 +251,10 @@ pub mod proto {
         }
     }
 
-    fn try_route(proto: outbound::HttpRoute) -> Result<Route, InvalidHttpRoute> {
+    fn try_route(
+        overrides: ClientPolicyOverrides,
+        proto: outbound::HttpRoute,
+    ) -> Result<Route, InvalidHttpRoute> {
         let outbound::HttpRoute {
             hosts,
             rules,
@@ -265,14 +272,15 @@ pub mod proto {
 
         let rules = rules
             .into_iter()
-            .map(|rule| try_rule(&meta, rule))
+            .map(|rule| try_rule(&meta, overrides, rule))
             .collect::<Result<Vec<_>, _>>()?;
 
         Ok(Route { hosts, rules })
     }
 
     fn try_rule(
         meta: &Arc<Meta>,
+        overrides: ClientPolicyOverrides,
         proto: outbound::http_route::Rule,
     ) -> Result<Rule, InvalidHttpRoute> {
         #[allow(deprecated)]
@@ -300,13 +308,8 @@ pub mod proto {
             .ok_or(InvalidHttpRoute::Missing("distribution"))?
             .try_into()?;
 
-        let export_hostname_labels = false;
-        let mut params = RouteParams::try_from_proto(
-            timeouts,
-            retry,
-            allow_l5d_request_headers,
-            export_hostname_labels,
-        )?;
+        let mut params =
+            RouteParams::try_from_proto(timeouts, retry, allow_l5d_request_headers, overrides)?;
         let legacy = request_timeout.map(TryInto::try_into).transpose()?;
         params.timeouts.request = params.timeouts.request.or(legacy);
 
@@ -326,7 +329,7 @@ pub mod proto {
             timeouts: Option<linkerd2_proxy_api::http_route::Timeouts>,
             retry: Option<http_route::Retry>,
             allow_l5d_request_headers: bool,
-            export_hostname_labels: bool,
+            overrides: ClientPolicyOverrides,
         ) -> Result<Self, InvalidHttpRoute> {
             Ok(Self {
                 retry: retry.map(Retry::try_from).transpose()?,
@@ -335,7 +338,7 @@ pub mod proto {
                     .transpose()?
                     .unwrap_or_default(),
                 allow_l5d_request_headers,
-                export_hostname_labels,
+                export_hostname_labels: overrides.export_hostname_labels,
             })
         }
     }