Add warning for COAT on NoDoubleUnderscoreSep (#36492)

Co-authored-by: iglendd <len.gamburg@datadoghq.com>

Author: pgimalac

comp/core/agenttelemetry/impl/agenttelemetry.go

@@ -489,6 +489,12 @@ func (a *atel) loadPayloads(profiles []*Profile) (*senderSession, error) {
 		a.logComp.Errorf("failed to get filtered telemetry metrics: %v", err)
 	}
 
+	// All metrics stored in the "pms" slice above must follow the format:
+	//    <subsystem>__<metric_name>
+	// The "subsystem" and "name" should be concatenated with a double underscore ("__") separator,
+	// e.g., "checks__execution_time". Therefore, the "Options.NoDoubleUnderscoreSep: true" option
+	// must not be used when creating metrics.
+
 	session := a.sender.startSession(a.cancelCtx)
 	for _, p := range profiles {
 		a.reportAgentMetrics(session, pms, p)

comp/core/agenttelemetry/impl/config.go

@@ -107,8 +107,10 @@ type Event struct {
 //
 // profiles[].metric.metrics[].name (required)
 // -------------------------------------------
-// Metric's full name typically in the form of "<metric group>.<metric name>".
-// It is required parameter to avoid emitting all available metrics unintentionally.
+// The full metric name is formatted as "<metric group>.<metric name>". In telemetry.NewGauge()
+// and similar APIs, "metric group" corresponds to the "subsystem" parameter, and "metric name"
+// corresponds to the "name" parameter. Do not use the "Options.NoDoubleUnderscoreSep" option
+// in these APIs, as it is not supported in agent telemetry.
 //
 // profiles[].metric.metrics[].aggregate_tags (optional)
 // -----------------------------------------------------
@@ -338,7 +340,9 @@ func compileMetric(p *Profile, m *MetricConfig) error {
 		return fmt.Errorf("profile '%s' 'metrics[].name' '(%s)' attribute should have two elements separated by '.'", p.Name, m.Name)
 	}
 
-	// Convert Datadog metric name to Prometheus metric name (used for quick(er) matching)
+	// Converts a Datadog metric name to a Prometheus metric name for quicker matching. Prometheus metrics
+	// (from the "telemetry" package) must be declared without setting Options.NoDoubleUnderscoreSep to true,
+	// ensuring the full metric name includes double underscores ("__"); otherwise, matching will fail.
 	promName := fmt.Sprintf("%s__%s", names[0], names[1])
 	p.metricsMap[promName] = m
 

comp/core/telemetry/options.go

@@ -13,6 +13,8 @@ import "fmt"
 type Options struct {
 	// NoDoubleUnderscoreSep is set to true when you don't want to
 	// separate the subsystem and the name with a double underscore separator.
+	//
+	// This option is not compatible with the cross-org agent telemetry
 	NoDoubleUnderscoreSep bool
 
 	// DefaultMetric exports metric by default via built-in agent_telemetry core check.