Datadog update env clobbering behavior (#3851)

* [exporter/datadog]: handle multiple env tag edge case

* [exporter/datadog]: fix typos in code comments

* Update exporter/datadogexporter/translate_traces_test.go

Co-authored-by: Pablo Baeyens <pablo.baeyens@datadoghq.com>

* Update exporter/datadogexporter/translate_traces.go

Co-authored-by: Pablo Baeyens <pablo.baeyens@datadoghq.com>

* [exporter/datadog]: linting and formatting

Co-authored-by: Pablo Baeyens <pablo.baeyens@datadoghq.com>

Author: ericmustin

exporter/datadogexporter/translate_traces.go

@@ -62,6 +62,7 @@ const (
 	// tagContainersTags specifies the name of the tag which holds key/value
 	// pairs representing information about the container (Docker, EC2, etc).
 	tagContainersTags = "_dd.tags.container"
+	tagDatadogEnv     = "env"
 )
 
 // converts Traces into an array of datadog trace payloads grouped by env
@@ -230,7 +231,7 @@ func spanToDatadogSpan(s pdata.Span,
 	tags := aggregateSpanTags(s, datadogTags)
 
 	// otel specification resource service.name takes precedence
-	// and configuration DD_ENV as fallback if it exists
+	// and configuration DD_SERVICE as fallback if it exists
 	if cfg.Service != "" {
 		// prefer the collector level service name over an empty string or otel default
 		if serviceName == "" || serviceName == tracetranslator.ResourceNoServiceName {
@@ -313,8 +314,8 @@ func resourceToDatadogServiceNameAndAttributeMap(
 	resource pdata.Resource,
 ) (serviceName string, datadogTags map[string]string) {
 	attrs := resource.Attributes()
-	// predefine capacity where possible with extra for _dd.tags.container payload
-	datadogTags = make(map[string]string, attrs.Len()+1)
+	// predefine capacity where possible with extra for _dd.tags.container payload and duplicate env tag
+	datadogTags = make(map[string]string, attrs.Len()+2)
 
 	if attrs.Len() == 0 {
 		return tracetranslator.ResourceNoServiceName, datadogTags
@@ -325,6 +326,17 @@ func resourceToDatadogServiceNameAndAttributeMap(
 		return true
 	})
 
+	// specification states that the resource level deployment.environment should be used for passing env,
+	// and also a number of Datadog UI components are hardcoded to point to /  look for / search with `env`.
+	// So we ensure that anytime deployment.environment is set, `env` is the same value.
+	// In the future, we can probably just set `env` using the deployment.environment value, but drop the duplicate
+	// deployment.environment tag afterward. Without knowing whether this would break existing user setups, its better
+	// to simply set both tags.
+	// https://github.com/open-telemetry/opentelemetry-specification/blob/v1.4.0/specification/resource/semantic_conventions/deployment_environment.md#deployment
+	if resourceEnv, ok := datadogTags[conventions.AttributeDeploymentEnvironment]; ok {
+		datadogTags[tagDatadogEnv] = utils.NormalizeTag(resourceEnv)
+	}
+
 	serviceName = extractDatadogServiceName(datadogTags)
 	return serviceName, datadogTags
 }

exporter/datadogexporter/translate_traces_test.go

@@ -374,7 +374,7 @@ func TestTracesTranslationErrorsAndResource(t *testing.T) {
 	// ensure that version gives resource service.version priority
 	assert.Equal(t, "test-version", datadogPayload.Traces[0].Spans[0].Meta["version"])
 
-	assert.Equal(t, 19, len(datadogPayload.Traces[0].Spans[0].Meta))
+	assert.Equal(t, 20, len(datadogPayload.Traces[0].Spans[0].Meta))
 
 	assert.Contains(t, datadogPayload.Traces[0].Spans[0].Meta[tagContainersTags], "container_id:3249847017410247")
 	assert.Contains(t, datadogPayload.Traces[0].Spans[0].Meta[tagContainersTags], "pod_name:example-pod-name")
@@ -617,7 +617,7 @@ func TestTracesTranslationOkStatus(t *testing.T) {
 	// ensure that version gives resource service.version priority
 	assert.Equal(t, "test-version", datadogPayload.Traces[0].Spans[0].Meta["version"])
 
-	assert.Equal(t, 19, len(datadogPayload.Traces[0].Spans[0].Meta))
+	assert.Equal(t, 20, len(datadogPayload.Traces[0].Spans[0].Meta))
 }
 
 // ensure that the datadog span uses the configured unified service tags
@@ -664,7 +664,7 @@ func TestTracesTranslationConfig(t *testing.T) {
 	// ensure that version gives resource service.version priority
 	assert.Equal(t, "test-version", datadogPayload.Traces[0].Spans[0].Meta["version"])
 
-	assert.Equal(t, 16, len(datadogPayload.Traces[0].Spans[0].Meta))
+	assert.Equal(t, 17, len(datadogPayload.Traces[0].Spans[0].Meta))
 }
 
 // ensure that the translation returns early if no resource instrumentation library spans
@@ -1399,3 +1399,54 @@ func TestSpanNameMapping(t *testing.T) {
 
 	assert.Equal(t, "bang.client", aggregatedTraces[0].Traces[0].Spans[0].Name)
 }
+
+// ensure that sanitization  of trace payloads occurs
+func TestSpanEnvClobbering(t *testing.T) {
+	mockTraceID := [16]byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F}
+	mockSpanID := [8]byte{0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8}
+	mockParentSpanID := [8]byte{0xEF, 0xEE, 0xED, 0xEC, 0xEB, 0xEA, 0xE9, 0xE8}
+	endTime := time.Now().Round(time.Second)
+	pdataEndTime := pdata.TimestampFromTime(endTime)
+	startTime := endTime.Add(-90 * time.Second)
+	pdataStartTime := pdata.TimestampFromTime(startTime)
+
+	denylister := newDenylister([]string{})
+	buildInfo := component.BuildInfo{
+		Version: "1.0",
+	}
+
+	traces := pdata.NewTraces()
+	traces.ResourceSpans().Resize(1)
+	rs := traces.ResourceSpans().At(0)
+	resource := rs.Resource()
+	resource.Attributes().InitFromMap(map[string]pdata.AttributeValue{
+		conventions.AttributeDeploymentEnvironment: pdata.NewAttributeValueString("correctenv"),
+		"env": pdata.NewAttributeValueString("incorrectenv"),
+	})
+
+	rs.InstrumentationLibrarySpans().Resize(1)
+	ilss := rs.InstrumentationLibrarySpans().At(0)
+	instrumentationLibrary := ilss.InstrumentationLibrary()
+	instrumentationLibrary.SetName("flash")
+	instrumentationLibrary.SetVersion("v1")
+	span := ilss.Spans().AppendEmpty()
+
+	traceID := pdata.NewTraceID(mockTraceID)
+	spanID := pdata.NewSpanID(mockSpanID)
+	parentSpanID := pdata.NewSpanID(mockParentSpanID)
+	span.SetTraceID(traceID)
+	span.SetSpanID(spanID)
+	span.SetParentSpanID(parentSpanID)
+	span.SetName("End-To-End Here")
+	span.SetKind(pdata.SpanKindServer)
+	span.SetStartTimestamp(pdataStartTime)
+	span.SetEndTimestamp(pdataEndTime)
+
+	config := config.Config{Traces: config.TracesConfig{SpanNameRemappings: map[string]string{"flash.server": "bang.client"}}}
+
+	outputTraces, _ := convertToDatadogTd(traces, "test-host", &config, denylister, buildInfo)
+
+	// Ensure the deployment.environment value is copied to both deployment.environment and env
+	assert.Equal(t, "correctenv", outputTraces[0].Traces[0].Spans[0].Meta["env"])
+	assert.Equal(t, "correctenv", outputTraces[0].Traces[0].Spans[0].Meta["deployment.environment"])
+}