APM: tracer setup with Disruption and Reconcile spans (#721)

Author: nathantournant

LICENSE-3rdparty.csv

@@ -4,6 +4,7 @@ github.com/AlecAivazis/survey/v2,github.com/AlecAivazis/survey/v2/core,MIT
 github.com/AlecAivazis/survey/v2,github.com/AlecAivazis/survey/v2/terminal,MIT
 github.com/DataDog/appsec-internal-go,github.com/DataDog/appsec-internal-go/httpsec,Apache-2.0
 github.com/DataDog/appsec-internal-go,github.com/DataDog/appsec-internal-go/netip,Apache-2.0
+github.com/DataDog/datadog-agent/pkg/obfuscate,github.com/DataDog/datadog-agent/pkg/obfuscate,Apache-2.0
 github.com/DataDog/datadog-agent/pkg/remoteconfig/state,github.com/DataDog/datadog-agent/pkg/remoteconfig/state,Apache-2.0
 github.com/DataDog/datadog-go,github.com/DataDog/datadog-go/statsd,MIT
 github.com/DataDog/datadog-go/v5,github.com/DataDog/datadog-go/v5/statsd,MIT
@@ -22,6 +23,12 @@ github.com/DataDog/go-tuf,github.com/DataDog/go-tuf/pkg/targets,BSD-3-Clause
 github.com/DataDog/go-tuf,github.com/DataDog/go-tuf/util,BSD-3-Clause
 github.com/DataDog/go-tuf,github.com/DataDog/go-tuf/verify,BSD-3-Clause
 github.com/DataDog/gostackparse,github.com/DataDog/gostackparse,Apache-2.0
+github.com/DataDog/sketches-go,github.com/DataDog/sketches-go/ddsketch,Apache-2.0
+github.com/DataDog/sketches-go,github.com/DataDog/sketches-go/ddsketch/encoding,Apache-2.0
+github.com/DataDog/sketches-go,github.com/DataDog/sketches-go/ddsketch/mapping,Apache-2.0
+github.com/DataDog/sketches-go,github.com/DataDog/sketches-go/ddsketch/pb/sketchpb,Apache-2.0
+github.com/DataDog/sketches-go,github.com/DataDog/sketches-go/ddsketch/stat,Apache-2.0
+github.com/DataDog/sketches-go,github.com/DataDog/sketches-go/ddsketch/store,Apache-2.0
 github.com/Microsoft/go-winio,github.com/Microsoft/go-winio,MIT
 github.com/Microsoft/go-winio,github.com/Microsoft/go-winio/backuptar,MIT
 github.com/Microsoft/go-winio,github.com/Microsoft/go-winio/internal/fs,MIT
@@ -157,13 +164,15 @@ github.com/docker/go-connections,github.com/docker/go-connections/sockets,Apache
 github.com/docker/go-connections,github.com/docker/go-connections/tlsconfig,Apache-2.0
 github.com/docker/go-events,github.com/docker/go-events,Apache-2.0
 github.com/docker/go-units,github.com/docker/go-units,Apache-2.0
+github.com/dustin/go-humanize,github.com/dustin/go-humanize,MIT
 github.com/emicklei/go-restful/v3,github.com/emicklei/go-restful/v3,MIT
 github.com/emicklei/go-restful/v3,github.com/emicklei/go-restful/v3/log,MIT
 github.com/evanphx/json-patch,github.com/evanphx/json-patch,BSD-3-Clause
 github.com/evanphx/json-patch/v5,github.com/evanphx/json-patch/v5,BSD-3-Clause
 github.com/fsnotify/fsnotify,github.com/fsnotify/fsnotify,BSD-3-Clause
 github.com/go-logr/logr,github.com/go-logr/logr,Apache-2.0
 github.com/go-logr/logr,github.com/go-logr/logr/funcr,Apache-2.0
+github.com/go-logr/stdr,github.com/go-logr/stdr,Apache-2.0
 github.com/go-openapi/jsonpointer,github.com/go-openapi/jsonpointer,Apache-2.0
 github.com/go-openapi/jsonreference,github.com/go-openapi/jsonreference,Apache-2.0
 github.com/go-openapi/jsonreference,github.com/go-openapi/jsonreference/internal,Apache-2.0
@@ -291,11 +300,15 @@ github.com/opencontainers/runtime-spec,github.com/opencontainers/runtime-spec/sp
 github.com/opencontainers/selinux,github.com/opencontainers/selinux/go-selinux,Apache-2.0
 github.com/opencontainers/selinux,github.com/opencontainers/selinux/go-selinux/label,Apache-2.0
 github.com/opencontainers/selinux,github.com/opencontainers/selinux/pkg/pwalkdir,Apache-2.0
+github.com/outcaste-io/ristretto,github.com/outcaste-io/ristretto,Apache-2.0
+github.com/outcaste-io/ristretto,github.com/outcaste-io/ristretto/z,Apache-2.0
+github.com/outcaste-io/ristretto,github.com/outcaste-io/ristretto/z/simd,Apache-2.0
 github.com/pelletier/go-toml/v2,github.com/pelletier/go-toml/v2,MIT
 github.com/pelletier/go-toml/v2,github.com/pelletier/go-toml/v2/internal/characters,MIT
 github.com/pelletier/go-toml/v2,github.com/pelletier/go-toml/v2/internal/danger,MIT
 github.com/pelletier/go-toml/v2,github.com/pelletier/go-toml/v2/internal/tracker,MIT
 github.com/pelletier/go-toml/v2,github.com/pelletier/go-toml/v2/unstable,MIT
+github.com/philhofer/fwd,github.com/philhofer/fwd,MIT
 github.com/pkg/errors,github.com/pkg/errors,BSD-2-Clause
 github.com/pmezard/go-difflib,github.com/pmezard/go-difflib/difflib,BSD-3-Clause
 github.com/prometheus/client_golang,github.com/prometheus/client_golang/prometheus,Apache-2.0
@@ -341,6 +354,7 @@ github.com/stretchr/testify,github.com/stretchr/testify/assert,MIT
 github.com/stretchr/testify,github.com/stretchr/testify/mock,MIT
 github.com/stretchr/testify,github.com/stretchr/testify/require,MIT
 github.com/subosito/gotenv,github.com/subosito/gotenv,MIT
+github.com/tinylib/msgp,github.com/tinylib/msgp/msgp,MIT
 github.com/vishvananda/netlink,github.com/vishvananda/netlink,Apache-2.0
 github.com/vishvananda/netlink,github.com/vishvananda/netlink/nl,Apache-2.0
 github.com/vishvananda/netns,github.com/vishvananda/netns,Apache-2.0
@@ -349,6 +363,18 @@ go.opencensus.io,go.opencensus.io/internal,Apache-2.0
 go.opencensus.io,go.opencensus.io/trace,Apache-2.0
 go.opencensus.io,go.opencensus.io/trace/internal,Apache-2.0
 go.opencensus.io,go.opencensus.io/trace/tracestate,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel/attribute,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel/baggage,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel/codes,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel/internal,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel/internal/attribute,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel/internal/baggage,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel/internal/global,Apache-2.0
+go.opentelemetry.io/otel,go.opentelemetry.io/otel/propagation,Apache-2.0
+go.opentelemetry.io/otel/metric,go.opentelemetry.io/otel/metric,Apache-2.0
+go.opentelemetry.io/otel/metric,go.opentelemetry.io/otel/metric/embedded,Apache-2.0
+go.opentelemetry.io/otel/trace,go.opentelemetry.io/otel/trace,Apache-2.0
 go.uber.org/atomic,go.uber.org/atomic,MIT
 go.uber.org/multierr,go.uber.org/multierr,MIT
 go.uber.org/zap,go.uber.org/zap,MIT
@@ -432,6 +458,8 @@ golang.org/x/tools,golang.org/x/tools/internal/pkgbits,BSD-3-Clause
 golang.org/x/tools,golang.org/x/tools/internal/tokeninternal,BSD-3-Clause
 golang.org/x/tools,golang.org/x/tools/internal/typeparams,BSD-3-Clause
 golang.org/x/tools,golang.org/x/tools/internal/typesinternal,BSD-3-Clause
+golang.org/x/xerrors,golang.org/x/xerrors,BSD-3-Clause
+golang.org/x/xerrors,golang.org/x/xerrors/internal,BSD-3-Clause
 gomodules.xyz/jsonpatch/v2,gomodules.xyz/jsonpatch/v2,Apache-2.0
 google.golang.org/appengine,google.golang.org/appengine/internal,Apache-2.0
 google.golang.org/appengine,google.golang.org/appengine/internal/base,Apache-2.0
@@ -525,6 +553,9 @@ google.golang.org/protobuf,google.golang.org/protobuf/types/known/emptypb,BSD-3-
 google.golang.org/protobuf,google.golang.org/protobuf/types/known/timestamppb,BSD-3-Clause
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/ddtrace,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/internal,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/opentelemetry,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/appsec,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo,Apache-2.0
@@ -533,7 +564,16 @@ gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/httpsec,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo/instrumentation/sharedsec,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/hostname,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/hostname/azure,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/hostname/cachedfetch,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/hostname/ec2,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/hostname/ecs,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/hostname/gce,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/hostname/httputils,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/hostname/validate,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/log,Apache-2.0
+gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/namingschema,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/osinfo,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/remoteconfig,Apache-2.0
 gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/internal/samplernames,Apache-2.0
@@ -548,7 +588,7 @@ gopkg.in/DataDog/dd-trace-go.v1,gopkg.in/DataDog/dd-trace-go.v1/profiler/interna
 gopkg.in/inf.v0,gopkg.in/inf.v0,BSD-3-Clause
 gopkg.in/ini.v1,gopkg.in/ini.v1,Apache-2.0
 gopkg.in/yaml.v2,gopkg.in/yaml.v2,Apache-2.0
-gopkg.in/yaml.v3,gopkg.in/yaml.v3,Apache-2.0
+gopkg.in/yaml.v3,gopkg.in/yaml.v3,MIT
 inet.af/netaddr,inet.af/netaddr,BSD-3-Clause
 k8s.io/api,k8s.io/api/admission/v1,Apache-2.0
 k8s.io/api,k8s.io/api/admission/v1beta1,Apache-2.0

api/v1beta1/disruption_spancontext.go

@@ -0,0 +1,58 @@
+// Unless explicitly stated otherwise all files in this repository are licensed
+// under the Apache License Version 2.0.
+// This product includes software developed at Datadog (https://www.datadoghq.com/).
+// Copyright 2023 Datadog, Inc.
+package v1beta1
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+)
+
+const (
+	annotationSpanContextKey = "SpanContext"
+)
+
+var (
+	// ErrNoSpanContext is the error returned when the annotation does not contains expected span context key
+	ErrNoSpanContext = fmt.Errorf("span context not found in disruption annotations")
+)
+
+// SpanContext extracts this disruption's span context, injects it in the given context, then returns it
+func (r *Disruption) SpanContext(ctx context.Context) (context.Context, error) {
+	var annotation propagation.MapCarrier
+
+	spanContext, ok := r.Annotations[annotationSpanContextKey]
+	if !ok {
+		return ctx, ErrNoSpanContext
+	}
+
+	err := json.Unmarshal([]byte(spanContext), &annotation)
+	if err != nil {
+		return ctx, fmt.Errorf("unable to unmarshal span context from annotation: %w", err)
+	}
+
+	ctx = otel.GetTextMapPropagator().Extract(ctx, annotation)
+
+	return ctx, nil
+}
+
+// SetSpanContext store provided spanContext into expected disruption annotation
+func (r *Disruption) SetSpanContext(ctx context.Context) error {
+	var annotation = make(propagation.MapCarrier)
+
+	otel.GetTextMapPropagator().Inject(ctx, annotation)
+
+	marshaledAnnotation, err := json.Marshal(annotation)
+	if err != nil {
+		return fmt.Errorf("unable to marshal span context: %w", err)
+	}
+
+	r.Annotations[annotationSpanContextKey] = string(marshaledAnnotation)
+
+	return nil
+}

chart/templates/configmap.yaml

@@ -14,6 +14,7 @@ data:
       leaderElection: {{ .Values.controller.leaderElection }}
       metricsSink: {{ .Values.controller.metricsSink | quote }}
       profilerSink: {{ .Values.controller.profilerSink | quote }}
+      tracerSink: {{ .Values.controller.tracerSink | quote }}
       enableSafeguards: {{ .Values.controller.enableSafeguards }}
       enableObserver: {{ .Values.controller.enableObserver }}
       notifiers:

chart/templates/webhook.yaml

@@ -152,6 +152,40 @@ webhooks:
     - disruptions/status
 {{- end }}
 ---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  annotations:
+  {{- if not .Values.controller.webhook.generateCert }}
+    cert-manager.io/inject-ca-from: {{ .Values.chaosNamespace }}/chaos-controller-serving-cert
+  {{- end }}
+  name: chaos-controller-disruption-span-context
+webhooks:
+- clientConfig:
+  {{- if not .Values.controller.webhook.generateCert }}
+    caBundle: Cg==
+  {{- else }}
+    caBundle: {{ b64enc $ca.Cert }}
+  {{- end }}
+    service:
+      name: chaos-controller-webhook-service
+      namespace: {{ .Values.chaosNamespace }}
+      path: /mutate-chaos-datadoghq-com-v1beta1-disruption-span-context
+  failurePolicy: Fail
+  name: chaos-controller-admission-webhook.{{ .Values.chaosNamespace }}.svc
+  sideEffects: NoneOnDryRun
+  admissionReviewVersions: ["v1", "v1beta1"]
+  rules:
+  - apiGroups:
+    - "chaos.datadoghq.com"
+    apiVersions:
+    - v1beta1
+    scope: "Namespaced"
+    operations:
+    - CREATE
+    resources:
+    - disruptions
+---
 {{- if not .Values.controller.webhook.generateCert }}
 apiVersion: cert-manager.io/v1
 kind: Certificate

chart/values.yaml

@@ -26,6 +26,7 @@ controller:
   enableObserver: true # enable observer on targets, notifying of target warning status and events
   metricsSink: noop
   profilerSink: noop
+  tracerSink: noop
   notifiers:
     common:
       clusterName: lima

config/config.go

@@ -39,6 +39,7 @@ type controllerConfig struct {
 	UserInfoHook             bool                            `json:"userInfoHook"`
 	SafeMode                 safeModeConfig                  `json:"safeMode"`
 	ProfilerSink             string                          `json:"profilerSink"`
+	TracerSink               string                          `json:"tracerSink"`
 }
 
 type controllerWebhookConfig struct {
@@ -381,6 +382,12 @@ func New(logger *zap.SugaredLogger, osArgs []string) (config, error) {
 		return cfg, err
 	}
 
+	mainFS.StringVar(&cfg.Controller.TracerSink, "tracer-sink", "noop", "tracer sink (datadog, or noop)")
+
+	if err := viper.BindPFlag("controller.tracerSink", mainFS.Lookup("tracer-sink")); err != nil {
+		return cfg, err
+	}
+
 	err := preConfigFS.Parse(osArgs)
 	if err != nil {
 		return cfg, fmt.Errorf("unable to retrieve configuration parse from provided flag: %w", err)

controllers/disruption_controller.go

@@ -17,11 +17,15 @@ import (
 	chaosapi "github.com/DataDog/chaos-controller/api"
 	"github.com/DataDog/chaos-controller/cloudservice"
 	"github.com/DataDog/chaos-controller/o11y/metrics"
+	"github.com/DataDog/chaos-controller/o11y/tracer"
 	"github.com/DataDog/chaos-controller/safemode"
 	"github.com/DataDog/chaos-controller/targetselector"
 	chaostypes "github.com/DataDog/chaos-controller/types"
 	"github.com/DataDog/chaos-controller/watchers"
 	"github.com/cenkalti/backoff"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"go.uber.org/zap"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
@@ -54,6 +58,7 @@ type DisruptionReconciler struct {
 	Scheme                                *runtime.Scheme
 	Recorder                              record.EventRecorder
 	MetricsSink                           metrics.Sink
+	TracerSink                            tracer.Sink
 	TargetSelector                        targetselector.TargetSelector
 	InjectorAnnotations                   map[string]string
 	InjectorLabels                        map[string]string
@@ -89,7 +94,7 @@ type CtxTuple struct {
 // +kubebuilder:rbac:groups=core,resources=pods/status,verbs=update;patch
 // +kubebuilder:rbac:groups=core,resources=nodes,verbs=list;watch
 // +kubebuilder:rbac:groups=core,resources=services,verbs=list;watch
-func (r *DisruptionReconciler) Reconcile(_ context.Context, req ctrl.Request) (res ctrl.Result, err error) {
+func (r *DisruptionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
 	instance := &chaosv1beta1.Disruption{}
 	tsStart := time.Now()
 
@@ -163,6 +168,29 @@ func (r *DisruptionReconciler) Reconcile(_ context.Context, req ctrl.Request) (r
 		r.log.Errorw("error during the creation of watchers", "error", err)
 	}
 
+	ctx, err = instance.SpanContext(ctx)
+	if err != nil {
+		r.log.Errorw("did not find span context", "err", err)
+	}
+
+	userInfo, err := instance.UserInfo()
+	if err != nil {
+		r.log.Errorw("error getting user info", "error", err)
+
+		userInfo.Username = "did-not-find-user-info@email.com"
+	}
+
+	ctx, reconcileSpan := otel.Tracer("").Start(ctx, "reconcile", trace.WithLinks(trace.LinkFromContext(ctx)),
+		trace.WithAttributes(
+			attribute.String("disruption_name", instance.Name),
+			attribute.String("disruption_namespace", instance.Namespace),
+			attribute.String("disruption_user", userInfo.Username),
+		))
+	defer reconcileSpan.End()
+
+	// allows to sync logs with traces
+	r.log = r.log.With(r.TracerSink.GetLoggableTraceContext(reconcileSpan)...)
+
 	// handle any chaos pods being deleted (either by the disruption deletion or by an external event)
 	if err := r.handleChaosPodsTermination(instance); err != nil {
 		return ctrl.Result{}, fmt.Errorf("error handling chaos pods termination: %w", err)
@@ -207,6 +235,17 @@ func (r *DisruptionReconciler) Reconcile(_ context.Context, req ctrl.Request) (r
 			r.handleMetricSinkError(r.MetricsSink.MetricDisruptionCompletedDuration(time.Since(instance.ObjectMeta.CreationTimestamp.Time), []string{"disruptionName:" + instance.Name, "namespace:" + instance.Namespace}))
 			r.emitKindCountMetrics(instance)
 
+			// close the ongoing disruption tracing Span
+			defer func() {
+				_, disruptionStopSpan := otel.Tracer("").Start(ctx, "disruption deletion", trace.WithAttributes(
+					attribute.String("disruption_name", instance.Name),
+					attribute.String("disruption_namespace", instance.Namespace),
+					attribute.String("disruption_user", userInfo.Username),
+				))
+
+				disruptionStopSpan.End()
+			}()
+
 			return ctrl.Result{}, nil
 		}
 	} else {