feat(tracing): improve migration (#449)

Author: mat-rumian

src/go/cmd/update-collection-v3/migrations/tracing-config/migrate.go

@@ -41,10 +41,34 @@ func migrate(inputValues *ValuesInput) (ValuesOutput, error) {
 	outputValues := ValuesOutput{
 		Rest: inputValues.Rest,
 	}
+
+	// otelcol-instrumentation migrations
 	// migrate otelcol source processor to otelcol-instrumentation
 	outputValues.OtelcolInstrumentation.Config.Processors.Source = inputValues.Otelcol.Config.Processors.Source
-	// migrate otelcol cascading_filter processor to tracesSampler
+
+	// tracesgateway (old otelgateway) migrations
+	// migrate deployment
+	outputValues.TracesGateway.Deployment = inputValues.Otelgateway.Deployment
+	// migrate loadbalancing exporter compression
+	outputValues.TracesGateway.Config.Exporters.LoadBalancing.Protocol.Otlp.Compression = inputValues.Otelgateway.Config.Exporters.LoadBalancing.Protocol.Otlp.Compression
+	// migration loadbalancing exporter num of consumers
+	outputValues.TracesGateway.Config.Exporters.LoadBalancing.Protocol.Otlp.SendingQueue.NumConsumers = inputValues.Otelgateway.Config.Exporters.LoadBalancing.Protocol.Otlp.SendingQueue.NumConsumers
+	// migration loadbalancing exporter queue size
+	outputValues.TracesGateway.Config.Exporters.LoadBalancing.Protocol.Otlp.SendingQueue.QueueSize = inputValues.Otelgateway.Config.Exporters.LoadBalancing.Protocol.Otlp.SendingQueue.QueueSize
+	// migrate batch processor
+	outputValues.TracesGateway.Config.Processors.Batch = inputValues.Otelgateway.Config.Processors.Batch
+	// migrate memory limiter processor
+	outputValues.TracesGateway.Config.Processors.MemoryLimiter = inputValues.Otelgateway.Config.Processors.MemoryLimiter
+
+	// tracessampler (old oltecol) migrations
+	// migrate deployment
+	outputValues.TracesSampler.Deployment = inputValues.Otelcol.Deployment
+	// migrate cascading_filter processor
 	outputValues.TracesSampler.Config.Processors.CascadingFilter = inputValues.Otelcol.Config.Processors.CascadingFilter
+	// migrate batch processor
+	outputValues.TracesSampler.Config.Processors.Batch = inputValues.Otelcol.Config.Processors.Batch
+	// migrate memory limiter processor
+	outputValues.TracesSampler.Config.Processors.MemoryLimiter = inputValues.Otelcol.Config.Processors.MemoryLimiter
 
 	return outputValues, nil
 }

src/go/cmd/update-collection-v3/migrations/tracing-config/values_input.go

@@ -2,19 +2,47 @@ package tracingconfig
 
 type ValuesInput struct {
 	Otelcol     Otelcol                `yaml:"otelcol,omitempty"`
+	Otelgateway Otelgateway            `yaml:"otelgateway,omitempty"`
 	Otelagent   map[string]interface{} `yaml:"otelagent,omitempty"`
-	Otelgateway map[string]interface{} `yaml:"otelgateway,omitempty"`
 	Rest        map[string]interface{} `yaml:",inline"`
 }
 
 type Otelcol struct {
-	Config struct {
+	Deployment map[string]interface{} `yaml:"deployment,omitempty"`
+	Config     struct {
 		Processors struct {
+			Batch           map[string]interface{} `yaml:"batch,omitempty"`
 			CascadingFilter map[string]interface{} `yaml:"cascading_filter,omitempty"`
+			MemoryLimiter   map[string]interface{} `yaml:"memory_lmiter,omitempty"`
 			Source          map[string]interface{} `yaml:"source,omitempty"`
 			Rest            map[string]interface{} `yaml:",inline"`
 		} `yaml:"processors,omitempty"`
 		Rest map[string]interface{} `yaml:",inline"`
 	} `yaml:"config,omitempty"`
 	Rest map[string]interface{} `yaml:",inline"`
 }
+
+type Otelgateway struct {
+	Deployment map[string]interface{} `yaml:"deployment,omitempty"`
+	Config     struct {
+		Exporters struct {
+			LoadBalancing struct {
+				Protocol struct {
+					Otlp struct {
+						Compression  string `yaml:"compression,omitempty"`
+						SendingQueue struct {
+							NumConsumers int `yaml:"num_consumers,omitempty"`
+							QueueSize    int `yaml:"queue_size,omitempty"`
+						} `yaml:"sending_queue,omitempty"`
+					} `yaml:"otlp,omitempty"`
+				} `yaml:"protocol,omitempty"`
+			} `yaml:"loadbalancing,omitempty"`
+			Rest map[string]interface{} `yaml:",inline"`
+		} `yaml:"exporters,omitempty"`
+		Processors struct {
+			Batch         map[string]interface{} `yaml:"batch,omitempty"`
+			MemoryLimiter map[string]interface{} `yaml:"memory_lmiter,omitempty"`
+		} `yaml:"processors,omitempty"`
+	} `yaml:"config,omitempty"`
+	Rest map[string]interface{} `yaml:",inline"`
+}

src/go/cmd/update-collection-v3/migrations/tracing-config/values_output.go

@@ -2,6 +2,7 @@ package tracingconfig
 
 type ValuesOutput struct {
 	OtelcolInstrumentation OtelcolInstrumentation `yaml:"otelcolInstrumentation,omitempty"`
+	TracesGateway          TracesGateway          `yaml:"tracesGateway,omitempty"`
 	TracesSampler          TracesSampler          `yaml:"tracesSampler,omitempty"`
 	Otelcol                map[string]interface{} `yaml:"-"`
 	Otelagent              map[string]interface{} `yaml:"-"`
@@ -18,11 +19,38 @@ type OtelcolInstrumentation struct {
 	} `yaml:"config,omitempty"`
 }
 
+type TracesGateway struct {
+	Config struct {
+		Exporters struct {
+			LoadBalancing struct {
+				Protocol struct {
+					Otlp struct {
+						Compression  string `yaml:"compression,omitempty"`
+						SendingQueue struct {
+							NumConsumers int `yaml:"num_consumers,omitempty"`
+							QueueSize    int `yaml:"queue_size,omitempty"`
+						} `yaml:"sending_queue,omitempty"`
+					} `yaml:"otlp,omitempty"`
+				} `yaml:"protocol,omitempty"`
+			} `yaml:"loadbalancing,omitempty"`
+			Rest map[string]interface{} `yaml:",inline"`
+		} `yaml:"exporters,omitempty"`
+		Processors struct {
+			Batch         map[string]interface{} `yaml:"batch,omitempty"`
+			MemoryLimiter map[string]interface{} `yaml:"memory_lmiter,omitempty"`
+		} `yaml:"processors,omitempty"`
+	} `yaml:"config,omitempty"`
+	Deployment map[string]interface{} `yaml:"deployment,omitempty"`
+}
+
 type TracesSampler struct {
 	Config struct {
 		Processors struct {
+			Batch           map[string]interface{} `yaml:"batch,omitempty"`
 			CascadingFilter map[string]interface{} `yaml:"cascading_filter,omitempty"`
+			MemoryLimiter   map[string]interface{} `yaml:"memory_lmiter,omitempty"`
 			Rest            map[string]interface{} `yaml:",inline"`
 		} `yaml:"processors,omitempty"`
 	} `yaml:"config,omitempty"`
+	Deployment map[string]interface{} `yaml:"deployment,omitempty"`
 }

src/go/cmd/update-collection-v3/testdata/tracing.input.yaml

@@ -10,11 +10,35 @@ otelagent:
       otlphttp/traces:
         endpoint: http://exporters.otlptraces.endpoint.replace:4318
 otelgateway:
+  deployment:
+    replicas: 7
+    resources:
+      limits:
+        memory: 3Gi
+        cpu: 2
+      requests:
+        memory: 2Gi
+        cpu: 1
   enabled: true
   config:
     exporters:
       loadbalancing:
+        protocol:
+          otlp:
+            compression: gzip
+            sending_queue:
+              num_consumers: 100
+              queue_size: 13000
 otelcol:
+  deployment:
+    replicas: 12
+    resources:
+      limits:
+        memory: 4Gi
+        cpu: 2000m
+      requests:
+        memory: 2Gi
+        cpu: 100m
   config:
     exporters:
       sumologic:
@@ -23,6 +47,8 @@ otelcol:
     processors:
       other:
         enabled: true
+      batch:
+        send_batch_size: 1024
       cascading_filter:
         num_spans: 200000
       source:

src/go/cmd/update-collection-v3/testdata/tracing.output.yaml

@@ -20,8 +20,38 @@ otelcolInstrumentation:
         source_category_replace_dash: processors.source.category_replace_dash.replace
         source_host: "%{k8s.pod.hostname}"
         source_name: processors.source.name.replace
+tracesGateway:
+  config:
+    exporters:
+      loadbalancing:
+        protocol:
+          otlp:
+            compression: gzip
+            sending_queue:
+              num_consumers: 100
+              queue_size: 13000
+  deployment:
+    replicas: 7
+    resources:
+      limits:
+        cpu: 2
+        memory: 3Gi
+      requests:
+        cpu: 1
+        memory: 2Gi
 tracesSampler:
   config:
     processors:
+      batch:
+        send_batch_size: 1024
       cascading_filter:
         num_spans: 200000
+  deployment:
+    replicas: 12
+    resources:
+      limits:
+        cpu: 2000m
+        memory: 4Gi
+      requests:
+        cpu: 100m
+        memory: 2Gi