Add integration tests for JMX. (#250)

Author: musa-asad

.github/workflows/operator-integration-test.yml

@@ -78,7 +78,7 @@ jobs:
           kubectl wait --for=condition=Ready pod --all -n default
           kubectl get pods -A 
           kubectl describe pods -n default
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/java/default_instrumentation_java_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/java/default_instrumentation_java_env_variables.json app_signals
 
       - name: Test for defined instrumentation resources for Java
         run: |
@@ -87,7 +87,7 @@ jobs:
           sleep 5
           kubectl wait --for=condition=Ready pod --all -n default
           
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json     
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals    
           kubectl delete instrumentation sample-instrumentation
 
       - name: Test for default instrumentation resources for python
@@ -97,15 +97,15 @@ jobs:
           kubectl wait --for=condition=Ready pod --all -n default
           kubectl get pods -A 
           kubectl describe pods -n default
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/python/default_instrumentation_python_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/python/default_instrumentation_python_env_variables.json app_signals
 
       - name: Test for defined instrumentation resources for python
         run: |
           kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
           kubectl delete pods --all -n default
           sleep 5
           kubectl wait --for=condition=Ready pod --all -n default
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
           kubectl delete instrumentation sample-instrumentation
 
       - name: Test for default instrumentation resources for dotnet
@@ -116,15 +116,15 @@ jobs:
           kubectl wait --for=condition=Available deployment/nginx -n default
           kubectl get pods -A 
           kubectl describe pods -n default
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/dotnet/default_instrumentation_dotnet_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/dotnet/default_instrumentation_dotnet_env_variables.json app_signals
 
       - name: Test for defined instrumentation resources for dotnet
         run: |
           kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
           kubectl delete pods --all -n default
           sleep 5
           kubectl wait --for=condition=Ready pod --all -n default  
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
           kubectl delete instrumentation sample-instrumentation
 
       - name: Test for default instrumentation resources for nodejs
@@ -137,7 +137,7 @@ jobs:
           kubectl wait --for=condition=Available deployment/nginx -n default
           kubectl get pods -A 
           kubectl describe pods -n default
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/nodejs/default_instrumentation_nodejs_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/nodejs/default_instrumentation_nodejs_env_variables.json app_signals
 
       - name: Test for defined instrumentation resources for nodejs
         run: |
@@ -146,7 +146,7 @@ jobs:
           sleep 5
           kubectl wait --for=condition=Available deployment/nginx -n default
           sleep 5
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
           kubectl delete instrumentation sample-instrumentation
 
       - name: Test for default instrumentation resources for all languages
@@ -155,8 +155,7 @@ jobs:
           sleep 5
           kubectl wait --for=condition=Available deployment/nginx -n default
           sleep 5
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/all-languages/default_instrumentation_all-languages_env_variables.json
-
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/all-languages/default_instrumentation_all-languages_env_variables.json app_signals
 
       - name: Test for defined instrumentation resources for all languages
         run: |
@@ -165,9 +164,69 @@ jobs:
           sleep 5
           kubectl wait --for=condition=Available deployment/nginx -n default
           sleep 5
-          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
           kubectl delete instrumentation sample-instrumentation
 
+      - name: Test for default instrumentation resources for jmx
+        run: |
+          kubectl delete pods --all -n default
+          sleep 5
+          cat integration-tests/jmx/sample-deployment-jmx.yaml
+          kubectl apply -f integration-tests/jmx/sample-deployment-jmx.yaml
+          sleep 5
+          kubectl wait --for=condition=Available deployment/nginx -n default
+          kubectl get pods -A 
+          kubectl describe pods -n default
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/jmx/default_instrumentation_jmx_env_variables.json app_signals
+
+      - name: Test for defined instrumentation resources for jmx
+        run: |
+          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
+          kubectl delete pods --all -n default
+          sleep 5
+          kubectl wait --for=condition=Available deployment/nginx -n default
+          sleep 5
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
+          kubectl delete instrumentation sample-instrumentation
+
+      - name: Test case for cloudwatch agent without app signals pod creation
+        run: |
+          kubectl delete daemonset cloudwatch-agent -n amazon-cloudwatch
+          sleep 5
+          kubectl apply -f integration-tests/manifests/cloudwatch-agent-daemonset-no-app-signals.yaml -n amazon-cloudwatch
+          sleep 20
+          kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch
+          
+          pod_name="$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/component=amazon-cloudwatch-agent,app.kubernetes.io/instance=amazon-cloudwatch.cloudwatch-agent -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')"
+          if [ -z "$pod_name" ]; then
+            echo "Pod $pod_name is not created. Exiting with ERROR."
+            echo "Pod is not created test case: FAIL"
+          else
+            echo "Pod $pod_name is created."
+            echo "Pod is created test case: PASS"
+          fi
+
+      - name: Test for default instrumentation resources for jmx without app signals
+        run: |
+          kubectl delete pods --all -n default
+          sleep 5
+          cat integration-tests/jmx/sample-deployment-jmx.yaml
+          kubectl apply -f integration-tests/jmx/sample-deployment-jmx.yaml
+          sleep 5
+          kubectl wait --for=condition=Available deployment/nginx -n default
+          kubectl get pods -A 
+          kubectl describe pods -n default
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/jmx/default_instrumentation_jmx_env_variables_no_app_signals.json no_app_signals
+
+      - name: Test for defined instrumentation resources for jmx without app signals
+        run: |
+          kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml
+          kubectl delete pods --all -n default
+          sleep 5
+          kubectl wait --for=condition=Available deployment/nginx -n default
+          sleep 5
+          go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/manifests/cmd/ns_instrumentation_env_variables.json app_signals
+          kubectl delete instrumentation sample-instrumentation
 
   DeploymentAnnotationsTest:
     name: DeploymentAnnotationsTest

integration-tests/all-languages/default_instrumentation_all-languages_env_variables.json

@@ -20,6 +20,7 @@
   "DOTNET_ADDITIONAL_DEPS": "/otel-auto-instrumentation-dotnet/AdditionalDeps",
   "OTEL_DOTNET_AUTO_HOME": "/otel-auto-instrumentation-dotnet",
   "DOTNET_SHARED_STORE": "/otel-auto-instrumentation-dotnet/store",
-  "CORECLR_ENABLE_PROFILING": "1"
-
-}
+  "CORECLR_ENABLE_PROFILING": "1",
+  "OTEL_AWS_JMX_EXPORTER_METRICS_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4314/v1/metrics",
+  "OTEL_JMX_TARGET_SYSTEM": "jvm,tomcat,kafka,kafka-consumer,kafka-producer"
+}

integration-tests/all-languages/sample-deployment-all-languages.yaml

@@ -14,6 +14,12 @@ spec:
         instrumentation.opentelemetry.io/inject-python: "true"
         instrumentation.opentelemetry.io/inject-java: "true"
         instrumentation.opentelemetry.io/inject-dotnet: "true"
+        instrumentation.opentelemetry.io/inject-nodejs: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-jvm: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-tomcat: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-kafka: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-kafka-consumer: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-kafka-producer: "true"
     spec:
       containers:
         - name: nginx

integration-tests/jmx/default_instrumentation_jmx_env_variables.json

@@ -0,0 +1,15 @@
+{
+  "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
+  "OTEL_METRICS_EXPORTER": "none",
+  "OTEL_LOGS_EXPORTER": "none",
+  "OTEL_AWS_APP_SIGNALS_ENABLED": "true",
+  "OTEL_AWS_APPLICATION_SIGNALS_ENABLED": "true",
+  "OTEL_TRACES_SAMPLER_ARG": "endpoint=http://cloudwatch-agent.amazon-cloudwatch:2000",
+  "OTEL_TRACES_SAMPLER": "xray",
+  "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4316/v1/traces",
+  "OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4316/v1/metrics",
+  "OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4316/v1/metrics",
+  "OTEL_AWS_JMX_EXPORTER_METRICS_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4314/v1/metrics",
+  "OTEL_JMX_TARGET_SYSTEM": "jvm,tomcat,kafka,kafka-consumer,kafka-producer",
+  "JAVA_TOOL_OPTIONS": "-javaagent:/otel-auto-instrumentation-java/javaagent.jar"
+}

integration-tests/jmx/default_instrumentation_jmx_env_variables_no_app_signals.json

@@ -0,0 +1,9 @@
+{
+  "OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf",
+  "OTEL_METRICS_EXPORTER": "none",
+  "OTEL_LOGS_EXPORTER": "none",
+  "OTEL_TRACES_EXPORTER": "none",
+  "OTEL_AWS_JMX_EXPORTER_METRICS_ENDPOINT": "http://cloudwatch-agent.amazon-cloudwatch:4314/v1/metrics",
+  "OTEL_JMX_TARGET_SYSTEM": "jvm,tomcat,kafka,kafka-consumer,kafka-producer",
+  "JAVA_TOOL_OPTIONS": "-javaagent:/otel-auto-instrumentation-java/javaagent.jar"
+}

integration-tests/jmx/sample-deployment-jmx.yaml

@@ -0,0 +1,25 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx
+spec:
+  selector:
+    matchLabels:
+      app: nginx
+  template:
+    metadata:
+      labels:
+        app: nginx
+      annotations:
+        instrumentation.opentelemetry.io/inject-java: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-jvm: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-tomcat: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-kafka: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-kafka-consumer: "true"
+        cloudwatch.aws.amazon.com/inject-jmx-kafka-producer: "true"
+    spec:
+      containers:
+        - name: nginx
+          image: nginx:1.14.2
+      restartPolicy: Always
+status: {}

integration-tests/manifests/cloudwatch-agent-daemonset-no-app-signals.yaml

@@ -0,0 +1,70 @@
+apiVersion: cloudwatch.aws.amazon.com/v1alpha1
+kind: AmazonCloudWatchAgent
+metadata:
+  name: cloudwatch-agent
+  namespace: amazon-cloudwatch
+spec:
+  mode: daemonset
+  serviceAccount: cloudwatch-agent
+  image: public.ecr.aws/cloudwatch-agent/cloudwatch-agent:latest
+  config: |
+    {
+      "agent": {
+        "region": "${REGION}",
+        "debug": true
+      }
+    }
+  volumeMounts:
+    - mountPath: /rootfs
+      name: rootfs
+      readOnly: true
+    - mountPath: /var/run/docker.sock
+      name: dockersock
+      readOnly: true
+    - mountPath: /run/containerd/containerd.sock
+      name: containerdsock
+    - mountPath: /var/lib/docker
+      name: varlibdocker
+      readOnly: true
+    - mountPath: /sys
+      name: sys
+      readOnly: true
+    - mountPath: /dev/disk
+      name: devdisk
+      readOnly: true
+  volumes:
+    - name: rootfs
+      hostPath:
+        path: /
+    - hostPath:
+        path: /var/run/docker.sock
+      name: dockersock
+    - hostPath:
+        path: /var/lib/docker
+      name: varlibdocker
+    - hostPath:
+        path: /run/containerd/containerd.sock
+      name: containerdsock
+    - hostPath:
+        path: /sys
+      name: sys
+    - hostPath:
+        path: /dev/disk/
+      name: devdisk
+  env:
+    - name: K8S_NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: HOST_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.hostIP
+    - name: HOST_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: K8S_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace

integration-tests/manifests/cmd/validate_instrumentation_vars.go

@@ -18,11 +18,21 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
+var appSignalsEnvVarKeys = []string{
+	"OTEL_AWS_APP_SIGNALS_ENABLED",
+	"OTEL_AWS_APPLICATION_SIGNALS_ENABLED",
+	"OTEL_TRACES_SAMPLER_ARG",
+	"OTEL_TRACES_SAMPLER",
+	"OTEL_EXPORTER_OTLP_TRACES_ENDPOINT",
+	"OTEL_AWS_APP_SIGNALS_EXPORTER_ENDPOINT",
+}
+
 func main() {
 
 	args := os.Args
 	namespace := args[1]
 	jsonFilePath := args[2]
+	appSignalsEnabled := args[3] == "app_signals"
 
 	userHomeDir, err := os.UserHomeDir()
 	if err != nil {
@@ -42,7 +52,7 @@ func main() {
 		fmt.Printf("error getting kubernetes config: %v\n\n", err)
 	}
 
-	success := verifyInstrumentationEnvVariables(clientSet, namespace, jsonFilePath)
+	success := verifyInstrumentationEnvVariables(clientSet, namespace, jsonFilePath, appSignalsEnabled)
 	if !success {
 		fmt.Println("Instrumentation Annotation Injection Test: FAIL")
 		os.Exit(1)
@@ -51,7 +61,7 @@ func main() {
 	}
 }
 
-func verifyInstrumentationEnvVariables(clientset *kubernetes.Clientset, namespace, jsonPath string) bool {
+func verifyInstrumentationEnvVariables(clientset *kubernetes.Clientset, namespace, jsonPath string, appSignalsEnabled bool) bool {
 	podList, err := clientset.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{
 		LabelSelector: "app=nginx",
 		FieldSelector: "status.phase!=Terminating",
@@ -89,6 +99,16 @@ func verifyInstrumentationEnvVariables(clientset *kubernetes.Clientset, namespac
 	}
 	fmt.Println("JSON data:", jsonData)
 
+	if !appSignalsEnabled {
+		fmt.Println("Checking if app signals environment variables exist")
+		for _, key := range appSignalsEnvVarKeys {
+			if _, exists := jsonData[key]; exists {
+				fmt.Printf("Error: Key '%s' should not exist in jsonData when app signals is not enabled\n", key)
+				return false
+			}
+		}
+	}
+
 	for key, value := range jsonData {
 		if val, ok := envMap[key]; ok {
 			if strings.ReplaceAll(val, " ", "") != strings.ReplaceAll(value, " ", "") {