From 8b03e772370dae23a399db9ecc9e604521ca6866 Mon Sep 17 00:00:00 2001 From: lisguo Date: Mon, 17 Jun 2024 13:58:19 -0400 Subject: [PATCH 1/3] Use kubectl wait to make the integration test more reliable --- .../workflows/operator-integration-test.yml | 38 ++++++++----------- 1 file changed, 16 insertions(+), 22 deletions(-) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index b81886a06..977957db4 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -57,8 +57,9 @@ jobs: - name: Test case for AmazonCloudWatchAgent pod creation run: | kubectl apply -f integration-tests/manifests/cloudwatch-agent-daemonset.yaml -n amazon-cloudwatch - sleep 60 - kubectl describe pods -n amazon-cloudwatch + sleep 5 + kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch + pod_name="$(kubectl get pods -n amazon-cloudwatch -l app.kubernetes.io/component=amazon-cloudwatch-agent,app.kubernetes.io/instance=amazon-cloudwatch.cloudwatch-agent -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}')" if [ -z "$pod_name" ]; then echo "Pod $pod_name is not created. Exiting with ERROR." @@ -72,18 +73,17 @@ jobs: run: | kubectl apply -f integration-tests/java/sample-deployment-java.yaml sleep 5 - kubectl get pods -A - kubectl describe pods -n default + kubectl wait --for=condition=Ready pod --all -n default + go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/java/default_instrumentation_java_env_variables.json - name: Test for defined instrumentation resources for Java run: | kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml - sleep 5 kubectl rollout restart deployment nginx sleep 5 - kubectl get pods -A - kubectl describe pods -n default + kubectl wait --for=condition=Ready pod --all -n default + cd integration-tests/manifests/cmd go run validate_instrumentation_vars.go default ns_instrumentation_env_variables.json kubectl delete instrumentation sample-instrumentation @@ -91,20 +91,19 @@ jobs: - name: Test for default instrumentation resources for python run: | kubectl apply -f integration-tests/python/sample-deployment-python.yaml - sleep 5 kubectl rollout restart deployment nginx sleep 5 - kubectl get pods -A - kubectl describe pods -n default + kubectl wait --for=condition=Ready pod --all -n default + go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/python/default_instrumentation_python_env_variables.json - name: Test for defined instrumentation resources for python run: | kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml - sleep 5 kubectl rollout restart deployment nginx sleep 5 - kubectl describe pods -n default + kubectl wait --for=condition=Ready pod --all -n default + cd integration-tests/manifests/cmd go run validate_instrumentation_vars.go default ns_instrumentation_env_variables.json kubectl delete instrumentation sample-instrumentation @@ -112,22 +111,19 @@ jobs: - name: Test for default instrumentation resources for python and java run: | kubectl apply -f integration-tests/python-java/sample-deployment-python-java.yaml - sleep 5 kubectl rollout restart deployment nginx sleep 5 - kubectl get pods -A - kubectl describe pods -n default - kubectl describe pods -n amazon-cloudwatch + kubectl wait --for=condition=Ready pod --all -n default + go run integration-tests/manifests/cmd/validate_instrumentation_vars.go default integration-tests/python-java/default_instrumentation_python-java_env_variables.json - name: Test for defined instrumentation resources for python and java run: | kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml - sleep 5 kubectl rollout restart deployment nginx sleep 5 - kubectl get pods -A - kubectl describe pods -n default + kubectl wait --for=condition=Ready pod --all -n default + cd integration-tests/manifests/cmd go run validate_instrumentation_vars.go default ns_instrumentation_env_variables.json kubectl delete instrumentation sample-instrumentation @@ -326,6 +322,4 @@ jobs: sleep 5 go test -v -run TestPythonOnlyNamespace ./integration-tests/manifests/annotations -timeout 30m sleep 5 - go test -v -run TestAlreadyAutoAnnotatedResourceShouldNotRestart ./integration-tests/manifests/annotations -timeout 30m - - + go test -v -run TestAlreadyAutoAnnotatedResourceShouldNotRestart ./integration-tests/manifests/annotations -timeout 30m \ No newline at end of file From 06a197814fb92e1dd9a26735a4cf8c106e56b351 Mon Sep 17 00:00:00 2001 From: lisguo Date: Mon, 17 Jun 2024 14:21:07 -0400 Subject: [PATCH 2/3] Delete pods instead of using kubectl rollout which staggers pods. Rename integ test to instrumentation test --- .github/workflows/operator-integration-test.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 977957db4..16a2f52ca 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -20,8 +20,8 @@ permissions: contents: read jobs: - AmazonCloudWatchAgentOperatorIntegrationTest: - name: AmazonCloudWatchAgentOperatorIntegrationTest + InstrumentationTest: + name: InstrumentationTest runs-on: ubuntu-latest permissions: id-token: write @@ -54,7 +54,7 @@ jobs: run: | make deploy - - name: Test case for AmazonCloudWatchAgent pod creation + - name: Test case for cloudwatch agent pod creation run: | kubectl apply -f integration-tests/manifests/cloudwatch-agent-daemonset.yaml -n amazon-cloudwatch sleep 5 @@ -80,7 +80,7 @@ jobs: - name: Test for defined instrumentation resources for Java run: | kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml - kubectl rollout restart deployment nginx + kubectl delete pods --all -n default sleep 5 kubectl wait --for=condition=Ready pod --all -n default @@ -91,7 +91,7 @@ jobs: - name: Test for default instrumentation resources for python run: | kubectl apply -f integration-tests/python/sample-deployment-python.yaml - kubectl rollout restart deployment nginx + kubectl delete pods --all -n default sleep 5 kubectl wait --for=condition=Ready pod --all -n default @@ -100,7 +100,7 @@ jobs: - name: Test for defined instrumentation resources for python run: | kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml - kubectl rollout restart deployment nginx + kubectl delete pods --all -n default sleep 5 kubectl wait --for=condition=Ready pod --all -n default @@ -111,7 +111,7 @@ jobs: - name: Test for default instrumentation resources for python and java run: | kubectl apply -f integration-tests/python-java/sample-deployment-python-java.yaml - kubectl rollout restart deployment nginx + kubectl delete pods --all -n default sleep 5 kubectl wait --for=condition=Ready pod --all -n default @@ -120,7 +120,7 @@ jobs: - name: Test for defined instrumentation resources for python and java run: | kubectl apply -f integration-tests/manifests/sample-instrumentation.yaml - kubectl rollout restart deployment nginx + kubectl delete pods --all -n default sleep 5 kubectl wait --for=condition=Ready pod --all -n default From 5e8bdd0764d702c1e52f7e1624531037c31d36d8 Mon Sep 17 00:00:00 2001 From: lisguo Date: Mon, 17 Jun 2024 15:27:00 -0400 Subject: [PATCH 3/3] Wait for operator pod to be ready --- .github/workflows/operator-integration-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/operator-integration-test.yml b/.github/workflows/operator-integration-test.yml index 16a2f52ca..32ee90c38 100644 --- a/.github/workflows/operator-integration-test.yml +++ b/.github/workflows/operator-integration-test.yml @@ -53,6 +53,7 @@ jobs: - name: Deploy operator to minikube run: | make deploy + kubectl wait --for=condition=Ready pod --all -n amazon-cloudwatch - name: Test case for cloudwatch agent pod creation run: |