Add retries to e2e tests CI jobs (#2131)

Author: L3n41c

.gitlab-ci.yml

@@ -191,30 +191,6 @@ build_bundle_image:
     - BUNDLE_IMG=$TARGET_IMAGE make bundle-build-push
     - if [ -n "$CI_COMMIT_TAG" ]; then docker buildx imagetools create $TARGET_IMAGE --tag $RELEASE_IMAGE; fi
 
-.new_e2e_template:
-  stage: e2e
-  tags: ["arch:amd64"]
-  image: $BUILD_DOCKER_REGISTRY/test-infra-definitions/runner:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
-  before_script:
-    # Setup AWS Credentials
-    - echo "Starting setup for E2E testing..."
-    - mkdir -p ~/.aws
-    - aws ssm get-parameter --region us-east-1 --name ci.datadog-operator.agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
-    - export AWS_PROFILE=agent-qa-ci
-    # Now all `aws` commands target the agent-qa profile
-    - echo "Retrieving SSH keys from AWS..."
-    - aws ssm get-parameter --region us-east-1 --name ci.datadog-operator.ssh_public_key --with-decryption --query "Parameter.Value" --out text > $E2E_AWS_PUBLIC_KEY_PATH
-    - aws ssm get-parameter --region us-east-1 --name ci.datadog-operator.ssh_private_key --with-decryption --query "Parameter.Value" --out text > $E2E_AWS_PRIVATE_KEY_PATH
-    # Use S3 backend to store stack status
-    - echo "Logging in to Pulumi with S3 backend..."
-    - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
-  variables:
-    E2E_AWS_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub
-    E2E_AWS_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key
-    E2E_KEY_PAIR_NAME: ci.datadog-operator
-    KUBERNETES_MEMORY_REQUEST: 12Gi
-    KUBERNETES_MEMORY_LIMIT: 16Gi
-
 .on_run_e2e:
   # Skip if only .md files are changed
   - if: $CI_COMMIT_BRANCH
@@ -250,8 +226,9 @@ trigger_e2e_operator_image:
     IMG_REGISTRIES: agent-qa
 
 e2e:
-  extends: .new_e2e_template
   stage: e2e
+  tags:
+    - "arch:amd64"
   needs:
     - "trigger_e2e_operator_image"
   rules: !reference [.on_run_e2e]
@@ -267,10 +244,30 @@ e2e:
           - "1.30"
           - "1.32"
   variables:
+    E2E_AWS_PUBLIC_KEY_PATH: /tmp/agent-qa-ssh-key.pub
+    E2E_AWS_PRIVATE_KEY_PATH: /tmp/agent-qa-ssh-key
+    E2E_KEY_PAIR_NAME: ci.datadog-operator
+    KUBERNETES_MEMORY_REQUEST: 12Gi
+    KUBERNETES_MEMORY_LIMIT: 16Gi
     TARGET_IMAGE: $E2E_DOCKER_REGISTRY:${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}
+  image: $BUILD_DOCKER_REGISTRY/test-infra-definitions/runner:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  before_script:
+    # Setup AWS Credentials
+    - echo "Starting setup for E2E testing..."
+    - mkdir -p ~/.aws
+    - aws ssm get-parameter --region us-east-1 --name ci.datadog-operator.agent-qa-profile --with-decryption --query "Parameter.Value" --out text >> ~/.aws/config
+    - export AWS_PROFILE=agent-qa-ci
+    # Now all `aws` commands target the agent-qa profile
+    - echo "Retrieving SSH keys from AWS..."
+    - aws ssm get-parameter --region us-east-1 --name ci.datadog-operator.ssh_public_key --with-decryption --query "Parameter.Value" --out text > $E2E_AWS_PUBLIC_KEY_PATH
+    - aws ssm get-parameter --region us-east-1 --name ci.datadog-operator.ssh_private_key --with-decryption --query "Parameter.Value" --out text > $E2E_AWS_PRIVATE_KEY_PATH
+    # Use S3 backend to store stack status
+    - echo "Logging in to Pulumi with S3 backend..."
+    - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
   script:
     - echo "Running e2e test with target image $TARGET_IMAGE"
     - IMAGE_PULL_PASSWORD=$(aws ecr get-login-password) IMG=$TARGET_IMAGE make e2e-tests
+  retry: 2
 
 publish_public_main:
   stage: release