CI | Data Buckets | Test AWS SDK Clients

1. Add to the CI a test that runs basic commands using the most updated AWS SDK client. In this PR it is AWS SDK Go V2 as a start.
2. Add log printings of 'pools_to_create.length' in coretest (helps in debugging in case the nodes/hosts were not created).

Signed-off-by: shirady <57721533+shirady@users.noreply.github.com>

Author: shirady

.github/workflows/test-aws-sdk-clients.yaml

@@ -0,0 +1,27 @@
+name: Test AWS SDK Clients
+on: 
+  schedule:
+    - cron: "0 1 * * *"
+  workflow_dispatch:
+
+jobs:
+  test-aws-sdk-clients:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: 'noobaa/noobaa-core'
+          path: 'noobaa-core'
+
+      - name: Run AWS SDK Clients
+        run: |
+          set -x
+          cd ./noobaa-core
+          make test-aws-sdk-clients
+
+

Makefile

@@ -229,6 +229,12 @@ build-ssl-postgres: tester
 	@echo "##\033[1;32m Build image postgres:ssl done.\033[0m"
 .PHONY: build-ssl-postgres
 
+build-aws-client: noobaa
+	@echo "\n##\033[1;32m Build image for AWS Client tests ...\033[0m"
+	$(CONTAINER_ENGINE) build $(CONTAINER_PLATFORM_FLAG) $(CPUSET) -f src/deploy/NVA_build/AWSClient.Dockerfile $(CACHE_FLAG) $(NETWORK_FLAG) -t noobaa-aws-client . $(REDIRECT_STDOUT)
+	@echo "\033[1;32mBuild image for AWS Client tests done.\033[0m"
+.PHONY: build-aws-client
+
 test: tester
 	@echo "\033[1;34mRunning tests with Mongo.\033[0m"
 	@$(call create_docker_network)
@@ -363,6 +369,17 @@ test-external-pg-sanity: build-ssl-postgres
 	@$(call remove_docker_network)
 .PHONY: test-external-pg-sanity
 
+test-aws-sdk-clients: build-aws-client
+	@echo "\033[1;34mRunning tests with Postgres.\033[0m"
+	@$(call create_docker_network)
+	@$(call run_postgres)
+	@echo "\033[1;34mRunning aws sdk clients tests\033[0m"
+	$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" --env "NOOBAA_LOG_LEVEL=all" -v $(PWD)/logs:/logs  noobaa-aws-client ./src/test/unit_tests/run_npm_test_on_test_container.sh -c ./node_modules/mocha/bin/mocha.js src/test/unit_tests/different_clients/test_go_sdkv2_script.js
+	@$(call stop_noobaa)
+	@$(call stop_postgres)
+	@$(call remove_docker_network)
+.PHONY: test-aws-sdk-clients
+
 clean:
 	@echo Stopping and Deleting containers
 	@$(CONTAINER_ENGINE) ps -a | grep noobaa_ | awk '{print $1}' | xargs $(CONTAINER_ENGINE) stop &> /dev/null

src/deploy/NVA_build/AWSClient.Dockerfile

@@ -0,0 +1,34 @@
+FROM noobaa
+
+USER 0:0
+
+ENV container=docker
+ENV TEST_CONTAINER=true
+
+WORKDIR /root/node_modules/noobaa-core/
+
+# check npm version
+RUN npm -v
+# install all dependencies (although we would like only the dev dependencies)
+RUN npm install
+
+############################################################################
+# Layers:
+#   Title: Install go and modules
+#           for testing with AWS SDK GO client with most updated version
+
+############################################################################
+
+# installing go
+RUN dnf install -y golang
+# verify go installation
+RUN go version
+# set the PATH for go
+ENV PATH="/usr/local/go/bin:$PATH"
+
+# install the needed modules
+# note: the files go.mod and go.sum will be automatically created after this step in the WORKDIR
+RUN go mod init src/test/unit_tests/different_clients
+RUN go mod tidy
+
+USER 10001:0

src/test/unit_tests/coretest.js

@@ -233,6 +233,7 @@ function setup(options = {}) {
         });
         rpc_client.options.auth_token = token;
         await overwrite_system_address(SYSTEM);
+        console.log('pools_to_create.length', pools_to_create.length);
         if (pools_to_create.length > 0) {
             await announce('setup_pools()');
             await setup_pools(pools_to_create);