Skip to content

Commit 95034f6

Browse files
romayalonromayalon
authored
Merge pull request #9123 from romayalon/romy-ci-mint
CI | Mint Integration with NooBaa
2 parents ae54f91 + f096fb5 commit 95034f6

File tree

13 files changed

+488
-41
lines changed

13 files changed

+488
-41
lines changed

‎.github/workflows/mint-nc-tests.yaml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
2+
name: Mint NC Tests
3+
on: [workflow_call]
4+
5+
jobs:
6+
mint-nc-tests:
7+
name: Mint NC Tests
8+
runs-on: ubuntu-latest
9+
timeout-minutes: 90
10+
permissions:
11+
actions: read # download-artifact
12+
contents: read # required for actions/checkout
13+
steps:
14+
- name: Checkout noobaa-core
15+
uses: actions/checkout@v4
16+
with:
17+
repository: 'noobaa/noobaa-core'
18+
path: 'noobaa-core'
19+
20+
- name: Download artifact
21+
uses: actions/download-artifact@v4
22+
with:
23+
name: noobaa-tester
24+
path: /tmp
25+
26+
- name: Load image
27+
run: docker load --input /tmp/noobaa-tester.tar
28+
29+
- name: Create Mint logs directory
30+
run: |
31+
set -x
32+
cd ./noobaa-core
33+
mkdir -p logs/mint-nc-test-logs
34+
chmod 777 logs/mint-nc-test-logs
35+
36+
- name: Run NC Mint tests
37+
run: |
38+
set -x
39+
cd ./noobaa-core
40+
make test-nc-mint -o tester
41+

‎.github/workflows/mint-tests.yaml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
name: Mint Tests
2+
on: [workflow_call]
3+
4+
jobs:
5+
mint-tests:
6+
name: Mint Tests
7+
runs-on: ubuntu-latest
8+
timeout-minutes: 90
9+
permissions:
10+
actions: read # download-artifact
11+
contents: read # required for actions/checkout
12+
steps:
13+
- name: Checkout noobaa-core
14+
uses: actions/checkout@v4
15+
with:
16+
repository: 'noobaa/noobaa-core'
17+
path: 'noobaa-core'
18+
19+
- name: Download artifact
20+
uses: actions/download-artifact@v4
21+
with:
22+
name: noobaa-tester
23+
path: /tmp
24+
25+
- name: Load image
26+
run: docker load --input /tmp/noobaa-tester.tar
27+
28+
- name: Create Mint logs directory
29+
run: |
30+
set -x
31+
cd ./noobaa-core
32+
mkdir -p logs/mint-test-logs
33+
chmod 777 logs/mint-test-logs
34+
35+
- name: Run Mint tests
36+
run: |
37+
set -x
38+
cd ./noobaa-core
39+
make test-mint -o tester
40+
41+

‎.github/workflows/run-pr-tests.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ jobs:
4141
needs: build-noobaa-image
4242
uses: ./.github/workflows/warp-nc-tests.yaml
4343

44+
mint-tests:
45+
needs: build-noobaa-image
46+
uses: ./.github/workflows/mint-tests.yaml
47+
48+
mint-nc-tests:
49+
needs: build-noobaa-image
50+
uses: ./.github/workflows/mint-nc-tests.yaml
51+
4452
build-noobaa-image:
4553
name: Build Noobaa Image
4654
runs-on: ubuntu-latest

‎Makefile

Lines changed: 63 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,10 +81,17 @@ ifdef testname
8181
endif
8282
endif
8383

84+
######################
85+
# S3SELECT VARIABLES #
86+
######################
87+
8488
BUILD_S3SELECT?=1
8589
BUILD_S3SELECT_PARQUET?=0
8690

87-
## RPM VARIABLES
91+
#################
92+
# RPM VARIABLES #
93+
#################
94+
8895
DATE := $(shell date +'%Y%m%d')
8996
NOOBAA_PKG_VERSION := $(shell jq -r '.version' < ./package.json)
9097
RPM_BASE_VERSION := noobaa-core-$(NOOBAA_PKG_VERSION)-${DATE}
@@ -96,6 +103,14 @@ endif
96103
RPM_FULL_PATH := $(RPM_BASE_VERSION).el${CENTOS_VER}.$(ARCH_SUFFIX).rpm
97104
install_rpm_and_deps_command := dnf install -y make && rpm -i $(RPM_FULL_PATH) && systemctl enable noobaa --now && systemctl status noobaa && systemctl stop noobaa
98105

106+
##################
107+
# MINT VARIABLES #
108+
##################
109+
110+
MINT_MOCK_ACCESS_KEY="aaaaaaaaaaaaaEXAMPLE"
111+
MINT_MOCK_SECRET_KEY="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaEXAMPLE"
112+
MINT_NOOBAA_HTTP_ENDPOINT_PORT=6001
113+
99114
###############
100115
# BUILD LOCAL #
101116
###############
@@ -353,6 +368,42 @@ test-nc-warp: tester
353368
$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/warp/run_nc_warp_on_test_container.sh"
354369
.PHONY: test-nc-warp
355370

371+
test-mint: tester
372+
@echo "\033[1;34mRunning mint tests with Postgres.\033[0m"
373+
@$(call create_docker_network)
374+
@$(call run_postgres)
375+
@echo "\033[1;34mRunning mint tests\033[0m"
376+
$(CONTAINER_ENGINE) run $(CPUSET) --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) -dit --network noobaa-net --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs/mint-test-logs/:/logs $(TESTER_TAG) bash -c "./src/test/system_tests/mint/run_mint_on_test_container.sh & tail -f /dev/null"
377+
sleep 180
378+
$(CONTAINER_ENGINE) run --name mint-$(GIT_COMMIT)-$(NAME_POSTFIX) --network noobaa-net -v $(PWD)/logs/mint-test-logs/:/mint/log --env SERVER_ENDPOINT=noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX):$(MINT_NOOBAA_HTTP_ENDPOINT_PORT) --env ACCESS_KEY=$(MINT_MOCK_ACCESS_KEY) --env SECRET_KEY=$(MINT_MOCK_SECRET_KEY) --env ENABLE_HTTPS=0 minio/mint minio-go s3cmd
379+
@echo "\033[1;34mPrinting noobaa configuration and logs\033[0m"
380+
$(CONTAINER_ENGINE) logs noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
381+
@echo "\033[1;34mPrinting mint results file\033[0m"
382+
cat $(PWD)/logs/mint-test-logs/log.json
383+
@$(call disconnect_container_from_noobaa_network, mint-$(GIT_COMMIT)-$(NAME_POSTFIX))
384+
$(CONTAINER_ENGINE) rm mint-$(GIT_COMMIT)-$(NAME_POSTFIX)
385+
@$(call stop_noobaa)
386+
@$(call stop_postgres)
387+
@$(call remove_docker_network)
388+
.PHONY: test-mint
389+
390+
391+
test-nc-mint: tester
392+
@echo "\033[1;34mRunning mint tests on NC environment\033[0m"
393+
@$(call create_docker_network)
394+
$(CONTAINER_ENGINE) run $(CPUSET) --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) -dit --privileged --user root --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --network noobaa-net -v $(PWD)/logs/mint-nc-test-logs/:/logs $(TESTER_TAG) bash -c "./src/test/system_tests/mint/run_nc_mint_on_test_container.sh; tail -f /dev/null"
395+
sleep 15
396+
$(CONTAINER_ENGINE) run --name mint-$(GIT_COMMIT)-$(NAME_POSTFIX) --network noobaa-net -v $(PWD)/logs/mint-nc-test-logs/:/mint/log --env RUN_ON_FAIL=0 --env SERVER_ENDPOINT=noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX):$(MINT_NOOBAA_HTTP_ENDPOINT_PORT) --env ACCESS_KEY=$(MINT_MOCK_ACCESS_KEY) --env SECRET_KEY=$(MINT_MOCK_SECRET_KEY) --env ENABLE_HTTPS=0 minio/mint minio-go s3cmd
397+
@echo "\033[1;34mPrinting noobaa configuration and logs\033[0m"
398+
$(CONTAINER_ENGINE) logs noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
399+
@echo "\033[1;34mPrinting mint results file\033[0m"
400+
cat $(PWD)/logs/mint-nc-test-logs/log.json
401+
@$(call disconnect_container_from_noobaa_network, mint-$(GIT_COMMIT)-$(NAME_POSTFIX))
402+
$(CONTAINER_ENGINE) rm mint-$(GIT_COMMIT)-$(NAME_POSTFIX)
403+
@$(call stop_noobaa)
404+
@$(call remove_docker_network)
405+
.PHONY: test-nc-mint
406+
356407
test-nsfs-cephs3: tester
357408
@echo "\033[1;34mRunning Ceph S3 tests on NSFS Standalone platform\033[0m"
358409
$(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh"
@@ -413,7 +464,7 @@ clean:
413464

414465
define stop_noobaa
415466
@echo "\033[1;34mStopping/removing test container\033[0m"
416-
$(CONTAINER_ENGINE) network disconnect noobaa-net noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
467+
$(call disconnect_container_from_noobaa_network, noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX))
417468
$(CONTAINER_ENGINE) stop noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
418469
$(CONTAINER_ENGINE) rm noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX)
419470
@echo "\033[1;32mRemoving test container done.\033[0m"
@@ -435,6 +486,12 @@ define remove_docker_network
435486
@echo "\033[1;32mRemove docker network done.\033[0m"
436487
endef
437488

489+
define disconnect_container_from_noobaa_network
490+
echo "\033[1;34mDisconnect container $(1) from noobaa network\033[0m"; \
491+
$(CONTAINER_ENGINE) network disconnect noobaa-net $(1); \
492+
echo "\033[1;34mDisconnect container $(1) from noobaa network done.\033[0m"
493+
endef
494+
438495
#########
439496
# MONGO #
440497
#########
@@ -447,7 +504,7 @@ endef
447504

448505
define stop_mongo
449506
@echo "\033[1;34mStopping/removing Mongo container\033[0m"
450-
$(CONTAINER_ENGINE) network disconnect noobaa-net coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)
507+
$(call disconnect_container_from_noobaa_network, coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX))
451508
$(CONTAINER_ENGINE) stop coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)
452509
$(CONTAINER_ENGINE) rm coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)
453510
@echo "\033[1;32mStop mongo done.\033[0m"
@@ -467,7 +524,7 @@ endef
467524

468525
define stop_postgres
469526
@echo "\033[1;34mStopping/removing Postgres container\033[0m"
470-
$(CONTAINER_ENGINE) network disconnect noobaa-net coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)
527+
$(call disconnect_container_from_noobaa_network, coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX))
471528
$(CONTAINER_ENGINE) stop coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)
472529
$(CONTAINER_ENGINE) rm coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)
473530
@echo "\033[1;32mStop postgres done.\033[0m"
@@ -487,7 +544,7 @@ endef
487544

488545
define stop_external_postgres
489546
@echo "\033[1;34mStopping/removing Postgres container\033[0m"
490-
$(CONTAINER_ENGINE) network disconnect noobaa-net ssl-pg-$(GIT_COMMIT)-$(NAME_POSTFIX)
547+
$(call disconnect_container_from_noobaa_network, ssl-pg-$(GIT_COMMIT)-$(NAME_POSTFIX))
491548
$(CONTAINER_ENGINE) stop ssl-pg-$(GIT_COMMIT)-$(NAME_POSTFIX)
492549
$(CONTAINER_ENGINE) rm ssl-pg-$(GIT_COMMIT)-$(NAME_POSTFIX)
493550
@echo "\033[1;32mStop postgres done.\033[0m"
@@ -518,7 +575,7 @@ define stop_blob_mock
518575
@echo "\033[1;34mStopping blob mock server if RUN_BLOB_MOCK=$(RUN_BLOB_MOCK) is true.\033[0m"
519576
@ if [ $(RUN_BLOB_MOCK) = true ]; then \
520577
echo "\033[1;34mStopping tests with Blob mock.\033[0m"; \
521-
$(CONTAINER_ENGINE) network disconnect noobaa-net blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX); \
578+
$(call disconnect_container_from_noobaa_network, blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)); \
522579
$(CONTAINER_ENGINE) stop blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX); \
523580
$(CONTAINER_ENGINE) rm blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX); \
524581
fi

‎docs/CI & Tests/Mint.md

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
# Mint Github Action, Tests and Tool
2+
3+
1. [Introduction](#introduction)
4+
2. [Mint GitHub actions](#mint-github-actions)
5+
3. [Mint Makefile Targets](#mint-makefile-targets)
6+
4. [Manual Mint Installation](#manual-mint-installation)
7+
8+
9+
10+
## Introduction
11+
12+
[Mint](https://github.com/minio/mint) is a testing framework for S3-compatible object storage systems, NooBaa CI runs Mint as correctness/benchmarking and stress tests for the NooBaa system on both containerized and Non Containerized flavors.
13+
Following are the SDKs/tools used in correctness tests.
14+
15+
- awscli
16+
- aws-sdk-go
17+
- aws-sdk-java
18+
- aws-sdk-java-v2
19+
- aws-sdk-php
20+
- aws-sdk-ruby
21+
- healthcheck
22+
- mc
23+
- minio-go
24+
- minio-java
25+
- minio-js
26+
- minio-py
27+
- s3cmd
28+
- s3select
29+
- versioning
30+
31+
## Mint GitHub actions
32+
33+
NooBaa CI contains 2 Github actions that build, configure and run Mint. These Github actions run automatically on every PR and on every push, and can run by workflow dispatch manually.
34+
* [Mint Tests](../../.github/workflows/mint-tests.yaml) - Based on NooBaa Tester image, runs Mint on standard NooBaa (db configuration).
35+
* [Mint NC Tests](../../.github/workflows/mint-nc-tests.yaml) - Based on NooBaa Tester image, runs Mint on non-containerized NooBaa (ConfigFS configuration).
36+
37+
Our next goal is to add longer Mint runs as part of NooBaa's nightly CI process.
38+
39+
## Mint Makefile Targets
40+
41+
One can run Mint tests on NooBaa using Mint Makefile targets -
42+
* `make test-mint` - Based on NooBaa Tester image, runs Mint on standard NooBaa (db configuration).
43+
* `make test-nc-mint` - Based on NooBaa Tester image, runs Mint on non-containerized NooBaa (ConfigFS configuration).
44+
45+
The above makefile targets, build NooBaa tester image, and later deploy NooBaa (DB/ConfigFS deployments), create default account and runs the supported sdks on Mint per the deployment type.
46+
47+
Currently, the supported mint test frameworks are:
48+
1. s3cmd
49+
2. minio-go
50+
51+
## Manual Mint Installation
52+
53+
NC deployment -
54+
1. Tab 1 - Install NooBaa
55+
2. Tab 2 - Create a NooBaa account.
56+
2. Tab 2 - Run Mint pointing to NooBaa endpoint -
57+
```
58+
docker run -e SERVER_ENDPOINT=<noobaa-endpoint-address>:<noobaa-endpoint-http-port> -e ACCESS_KEY=<pre-existing-account-access-key> -e SECRET_KEY=<pre-existing-account-secret-key> -e ENABLE_HTTPS=0 minio/mint <sdk-or-tool-name>
59+
```
60+
61+
Developer notes -
62+
63+
64+
To manually run a MinIO Mint container that connects to a noobaa-tester container, ensure both containers are on the same Docker network (noobaa-net).
65+
66+
67+
If the noobaa-tester is already connected to noobaa-net, your Mint run command should look like this:
68+
```
69+
docker run -e SERVER_ENDPOINT=<noobaa-tester-container-id-or-name>:<noobaa-http-endpoint-port> -e ACCESS_KEY=<pre-existing-account-access-key> -e SECRET_KEY=<pre-existing-account-secret-key> -e ENABLE_HTTPS=0 --network noobaa-net minio/mint <sdk-or-tool-name>
70+
```
71+
72+
Replace <sdk-or-tool-name> with the specific SDK or tool you want to test (e.g., aws-sdk-java, minio-go, s3cmd, etc.).
73+
74+
## Debugging Mint on NooBaa locally -
75+
Running `make test-mint` will generate the following debug log files -
76+
77+
```bash
78+
noobaa-core % tree logs/mint-test-logs
79+
logs/mint-test-logs
80+
├── log.json // contains the Mint run results
81+
├── minio-go
82+
│   └── error.log // contains errors coming from minio-go run
83+
├── mint-test-logs
84+
│   ├── backingstore1.log // contains noobaa backingstore logs
85+
│   ├── bg.log // contains noobaa BG workers logs
86+
│   ├── hosted_agents.log // contains noobaa hosted agents logs
87+
│   ├── s3.log // contains noobaa endpoint logs
88+
│   └── web.log // contains noobaa webserver logs
89+
└── s3cmd
90+
└── error.log // contains errors coming from s3-cmd run
91+
92+
```
93+
94+
Running `make test-nc-mint` will generate the following debug log files -
95+
96+
```bash
97+
noobaa-core % tree /mint-nc-test-logs
98+
logs/mint-nc-test-logs
99+
├── log.json // contains the Mint run results
100+
├── minio-go
101+
│   └── error.log // contains errors coming from minio-go run
102+
├── mint-nc-test-logs
103+
│   └── nsfs.log // contains NC noobaa endpoint logs
104+
└── s3cmd
105+
└── error.log // contains errors coming from s3-cmd run
106+
107+
```

‎src/endpoint/s3/s3_errors.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ S3Error.NoLoggingStatusForKey = Object.freeze({
332332
});
333333
S3Error.NoSuchBucket = Object.freeze({
334334
code: 'NoSuchBucket',
335-
message: 'The specified bucket does not exist.',
335+
message: 'The specified bucket does not exist',
336336
http_code: 404,
337337
});
338338
S3Error.NoSuchKey = Object.freeze({

0 commit comments

Comments
 (0)