diff --git a/.eslintignore b/.eslintignore index f14c482532..86ae2f29d3 100644 --- a/.eslintignore +++ b/.eslintignore @@ -1 +1 @@ -src/test/system_tests/ceph_s3_tests/s3-tests/** +src/test/external_tests/ceph_s3_tests/s3-tests/** diff --git a/.github/workflows/nightly-tests.yaml b/.github/workflows/nightly-tests.yaml index e41c8488b7..90b91d75da 100644 --- a/.github/workflows/nightly-tests.yaml +++ b/.github/workflows/nightly-tests.yaml @@ -15,4 +15,4 @@ jobs: -e NEWAWSPROJSECRET=${{ secrets.NEWAWSPROJSECRET }} \ -e NEWAZUREPROJKEY=${{ secrets.NEWAZUREPROJKEY }} \ -e NEWAZUREPROJSECRET=${{ secrets.NEWAZUREPROJSECRET }} \ - --name test1 noobaa-tester ./src/test/unit_tests/run_npm_test_on_test_container.sh -s test_s3_ops.js + --name test1 noobaa-tester ./src/test/framework/run_npm_test_on_test_container.sh -s test_s3_ops.js diff --git a/Makefile b/Makefile index 72675f004f..2a5458b678 100644 --- a/Makefile +++ b/Makefile @@ -268,7 +268,7 @@ root-perm-test: tester @$(call create_docker_network) @$(call run_mongo) @echo "\033[1;34mRunning root permission tests\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "DB_TYPE=mongodb" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s sudo_index.js + $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "DB_TYPE=mongodb" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/framework/run_npm_test_on_test_container.sh -s utils/index/sudo_index.js @$(call stop_noobaa) @$(call stop_mongo) @$(call remove_docker_network) @@ -280,7 +280,7 @@ run-single-test: tester @$(call run_mongo) @$(call run_blob_mock) @echo "\033[1;34mRunning tests\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "DB_TYPE=mongodb" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "BLOB_HOST=blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "NOOBAA_LOG_LEVEL=all" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s $(testname) + $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "DB_TYPE=mongodb" --env "MONGODB_URL=mongodb://noobaa:noobaa@coretest-mongo-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "BLOB_HOST=blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "NOOBAA_LOG_LEVEL=all" $(TESTER_TAG) ./src/test/framework/run_npm_test_on_test_container.sh -s $(testname) @$(call stop_noobaa) @$(call stop_blob_mock) @$(call stop_mongo) @@ -290,7 +290,7 @@ run-single-test: tester run-nc-tests: tester @$(call create_docker_network) @echo "\033[1;34mRunning nc tests\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "NC_CORETEST=true" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s nc_index.js + $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "NC_CORETEST=true" $(TESTER_TAG) ./src/test/framework/run_npm_test_on_test_container.sh -s utils/index/nc_index.js @$(call stop_noobaa) @$(call remove_docker_network) .PHONY: run-nc-tests @@ -301,7 +301,7 @@ run-single-test-postgres: tester @$(call run_postgres) @$(call run_blob_mock) @echo "\033[1;34mRunning tests\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "PG_ENABLE_QUERY_LOG=true" --env "PG_EXPLAIN_QUERIES=true" --env "BLOB_HOST=blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/unit_tests/run_npm_test_on_test_container.sh -s $(testname) + $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "PG_ENABLE_QUERY_LOG=true" --env "PG_EXPLAIN_QUERIES=true" --env "BLOB_HOST=blob-mock-$(GIT_COMMIT)-$(NAME_POSTFIX)" $(TESTER_TAG) ./src/test/framework/run_npm_test_on_test_container.sh -s $(testname) @$(call stop_noobaa) @$(call stop_postgres) @$(call stop_blob_mock) @@ -346,7 +346,7 @@ test-cephs3: tester @$(call create_docker_network) @$(call run_postgres) @echo "\033[1;34mRunning tests\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh" + $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh" @$(call stop_noobaa) @$(call stop_postgres) @$(call remove_docker_network) @@ -357,7 +357,7 @@ test-warp: tester @$(call create_docker_network) @$(call run_postgres) @echo "\033[1;34mRunning warp tests\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/warp/run_warp_on_test_container.sh" + $(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/warp/run_warp_on_test_container.sh" @$(call stop_noobaa) @$(call stop_postgres) @$(call remove_docker_network) @@ -365,7 +365,7 @@ test-warp: tester test-nc-warp: tester @echo "\033[1;34mRunning warp tests on NC environment\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/warp/run_nc_warp_on_test_container.sh" + $(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/warp/run_nc_warp_on_test_container.sh" .PHONY: test-nc-warp test-mint: tester @@ -373,7 +373,7 @@ test-mint: tester @$(call create_docker_network) @$(call run_postgres) @echo "\033[1;34mRunning mint tests\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) -dit --network noobaa-net --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs/mint-test-logs/:/logs $(TESTER_TAG) bash -c "./src/test/system_tests/mint/run_mint_on_test_container.sh & tail -f /dev/null" + $(CONTAINER_ENGINE) run $(CPUSET) --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) -dit --network noobaa-net --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" -v $(PWD)/logs/mint-test-logs/:/logs $(TESTER_TAG) bash -c "./src/test/external_tests/mint/run_mint_on_test_container.sh & tail -f /dev/null" sleep 180 $(CONTAINER_ENGINE) run --name mint-$(GIT_COMMIT)-$(NAME_POSTFIX) --network noobaa-net -v $(PWD)/logs/mint-test-logs/:/mint/log --env SERVER_ENDPOINT=noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX):$(MINT_NOOBAA_HTTP_ENDPOINT_PORT) --env ACCESS_KEY=$(MINT_MOCK_ACCESS_KEY) --env SECRET_KEY=$(MINT_MOCK_SECRET_KEY) --env ENABLE_HTTPS=0 minio/mint minio-go s3cmd @echo "\033[1;34mPrinting noobaa configuration and logs\033[0m" @@ -391,7 +391,7 @@ test-mint: tester test-nc-mint: tester @echo "\033[1;34mRunning mint tests on NC environment\033[0m" @$(call create_docker_network) - $(CONTAINER_ENGINE) run $(CPUSET) --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) -dit --privileged --user root --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --network noobaa-net -v $(PWD)/logs/mint-nc-test-logs/:/logs $(TESTER_TAG) bash -c "./src/test/system_tests/mint/run_nc_mint_on_test_container.sh; tail -f /dev/null" + $(CONTAINER_ENGINE) run $(CPUSET) --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) -dit --privileged --user root --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --network noobaa-net -v $(PWD)/logs/mint-nc-test-logs/:/logs $(TESTER_TAG) bash -c "./src/test/external_tests/mint/run_nc_mint_on_test_container.sh; tail -f /dev/null" sleep 15 $(CONTAINER_ENGINE) run --name mint-$(GIT_COMMIT)-$(NAME_POSTFIX) --network noobaa-net -v $(PWD)/logs/mint-nc-test-logs/:/mint/log --env RUN_ON_FAIL=0 --env SERVER_ENDPOINT=noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX):$(MINT_NOOBAA_HTTP_ENDPOINT_PORT) --env ACCESS_KEY=$(MINT_MOCK_ACCESS_KEY) --env SECRET_KEY=$(MINT_MOCK_SECRET_KEY) --env ENABLE_HTTPS=0 minio/mint minio-go s3cmd @echo "\033[1;34mPrinting noobaa configuration and logs\033[0m" @@ -406,7 +406,7 @@ test-nc-mint: tester test-nsfs-cephs3: tester @echo "\033[1;34mRunning Ceph S3 tests on NSFS Standalone platform\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh" + $(CONTAINER_ENGINE) run $(CPUSET) --privileged --user root --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" -v $(PWD)/logs:/logs $(TESTER_TAG) "./src/test/external_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh" .PHONY: test-nsfs-cephs3 test-sanity: tester @@ -441,7 +441,7 @@ test-aws-sdk-clients: build-aws-client @$(call create_docker_network) @$(call run_postgres) @echo "\033[1;34mRunning aws sdk clients tests\033[0m" - $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" --env "NOOBAA_LOG_LEVEL=all" -v $(PWD)/logs:/logs noobaa-aws-client ./src/test/unit_tests/run_npm_test_on_test_container.sh -c ./node_modules/mocha/bin/mocha.js src/test/unit_tests/different_clients/test_go_sdkv2_script.js + $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" --env "NOOBAA_LOG_LEVEL=all" -v $(PWD)/logs:/logs noobaa-aws-client ./src/test/unit_tests/run_npm_test_on_test_container.sh -c ./node_modules/mocha/bin/mocha.js src/test/external_tests/test_go_sdkv2_script.js @$(call stop_noobaa) @$(call stop_postgres) @$(call remove_docker_network) diff --git a/docs/NooBaaNonContainerized/CI&Tests.md b/docs/NooBaaNonContainerized/CI&Tests.md index 6691057761..09e1493d7d 100644 --- a/docs/NooBaaNonContainerized/CI&Tests.md +++ b/docs/NooBaaNonContainerized/CI&Tests.md @@ -76,10 +76,10 @@ Run `NC mocha tests` with root permissions - * Run `NC mocha` tests **locally** - **Warning:** Running tests locally will do changes to your file system. * Run **all** NC mocha tests locally - - Command: `sudo NC_CORETEST=true node node_modules/mocha/bin/mocha src/test/unit_tests/nc_index.js`. + Command: `sudo NC_CORETEST=true node node_modules/mocha/bin/mocha src/test/utils/index/nc_index.js`. * Run **a single** mocha test locally - - Command: `sudo NC_CORETEST=true node node_modules/mocha/bin/mocha src/test/unit_tests/{test_name}.js`. + Command: `sudo NC_CORETEST=true node node_modules/mocha/bin/mocha src/test/{test_type}/{test_name}.js`. * Run `NC jest tests` - * Command: `sudo jest --testRegex=jest_tests/test_nc`. @@ -162,10 +162,10 @@ Run `NSFS tests` with root permissions - * Run `NSFS mocha` tests **locally** - **Warning:** Running tests locally will do changes to your file system. * Run **all** NSFS mocha tests locally - - Command: `sudo node node_modules/mocha/bin/mocha src/test/unit_tests/sudo_index.js`. + Command: `sudo node node_modules/mocha/bin/mocha src/test/utils/index/sudo_index.js`. * Run **a single** mocha test locally - - Command: `sudo node node_modules/mocha/bin/mocha src/test/unit_tests/{test_name}.js`. + Command: `sudo node node_modules/mocha/bin/mocha src/test/{test_type}/{test_name}.js`. #### NSFS Tests Files List diff --git a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_guide.md b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_guide.md index 13c564e442..6f3da16f73 100644 --- a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_guide.md +++ b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_guide.md @@ -85,7 +85,7 @@ Following the 'General Settings For Ceph S3 Tests' steps. ### 2) Deploy The Tests Job (Noobaa-Core Tab): ```bash -kubectl apply -f src/test/system_tests/ceph_s3_tests/test_ceph_s3_job.yml +kubectl apply -f src/test/external_tests/ceph_s3_tests/test_ceph_s3_job.yml ``` ### 3) View Logs of The Tester Job (New Tab): ```bash @@ -96,7 +96,7 @@ Running all the tests on a local machine takes about 20 minutes. With the curren #### Skipped tests -We run all the tests except the tests that appear in the lists `src/test/system_tests/ceph_s3_tests/s3-tests-lists` if you would like to add or remove a test you can edit those files (and then repeat the steps starting from 'Build Core And Tester Images (Noobaa-Core)' above). +We run all the tests except the tests that appear in the lists `src/test/external_tests/ceph_s3_tests/s3-tests-lists` if you would like to add or remove a test you can edit those files (and then repeat the steps starting from 'Build Core And Tester Images (Noobaa-Core)' above). Some tests are marked to be skipped in the code of ceph/s3-tests repository. Usually, because the environment doesn't support certain criteria. Those tests will appear with a `[WARN]` tag and will be marked as "Test Skipped", for example: `[WARN] CONSOLE:: Test skipped: s3tests_boto3/functional/test_s3.py::test_lifecycle_transition` @@ -128,7 +128,7 @@ Tip: If there is an existing printing in a higher level than 1 and you only want Use the `test_ceph_s3_deployment.yml` file to install the tester pod. Applying this file will result in the deployment of the noobaa tester image in a pod so a developer will be able to run and configure the test from inside of the pod. ```bash -kubectl apply -f src/test/system_tests/ceph_s3_tests/test_ceph_s3_deployment.yml +kubectl apply -f src/test/external_tests/ceph_s3_tests/test_ceph_s3_deployment.yml ``` ### 4) Setup Test Config (Inside The Tester Pod) @@ -148,19 +148,19 @@ cd /root/node_modules/noobaa-core/ Run the script that will create the necessary accounts in noobaa and update the Ceph S3 tests config file accordingly: ```bash -node ./src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_setup.js +node ./src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_setup.js ``` For example: ```bash -S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none +S3TEST_CONF=${PWD}/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/external_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none ``` ### 5) Run a Test (Inside The Tester Pod) To run a test, from noobaa working directory: ```bash -S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/ +S3TEST_CONF=${PWD}/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini ${PWD}/src/test/external_tests/ceph_s3_tests/s3-tests/ ``` This should run the test on the noobaa deployment we've set up. @@ -178,12 +178,12 @@ Note that every flag that comes after `--` is passed to pytest from tox. so if t for example to add --disable-pytest-warnings to the command: ```bash -S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- -m 'not fails_on_rgw' ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none +S3TEST_CONF=${PWD}/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini -- -m 'not fails_on_rgw' ${PWD}/src/test/external_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none ``` it should be: ```bash -S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- -m 'not fails_on_rgw' --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none +S3TEST_CONF=${PWD}/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini -- -m 'not fails_on_rgw' --disable-pytest-warnings ${PWD}/src/test/external_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none ``` ## Debug a Single Test (Inside The Tester Pod) @@ -191,7 +191,7 @@ S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf ### 1) Prerequisites: Following the 'Run a Single Ceph S3 Test' steps. ### 2) View The Test Content -You can view the test by going to the test file and searching for the test function. e.g. if you are working on test `s3tests_boto3.functional.test_s3:test_set_bucket_tagging` then you should `vim ./src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py` and search for the function `test_set_bucket_tagging`. +You can view the test by going to the test file and searching for the test function. e.g. if you are working on test `s3tests_boto3.functional.test_s3:test_set_bucket_tagging` then you should `vim ./src/test/external_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py` and search for the function `test_set_bucket_tagging`. The best place to start investigating is noobaa endpoint pod logs. if you are running with debug level that is higher than 1, you should see log messages of the S3 requests with the prefix `S3 REQUEST`. S3 replies will be with the prefix `HTTP REPLY`. @@ -204,7 +204,7 @@ You can edit a test by going to the test file and editing the test function. See #### B. Permanent change - this change will be saved in a repo, it is for continues investigating. 1) Fork and clone the repository [ceph/s3-test](https://github.com/ceph/s3-tests). -2) Create a new branch from the hash number that was set in the file `./src/test/system_tests/ceph_s3_tests/test_ceph_s3_deploy.sh`. +2) Create a new branch from the hash number that was set in the file `./src/test/external_tests/ceph_s3_tests/test_ceph_s3_deploy.sh`. 3) Change the code, commit, and push to the remote branch. 4) Inside the file `test_ceph_s3_deploy.sh` (mentioned above) Change the values of `CEPH_LINK` to your remote repository and the `CEPH_TESTS_VERSION` to the newest commit in your repository. 5) Build the tester image again, deploy noobaa, and run the test (repeat the steps starting from 'Build Core And Tester Images (Noobaa-Core)' above). @@ -217,11 +217,11 @@ Following the 'Run a Single Ceph S3 Test' steps until 'Deploy The Tester Deploym In this section we will do some manual changes that will allow you to check AWS response for a specific test (tests that do not use neither ACL nor tenant group). 1) Copy configuration file - this will allow us to run a test on AWS and then back to NooBaa just by changing the configuration file (we would have 2 configuration files: `test_ceph_s3_config.conf` and `test_ceph_s3_config_aws.conf`): ```bash - cp src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_aws.conf + cp src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_aws.conf ``` 2) Change the new configuration file to match AWS details: ```bash -vim src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_aws.conf +vim src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_aws.conf ``` * host = s3.amazonaws.com * bucket prefix = choose_name @@ -232,7 +232,7 @@ vim src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_aws.conf * access_key, secret_key appears 3 times each in the file. 3) Running tests with the new configuration files will run against AWS: ```bash -S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_aws.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/ +S3TEST_CONF=${PWD}/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_aws.conf tox -c src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini ${PWD}/src/test/external_tests/ceph_s3_tests/s3-tests/ ``` ## Examples @@ -264,25 +264,25 @@ Following the 'Run a Single Ceph S3 Test' steps. ### 1) Test Pass For example: ```bash -S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none +S3TEST_CONF=${PWD}/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/external_tests/ceph_s3_tests/s3-tests/s3tests/functional/test_headers.py::test_bucket_create_contentlength_none ``` ![test pass screenshot](images/tox_test_pass.png) Note that there is the warning: `WARNING: could not copy distfile to //.tox/distshare` -this warning is for tox to use the same dependancies between projects. this feature is deprecated and not used on this project. In order to remove the warning you can modify `src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini` to include the following line in the `[tox]` section: `distshare = /root/node_modules/noobaa-core/.tox/distshare` +this warning is for tox to use the same dependancies between projects. this feature is deprecated and not used on this project. In order to remove the warning you can modify `src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini` to include the following line in the `[tox]` section: `distshare = /root/node_modules/noobaa-core/.tox/distshare` ### 2) Test Fail For example: ```bash -S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_account_usage +S3TEST_CONF=${PWD}/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/external_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_account_usage ``` ![test failed screenshot](images/tox_test_failed.png) ### 3) Test Skipped For example: ```bash -S3TEST_CONF=${PWD}/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/system_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/system_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_bucket_get_location +S3TEST_CONF=${PWD}/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf tox -c src/test/external_tests/ceph_s3_tests/s3-tests/tox.ini -- --disable-pytest-warnings ${PWD}/src/test/external_tests/ceph_s3_tests/s3-tests/s3tests_boto3/functional/test_s3.py::test_bucket_get_location ``` ![test skipped screenshot](images/tox_test_skipped.png) diff --git a/package.json b/package.json index d919e8c3c1..65e0059f89 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,7 @@ "test": "npm run lint && npm run mocha", "ts": "tsc", "lint": "eslint src --quiet", - "mocha": "node --allow-natives-syntax ./node_modules/.bin/_mocha src/test/unit_tests/index.js", + "mocha": "node --allow-natives-syntax ./node_modules/.bin/_mocha src/test/utils/index/index.js", "jest": "sudo npx jest", "----": "-----------------------------------------------------------------", "start": "node --unhandled-rejections=warn src/cmd", diff --git a/src/deploy/NVA_build/Tests.Dockerfile b/src/deploy/NVA_build/Tests.Dockerfile index b60d2dc332..f945a07a9a 100644 --- a/src/deploy/NVA_build/Tests.Dockerfile +++ b/src/deploy/NVA_build/Tests.Dockerfile @@ -30,9 +30,9 @@ WORKDIR /root/node_modules/noobaa-core/ # Size: ~ 83.9 MB # ############################################################## -RUN ./src/test/system_tests/ceph_s3_tests/test_ceph_s3_deploy.sh $(pwd) +RUN ./src/test/external_tests/ceph_s3_tests/test_ceph_s3_deploy.sh $(pwd) # add group permissions to s3-tests directory (tox needs it in order to run) -RUN cd ./src/test/system_tests/ceph_s3_tests/ && \ +RUN cd ./src/test/external_tests/ceph_s3_tests/ && \ chgrp -R 0 s3-tests && \ chmod -R g=u s3-tests @@ -51,10 +51,10 @@ RUN npm install # ############################################################## COPY ./src/deploy/NVA_build/standalone_deploy.sh ./src/deploy/NVA_build/standalone_deploy.sh -COPY ./src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh ./src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh +COPY ./src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh ./src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh COPY ./src/deploy/NVA_build/standalone_deploy_nsfs.sh ./src/deploy/NVA_build/standalone_deploy_nsfs.sh -COPY ./src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh ./src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh -RUN chmod +x ./src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh +COPY ./src/test/external_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh ./src/test/external_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh +RUN chmod +x ./src/test/external_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh COPY ./src/test/system_tests/run_sanity_test_on_test_container.sh ./src/test/system_tests/run_sanity_test_on_test_container.sh COPY .eslintrc.js /root/node_modules/noobaa-core @@ -72,4 +72,4 @@ RUN mkdir -p /.npm && \ chmod -R g=u /.npm USER 10001:0 -CMD ["./src/test/unit_tests/run_npm_test_on_test_container.sh"] +CMD ["./src/test/framework/run_npm_test_on_test_container.sh"] diff --git a/src/test/unit_tests/different_clients/go_aws_sdkv2_client.go b/src/test/different_clients/go_aws_sdkv2_client.go similarity index 100% rename from src/test/unit_tests/different_clients/go_aws_sdkv2_client.go rename to src/test/different_clients/go_aws_sdkv2_client.go diff --git a/src/test/unit_tests/different_clients/run_go_sdkv2_client_script.js b/src/test/different_clients/run_go_sdkv2_client_script.js similarity index 100% rename from src/test/unit_tests/different_clients/run_go_sdkv2_client_script.js rename to src/test/different_clients/run_go_sdkv2_client_script.js diff --git a/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh b/src/test/external_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh similarity index 95% rename from src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh rename to src/test/external_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh index 7962a9030c..ee4c786784 100644 --- a/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh +++ b/src/test/external_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh @@ -54,4 +54,4 @@ mkdir -p ${CEPH_TEST_LOGS_DIR} # ==================================================================================== # Run the tests -./src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_and_run_s3_tests.sh +./src/test/external_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_and_run_s3_tests.sh diff --git a/src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh b/src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh similarity index 96% rename from src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh rename to src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh index 9dfa1b4086..1b1ef28b81 100755 --- a/src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh +++ b/src/test/external_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh @@ -51,6 +51,6 @@ mkdir -p ${CEPH_TEST_LOGS_DIR} # ==================================================================================== # Run the tests -./src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh +./src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh # ==================================================================================== diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt b/src/test/external_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt similarity index 100% rename from src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt rename to src/test/external_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_black_list.txt diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_pending_list.txt b/src/test/external_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_pending_list.txt similarity index 100% rename from src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_pending_list.txt rename to src/test/external_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_pending_list.txt diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt b/src/test/external_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt similarity index 100% rename from src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt rename to src/test/external_tests/ceph_s3_tests/s3-tests-lists/s3_tests_black_list.txt diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/external_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt similarity index 100% rename from src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt rename to src/test/external_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_and_run_s3_tests.sh b/src/test/external_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_and_run_s3_tests.sh similarity index 91% rename from src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_and_run_s3_tests.sh rename to src/test/external_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_and_run_s3_tests.sh index 6978fcae64..f9c60e9499 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_and_run_s3_tests.sh +++ b/src/test/external_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_and_run_s3_tests.sh @@ -3,7 +3,7 @@ #!/bin/bash set -x -CEPH_S3_DIRECTORY="src/test/system_tests/ceph_s3_tests" +CEPH_S3_DIRECTORY="src/test/external_tests/ceph_s3_tests" CEPH_S3_TESTS_CONFIG="${CEPH_S3_DIRECTORY}/test_ceph_nsfs_s3_config_setup.js" CEPH_S3_RUN_TESTS="${CEPH_S3_DIRECTORY}/test_ceph_s3.js" S3_CEPH_TEST_BLACKLIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/nsfs_s3_tests_black_list.txt" diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js b/src/test/external_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js similarity index 97% rename from src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js rename to src/test/external_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js index e17fb6b2d2..928d8f8033 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js +++ b/src/test/external_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js @@ -13,7 +13,7 @@ dbg.set_process_name('test_ceph_s3'); const fs = require('fs'); const os_utils = require('../../../util/os_utils'); const { CEPH_TEST } = require('./test_ceph_s3_constants.js'); -const { get_access_keys, create_account } = require('../nc_test_utils'); +const { get_access_keys, create_account } = require('../../utils/nc_test_utils'); async function main() { try { diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3.js b/src/test/external_tests/ceph_s3_tests/test_ceph_s3.js similarity index 100% rename from src/test/system_tests/ceph_s3_tests/test_ceph_s3.js rename to src/test/external_tests/ceph_s3_tests/test_ceph_s3.js diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf similarity index 100% rename from src/test/system_tests/ceph_s3_tests/test_ceph_s3_config.conf rename to src/test/external_tests/ceph_s3_tests/test_ceph_s3_config.conf diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh similarity index 92% rename from src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh rename to src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh index a5aaf7d828..10f3cd6aaf 100755 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh +++ b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_and_run_s3_tests.sh @@ -5,7 +5,7 @@ #!/bin/bash set -x -CEPH_S3_DIRECTORY="src/test/system_tests/ceph_s3_tests" +CEPH_S3_DIRECTORY="src/test/external_tests/ceph_s3_tests" CEPH_S3_TESTS_CONFIG="${CEPH_S3_DIRECTORY}/test_ceph_s3_config_setup.js" CEPH_S3_RUN_TESTS="${CEPH_S3_DIRECTORY}/test_ceph_s3.js" S3_CEPH_TEST_BLACKLIST="${CEPH_S3_DIRECTORY}/s3-tests-lists/s3_tests_black_list.txt" diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_setup.js b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_setup.js similarity index 100% rename from src/test/system_tests/ceph_s3_tests/test_ceph_s3_config_setup.js rename to src/test/external_tests/ceph_s3_tests/test_ceph_s3_config_setup.js diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_constants.js similarity index 96% rename from src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js rename to src/test/external_tests/ceph_s3_tests/test_ceph_s3_constants.js index c7c3869d6a..6960b9b3a6 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js +++ b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_constants.js @@ -9,7 +9,7 @@ const FS_ROOT_1 = '/tmp/nsfs_root1/'; const FS_ROOT_2 = '/tmp/nsfs_root2/'; const CEPH_TEST = { - test_dir: 'src/test/system_tests/ceph_s3_tests/', + test_dir: 'src/test/external_tests/ceph_s3_tests/', s3_test_dir: 's3-tests/', ceph_config: 'test_ceph_s3_config.conf', tox_config: 'tox.ini', diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_deploy.sh b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_deploy.sh similarity index 96% rename from src/test/system_tests/ceph_s3_tests/test_ceph_s3_deploy.sh rename to src/test/external_tests/ceph_s3_tests/test_ceph_s3_deploy.sh index 5f506be87e..b432093bf4 100755 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_deploy.sh +++ b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_deploy.sh @@ -9,7 +9,7 @@ else NOOBAA_DIR=${1} fi -CEPH_S3_TESTS_PATH="src/test/system_tests/ceph_s3_tests" +CEPH_S3_TESTS_PATH="src/test/external_tests/ceph_s3_tests" mkdir -p /${NOOBAA_DIR}/${CEPH_S3_TESTS_PATH} cd /${NOOBAA_DIR}/${CEPH_S3_TESTS_PATH} diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_deployment.yml b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_deployment.yml similarity index 100% rename from src/test/system_tests/ceph_s3_tests/test_ceph_s3_deployment.yml rename to src/test/external_tests/ceph_s3_tests/test_ceph_s3_deployment.yml diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_job.yml b/src/test/external_tests/ceph_s3_tests/test_ceph_s3_job.yml similarity index 100% rename from src/test/system_tests/ceph_s3_tests/test_ceph_s3_job.yml rename to src/test/external_tests/ceph_s3_tests/test_ceph_s3_job.yml diff --git a/src/test/system_tests/external_tests_utils.js b/src/test/external_tests/external_tests_utils.js similarity index 99% rename from src/test/system_tests/external_tests_utils.js rename to src/test/external_tests/external_tests_utils.js index 676eea8c3d..bbf5291709 100644 --- a/src/test/system_tests/external_tests_utils.js +++ b/src/test/external_tests/external_tests_utils.js @@ -3,7 +3,7 @@ const api = require('../../api'); const SensitiveString = require('../../util/sensitive_string'); -const { get_account, create_account, create_bucket } = require('./nc_test_utils'); +const { get_account, create_account, create_bucket } = require('../utils/nc_test_utils'); /** * get_global_rpc_client returns a global RPC client. diff --git a/src/test/system_tests/mint/configure_mint.js b/src/test/external_tests/mint/configure_mint.js similarity index 100% rename from src/test/system_tests/mint/configure_mint.js rename to src/test/external_tests/mint/configure_mint.js diff --git a/src/test/system_tests/mint/mint_constants.js b/src/test/external_tests/mint/mint_constants.js similarity index 100% rename from src/test/system_tests/mint/mint_constants.js rename to src/test/external_tests/mint/mint_constants.js diff --git a/src/test/system_tests/mint/run_mint_on_test_container.sh b/src/test/external_tests/mint/run_mint_on_test_container.sh similarity index 96% rename from src/test/system_tests/mint/run_mint_on_test_container.sh rename to src/test/external_tests/mint/run_mint_on_test_container.sh index 229645e210..19b81d114a 100644 --- a/src/test/system_tests/mint/run_mint_on_test_container.sh +++ b/src/test/external_tests/mint/run_mint_on_test_container.sh @@ -50,4 +50,4 @@ mkdir -p ${CEPH_TEST_LOGS_DIR} cd /root/node_modules/noobaa-core/ # Configure the mint test -node ./src/test/system_tests/mint/configure_mint.js +node ./src/test/external_tests/mint/configure_mint.js diff --git a/src/test/system_tests/mint/run_nc_mint_on_test_container.sh b/src/test/external_tests/mint/run_nc_mint_on_test_container.sh similarity index 97% rename from src/test/system_tests/mint/run_nc_mint_on_test_container.sh rename to src/test/external_tests/mint/run_nc_mint_on_test_container.sh index 9aa09fb9ce..e7afb4edf9 100644 --- a/src/test/system_tests/mint/run_nc_mint_on_test_container.sh +++ b/src/test/external_tests/mint/run_nc_mint_on_test_container.sh @@ -54,4 +54,4 @@ chmod 777 ${CEPH_TEST_LOGS_DIR} cd /root/node_modules/noobaa-core/ # Configure the mint test -node ./src/test/system_tests/mint/configure_mint.js +node ./src/test/external_tests/mint/configure_mint.js diff --git a/src/test/unit_tests/different_clients/test_go_sdkv2_script.js b/src/test/external_tests/test_go_sdkv2_script.js similarity index 92% rename from src/test/unit_tests/different_clients/test_go_sdkv2_script.js rename to src/test/external_tests/test_go_sdkv2_script.js index a6e52359bb..0c75ec4d22 100644 --- a/src/test/unit_tests/different_clients/test_go_sdkv2_script.js +++ b/src/test/external_tests/test_go_sdkv2_script.js @@ -5,14 +5,14 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; // setup coretest first to prepare the env -const coretest = require('../coretest'); +const coretest = require('../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); const { rpc_client, EMAIL } = coretest; const mocha = require('mocha'); const assert = require('assert'); -const config = require('../../../../config'); -const { run_go_sdk_v2_client_script } = require('./run_go_sdkv2_client_script'); +const config = require('../../../config'); +const { run_go_sdk_v2_client_script } = require('../different_clients/run_go_sdkv2_client_script'); mocha.describe('Go AWS SDK V2 Client script execution', function() { diff --git a/src/test/system_tests/warp/configure_warp.js b/src/test/external_tests/warp/configure_warp.js similarity index 100% rename from src/test/system_tests/warp/configure_warp.js rename to src/test/external_tests/warp/configure_warp.js diff --git a/src/test/system_tests/warp/run_nc_warp_on_test_container.sh b/src/test/external_tests/warp/run_nc_warp_on_test_container.sh similarity index 95% rename from src/test/system_tests/warp/run_nc_warp_on_test_container.sh rename to src/test/external_tests/warp/run_nc_warp_on_test_container.sh index 1861f2fb36..1c3cbfda31 100644 --- a/src/test/system_tests/warp/run_nc_warp_on_test_container.sh +++ b/src/test/external_tests/warp/run_nc_warp_on_test_container.sh @@ -65,11 +65,11 @@ warp --version cd /root/node_modules/noobaa-core/ # Configure the warp test -node ./src/test/system_tests/warp/configure_warp.js +node ./src/test/external_tests/warp/configure_warp.js # ==================================================================================== # Run the warp tests -node ./src/test/system_tests/warp/run_warp.js +node ./src/test/external_tests/warp/run_warp.js # ==================================================================================== diff --git a/src/test/system_tests/warp/run_warp.js b/src/test/external_tests/warp/run_warp.js similarity index 100% rename from src/test/system_tests/warp/run_warp.js rename to src/test/external_tests/warp/run_warp.js diff --git a/src/test/system_tests/warp/run_warp_on_test_container.sh b/src/test/external_tests/warp/run_warp_on_test_container.sh similarity index 95% rename from src/test/system_tests/warp/run_warp_on_test_container.sh rename to src/test/external_tests/warp/run_warp_on_test_container.sh index 54bb943e22..0d01582148 100644 --- a/src/test/system_tests/warp/run_warp_on_test_container.sh +++ b/src/test/external_tests/warp/run_warp_on_test_container.sh @@ -60,7 +60,7 @@ mkdir -p ${CEPH_TEST_LOGS_DIR} cd /root/node_modules/noobaa-core/ # Configure the warp test -node ./src/test/system_tests/warp/configure_warp.js +node ./src/test/external_tests/warp/configure_warp.js # ==================================================================================== @@ -70,6 +70,6 @@ sleep 90 # ==================================================================================== # Run the warp tests -node ./src/test/system_tests/warp/run_warp.js +node ./src/test/external_tests/warp/run_warp.js # ==================================================================================== diff --git a/src/test/system_tests/warp/warp_constants.js b/src/test/external_tests/warp/warp_constants.js similarity index 100% rename from src/test/system_tests/warp/warp_constants.js rename to src/test/external_tests/warp/warp_constants.js diff --git a/src/test/unit_tests/run_npm_test_on_test_container.sh b/src/test/framework/run_npm_test_on_test_container.sh similarity index 97% rename from src/test/unit_tests/run_npm_test_on_test_container.sh rename to src/test/framework/run_npm_test_on_test_container.sh index 8df8be98e9..7e342ba8ec 100755 --- a/src/test/unit_tests/run_npm_test_on_test_container.sh +++ b/src/test/framework/run_npm_test_on_test_container.sh @@ -38,7 +38,7 @@ do command=${*} command_array=(${command}) shift ${#command_array[@]};; - -s|--single) command="./node_modules/mocha/bin/mocha.js src/test/unit_tests/${2}" + -s|--single) command="./node_modules/mocha/bin/mocha.js src/test/${2}" shift 2;; *) usage;; esac diff --git a/src/test/unit_tests/test_nc_iam_basic_integration.js b/src/test/integration_tests/api/iam/test_nc_iam_basic_integration.js similarity index 98% rename from src/test/unit_tests/test_nc_iam_basic_integration.js rename to src/test/integration_tests/api/iam/test_nc_iam_basic_integration.js index 8cf608d077..88f7338e69 100644 --- a/src/test/unit_tests/test_nc_iam_basic_integration.js +++ b/src/test/integration_tests/api/iam/test_nc_iam_basic_integration.js @@ -7,9 +7,9 @@ const path = require('path'); const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); +const fs_utils = require('../../../../util/fs_utils'); const { TMP_PATH, generate_nsfs_account, get_new_buckets_path_by_test_env, generate_iam_client, - get_coretest_path } = require('../system_tests/test_utils'); + require_coretest } = require('../../../system_tests/test_utils'); const { ListUsersCommand, CreateUserCommand, GetUserCommand, UpdateUserCommand, DeleteUserCommand, ListAccessKeysCommand, CreateAccessKeyCommand, GetAccessKeyLastUsedCommand, UpdateAccessKeyCommand, DeleteAccessKeyCommand, @@ -23,12 +23,11 @@ const { ListUsersCommand, CreateUserCommand, GetUserCommand, UpdateUserCommand, ListServerCertificateTagsCommand, ListServiceSpecificCredentialsCommand, ListSigningCertificatesCommand, ListSSHPublicKeysCommand, ListUserPoliciesCommand, ListUserTagsCommand, ListVirtualMFADevicesCommand } = require('@aws-sdk/client-iam'); -const { ACCESS_KEY_STATUS_ENUM } = require('../../endpoint/iam/iam_constants'); -const IamError = require('../../endpoint/iam/iam_errors').IamError; +const { ACCESS_KEY_STATUS_ENUM } = require('../../../../endpoint/iam/iam_constants'); +const IamError = require('../../../../endpoint/iam/iam_errors').IamError; -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const coretest = require_coretest(); const setup_options = { should_run_iam: true, https_port_iam: 7005, debug: 5 }; coretest.setup(setup_options); const { rpc_client, EMAIL, get_current_setup_options, stop_nsfs_process, start_nsfs_process } = coretest; diff --git a/src/test/unit_tests/test_bucket_logging.js b/src/test/integration_tests/api/s3/test_bucket_logging.js similarity index 95% rename from src/test/unit_tests/test_bucket_logging.js rename to src/test/integration_tests/api/s3/test_bucket_logging.js index 491e78dfbe..5bc8ac0bcf 100644 --- a/src/test/unit_tests/test_bucket_logging.js +++ b/src/test/integration_tests/api/s3/test_bucket_logging.js @@ -4,10 +4,10 @@ const mocha = require('mocha'); const assert = require('assert'); const _ = require('lodash'); -const P = require('../../util/promise'); -const coretest = require('./coretest'); +const P = require('../../../../util/promise'); +const coretest = require('../../../utils/coretest/coretest'); const { rpc_client } = coretest; //, PASSWORD, SYSTEM -const { BucketLogUploader } = require('../../server/bg_services/bucket_logs_upload'); +const { BucketLogUploader } = require('../../../../server/bg_services/bucket_logs_upload'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); diff --git a/src/test/unit_tests/test_bucket_replication.js b/src/test/integration_tests/api/s3/test_bucket_replication.js similarity index 98% rename from src/test/unit_tests/test_bucket_replication.js rename to src/test/integration_tests/api/s3/test_bucket_replication.js index f7bf551c1c..11a0bc80fc 100644 --- a/src/test/unit_tests/test_bucket_replication.js +++ b/src/test/integration_tests/api/s3/test_bucket_replication.js @@ -4,18 +4,18 @@ const mocha = require('mocha'); const assert = require('assert'); const _ = require('lodash'); -const P = require('../../util/promise'); -const coretest = require('./coretest'); +const P = require('../../../../util/promise'); +const coretest = require('../../../utils/coretest/coretest'); const { rpc_client, EMAIL } = coretest; //, PASSWORD, SYSTEM const util = require('util'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); const { S3 } = require('@aws-sdk/client-s3'); const http = require('http'); -const cloud_utils = require('../../util/cloud_utils'); -const { ReplicationScanner } = require('../../server/bg_services/replication_scanner'); -const { get_object } = require('../system_tests/test_utils'); +const cloud_utils = require('../../../../util/cloud_utils'); +const { ReplicationScanner } = require('../../../../server/bg_services/replication_scanner'); +const { get_object } = require('../../../system_tests/test_utils'); -const db_client = require('../../util/db_client').instance(); +const db_client = require('../../../../util/db_client').instance(); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); mocha.describe('replication configuration validity tests', function() { @@ -163,7 +163,7 @@ mocha.describe('replication collection tests', function() { const buckets = [bucket1, bucket2, bucket3]; let replication_policy_id; // eslint-disable-next-line global-require - const replication_store = require('../../server/system_services/replication_store').instance(); + const replication_store = require('../../../../server/system_services/replication_store').instance(); mocha.before('create buckets', async function() { await P.all(_.map(buckets, async bucket_name => { @@ -384,7 +384,7 @@ mocha.describe('replication configuration bg worker tests', function() { // delete object from dst await s3_owner.deleteObject({ Bucket: bucket_for_replications, Key: contents[0].Key }); - await _list_objects_and_wait(s3_owner, bucket_for_replications, uploaded_prefix_objects_count - 1); //Verify that one object was deleted + await _list_objects_and_wait(s3_owner, bucket_for_replications, uploaded_prefix_objects_count - 1); //Verify that one object was deleted // sync again res1 = await scanner.run_batch(); console.log('waiting for replication objects - one rule one prefix', res1); @@ -397,7 +397,7 @@ mocha.describe('replication configuration bg worker tests', function() { console.log('objs override dst1', dst_obj1, dst_obj2); assert.deepStrictEqual(dst_obj2.body, 'lalalala'); // dst object was updated correctly - assert.notDeepStrictEqual(dst_obj1.body, dst_obj2.body); // new dst object was updated and + assert.notDeepStrictEqual(dst_obj1.body, dst_obj2.body); // new dst object was updated and // now is diff from the original dst object // sync again - should not replicate since dst bucket is last modified @@ -407,7 +407,7 @@ mocha.describe('replication configuration bg worker tests', function() { const dst_obj3 = await get_object(s3_owner, bucket_for_replications, key1); const src_obj = await get_object(s3_owner, bucket1, key1); console.log('objs override dst2', src_obj, dst_obj3); - assert.notDeepStrictEqual(src_obj.body, dst_obj3.body); // object in src !== object in dst + assert.notDeepStrictEqual(src_obj.body, dst_obj3.body); // object in src !== object in dst assert.deepStrictEqual(dst_obj3.body, 'lalalala'); // dst object was not overridden by the object in the source @@ -687,7 +687,7 @@ mocha.describe('Replication pagination test', function() { await delete_object(s3_owner, bucket_name, target_bucket_keys[i]); } await rpc_client.bucket.delete_bucket({ name: bucket_name }); - // Revert the replication scanner's + // Revert the replication scanner's delete process.env.REPLICATION_MAX_KEYS; })); }); diff --git a/src/test/unit_tests/test_lifecycle.js b/src/test/integration_tests/api/s3/test_lifecycle.js similarity index 98% rename from src/test/unit_tests/test_lifecycle.js rename to src/test/integration_tests/api/s3/test_lifecycle.js index 96aa77aded..e0f16fad3a 100644 --- a/src/test/unit_tests/test_lifecycle.js +++ b/src/test/integration_tests/api/s3/test_lifecycle.js @@ -15,16 +15,16 @@ const crypto = require('crypto'); const stream = require('stream'); const moment = require('moment'); -const ObjectIO = require('../../sdk/object_io'); -const P = require('../../util/promise'); -const config = require('../../../config'); -const MDStore = require('../../server/object_services/md_store').MDStore; -const coretest = require('./coretest'); +const ObjectIO = require('../../../../sdk/object_io'); +const P = require('../../../../util/promise'); +const config = require('../../../../../config'); +const MDStore = require('../../../../server/object_services/md_store').MDStore; +const coretest = require('../../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); -const lifecycle = require('../../server/bg_services/lifecycle'); -const http_utils = require('../../util/http_utils'); -const test_utils = require('../../../src/test/system_tests/test_utils'); -const commonTests = require('../lifecycle/common'); +const lifecycle = require('../../../../server/bg_services/lifecycle'); +const http_utils = require('../../../../util/http_utils'); +const test_utils = require('../../../system_tests/test_utils'); +const commonTests = require('../../../lifecycle/common'); const seed = crypto.randomBytes(16); const generator = crypto.createCipheriv('aes-128-gcm', seed, Buffer.alloc(12)); diff --git a/src/test/unit_tests/test_namespace_auth.js b/src/test/integration_tests/api/s3/test_namespace_auth.js similarity index 93% rename from src/test/unit_tests/test_namespace_auth.js rename to src/test/integration_tests/api/s3/test_namespace_auth.js index 1dabf13555..343ba550ac 100644 --- a/src/test/unit_tests/test_namespace_auth.js +++ b/src/test/integration_tests/api/s3/test_namespace_auth.js @@ -2,12 +2,12 @@ /* eslint max-lines-per-function: ['error', 700] */ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); -const buffer_utils = require('../../util/buffer_utils'); +const coretest = require('../../../utils/coretest/coretest'); +const buffer_utils = require('../../../../util/buffer_utils'); coretest.setup({ pools_to_create: coretest.POOL_LIST }); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); -const http_utils = require('../../util/http_utils'); +const http_utils = require('../../../../util/http_utils'); const mocha = require('mocha'); const assert = require('assert'); @@ -18,7 +18,7 @@ const NAMESPACE_RESOURCE_NAME = 'nsr_auth'; const NS_BUCKET = 'nsb'; const BODY = "THE_MAJESTIC_SLOTH"; const FKEY = 'ns_auth_file'; -const config = require('../../../config'); +const config = require('../../../../../config'); mocha.describe('Namespace Auth', function() { diff --git a/src/test/unit_tests/test_notifications.js b/src/test/integration_tests/api/s3/test_notifications.js similarity index 96% rename from src/test/unit_tests/test_notifications.js rename to src/test/integration_tests/api/s3/test_notifications.js index b3a220efe2..979c30450e 100644 --- a/src/test/unit_tests/test_notifications.js +++ b/src/test/integration_tests/api/s3/test_notifications.js @@ -7,12 +7,11 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const fs = require('fs'); const path = require('path'); -const config = require('../../../config'); +const config = require('../../../../../config'); // setup coretest first to prepare the env -const { get_coretest_path, generate_s3_client, TMP_PATH } = require('../system_tests/test_utils'); -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const { require_coretest, generate_s3_client, TMP_PATH } = require('../../../system_tests/test_utils'); +const coretest = require_coretest(); const { rpc_client, EMAIL, POOL_LIST } = coretest; @@ -29,7 +28,7 @@ const mocha = require('mocha'); const http = require('http'); const assert = require('assert'); const timer = require('node:timers/promises'); -const notifications_util = require('../../util/notifications_util'); +const notifications_util = require('../../../../util/notifications_util'); const http_connect_filename = 'http_connect.json'; const http_connect_path = path.join(tmp_connect, http_connect_filename); diff --git a/src/test/unit_tests/test_nsfs_versioning_gpfs.js b/src/test/integration_tests/api/s3/test_nsfs_versioning_gpfs.js similarity index 97% rename from src/test/unit_tests/test_nsfs_versioning_gpfs.js rename to src/test/integration_tests/api/s3/test_nsfs_versioning_gpfs.js index 3cfcbf2fc5..d4f608f91f 100644 --- a/src/test/unit_tests/test_nsfs_versioning_gpfs.js +++ b/src/test/integration_tests/api/s3/test_nsfs_versioning_gpfs.js @@ -7,12 +7,12 @@ const path = require('path'); const mocha = require('mocha'); const crypto = require('crypto'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); -const nb_native = require('../../util/nb_native'); -const size_utils = require('../../util/size_utils'); -const NamespaceFS = require('../../sdk/namespace_fs'); -const buffer_utils = require('../../util/buffer_utils'); -const { get_process_fs_context } = require('../../util/native_fs_utils'); +const fs_utils = require('../../../../util/fs_utils'); +const nb_native = require('../../../../util/nb_native'); +const size_utils = require('../../../../util/size_utils'); +const NamespaceFS = require('../../../../sdk/namespace_fs'); +const buffer_utils = require('../../../../util/buffer_utils'); +const { get_process_fs_context } = require('../../../../util/native_fs_utils'); const XATTR_VERSION_ID = 'user.noobaa.version_id'; const XATTR_PREV_VERSION_ID = 'user.noobaa.prev_version_id'; diff --git a/src/test/unit_tests/test_public_access_block.js b/src/test/integration_tests/api/s3/test_public_access_block.js similarity index 96% rename from src/test/unit_tests/test_public_access_block.js rename to src/test/integration_tests/api/s3/test_public_access_block.js index 6a52dfd301..d5c64acc98 100644 --- a/src/test/unit_tests/test_public_access_block.js +++ b/src/test/integration_tests/api/s3/test_public_access_block.js @@ -8,11 +8,10 @@ const http = require('http'); const crypto = require('crypto'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); -const { get_coretest_path } = require('../system_tests/test_utils'); -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); -const { S3Error } = require('../../endpoint/s3/s3_errors'); -const config = require('../../../config'); +const { require_coretest } = require('../../../system_tests/test_utils'); +const coretest = require_coretest(); +const { S3Error } = require('../../../../endpoint/s3/s3_errors'); +const config = require('../../../../../config'); const { rpc_client, EMAIL } = coretest; //, PASSWORD, SYSTEM coretest.setup({ pools_to_create: process.env.NC_CORETEST ? undefined : [coretest.POOL_LIST[1]] }); diff --git a/src/test/unit_tests/test_s3_bucket_policy.js b/src/test/integration_tests/api/s3/test_s3_bucket_policy.js similarity index 99% rename from src/test/unit_tests/test_s3_bucket_policy.js rename to src/test/integration_tests/api/s3/test_s3_bucket_policy.js index 580f62c245..59732cf57f 100644 --- a/src/test/unit_tests/test_s3_bucket_policy.js +++ b/src/test/integration_tests/api/s3/test_s3_bucket_policy.js @@ -4,23 +4,22 @@ 'use strict'; // setup coretest first to prepare the env -const { get_coretest_path, TMP_PATH, is_nc_coretest } = require('../system_tests/test_utils'); -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const { require_coretest, TMP_PATH, is_nc_coretest } = require('../../../system_tests/test_utils'); +const coretest = require_coretest(); const { rpc_client, EMAIL, POOL_LIST, anon_rpc_client } = coretest; -const MDStore = require('../../server/object_services/md_store').MDStore; +const MDStore = require('../../../../server/object_services/md_store').MDStore; coretest.setup({ pools_to_create: process.env.NC_CORETEST ? undefined : [POOL_LIST[1]] }); const path = require('path'); const _ = require('lodash'); -const fs_utils = require('../../util/fs_utils'); +const fs_utils = require('../../../../util/fs_utils'); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); const http = require('http'); const mocha = require('mocha'); const assert = require('assert'); -const { S3Error } = require('../../endpoint/s3/s3_errors'); -const config = require('../../../config'); +const { S3Error } = require('../../../../endpoint/s3/s3_errors'); +const config = require('../../../../../config'); async function assert_throws_async(promise, expected_message = 'Access Denied') { try { @@ -1613,7 +1612,7 @@ mocha.describe('s3_bucket_policy', function() { mocha.it.skip('should be able to use notPrincipal', async function() { // This test is broken - Effect Allow can't be used with NotPrincipal - // copied from the docs: + // copied from the docs: // "NotPrincipal must be used with "Effect":"Deny". // Using it with "Effect":"Allow" is not supported." // source: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_notprincipal.html diff --git a/src/test/unit_tests/test_s3_encryption.js b/src/test/integration_tests/api/s3/test_s3_encryption.js similarity index 99% rename from src/test/unit_tests/test_s3_encryption.js rename to src/test/integration_tests/api/s3/test_s3_encryption.js index 84de263a0b..7a6925a516 100644 --- a/src/test/unit_tests/test_s3_encryption.js +++ b/src/test/integration_tests/api/s3/test_s3_encryption.js @@ -2,7 +2,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); const { rpc_client, EMAIL } = coretest; @@ -19,7 +19,7 @@ const FILE_NAME_COPY = 'sloth-file-copy.txt'; const SSECustomerKeyOrig = '123456789012345678901234567890AB'; const SSECustomerKeyCopy = '123456789012345678901234567890AC'; const SKIP_TEST = !process.env.AWS_ACCESS_KEY_ID || !process.env.AWS_SECRET_ACCESS_KEY; -const config = require('../../../config'); +const config = require('../../../../../config'); async function get_s3_instances() { const account_info = await rpc_client.account.read_account({ diff --git a/src/test/unit_tests/test_s3_list_buckets.js b/src/test/integration_tests/api/s3/test_s3_list_buckets.js similarity index 96% rename from src/test/unit_tests/test_s3_list_buckets.js rename to src/test/integration_tests/api/s3/test_s3_list_buckets.js index e6d2bb3094..04b62fa793 100644 --- a/src/test/unit_tests/test_s3_list_buckets.js +++ b/src/test/integration_tests/api/s3/test_s3_list_buckets.js @@ -4,12 +4,12 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../../utils/coretest/coretest'); coretest.setup({ pools_to_create: coretest.POOL_LIST }); -const config = require('../../../config'); +const config = require('../../../../../config'); const { S3Client, ListBucketsCommand, CreateBucketCommand, DeleteBucketCommand } = require("@aws-sdk/client-s3"); const { NodeHttpHandler } = require("@smithy/node-http-handler"); -const http_utils = require('../../util/http_utils'); +const http_utils = require('../../../../util/http_utils'); const mocha = require('mocha'); const assert = require('assert'); diff --git a/src/test/unit_tests/test_s3_list_objects.js b/src/test/integration_tests/api/s3/test_s3_list_objects.js similarity index 99% rename from src/test/unit_tests/test_s3_list_objects.js rename to src/test/integration_tests/api/s3/test_s3_list_objects.js index 31700c34e4..676fff5182 100644 --- a/src/test/unit_tests/test_s3_list_objects.js +++ b/src/test/integration_tests/api/s3/test_s3_list_objects.js @@ -3,15 +3,15 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../../utils/coretest/coretest'); coretest.setup(); const _ = require('lodash'); const util = require('util'); const mocha = require('mocha'); -const P = require('../../util/promise'); -const ObjectIO = require('../../sdk/object_io'); +const P = require('../../../../util/promise'); +const ObjectIO = require('../../../../sdk/object_io'); const { rpc_client } = coretest; const object_io = new ObjectIO(); diff --git a/src/test/unit_tests/test_s3_ops.js b/src/test/integration_tests/api/s3/test_s3_ops.js similarity index 99% rename from src/test/unit_tests/test_s3_ops.js rename to src/test/integration_tests/api/s3/test_s3_ops.js index 52c3237a95..7f939b86ad 100644 --- a/src/test/unit_tests/test_s3_ops.js +++ b/src/test/integration_tests/api/s3/test_s3_ops.js @@ -5,15 +5,15 @@ 'use strict'; const _ = require('lodash'); // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); -const config = require('../../../config'); +const config = require('../../../../../config'); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); -const http_utils = require('../../util/http_utils'); +const http_utils = require('../../../../util/http_utils'); const mocha = require('mocha'); const assert = require('assert'); -const P = require('../../util/promise'); +const P = require('../../../../util/promise'); const azure_storage = require('@azure/storage-blob'); // If any of these variables are not defined, diff --git a/src/test/unit_tests/test_s3_worm.js b/src/test/integration_tests/api/s3/test_s3_worm.js similarity index 99% rename from src/test/unit_tests/test_s3_worm.js rename to src/test/integration_tests/api/s3/test_s3_worm.js index b113c9f633..0491124008 100644 --- a/src/test/unit_tests/test_s3_worm.js +++ b/src/test/integration_tests/api/s3/test_s3_worm.js @@ -2,7 +2,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../../utils/coretest/coretest'); coretest.setup({ pools_to_create: coretest.POOL_LIST }); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); diff --git a/src/test/unit_tests/test_sts.js b/src/test/integration_tests/api/sts/test_sts.js similarity index 98% rename from src/test/unit_tests/test_sts.js rename to src/test/integration_tests/api/sts/test_sts.js index 8d727f03df..59b35b1a67 100644 --- a/src/test/unit_tests/test_sts.js +++ b/src/test/integration_tests/api/sts/test_sts.js @@ -2,19 +2,19 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../../utils/coretest/coretest'); coretest.setup(); const AWS = require('aws-sdk'); const https = require('https'); const mocha = require('mocha'); const assert = require('assert'); -const stsErr = require('../../endpoint/sts/sts_errors').StsError; -const http_utils = require('../../util/http_utils'); -const dbg = require('../../util/debug_module')(__filename); -const cloud_utils = require('../../util/cloud_utils'); -const jwt_utils = require('../../util/jwt_utils'); -const config = require('../../../config'); -const { S3Error } = require('../../endpoint/s3/s3_errors'); +const stsErr = require('../../../../endpoint/sts/sts_errors').StsError; +const http_utils = require('../../../../util/http_utils'); +const dbg = require('../../../../util/debug_module')(__filename); +const cloud_utils = require('../../../../util/cloud_utils'); +const jwt_utils = require('../../../../util/jwt_utils'); +const config = require('../../../../../config'); +const { S3Error } = require('../../../../endpoint/s3/s3_errors'); const defualt_expiry_seconds = Math.ceil(config.STS_DEFAULT_SESSION_TOKEN_EXPIRY_MS / 1000); diff --git a/src/test/unit_tests/test_md_aggregator_unit.js b/src/test/integration_tests/db/test_md_aggregator_unit.js similarity index 98% rename from src/test/unit_tests/test_md_aggregator_unit.js rename to src/test/integration_tests/db/test_md_aggregator_unit.js index 4969cd9c17..2420bb2e67 100644 --- a/src/test/unit_tests/test_md_aggregator_unit.js +++ b/src/test/integration_tests/db/test_md_aggregator_unit.js @@ -3,7 +3,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup(); const _ = require('lodash'); @@ -12,10 +12,10 @@ const mocha = require('mocha'); const assert = require('assert'); // const mongodb = require('mongodb'); -const P = require('../../util/promise'); -const config = require('../../../config.js'); -const MDStore = require('../../server/object_services/md_store').MDStore; -const md_aggregator = require('../../server/bg_services/md_aggregator.js'); +const P = require('../../../util/promise'); +const config = require('../../../../config.js'); +const MDStore = require('../../../server/object_services/md_store').MDStore; +const md_aggregator = require('../../../server/bg_services/md_aggregator.js'); function make_test_system_store(last_update, md_store) { diff --git a/src/test/unit_tests/test_md_store.js b/src/test/integration_tests/db/test_md_store.js similarity index 98% rename from src/test/unit_tests/test_md_store.js rename to src/test/integration_tests/db/test_md_store.js index 2c4a2d45ce..3bc1c4fbe8 100644 --- a/src/test/unit_tests/test_md_store.js +++ b/src/test/integration_tests/db/test_md_store.js @@ -2,16 +2,16 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup(); const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const config = require('./../../../config'); +const config = require('../../../../config'); // const P = require('../../util/promise'); -const MDStore = require('../../server/object_services/md_store').MDStore; +const MDStore = require('../../../server/object_services/md_store').MDStore; mocha.describe('md_store', function() { diff --git a/src/test/unit_tests/test_mdsequence.js b/src/test/integration_tests/db/test_mdsequence.js similarity index 86% rename from src/test/unit_tests/test_mdsequence.js rename to src/test/integration_tests/db/test_mdsequence.js index 0c98980622..629d7cdda6 100644 --- a/src/test/unit_tests/test_mdsequence.js +++ b/src/test/integration_tests/db/test_mdsequence.js @@ -1,14 +1,14 @@ /* Copyright (C) 2022 NooBaa */ 'use strict'; -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup(); const mocha = require('mocha'); const assert = require('assert'); -const config = require('../../../config'); -const { MongoSequence } = require('../../util/mongo_client'); -const { PostgresClient } = require('../../util/postgres_client'); +const config = require('../../../../config'); +const { MongoSequence } = require('../../../util/mongo_client'); +const { PostgresClient } = require('../../../util/postgres_client'); async function get_postgres_client(params) { const pgc = new PostgresClient(params); diff --git a/src/test/unit_tests/test_nodes_store.js b/src/test/integration_tests/db/test_nodes_store.js similarity index 77% rename from src/test/unit_tests/test_nodes_store.js rename to src/test/integration_tests/db/test_nodes_store.js index 61f4f2605f..5d149afd82 100644 --- a/src/test/unit_tests/test_nodes_store.js +++ b/src/test/integration_tests/db/test_nodes_store.js @@ -2,13 +2,13 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup(); // const _ = require('lodash'); const mocha = require('mocha'); // const assert = require('assert'); -const NodesStore = require('../../server/node_services/nodes_store').NodesStore; +const NodesStore = require('../../../server/node_services/nodes_store').NodesStore; mocha.describe('nodes_store', function() { diff --git a/src/test/unit_tests/test_postgres_upgrade.js b/src/test/integration_tests/db/test_postgres_upgrade.js similarity index 98% rename from src/test/unit_tests/test_postgres_upgrade.js rename to src/test/integration_tests/db/test_postgres_upgrade.js index 2f0eaa5b3d..24f8263d85 100644 --- a/src/test/unit_tests/test_postgres_upgrade.js +++ b/src/test/integration_tests/db/test_postgres_upgrade.js @@ -4,11 +4,11 @@ // setup coretest first to prepare the env // const coretest = require('./coretest'); const mocha = require('mocha'); -const { Migrator } = require('../../upgrade/migrator'); +const { Migrator } = require('../../../upgrade/migrator'); const assert = require('assert'); const EventEmitter = require('events').EventEmitter; const _ = require('lodash'); -const postgres_client = require('../../util/postgres_client'); +const postgres_client = require('../../../util/postgres_client'); const wtf = require('wtfnode'); diff --git a/src/test/unit_tests/test_system_store.js b/src/test/integration_tests/db/test_system_store.js similarity index 95% rename from src/test/unit_tests/test_system_store.js rename to src/test/integration_tests/db/test_system_store.js index 0000f8c23f..16662cc71e 100644 --- a/src/test/unit_tests/test_system_store.js +++ b/src/test/integration_tests/db/test_system_store.js @@ -3,8 +3,8 @@ const mocha = require('mocha'); const assert = require('assert'); -const coretest = require('./coretest'); -const db_client = require('../../util/db_client'); +const coretest = require('../../utils/coretest/coretest'); +const db_client = require('../../../util/db_client'); // setup coretest first to prepare the env coretest.setup(); @@ -14,7 +14,7 @@ mocha.describe('system_store', function() { this.timeout(90000); // eslint-disable-line no-invalid-this // eslint-disable-next-line global-require - const system_store = require('../../server/system_services/system_store').get_instance(); + const system_store = require('../../../server/system_services/system_store').get_instance(); // eslint-disable-next-line no-undef afterEach(async function() { diff --git a/src/test/unit_tests/test_agent_blocks_reclaimer.js b/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js similarity index 96% rename from src/test/unit_tests/test_agent_blocks_reclaimer.js rename to src/test/integration_tests/internal/test_agent_blocks_reclaimer.js index 9406848b65..bf314a0adc 100644 --- a/src/test/unit_tests/test_agent_blocks_reclaimer.js +++ b/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js @@ -2,7 +2,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); const _ = require('lodash'); @@ -11,14 +11,14 @@ const assert = require('assert'); const crypto = require('crypto'); const mongodb = require('mongodb'); -const P = require('../../util/promise'); -const config = require('../../../config'); -const db_client = require('../../util/db_client'); -const schema_utils = require('../../util/schema_utils'); -const MDStore = require('../../server/object_services/md_store').MDStore; -const ObjectIO = require('../../sdk/object_io'); -const SliceReader = require('../../util/slice_reader'); -const AgentBlocksReclaimer = require('../../server/bg_services/agent_blocks_reclaimer').AgentBlocksReclaimer; +const P = require('../../../util/promise'); +const config = require('../../../../config'); +const db_client = require('../../../util/db_client'); +const schema_utils = require('../../../util/schema_utils'); +const MDStore = require('../../../server/object_services/md_store').MDStore; +const ObjectIO = require('../../../sdk/object_io'); +const SliceReader = require('../../../util/slice_reader'); +const AgentBlocksReclaimer = require('../../../server/bg_services/agent_blocks_reclaimer').AgentBlocksReclaimer; class ReclaimerMock extends AgentBlocksReclaimer { diff --git a/src/test/unit_tests/test_agent_blocks_verifier.js b/src/test/integration_tests/internal/test_agent_blocks_verifier.js similarity index 94% rename from src/test/unit_tests/test_agent_blocks_verifier.js rename to src/test/integration_tests/internal/test_agent_blocks_verifier.js index 6ffbf01c4d..635fdcbf45 100644 --- a/src/test/unit_tests/test_agent_blocks_verifier.js +++ b/src/test/integration_tests/internal/test_agent_blocks_verifier.js @@ -1,21 +1,21 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -/** @typedef {typeof import('../../sdk/nb')} nb */ +/** @typedef {typeof import('../../../sdk/nb')} nb */ // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup(); const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const AgentBlocksVerifier = require('../../server/bg_services/agent_blocks_verifier').AgentBlocksVerifier; -const db_client = require('../../util/db_client'); -const schema_utils = require('../../util/schema_utils'); +const AgentBlocksVerifier = require('../../../server/bg_services/agent_blocks_verifier').AgentBlocksVerifier; +const db_client = require('../../../util/db_client'); +const schema_utils = require('../../../util/schema_utils'); const mongodb = require('mongodb'); -const config = require('../../../config'); -const { ChunkDB, BlockDB } = require('../../server/object_services/map_db_types'); +const config = require('../../../../config'); +const { ChunkDB, BlockDB } = require('../../../server/object_services/map_db_types'); class VerifierMock extends AgentBlocksVerifier { /** @@ -72,7 +72,7 @@ class VerifierMock extends AgentBlocksVerifier { const chunks_idmap = _.keyBy(this.chunks, '_id'); const nodes_idmap = _.keyBy(nodes, '_id'); const pools_name = _.keyBy(this.pools, 'name'); - const db_blocks = blocks.map(block => { + const db_blocks = blocks?.map(block => { const chunk_id = _.get(block, 'chunk'); const chunk = new ChunkDB(chunks_idmap[chunk_id.toHexString()]); const frag = chunk.frags[0]; diff --git a/src/test/unit_tests/test_dedup.js b/src/test/integration_tests/internal/test_dedup.js similarity index 88% rename from src/test/unit_tests/test_dedup.js rename to src/test/integration_tests/internal/test_dedup.js index 92db7ae5c3..2fc5398a68 100644 --- a/src/test/unit_tests/test_dedup.js +++ b/src/test/integration_tests/internal/test_dedup.js @@ -2,15 +2,15 @@ /* eslint max-lines-per-function: ['error', 700] */ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: coretest.POOL_LIST }); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); -const http_utils = require('../../util/http_utils'); +const http_utils = require('../../../util/http_utils'); const mocha = require('mocha'); -const MDStore = require('../../server/object_services/md_store').MDStore; +const MDStore = require('../../../server/object_services/md_store').MDStore; const assert = require('assert'); -const config = require('../../../config'); +const config = require('../../../../config'); const { rpc_client, EMAIL } = coretest; const BKT = 'dedup-sloth-bucket'; diff --git a/src/test/unit_tests/test_encryption.js b/src/test/integration_tests/internal/test_encryption.js similarity index 99% rename from src/test/unit_tests/test_encryption.js rename to src/test/integration_tests/internal/test_encryption.js index d27c1cfeff..572e226d1d 100644 --- a/src/test/unit_tests/test_encryption.js +++ b/src/test/integration_tests/internal/test_encryption.js @@ -3,14 +3,14 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); -const system_store = require('../../server/system_services/system_store').get_instance(); -const auth_server = require('../../server/common_services/auth_server'); -const SensitiveString = require('../../util/sensitive_string'); -const db_client = require('../../util/db_client').instance(); -const P = require('../../util/promise'); +const system_store = require('../../../server/system_services/system_store').get_instance(); +const auth_server = require('../../../server/common_services/auth_server'); +const SensitiveString = require('../../../util/sensitive_string'); +const db_client = require('../../../util/db_client').instance(); +const P = require('../../../util/promise'); const assert = require('assert'); const mocha = require('mocha'); const { S3 } = require('@aws-sdk/client-s3'); @@ -19,8 +19,8 @@ const _ = require('lodash'); const http = require('http'); const fs = require('fs'); const crypto = require('crypto'); -const config = require('../../../config.js'); -const { KeyRotator } = require('../../server/bg_services/key_rotator'); +const config = require('../../../../config.js'); +const { KeyRotator } = require('../../../server/bg_services/key_rotator'); let s3; let coretest_access_key; diff --git a/src/test/unit_tests/test_host_server.js b/src/test/integration_tests/internal/test_host_server.js similarity index 94% rename from src/test/unit_tests/test_host_server.js rename to src/test/integration_tests/internal/test_host_server.js index bf760ada26..691ae28e77 100644 --- a/src/test/unit_tests/test_host_server.js +++ b/src/test/integration_tests/internal/test_host_server.js @@ -5,7 +5,7 @@ const mocha = require('mocha'); const assert = require('assert'); // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); mocha.describe('host_server', function() { diff --git a/src/test/unit_tests/test_map_builder.js b/src/test/integration_tests/internal/test_map_builder.js similarity index 93% rename from src/test/unit_tests/test_map_builder.js rename to src/test/integration_tests/internal/test_map_builder.js index 5293d3987e..686ae60702 100644 --- a/src/test/unit_tests/test_map_builder.js +++ b/src/test/integration_tests/internal/test_map_builder.js @@ -1,10 +1,10 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -/** @typedef {typeof import('../../sdk/nb')} nb */ +/** @typedef {typeof import('../../../sdk/nb')} nb */ // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); const _ = require('lodash'); @@ -12,19 +12,19 @@ const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); const crypto = require('crypto'); -const config = require('../../../config'); +const config = require('../../../../config'); -const MDStore = require('../../server/object_services/md_store').MDStore; -const ObjectIO = require('../../sdk/object_io'); +const MDStore = require('../../../server/object_services/md_store').MDStore; +const ObjectIO = require('../../../sdk/object_io'); // const map_writer = require('../../server/object_services/map_writer'); -const { MapBuilder } = require('../../server/object_services/map_builder'); -const { ChunkDB } = require('../../server/object_services/map_db_types'); -const { get_all_chunks_blocks } = require('../../sdk/map_api_types'); -const map_deleter = require('../../server/object_services/map_deleter'); -const map_reader = require('../../server/object_services/map_reader'); -const SliceReader = require('../../util/slice_reader'); -const system_store = require('../../server/system_services/system_store').get_instance(); -const { STORAGE_CLASS_GLACIER_IR } = require('../../endpoint/s3/s3_utils'); +const { MapBuilder } = require('../../../server/object_services/map_builder'); +const { ChunkDB } = require('../../../server/object_services/map_db_types'); +const { get_all_chunks_blocks } = require('../../../sdk/map_api_types'); +const map_deleter = require('../../../server/object_services/map_deleter'); +const map_reader = require('../../../server/object_services/map_reader'); +const SliceReader = require('../../../util/slice_reader'); +const system_store = require('../../../server/system_services/system_store').get_instance(); +const { STORAGE_CLASS_GLACIER_IR } = require('../../../endpoint/s3/s3_utils'); const { rpc_client } = coretest; const object_io = new ObjectIO(); diff --git a/src/test/unit_tests/test_map_client.js b/src/test/integration_tests/internal/test_map_client.js similarity index 97% rename from src/test/unit_tests/test_map_client.js rename to src/test/integration_tests/internal/test_map_client.js index c91e2ead8b..66ddc66aa2 100644 --- a/src/test/unit_tests/test_map_client.js +++ b/src/test/integration_tests/internal/test_map_client.js @@ -1,20 +1,20 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -/** @typedef {typeof import('../../sdk/nb')} nb */ +/** @typedef {typeof import('../../../sdk/nb')} nb */ // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const { MapClient } = require('../../sdk/map_client'); -const system_store = require('../../server/system_services/system_store').get_instance(); -const db_client = require('../../util/db_client'); -const { ChunkAPI } = require('../../sdk/map_api_types'); +const { MapClient } = require('../../../sdk/map_client'); +const system_store = require('../../../server/system_services/system_store').get_instance(); +const db_client = require('../../../util/db_client'); +const { ChunkAPI } = require('../../../sdk/map_api_types'); /* eslint max-lines-per-function: ["error", 600]*/ coretest.describe_mapper_test_case({ diff --git a/src/test/unit_tests/test_map_deleter.js b/src/test/integration_tests/internal/test_map_deleter.js similarity index 91% rename from src/test/unit_tests/test_map_deleter.js rename to src/test/integration_tests/internal/test_map_deleter.js index e434890d73..8592305e26 100644 --- a/src/test/unit_tests/test_map_deleter.js +++ b/src/test/integration_tests/internal/test_map_deleter.js @@ -2,7 +2,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup(); // const _ = require('lodash'); @@ -14,7 +14,7 @@ const mocha = require('mocha'); // const P = require('../../util/promise'); // const MDStore = require('../../server/object_services/md_store').MDStore; // const map_writer = require('../../server/object_services/map_writer'); -const map_deleter = require('../../server/object_services/map_deleter'); +const map_deleter = require('../../../server/object_services/map_deleter'); // const system_store = require('../../server/system_services/system_store').get_instance(); coretest.describe_mapper_test_case({ diff --git a/src/test/unit_tests/test_map_reader.js b/src/test/integration_tests/internal/test_map_reader.js similarity index 93% rename from src/test/unit_tests/test_map_reader.js rename to src/test/integration_tests/internal/test_map_reader.js index 188681fc0a..cdef8ac62d 100644 --- a/src/test/unit_tests/test_map_reader.js +++ b/src/test/integration_tests/internal/test_map_reader.js @@ -2,7 +2,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); // const _ = require('lodash'); @@ -14,7 +14,7 @@ const mongodb = require('mongodb'); // const P = require('../../util/promise'); // const MDStore = require('../../server/object_services/md_store').MDStore; // const map_writer = require('../../server/object_services/map_writer'); -const map_reader = require('../../server/object_services/map_reader'); +const map_reader = require('../../../server/object_services/map_reader'); // const system_store = require('../../server/system_services/system_store').get_instance(); coretest.describe_mapper_test_case({ diff --git a/src/test/unit_tests/test_node_allocator.js b/src/test/integration_tests/internal/test_node_allocator.js similarity index 89% rename from src/test/unit_tests/test_node_allocator.js rename to src/test/integration_tests/internal/test_node_allocator.js index 49154c5e05..f8bb9809bf 100644 --- a/src/test/unit_tests/test_node_allocator.js +++ b/src/test/integration_tests/internal/test_node_allocator.js @@ -2,15 +2,15 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); const _ = require('lodash'); const mocha = require('mocha'); -const system_store = require('../../server/system_services/system_store').get_instance(); +const system_store = require('../../../server/system_services/system_store').get_instance(); const assert = require('assert'); -const config = require('../../../config'); -const P = require('../../util/promise'); +const config = require('../../../../config'); +const P = require('../../../util/promise'); const POOL = coretest.POOL_LIST[0].name; const NODE_FIELDS_FOR_MAP = [ diff --git a/src/test/unit_tests/test_node_server.js b/src/test/integration_tests/internal/test_node_server.js similarity index 96% rename from src/test/unit_tests/test_node_server.js rename to src/test/integration_tests/internal/test_node_server.js index 767c70ea30..f806799535 100644 --- a/src/test/unit_tests/test_node_server.js +++ b/src/test/integration_tests/internal/test_node_server.js @@ -2,7 +2,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); // const _ = require('lodash'); diff --git a/src/test/unit_tests/test_object_io.js b/src/test/integration_tests/internal/test_object_io.js similarity index 99% rename from src/test/unit_tests/test_object_io.js rename to src/test/integration_tests/internal/test_object_io.js index 435e6ed8dc..e29bdd0a6c 100644 --- a/src/test/unit_tests/test_object_io.js +++ b/src/test/integration_tests/internal/test_object_io.js @@ -3,7 +3,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); const _ = require('lodash'); @@ -13,8 +13,8 @@ const crypto = require('crypto'); const assert = require('assert'); const stream = require('stream'); -const ObjectIO = require('../../sdk/object_io'); -const { crypto_random_string } = require('../../util/string_utils'); +const ObjectIO = require('../../../sdk/object_io'); +const { crypto_random_string } = require('../../../util/string_utils'); const { rpc_client } = coretest; const object_io = new ObjectIO(); diff --git a/src/test/unit_tests/test_system_servers.js b/src/test/integration_tests/internal/test_system_servers.js similarity index 98% rename from src/test/unit_tests/test_system_servers.js rename to src/test/integration_tests/internal/test_system_servers.js index 39df6a7c2f..e44e8f30bb 100644 --- a/src/test/unit_tests/test_system_servers.js +++ b/src/test/integration_tests/internal/test_system_servers.js @@ -3,7 +3,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup(); const _ = require('lodash'); @@ -11,10 +11,10 @@ const mocha = require('mocha'); const assert = require('assert'); const S3Auth = require('aws-sdk/lib/signers/s3'); -const P = require('../../util/promise'); -const config = require('../../../config'); +const P = require('../../../util/promise'); +const config = require('../../../../config'); -const check_deletion_ownership = require('../../server/system_services/pool_server').check_deletion_ownership; +const check_deletion_ownership = require('../../../server/system_services/pool_server').check_deletion_ownership; mocha.describe('system_servers', function() { diff --git a/src/test/unit_tests/test_tiering_ttl_worker.js b/src/test/integration_tests/internal/test_tiering_ttl_worker.js similarity index 89% rename from src/test/unit_tests/test_tiering_ttl_worker.js rename to src/test/integration_tests/internal/test_tiering_ttl_worker.js index b49782156f..896a9f2edd 100644 --- a/src/test/unit_tests/test_tiering_ttl_worker.js +++ b/src/test/integration_tests/internal/test_tiering_ttl_worker.js @@ -2,20 +2,20 @@ 'use strict'; // setup coretest first to prepare the env -const { setup, rpc_client, POOL_LIST } = require('./coretest'); +const { setup, rpc_client, POOL_LIST } = require('../../utils/coretest/coretest'); setup({ pools_to_create: [POOL_LIST[1]] }); // const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); const crypto = require('crypto'); -const config = require('./../../../config'); -const SliceReader = require('../../util/slice_reader'); -const ObjectIO = require('../../sdk/object_io'); -const P = require('../../util/promise'); -const { TieringTTLWorker } = require('../../server/bg_services/tier_ttl_worker'); -const { MDStore } = require('../../server/object_services/md_store'); -const { ChunkDB } = require('../../server/object_services/map_db_types'); +const config = require('../../../../config'); +const SliceReader = require('../../../util/slice_reader'); +const ObjectIO = require('../../../sdk/object_io'); +const P = require('../../../util/promise'); +const { TieringTTLWorker } = require('../../../server/bg_services/tier_ttl_worker'); +const { MDStore } = require('../../../server/object_services/md_store'); +const { ChunkDB } = require('../../../server/object_services/map_db_types'); const object_io = new ObjectIO(); object_io.set_verification_mode(); diff --git a/src/test/unit_tests/test_tiering_upload.js b/src/test/integration_tests/internal/test_tiering_upload.js similarity index 95% rename from src/test/unit_tests/test_tiering_upload.js rename to src/test/integration_tests/internal/test_tiering_upload.js index f36d639d29..caa7bff7b7 100644 --- a/src/test/unit_tests/test_tiering_upload.js +++ b/src/test/integration_tests/internal/test_tiering_upload.js @@ -2,7 +2,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.setup({ pools_to_create: coretest.POOL_LIST }); const _ = require('lodash'); @@ -12,11 +12,11 @@ const crypto = require('crypto'); const assert = require('assert'); // const P = require('../../util/promise'); -const config = require('../../../config.js'); -const ObjectIO = require('../../sdk/object_io'); -const SliceReader = require('../../util/slice_reader'); -const system_store = require('../../server/system_services/system_store').get_instance(); -const node_allocator = require('../../server/node_services/node_allocator'); +const config = require('../../../../config.js'); +const ObjectIO = require('../../../sdk/object_io'); +const SliceReader = require('../../../util/slice_reader'); +const system_store = require('../../../server/system_services/system_store').get_instance(); +const node_allocator = require('../../../server/node_services/node_allocator'); const { rpc_client } = coretest; const object_io = new ObjectIO(); @@ -25,7 +25,7 @@ object_io.set_verification_mode(); const seed = crypto.randomBytes(16); const generator = crypto.createCipheriv('aes-128-gcm', seed, Buffer.alloc(12)); const core_agent_control = {}; // TODO require('./core_agent_control'); -const node_server = require('../../server/node_services/node_server'); +const node_server = require('../../../server/node_services/node_server'); mocha.describe('tiering upload', function() { diff --git a/src/test/unit_tests/test_upgrade_scripts.js b/src/test/integration_tests/internal/test_upgrade_scripts.js similarity index 90% rename from src/test/unit_tests/test_upgrade_scripts.js rename to src/test/integration_tests/internal/test_upgrade_scripts.js index e8a5a1ea76..673e8c989f 100644 --- a/src/test/unit_tests/test_upgrade_scripts.js +++ b/src/test/integration_tests/internal/test_upgrade_scripts.js @@ -2,19 +2,19 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); const { rpc_client, EMAIL } = coretest; coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); const http = require('http'); -const system_store = require('../../server/system_services/system_store').get_instance(); -const upgrade_bucket_policy = require('../../upgrade/upgrade_scripts/5.15.6/upgrade_bucket_policy'); -const upgrade_bucket_cors = require('../../upgrade/upgrade_scripts/5.19.0/upgrade_bucket_cors'); -const dbg = require('../../util/debug_module')(__filename); +const system_store = require('../../../server/system_services/system_store').get_instance(); +const upgrade_bucket_policy = require('../../../upgrade/upgrade_scripts/5.15.6/upgrade_bucket_policy'); +const upgrade_bucket_cors = require('../../../upgrade/upgrade_scripts/5.19.0/upgrade_bucket_cors'); +const dbg = require('../../../util/debug_module')(__filename); const assert = require('assert'); const mocha = require('mocha'); -const config = require('../../../config'); +const config = require('../../../../config'); const BKT = 'test-bucket'; /** @type {S3} */ diff --git a/src/test/unit_tests/jest_tests/test_nc_account_cli.test.js b/src/test/integration_tests/nc/cli/test_nc_account_cli.test.js similarity index 99% rename from src/test/unit_tests/jest_tests/test_nc_account_cli.test.js rename to src/test/integration_tests/nc/cli/test_nc_account_cli.test.js index cb6857685d..4a5e1feb8d 100644 --- a/src/test/unit_tests/jest_tests/test_nc_account_cli.test.js +++ b/src/test/integration_tests/nc/cli/test_nc_account_cli.test.js @@ -9,18 +9,18 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const fs = require('fs'); const _ = require('lodash'); const path = require('path'); -const os_util = require('../../../util/os_utils'); -const fs_utils = require('../../../util/fs_utils'); -const { ConfigFS } = require('../../../sdk/config_fs'); +const os_util = require('../../../../util/os_utils'); +const fs_utils = require('../../../../util/fs_utils'); +const { ConfigFS } = require('../../../../sdk/config_fs'); const { set_path_permissions_and_owner, create_fs_user_by_platform, CLI_UNSET_EMPTY_STRING, - delete_fs_user_by_platform, TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); -const { TYPES, ACTIONS, ANONYMOUS } = require('../../../manage_nsfs/manage_nsfs_constants'); -const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; -const ManageCLIResponse = require('../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; + delete_fs_user_by_platform, TMP_PATH, set_nc_config_dir_in_config } = require('../../../system_tests/test_utils'); +const { TYPES, ACTIONS, ANONYMOUS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const ManageCLIError = require('../../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; +const ManageCLIResponse = require('../../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; const tmp_fs_path = path.join(TMP_PATH, 'test_nc_account_cli'); const timeout = 50000; -const config = require('../../../../config'); +const config = require('../../../../../config'); const config_fs_account_options = { show_secrets: true, decrypt_secret_key: true }; const quoted_type = Object.freeze({ diff --git a/src/test/unit_tests/jest_tests/test_nc_anonymous_cli.test.js b/src/test/integration_tests/nc/cli/test_nc_anonymous_cli.test.js similarity index 97% rename from src/test/unit_tests/jest_tests/test_nc_anonymous_cli.test.js rename to src/test/integration_tests/nc/cli/test_nc_anonymous_cli.test.js index 9d81d6ab59..69b2c728c8 100644 --- a/src/test/unit_tests/jest_tests/test_nc_anonymous_cli.test.js +++ b/src/test/integration_tests/nc/cli/test_nc_anonymous_cli.test.js @@ -5,16 +5,16 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const path = require('path'); -const os_util = require('../../../util/os_utils'); -const fs_utils = require('../../../util/fs_utils'); -const { ConfigFS } = require('../../../sdk/config_fs'); -const { TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); -const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); -const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; -const ManageCLIResponse = require('../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; +const os_util = require('../../../../util/os_utils'); +const fs_utils = require('../../../../util/fs_utils'); +const { ConfigFS } = require('../../../../sdk/config_fs'); +const { TMP_PATH, set_nc_config_dir_in_config } = require('../../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const ManageCLIError = require('../../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; +const ManageCLIResponse = require('../../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; const tmp_fs_path = path.join(TMP_PATH, 'test_nc_nsfs_anon_account_cli'); -const config = require('../../../../config'); +const config = require('../../../../../config'); const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); const config_fs = new ConfigFS(config_root); diff --git a/src/test/unit_tests/jest_tests/test_nc_bucket_cli.test.js b/src/test/integration_tests/nc/cli/test_nc_bucket_cli.test.js similarity index 98% rename from src/test/unit_tests/jest_tests/test_nc_bucket_cli.test.js rename to src/test/integration_tests/nc/cli/test_nc_bucket_cli.test.js index 171b835c92..0043dbe617 100644 --- a/src/test/unit_tests/jest_tests/test_nc_bucket_cli.test.js +++ b/src/test/integration_tests/nc/cli/test_nc_bucket_cli.test.js @@ -6,15 +6,15 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const fs = require('fs'); const path = require('path'); -const os_util = require('../../../util/os_utils'); -const fs_utils = require('../../../util/fs_utils'); -const { ConfigFS } = require('../../../sdk/config_fs'); +const os_util = require('../../../../util/os_utils'); +const fs_utils = require('../../../../util/fs_utils'); +const { ConfigFS } = require('../../../../sdk/config_fs'); const { set_path_permissions_and_owner, TMP_PATH, generate_s3_policy, CLI_UNSET_EMPTY_STRING, - set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); -const { ACTIONS, TYPES } = require('../../../manage_nsfs/manage_nsfs_constants'); -const { get_process_fs_context, is_path_exists, get_bucket_tmpdir_full_path } = require('../../../util/native_fs_utils'); -const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; -const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses'); + set_nc_config_dir_in_config } = require('../../../system_tests/test_utils'); +const { ACTIONS, TYPES } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const { get_process_fs_context, is_path_exists, get_bucket_tmpdir_full_path } = require('../../../../util/native_fs_utils'); +const ManageCLIError = require('../../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; +const { ManageCLIResponse } = require('../../../../manage_nsfs/manage_nsfs_cli_responses'); const tmp_fs_path = path.join(TMP_PATH, 'test_nc_nsfs_bucket_cli'); const DEFAULT_FS_CONFIG = get_process_fs_context(); @@ -1061,7 +1061,7 @@ async function create_json_file(path_to_dir, file_name, data) { * assert_bucket will verify the fields of the buckets (only required fields) * @param {object} bucket * @param {object} bucket_options - * @param {import('../../../sdk/config_fs').ConfigFS} config_fs + * @param {import('../../../../sdk/config_fs').ConfigFS} config_fs */ async function assert_bucket(bucket, bucket_options, config_fs) { expect(bucket.name).toEqual(bucket_options.name); diff --git a/src/test/unit_tests/test_nc_cli.js b/src/test/integration_tests/nc/cli/test_nc_cli.js similarity index 98% rename from src/test/unit_tests/test_nc_cli.js rename to src/test/integration_tests/nc/cli/test_nc_cli.js index 34729189f9..b71a1e57b7 100644 --- a/src/test/unit_tests/test_nc_cli.js +++ b/src/test/integration_tests/nc/cli/test_nc_cli.js @@ -6,17 +6,17 @@ const path = require('path'); const mocha = require('mocha'); const assert = require('assert'); -const config = require('../../../config'); -const fs_utils = require('../../util/fs_utils'); -const nb_native = require('../../util/nb_native'); -const { crypto_random_string } = require('../../util/string_utils'); -const { CONFIG_SUBDIRS, JSON_SUFFIX, SYMLINK_SUFFIX, ConfigFS } = require('../../sdk/config_fs'); -const { get_process_fs_context } = require('../../util/native_fs_utils'); -const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); -const { ManageCLIResponse } = require('../../manage_nsfs/manage_nsfs_cli_responses'); +const config = require('../../../../../config'); +const fs_utils = require('../../../../util/fs_utils'); +const nb_native = require('../../../../util/nb_native'); +const { crypto_random_string } = require('../../../../util/string_utils'); +const { CONFIG_SUBDIRS, JSON_SUFFIX, SYMLINK_SUFFIX, ConfigFS } = require('../../../../sdk/config_fs'); +const { get_process_fs_context } = require('../../../../util/native_fs_utils'); +const { ManageCLIError } = require('../../../../manage_nsfs/manage_nsfs_cli_errors'); +const { ManageCLIResponse } = require('../../../../manage_nsfs/manage_nsfs_cli_responses'); const { exec_manage_cli, generate_s3_policy, create_fs_user_by_platform, delete_fs_user_by_platform, - set_path_permissions_and_owner, TMP_PATH, set_nc_config_dir_in_config } = require('../system_tests/test_utils'); -const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); + set_path_permissions_and_owner, TMP_PATH, set_nc_config_dir_in_config } = require('../../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); const tmp_fs_path = path.join(TMP_PATH, 'test_bucketspace_fs'); const DEFAULT_FS_CONFIG = get_process_fs_context(); diff --git a/src/test/unit_tests/jest_tests/test_nc_connection_cli.test.js b/src/test/integration_tests/nc/cli/test_nc_connection_cli.test.js similarity index 95% rename from src/test/unit_tests/jest_tests/test_nc_connection_cli.test.js rename to src/test/integration_tests/nc/cli/test_nc_connection_cli.test.js index 87ed0cb371..fca9813cd4 100644 --- a/src/test/unit_tests/jest_tests/test_nc_connection_cli.test.js +++ b/src/test/integration_tests/nc/cli/test_nc_connection_cli.test.js @@ -9,13 +9,13 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const _ = require('lodash'); const fs = require('fs'); const path = require('path'); -const os_util = require('../../../util/os_utils'); -const fs_utils = require('../../../util/fs_utils'); -const { ConfigFS } = require('../../../sdk/config_fs'); -const { TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); -const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); +const os_util = require('../../../../util/os_utils'); +const fs_utils = require('../../../../util/fs_utils'); +const { ConfigFS } = require('../../../../sdk/config_fs'); +const { TMP_PATH, set_nc_config_dir_in_config } = require('../../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); -const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; +const ManageCLIError = require('../../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const tmp_fs_path = path.join(TMP_PATH, 'test_nc_connection_cli.test'); const timeout = 5000; diff --git a/src/test/unit_tests/test_nc_lifecycle_expiration.js b/src/test/integration_tests/nc/lifecycle/test_nc_lifecycle_expiration.js similarity index 94% rename from src/test/unit_tests/test_nc_lifecycle_expiration.js rename to src/test/integration_tests/nc/lifecycle/test_nc_lifecycle_expiration.js index 7f73b0d626..847c8bf610 100644 --- a/src/test/unit_tests/test_nc_lifecycle_expiration.js +++ b/src/test/integration_tests/nc/lifecycle/test_nc_lifecycle_expiration.js @@ -4,13 +4,12 @@ const path = require('path'); const mocha = require('mocha'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); -const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); -const { TMP_PATH, set_path_permissions_and_owner, invalid_nsfs_root_permissions, generate_s3_client, get_coretest_path, - generate_lifecycle_rule, validate_expiration_header, update_file_mtime, exec_manage_cli } = require('../system_tests/test_utils'); +const fs_utils = require('../../../../util/fs_utils'); +const { TYPES, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const { TMP_PATH, set_path_permissions_and_owner, invalid_nsfs_root_permissions, generate_s3_client, require_coretest, + generate_lifecycle_rule, validate_expiration_header, update_file_mtime, exec_manage_cli } = require('../../../system_tests/test_utils'); -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const coretest = require_coretest(); const { rpc_client, EMAIL, get_admin_mock_account_details } = coretest; coretest.setup({}); diff --git a/src/test/unit_tests/test_nc_with_a_couple_of_forks.js b/src/test/integration_tests/nc/test_nc_with_a_couple_of_forks.js similarity index 95% rename from src/test/unit_tests/test_nc_with_a_couple_of_forks.js rename to src/test/integration_tests/nc/test_nc_with_a_couple_of_forks.js index 45198b4eef..eb8a2e38e1 100644 --- a/src/test/unit_tests/test_nc_with_a_couple_of_forks.js +++ b/src/test/integration_tests/nc/test_nc_with_a_couple_of_forks.js @@ -5,19 +5,18 @@ const path = require('path'); const _ = require('lodash'); const fs = require('fs'); -const P = require('../../util/promise'); +const P = require('../../../util/promise'); const mocha = require('mocha'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); -const { TMP_PATH, generate_nsfs_account, get_new_buckets_path_by_test_env, generate_s3_client, get_coretest_path, exec_manage_cli, set_health_mock_functions, create_system_json } = require('../system_tests/test_utils'); -const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); -const { NSFSHealth } = require('../../manage_nsfs/health'); -const { ConfigFS } = require('../../sdk/config_fs'); -const config = require('../../../config'); -const ManageCLIResponse = require('../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; +const fs_utils = require('../../../util/fs_utils'); +const { TMP_PATH, generate_nsfs_account, get_new_buckets_path_by_test_env, generate_s3_client, require_coretest, exec_manage_cli, set_health_mock_functions, create_system_json } = require('../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); +const { NSFSHealth } = require('../../../manage_nsfs/health'); +const { ConfigFS } = require('../../../sdk/config_fs'); +const config = require('../../../../config'); +const ManageCLIResponse = require('../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const coretest = require_coretest(); const setup_options = { forks: 2, debug: 5 }; coretest.setup(setup_options); const { rpc_client, EMAIL, get_current_setup_options, stop_nsfs_process, start_nsfs_process, diff --git a/src/test/unit_tests/jest_tests/test_cli_diagnose.test.js b/src/test/integration_tests/nc/tools/test_cli_diagnose.test.js similarity index 89% rename from src/test/unit_tests/jest_tests/test_cli_diagnose.test.js rename to src/test/integration_tests/nc/tools/test_cli_diagnose.test.js index c0edd959e4..3b3406fbb6 100644 --- a/src/test/unit_tests/jest_tests/test_cli_diagnose.test.js +++ b/src/test/integration_tests/nc/tools/test_cli_diagnose.test.js @@ -6,12 +6,12 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const path = require('path'); const express = require('express'); -const config = require('../../../../config'); -const { folder_delete } = require('../../../util/fs_utils'); -const { exec_manage_cli, TMP_PATH } = require('../../system_tests/test_utils'); -const { TYPES, DIAGNOSE_ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); -const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; -const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses'); +const config = require('../../../../../config'); +const { folder_delete } = require('../../../../util/fs_utils'); +const { exec_manage_cli, TMP_PATH } = require('../../../system_tests/test_utils'); +const { TYPES, DIAGNOSE_ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const ManageCLIError = require('../../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; +const { ManageCLIResponse } = require('../../../../manage_nsfs/manage_nsfs_cli_responses'); const config_root = path.join(TMP_PATH, 'test_cli_diagnose'); const metrics_url = `http://127.0.0.1:${config.EP_METRICS_SERVER_PORT}/metrics/nsfs_stats`; diff --git a/src/test/unit_tests/jest_tests/test_cli_upgrade.test.js b/src/test/integration_tests/nc/tools/test_cli_upgrade.test.js similarity index 97% rename from src/test/unit_tests/jest_tests/test_cli_upgrade.test.js rename to src/test/integration_tests/nc/tools/test_cli_upgrade.test.js index 495af8d6b4..0e34710f19 100644 --- a/src/test/unit_tests/jest_tests/test_cli_upgrade.test.js +++ b/src/test/integration_tests/nc/tools/test_cli_upgrade.test.js @@ -6,15 +6,15 @@ process.env.DISABLE_INIT_RANDOM_SEED = 'true'; const os = require('os'); const path = require('path'); -const config = require('../../../../config'); -const pkg = require('../../../../package.json'); -const fs_utils = require('../../../util/fs_utils'); -const { ConfigFS } = require('../../../sdk/config_fs'); -const { TMP_PATH, exec_manage_cli, clean_config_dir, fail_test_if_default_config_dir_exists, TEST_TIMEOUT } = require('../../system_tests/test_utils'); -const { ManageCLIError } = require('../../../manage_nsfs/manage_nsfs_cli_errors'); -const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses'); -const { TYPES, UPGRADE_ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); -const { CONFIG_DIR_PHASES } = require('../../../sdk/config_fs'); +const config = require('../../../../../config'); +const pkg = require('../../../../../package.json'); +const fs_utils = require('../../../../util/fs_utils'); +const { ConfigFS } = require('../../../../sdk/config_fs'); +const { TMP_PATH, exec_manage_cli, clean_config_dir, fail_test_if_default_config_dir_exists, TEST_TIMEOUT } = require('../../../system_tests/test_utils'); +const { ManageCLIError } = require('../../../../manage_nsfs/manage_nsfs_cli_errors'); +const { ManageCLIResponse } = require('../../../../manage_nsfs/manage_nsfs_cli_responses'); +const { TYPES, UPGRADE_ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const { CONFIG_DIR_PHASES } = require('../../../../sdk/config_fs'); const config_root = path.join(TMP_PATH, 'config_root_cli_upgrade_test'); const config_fs = new ConfigFS(config_root); diff --git a/src/test/unit_tests/test_nc_bucket_logging.js b/src/test/integration_tests/nc/tools/test_nc_bucket_logging.js similarity index 80% rename from src/test/unit_tests/test_nc_bucket_logging.js rename to src/test/integration_tests/nc/tools/test_nc_bucket_logging.js index ee52fd068f..a5e329905a 100644 --- a/src/test/unit_tests/test_nc_bucket_logging.js +++ b/src/test/integration_tests/nc/tools/test_nc_bucket_logging.js @@ -4,17 +4,16 @@ const path = require('path'); const mocha = require('mocha'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); -const nb_native = require('../../util/nb_native'); -const { get_process_fs_context } = require('../../util/native_fs_utils'); -const { ManageCLIResponse } = require('../../manage_nsfs/manage_nsfs_cli_responses'); -const { exec_manage_cli, get_coretest_path, TMP_PATH } = require('../system_tests/test_utils'); -const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); +const fs_utils = require('../../../../util/fs_utils'); +const nb_native = require('../../../../util/nb_native'); +const { get_process_fs_context } = require('../../../../util/native_fs_utils'); +const { ManageCLIResponse } = require('../../../../manage_nsfs/manage_nsfs_cli_responses'); +const { exec_manage_cli, require_coretest, TMP_PATH } = require('../../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); const DEFAULT_FS_CONFIG = get_process_fs_context(); -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const coretest = require_coretest(); coretest.setup({}); mocha.describe('cli logging flow', async function() { diff --git a/src/test/unit_tests/test_nc_health.js b/src/test/integration_tests/nc/tools/test_nc_health.js similarity index 98% rename from src/test/unit_tests/test_nc_health.js rename to src/test/integration_tests/nc/tools/test_nc_health.js index 562fab03cc..8ac4274a20 100644 --- a/src/test/unit_tests/test_nc_health.js +++ b/src/test/integration_tests/nc/tools/test_nc_health.js @@ -7,20 +7,20 @@ const path = require('path'); const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const config = require('../../../config'); -const pkg = require('../../../package.json'); +const config = require('../../../../../config'); +const pkg = require('../../../../../package.json'); const fs = require('fs'); -const fs_utils = require('../../util/fs_utils'); -const nb_native = require('../../util/nb_native'); -const { ConfigFS } = require('../../sdk/config_fs'); -const mongo_utils = require('../../util/mongo_utils'); -const test_utils = require('../system_tests/test_utils'); -const NSFSHealth = require('../../manage_nsfs/health').NSFSHealth; -const { get_process_fs_context } = require('../../util/native_fs_utils'); -const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); -const { TYPES, DIAGNOSE_ACTIONS, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); -const { TMP_PATH, create_fs_user_by_platform, delete_fs_user_by_platform, exec_manage_cli, TEST_TIMEOUT } = require('../system_tests/test_utils'); -const { CONFIG_DIR_PHASES } = require('../../sdk/config_fs'); +const fs_utils = require('../../../../util/fs_utils'); +const nb_native = require('../../../../util/nb_native'); +const { ConfigFS } = require('../../../../sdk/config_fs'); +const mongo_utils = require('../../../../util/mongo_utils'); +const test_utils = require('../../../system_tests/test_utils'); +const NSFSHealth = require('../../../../manage_nsfs/health').NSFSHealth; +const { get_process_fs_context } = require('../../../../util/native_fs_utils'); +const { ManageCLIError } = require('../../../../manage_nsfs/manage_nsfs_cli_errors'); +const { TYPES, DIAGNOSE_ACTIONS, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const { TMP_PATH, create_fs_user_by_platform, delete_fs_user_by_platform, exec_manage_cli, TEST_TIMEOUT } = require('../../../system_tests/test_utils'); +const { CONFIG_DIR_PHASES } = require('../../../../sdk/config_fs'); const tmp_fs_path = path.join(TMP_PATH, 'test_nc_health'); const DEFAULT_FS_CONFIG = get_process_fs_context(); diff --git a/src/test/unit_tests/jest_tests/test_nc_online_upgrade_cli_integrations.js b/src/test/integration_tests/nc/tools/test_nc_online_upgrade_cli_integrations.js similarity index 95% rename from src/test/unit_tests/jest_tests/test_nc_online_upgrade_cli_integrations.js rename to src/test/integration_tests/nc/tools/test_nc_online_upgrade_cli_integrations.js index 1501f9f9b0..d780081a0a 100644 --- a/src/test/unit_tests/jest_tests/test_nc_online_upgrade_cli_integrations.js +++ b/src/test/integration_tests/nc/tools/test_nc_online_upgrade_cli_integrations.js @@ -5,15 +5,15 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const path = require('path'); -const fs_utils = require('../../../util/fs_utils'); -const { exec_manage_cli, TMP_PATH, create_system_json } = require('../../system_tests/test_utils'); -const { folder_delete, create_fresh_path } = require('../../../util/fs_utils'); -const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); -const { ManageCLIError } = require('../../../manage_nsfs/manage_nsfs_cli_errors'); -const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses'); +const fs_utils = require('../../../../util/fs_utils'); +const { exec_manage_cli, TMP_PATH, create_system_json } = require('../../../system_tests/test_utils'); +const { folder_delete, create_fresh_path } = require('../../../../util/fs_utils'); +const { TYPES, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const { ManageCLIError } = require('../../../../manage_nsfs/manage_nsfs_cli_errors'); +const { ManageCLIResponse } = require('../../../../manage_nsfs/manage_nsfs_cli_responses'); const config_root = path.join(TMP_PATH, 'test_online_upgrade_cli_integrations'); -const { ConfigFS } = require('../../../sdk/config_fs'); +const { ConfigFS } = require('../../../../sdk/config_fs'); const config_fs = new ConfigFS(config_root); const default_new_buckets_path = path.join(TMP_PATH, 'default_new_buckets_path_online_upgrade_cli_integrations_test'); const bucket_storage_path = path.join(default_new_buckets_path, 'bucket1'); diff --git a/src/test/unit_tests/test_nc_online_upgrade_s3_integrations.js b/src/test/integration_tests/nc/tools/test_nc_online_upgrade_s3_integrations.js similarity index 96% rename from src/test/unit_tests/test_nc_online_upgrade_s3_integrations.js rename to src/test/integration_tests/nc/tools/test_nc_online_upgrade_s3_integrations.js index 4af162f1f3..5b2fb586fa 100644 --- a/src/test/unit_tests/test_nc_online_upgrade_s3_integrations.js +++ b/src/test/integration_tests/nc/tools/test_nc_online_upgrade_s3_integrations.js @@ -5,13 +5,12 @@ const path = require('path'); const http = require('http'); const mocha = require('mocha'); const assert = require('assert'); -const config = require('../../../config'); -const { get_coretest_path, TMP_PATH, TEST_TIMEOUT, update_system_json } = require('../system_tests/test_utils'); +const config = require('../../../../../config'); +const { require_coretest, TMP_PATH, TEST_TIMEOUT, update_system_json } = require('../../../system_tests/test_utils'); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); -const { folder_delete } = require('../../util/fs_utils'); -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const { folder_delete } = require('../../../../util/fs_utils'); +const coretest = require_coretest(); const { rpc_client, EMAIL, NC_CORETEST_CONFIG_FS } = coretest; coretest.setup({}); diff --git a/src/test/unit_tests/test_bucketspace_versioning.js b/src/test/integration_tests/nsfs/test_bucketspace_versioning.js similarity index 99% rename from src/test/unit_tests/test_bucketspace_versioning.js rename to src/test/integration_tests/nsfs/test_bucketspace_versioning.js index a3dd987eaf..fd916661f1 100644 --- a/src/test/unit_tests/test_bucketspace_versioning.js +++ b/src/test/integration_tests/nsfs/test_bucketspace_versioning.js @@ -8,18 +8,17 @@ const fs = require('fs'); const path = require('path'); const mocha = require('mocha'); const assert = require('assert'); -const P = require('../../util/promise'); -const fs_utils = require('../../util/fs_utils'); -const nb_native = require('../../util/nb_native'); -const size_utils = require('../../util/size_utils'); +const P = require('../../../util/promise'); +const fs_utils = require('../../../util/fs_utils'); +const nb_native = require('../../../util/nb_native'); +const size_utils = require('../../../util/size_utils'); const { TMP_PATH, IS_GPFS, is_nc_coretest, set_path_permissions_and_owner, generate_nsfs_account, get_new_buckets_path_by_test_env, - invalid_nsfs_root_permissions, generate_s3_client, get_coretest_path } = require('../system_tests/test_utils'); -const { get_process_fs_context } = require('../../util/native_fs_utils'); + invalid_nsfs_root_permissions, generate_s3_client, require_coretest } = require('../../system_tests/test_utils'); +const { get_process_fs_context } = require('../../../util/native_fs_utils'); const _ = require('lodash'); -const config = require('../../../config'); +const config = require('../../../../config'); -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const coretest = require_coretest(); const { rpc_client, EMAIL, get_admin_mock_account_details } = coretest; coretest.setup({}); diff --git a/src/test/integration_tests/nsfs/test_nc_versioning_integration.js b/src/test/integration_tests/nsfs/test_nc_versioning_integration.js new file mode 100644 index 0000000000..3657dd6006 --- /dev/null +++ b/src/test/integration_tests/nsfs/test_nc_versioning_integration.js @@ -0,0 +1,4669 @@ +/* Copyright (C) 2020 NooBaa */ +/* eslint-disable max-lines-per-function */ +/*eslint max-lines: ["error",5500]*/ +/* eslint-disable max-statements */ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const mocha = require('mocha'); +const assert = require('assert'); +const P = require('../../../util/promise'); +const fs_utils = require('../../../util/fs_utils'); +const nb_native = require('../../../util/nb_native'); +const size_utils = require('../../../util/size_utils'); +const { TMP_PATH, IS_GPFS, is_nc_coretest, set_path_permissions_and_owner, generate_nsfs_account, get_new_buckets_path_by_test_env, + invalid_nsfs_root_permissions, generate_s3_client, require_coretest } = require('../../../unit_tests/utils/test_utils'); +const { get_process_fs_context } = require('../../../util/native_fs_utils'); +const _ = require('lodash'); +const config = require('../../../../config'); + +const coretest = require_coretest(); +const { rpc_client, EMAIL, get_admin_mock_account_details } = coretest; +coretest.setup({}); + +const XATTR_INTERNAL_NOOBAA_PREFIX = 'user.noobaa.'; +const XATTR_VERSION_ID = XATTR_INTERNAL_NOOBAA_PREFIX + 'version_id'; +const XATTR_DELETE_MARKER = XATTR_INTERNAL_NOOBAA_PREFIX + 'delete_marker'; +const XATTR_NON_CURRENT_TIMESTASMP = XATTR_INTERNAL_NOOBAA_PREFIX + 'non_current_timestamp'; +const XATTR_DIR_CONTENT = XATTR_INTERNAL_NOOBAA_PREFIX + 'dir_content'; +const XATTR_USER_PREFIX = 'user.'; +const NULL_VERSION_ID = 'null'; +const HIDDEN_VERSIONS_PATH = '.versions'; +const NSFS_FOLDER_OBJECT_NAME = '.folder'; + +const DEFAULT_FS_CONFIG = get_process_fs_context(IS_GPFS ? 'GPFS' : ''); +let CORETEST_ENDPOINT; + +const tmp_fs_root = path.join(TMP_PATH, 'test_bucket_namespace_fs_versioning'); +// on NC - new_buckets_path is full absolute path +// on Containerized - new_buckets_path is the directory +const new_bucket_path_param = get_new_buckets_path_by_test_env(tmp_fs_root, '/'); + +mocha.describe('bucketspace namespace_fs - versioning', function() { + const nsr = 'versioned-nsr'; + const bucket_name = 'versioned-enabled-bucket'; + const disabled_bucket_name = 'disabled-bucket'; // be aware that this bucket would become versioned in the copy object tests + const suspended_bucket_name = 'suspended-bucket'; + const nested_keys_bucket_name = 'bucket-with-nested-keys'; + const content_dir_bucket_name = 'content-dir-bucket'; + + const bucket_path = '/bucket'; + const full_path = tmp_fs_root + bucket_path; + const disabled_bucket_path = '/disabled_bucket'; + const disabled_full_path = tmp_fs_root + disabled_bucket_path; + const suspended_bucket_path = '/suspended_bucket'; + const suspended_full_path = tmp_fs_root + suspended_bucket_path; + const nested_keys_bucket_path = '/bucket_with_nested_keys'; + const nested_keys_full_path = path.join(tmp_fs_root, nested_keys_bucket_path); + const content_dir_bucket_path = '/content_dir_bucket'; + const content_dir_full_path = tmp_fs_root + content_dir_bucket_path; + const versions_path = path.join(full_path, '.versions/'); + const suspended_versions_path = path.join(suspended_full_path, '.versions/'); + let s3_uid1055; + let s3_uid6; + let s3_admin; + const accounts = []; + const disabled_key = 'disabled_key.txt'; + const disabled_key2 = 'disabled_key2.txt'; + const key1 = 'key1.txt'; + const copied_key1 = 'copied_key1.txt'; + const copied_key5 = 'copied_key5.txt'; + + const dir1 = 'dir1/'; + const nested_key1 = dir1 + key1; + const body1 = 'AAAAABBBBBCCCCC'; + const body2 = body1 + 'DDDDD'; + const body3 = body2 + 'EEEEE'; + + let key1_ver1; + let key1_cur_ver; + const mpu_key1 = 'mpu_key1.txt'; + const dir1_versions_path = path.join(full_path, dir1, '.versions/'); + const suspended_dir1_versions_path = path.join(suspended_full_path, dir1, '.versions/'); + + const dir_path_nested = 'photos/animals/January/'; + const dir_path_complete = 'animal/mammals/dog/'; + const nested_key_level3 = path.join(dir_path_nested, 'cat.jpeg'); + + mocha.before(async function() { + this.timeout(0); // eslint-disable-line no-invalid-this + if (invalid_nsfs_root_permissions()) this.skip(); // eslint-disable-line no-invalid-this + // create paths + await fs_utils.create_fresh_path(tmp_fs_root, 0o777); + await fs_utils.create_fresh_path(full_path, 0o770); + await fs_utils.file_must_exist(full_path); + await fs_utils.create_fresh_path(disabled_full_path, 0o770); + await fs_utils.file_must_exist(disabled_full_path); + await fs_utils.create_fresh_path(suspended_full_path, 0o770); + await fs_utils.file_must_exist(suspended_full_path); + await fs_utils.create_fresh_path(nested_keys_full_path, 0o770); + await fs_utils.file_must_exist(nested_keys_full_path); + await fs_utils.create_fresh_path(content_dir_full_path, 0o770); + await fs_utils.file_must_exist(content_dir_full_path); + if (is_nc_coretest) { + const { uid, gid } = get_admin_mock_account_details(); + await set_path_permissions_and_owner(full_path, { uid, gid }, 0o700); + await set_path_permissions_and_owner(disabled_full_path, { uid, gid }, 0o700); + await set_path_permissions_and_owner(suspended_full_path, { uid, gid }, 0o700); + await set_path_permissions_and_owner(nested_keys_full_path, { uid, gid }, 0o700); + await set_path_permissions_and_owner(content_dir_full_path, { uid, gid }, 0o700); + } + // export dir as a bucket + await rpc_client.pool.create_namespace_resource({ + name: nsr, + nsfs_config: { + fs_root_path: tmp_fs_root, + } + }); + const obj_nsr = { resource: nsr, path: bucket_path }; + await rpc_client.bucket.create_bucket({ + name: bucket_name, + namespace: { + read_resources: [obj_nsr], + write_resource: obj_nsr + } + }); + const disabled_nsr = { resource: nsr, path: disabled_bucket_path }; + await rpc_client.bucket.create_bucket({ + name: disabled_bucket_name, + namespace: { + read_resources: [disabled_nsr], + write_resource: disabled_nsr + } + }); + const suspended_nsr = { resource: nsr, path: suspended_bucket_path }; + await rpc_client.bucket.create_bucket({ + name: suspended_bucket_name, + namespace: { + read_resources: [suspended_nsr], + write_resource: suspended_nsr + } + }); + const nested_keys_nsr = { resource: nsr, path: nested_keys_bucket_path }; + await rpc_client.bucket.create_bucket({ + name: nested_keys_bucket_name, + namespace: { + read_resources: [nested_keys_nsr], + write_resource: nested_keys_nsr + } + }); + const content_dir_nsr = { resource: nsr, path: content_dir_bucket_path }; + await rpc_client.bucket.create_bucket({ + name: content_dir_bucket_name, + namespace: { + read_resources: [content_dir_nsr], + write_resource: content_dir_nsr + } + }); + const policy = { + Version: '2012-10-17', + Statement: [{ + Sid: 'id-1', + Effect: 'Allow', + Principal: { AWS: "*" }, + Action: ['s3:*'], + Resource: [`arn:aws:s3:::*`] + } + ] + }; + CORETEST_ENDPOINT = coretest.get_http_address(); + // create accounts + let res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path_param, { admin: true }); + s3_admin = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await s3_admin.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(policy) + }); + + await s3_admin.putBucketPolicy({ + Bucket: disabled_bucket_name, + Policy: JSON.stringify(policy) + }); + + await s3_admin.putBucketPolicy({ + Bucket: suspended_bucket_name, + Policy: JSON.stringify(policy) + }); + + await s3_admin.putBucketPolicy({ + Bucket: nested_keys_bucket_name, + Policy: JSON.stringify(policy) + }); + + await s3_admin.putBucketPolicy({ + Bucket: content_dir_bucket_name, + Policy: JSON.stringify(policy) + }); + + res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path_param, { uid: 1055, gid: 1055 }); + s3_uid1055 = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + accounts.push(res.email); + + res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path_param); + s3_uid6 = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + accounts.push(res.email); + }); + + mocha.after(async () => { + this.timeout(0); // eslint-disable-line no-invalid-this + fs_utils.folder_delete(tmp_fs_root); + for (const email of accounts) { + await rpc_client.account.delete_account({ email }); + } + }); + + mocha.describe('put/get versioning', function() { + mocha.it('put object - versioning disabled - to be enabled', async function() { + await s3_uid6.putObject({ Bucket: bucket_name, Key: disabled_key, Body: body1 }); + }); + + mocha.it('put object - versioning disabled - to be suspended', async function() { + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: disabled_key, Body: body1 }); + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: disabled_key2, Body: body1 }); + }); + + mocha.it('put object - versioning disabled bucket', async function() { + await s3_uid6.putObject({ Bucket: disabled_bucket_name, Key: disabled_key, Body: body1 }); + }); + + mocha.it('set bucket versioning - Enabled - should fail - no permissions', async function() { + try { + await s3_uid1055.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + assert.fail(`put bucket versioning succeeded for account without permissions`); + } catch (err) { + assert.equal(err.Code, 'AccessDenied'); + } + }); + + mocha.it('set bucket versioning - Enabled - admin - should fail - no permissions', async function() { + // on NC env - s3_admin is a regular account created for being the owner of buckets created using rpc calls (which are coverted to cli) + if (is_nc_coretest) return; + try { + await s3_admin.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + assert.fail(`put bucket versioning succeeded for account without permissions`); + } catch (err) { + assert.equal(err.Code, 'AccessDenied'); + } + }); + + mocha.it('set bucket versioning - Enabled', async function() { + await s3_uid6.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const res = await s3_uid6.getBucketVersioning({ Bucket: bucket_name }); + assert.equal(res.Status, 'Enabled'); + }); + + mocha.it('set bucket versioning - Suspended - should fail - no permissions', async function() { + try { + await s3_uid1055.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + assert.fail(`put bucket versioning succeeded for account without permissions`); + } catch (err) { + assert.equal(err.Code, 'AccessDenied'); + } + }); + + mocha.it('set bucket versioning - Suspended - admin - should fail - no permissions', async function() { + // on NC env - s3_admin is a regular account created for being the owner of buckets created via rpc calls (which are coverted to cli) + if (is_nc_coretest) return; + try { + await s3_admin.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + assert.fail(`put bucket versioning succeeded for account without permissions`); + } catch (err) { + assert.equal(err.Code, 'AccessDenied'); + } + }); + + mocha.it('set bucket versioning - Suspended', async function() { + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_uid6.getBucketVersioning({ Bucket: suspended_bucket_name }); + assert.equal(res.Status, 'Suspended'); + }); + }); + + mocha.describe('versioning enabled', function() { + + mocha.describe('put object', function() { + + mocha.it('put object 1st time - no existing objects in bucket', async function() { + const res = await s3_uid6.putObject({ Bucket: bucket_name, Key: key1, Body: body1 }); + const comp_res = await compare_version_ids(full_path, key1, res.VersionId); + assert.ok(comp_res); + await fs_utils.file_must_not_exist(versions_path); + key1_ver1 = res.VersionId; + }); + + mocha.it('put object 2nd time - versioning enabled', async function() { + const prev_version_id = await stat_and_get_version_id(full_path, key1); + const res = await s3_uid6.putObject({ Bucket: bucket_name, Key: key1, Body: body2 }); + const comp_res = await compare_version_ids(full_path, key1, res.VersionId, prev_version_id); + assert.ok(comp_res); + const exist = await version_file_exists(full_path, key1, '', prev_version_id); + assert.ok(exist); + key1_cur_ver = res.VersionId; + }); + + mocha.it('put object 2nd time - versioning enabled - disabled key', async function() { + const prev_version_id = await stat_and_get_version_id(full_path, disabled_key); + const res = await s3_uid6.putObject({ Bucket: bucket_name, Key: disabled_key, Body: body2 }); + const comp_res = await compare_version_ids(full_path, disabled_key, res.VersionId, prev_version_id); + assert.ok(comp_res); + const exist = await version_file_exists(full_path, disabled_key, '', 'null'); + assert.ok(exist); + }); + + mocha.it('put object 3rd time - versioning enabled - disabled key', async function() { + const prev_version_id = await stat_and_get_version_id(full_path, disabled_key); + const res = await s3_uid6.putObject({ Bucket: bucket_name, Key: disabled_key, Body: body3 }); + const comp_res = await compare_version_ids(full_path, disabled_key, res.VersionId, prev_version_id); + assert.ok(comp_res); + let exist = await version_file_exists(full_path, disabled_key, '', prev_version_id); + assert.ok(exist); + exist = await version_file_exists(full_path, disabled_key, '', 'null'); + assert.ok(exist); + }); + mocha.it('put object 1st time - versioning enabled - nested', async function() { + const res = await s3_uid6.putObject({ Bucket: bucket_name, Key: nested_key1, Body: body1 }); + await fs_utils.file_must_not_exist(dir1_versions_path); + const comp_res = await compare_version_ids(full_path, nested_key1, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('put object 2nd time - versioning enabled - nested', async function() { + const prev_version_id = await stat_and_get_version_id(full_path, nested_key1); + const res = await s3_uid6.putObject({ Bucket: bucket_name, Key: nested_key1, Body: body2 }); + const version_path_nested = path.join(full_path, dir1, HIDDEN_VERSIONS_PATH); + const nested_versions_dir_exist = await fs_utils.file_exists(version_path_nested); + assert.ok(nested_versions_dir_exist); + const comp_res = await compare_version_ids(full_path, nested_key1, res.VersionId, prev_version_id); + assert.ok(comp_res); + const exist = await version_file_exists(full_path, key1, dir1, prev_version_id); + assert.ok(exist); + }); + + mocha.it('put object - versioning enabled - nested key (more than 1 level)', async function() { + await s3_uid6.putBucketVersioning({ Bucket: nested_keys_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const res_put_version_ids = new Set(); // array for the versions we expect in .version/ directory + let res = await s3_uid6.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level3, Body: body1 }); + res_put_version_ids.add(res.VersionId); + // only after the second PUT object we expect to have the .versions under the parent directory of the file + res = await s3_uid6.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level3, Body: body2}); + res_put_version_ids.add(res.VersionId); + await s3_uid6.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level3, Body: body3}); // latest version + const version_path_nested = path.join(nested_keys_full_path, dir_path_nested, HIDDEN_VERSIONS_PATH); + const exist = await fs_utils.file_exists(version_path_nested); + assert.ok(exist); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, version_path_nested); + const prefix = 'cat.jpeg_'; + const same_prefix = versions.every(version => version.name.includes(prefix)); + assert.ok(same_prefix); + // compare the response version_id to the file names in .version/ directory + const res_version_ids = new Set(); + for (const version of versions) { + const version_id = version.name.slice(prefix.length); + res_version_ids.add(version_id); + } + assert.deepEqual(res_version_ids, res_put_version_ids); + }); + + // dir_content is not supported in versioning, but we want to make sure there are no errors + mocha.it('put object - versioning enabled - directory content', async function() { + await s3_uid6.putBucketVersioning({ Bucket: nested_keys_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key_as_dir_content = '/a/b/c/'; + const size = 4; // in the original issue the error started on the 3rd PUT of dir content + const res_put_object = []; + for (let i = 0; i < size; i++) { + const res = await s3_uid6.putObject({ Bucket: nested_keys_bucket_name, Key: key_as_dir_content, Body: body1 }); + res_put_object.push(res); + } + assert(res_put_object.length === size); + const check_version_id_exists = res_put_object.every(res => res.VersionId !== undefined); + assert.ok(check_version_id_exists); + }); + }); + + mocha.describe('mpu object', function() { + mocha.it('mpu object 1st time - versioning enabled', async function() { + const mpu_res = await s3_uid6.createMultipartUpload({ Bucket: bucket_name, Key: mpu_key1 }); + const upload_id = mpu_res.UploadId; + const part1 = await s3_uid6.uploadPart({ + Bucket: bucket_name, Key: mpu_key1, Body: body1, UploadId: upload_id, PartNumber: 1 }); + const res = await s3_uid6.completeMultipartUpload({ + Bucket: bucket_name, + Key: mpu_key1, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }] + } + }); + const comp_res = await compare_version_ids(full_path, mpu_key1, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('mpu object 2nd time - versioning enabled', async function() { + const prev_version_id = await stat_and_get_version_id(full_path, mpu_key1); + const mpu_res = await s3_uid6.createMultipartUpload({ Bucket: bucket_name, Key: mpu_key1 }); + const upload_id = mpu_res.UploadId; + const part1 = await s3_uid6.uploadPart({ + Bucket: bucket_name, Key: mpu_key1, Body: body1, UploadId: upload_id, PartNumber: 1 }); + const part2 = await s3_uid6.uploadPart({ + Bucket: bucket_name, Key: mpu_key1, Body: body2, UploadId: upload_id, PartNumber: 2 }); + const res = await s3_uid6.completeMultipartUpload({ + Bucket: bucket_name, + Key: mpu_key1, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }, + { + ETag: part2.ETag, + PartNumber: 2 + }] + } + }); + const comp_res = await compare_version_ids(full_path, mpu_key1, res.VersionId, prev_version_id); + assert.ok(comp_res); + const exist = await version_file_exists(full_path, mpu_key1, '', prev_version_id); + assert.ok(exist); + }); + }); + }); + + mocha.describe('content directory', function() { + const put_object_key_new = 'put_object_key_new/'; + const put_object_key = 'put_object_key/'; + const put_object_empty_key = 'put_object_empty_key/'; + const put_object_key_suspended = 'put_object_key_suspended/'; + const put_object_empty_key_suspended = 'put_object_empty_key_suspended/'; + const head_object_key = 'head_object_key/'; + const get_object_key = 'get_object_key/'; + const delete_latest_object_key = 'delete_latest_object_key/'; + const delete_latest_object_key_suspended = 'delete_latest_object_key_suspended/'; + + mocha.before('put content directory on version disabled bucket', async function() { + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: put_object_key, Body: body1 }); + await create_empty_content_dir(DEFAULT_FS_CONFIG, content_dir_full_path, put_object_empty_key); + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: put_object_key_suspended, Body: body1 }); + await create_empty_content_dir(DEFAULT_FS_CONFIG, content_dir_full_path, put_object_empty_key_suspended); + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: head_object_key, Body: body1 }); + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: get_object_key, Body: body1 }); + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: delete_latest_object_key, Body: body1 }); + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: delete_latest_object_key_suspended, Body: body1 }); + + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('content directory - put object 1st put - versioning enabled', async function() { + const res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: put_object_key_new, Body: body1 }); + const comp_res = await compare_version_ids(content_dir_full_path, put_object_key_new + '.folder', res.VersionId); + assert.ok(comp_res); + await fs_utils.file_must_not_exist(path.join(content_dir_full_path, put_object_key_new, '.versions')); + }); + + mocha.it('content directory - put object 2nd put - versioning enabled', async function() { + const prev_version_id = await stat_and_get_version_id(content_dir_full_path, put_object_key_new + '.folder'); + const res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: put_object_key_new, Body: body2 }); + const comp_res = await compare_version_ids(content_dir_full_path, put_object_key_new + '.folder', res.VersionId, prev_version_id); + assert.ok(comp_res); + const exist = await version_file_exists(content_dir_full_path, '.folder', put_object_key_new, prev_version_id); + assert.ok(exist); + }); + + mocha.it('content directory - put object after disabled conent directory - versioning enabled', async function() { + const prev_version_id = 'null'; + const res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: put_object_key, Body: body2 }); + const comp_res = await compare_version_ids(content_dir_full_path, put_object_key + '.folder', res.VersionId, prev_version_id); + assert.ok(comp_res); + const exist = await version_file_exists(content_dir_full_path, '.folder', put_object_key, prev_version_id); + assert.ok(exist); + const dir_xattr_res = await check_no_user_attributes(content_dir_full_path, put_object_key); + assert.ok(dir_xattr_res); + }); + + mocha.it('content directory - put object after disabled empty content directory - versioning enabled', async function() { + const prev_version_id = 'null'; + const res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: put_object_empty_key, Body: body2 }); + const comp_res = await compare_version_ids(content_dir_full_path, put_object_empty_key + '.folder', res.VersionId, prev_version_id); + assert.ok(comp_res); + const exist = await version_file_exists(content_dir_full_path, '.folder', put_object_empty_key, prev_version_id); + assert.ok(exist); + const dir_xattr_res = await check_no_user_attributes(content_dir_full_path, put_object_empty_key); + assert.ok(dir_xattr_res); + }); + + mocha.it('content directory - should upload multipart object with versioning enabled', async function() { + const key = 'mpu_key/'; + const res_mpu = await s3_uid6.createMultipartUpload({ Bucket: content_dir_bucket_name, Key: key }); + const upload_id = res_mpu.UploadId; + const part1 = await s3_uid6.uploadPart({ + Bucket: content_dir_bucket_name, Key: key, Body: body1, UploadId: upload_id, PartNumber: 1 }); + const part2 = await s3_uid6.uploadPart({ + Bucket: content_dir_bucket_name, Key: key, Body: body2, UploadId: upload_id, PartNumber: 2 }); + const res_cmpu = await s3_uid6.completeMultipartUpload({ + Bucket: content_dir_bucket_name, + Key: key, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }, + { + ETag: part2.ETag, + PartNumber: 2 + }] + } + }); + + const comp_res = await compare_version_ids(content_dir_full_path, key + '.folder', res_cmpu.VersionId); + assert.ok(comp_res); + await fs_utils.file_must_not_exist(path.join(content_dir_full_path, key, '.versions')); + }); + + mocha.it('content directory - put object after disabled conent directory - versioning suspended', async function() { + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + + const prev_version_id = 'null'; + const res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: put_object_key_suspended, Body: body2 }); + const comp_res = await compare_version_ids(content_dir_full_path, put_object_key_suspended + '.folder', res.VersionId, undefined, false); + assert.ok(comp_res); + const not_exist = await version_file_must_not_exists(content_dir_full_path, '.folder', put_object_key_suspended, prev_version_id); + assert.ok(not_exist); + const dir_xattr_res = await check_no_user_attributes(content_dir_full_path, put_object_key_suspended); + assert.ok(dir_xattr_res); + }); + + mocha.it('content directory - put object after disabled empty conent directory - versioning suspended', async function() { + const prev_version_id = 'null'; + const res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: put_object_empty_key_suspended, Body: body2 }); + const comp_res = await compare_version_ids(content_dir_full_path, put_object_empty_key_suspended + '.folder', res.VersionId, undefined, false); + assert.ok(comp_res); + const not_exist = await version_file_must_not_exists(content_dir_full_path, '.folder', put_object_empty_key_suspended, prev_version_id); + assert.ok(not_exist); + const dir_xattr_res = await check_no_user_attributes(content_dir_full_path, put_object_empty_key_suspended); + assert.ok(dir_xattr_res); + }); + + mocha.it('head object - content directory - versioning enabled', async function() { + // Enable versioning + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + + let res = await s3_uid6.headObject({ Bucket: content_dir_bucket_name, Key: head_object_key }); + assert.equal(res.VersionId, NULL_VERSION_ID); + // Put object again to create a new version + const put_res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: head_object_key, Body: body2 }); + + // Get the latest version + res = await s3_uid6.headObject({ Bucket: content_dir_bucket_name, Key: head_object_key }); + assert.equal(res.VersionId, put_res.VersionId); + + // Get the previous version + res = await s3_uid6.headObject({ Bucket: content_dir_bucket_name, Key: head_object_key, VersionId: NULL_VERSION_ID }); + assert.equal(res.VersionId, NULL_VERSION_ID); + + // Get latest by version id + res = await s3_uid6.headObject({ Bucket: content_dir_bucket_name, Key: head_object_key, VersionId: put_res.VersionId }); + assert.equal(res.VersionId, put_res.VersionId); + + }); + + mocha.it('content directory - get object - versioning enabled', async function() { + let res = await s3_uid6.getObject({ Bucket: content_dir_bucket_name, Key: get_object_key }); + let body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, body1); + // Put object again to create a new version + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: get_object_key, Body: body2 }); + + // Get the latest version + res = await s3_uid6.getObject({ Bucket: content_dir_bucket_name, Key: get_object_key }); + body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, body2); + + // Get the previous version + res = await s3_uid6.getObject({ Bucket: content_dir_bucket_name, Key: get_object_key, VersionId: NULL_VERSION_ID }); + body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, body1); + + // Put object again to create another new version + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: get_object_key, Body: body3 }); + + // Get the latest version + res = await s3_uid6.getObject({ Bucket: content_dir_bucket_name, Key: get_object_key }); + body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, body3); + }); + + mocha.it('content directory - delete latest object - versioning enabled', async function() { + const res = await s3_uid6.deleteObject({ Bucket: content_dir_bucket_name, Key: delete_latest_object_key }); + assert.equal(res.DeleteMarker, true); + + await fs_utils.file_must_not_exist(path.join(content_dir_full_path, delete_latest_object_key + '.folder')); + const exist = await version_file_exists(content_dir_full_path, '.folder', delete_latest_object_key, NULL_VERSION_ID); + assert.ok(exist); + const max_version = await find_max_version_past(content_dir_full_path, '.folder', delete_latest_object_key); + assert.equal(max_version, res.VersionId); + const is_dm = await is_delete_marker(content_dir_full_path, delete_latest_object_key, '.folder', max_version); + assert.ok(is_dm); + }); + + mocha.it('content directory - delete latest object - versioning suspended', async function() { + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + + const res = await s3_uid6.deleteObject({ Bucket: content_dir_bucket_name, Key: delete_latest_object_key_suspended }); + assert.equal(res.DeleteMarker, true); + + await fs_utils.file_must_not_exist(path.join(content_dir_full_path, delete_latest_object_key_suspended + '.folder')); + const exist = await version_file_exists(content_dir_full_path, '.folder', delete_latest_object_key_suspended, NULL_VERSION_ID); + assert.ok(exist); + }); + + mocha.it('content directory - delete specific version', async function() { + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + + const key = "delete_specific_version_key/"; + const res1 = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: key, Body: body1 }); + const res2 = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: key, Body: body1 }); + const res3 = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: key, Body: body1 }); + + const ver1_path = path.join(content_dir_full_path, key, '.versions/.folder_' + res1.VersionId); + await fs_utils.file_must_exist(ver1_path); + + await s3_uid6.deleteObject({ Bucket: content_dir_bucket_name, Key: key, VersionId: res1.VersionId }); + + await fs_utils.file_must_not_exist(ver1_path); + + const delete_res = await s3_uid6.deleteObject({ Bucket: content_dir_bucket_name, Key: key, VersionId: res3.VersionId }); + assert.equal(delete_res.VersionId, res3.VersionId); + + const get_res = await s3_uid6.getObject({ Bucket: content_dir_bucket_name, Key: key }); + assert.equal(get_res.VersionId, res2.VersionId); + }); + + mocha.it('content directory - object tagging - versioning enabled', async function() { + const dir_tagging_key = 'dir_tagging/'; + const tag_set1 = { TagSet: [{ Key: "key1", Value: "Value1" }] }; + const tag_set2 = { TagSet: [{ Key: "key2", Value: "Value2" }] }; + + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const res_put = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, Body: body1 }); + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, Body: body1 }); + const version_id = res_put.VersionId; + + await s3_uid6.putObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, Tagging: tag_set1 }); + let res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key }); + assert.deepEqual(res.TagSet, tag_set1.TagSet); + + await s3_uid6.putObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, + Tagging: tag_set2, VersionId: version_id }); + res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key }); + assert.notDeepEqual(res.TagSet, tag_set2); + + const version_res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, + Key: dir_tagging_key, VersionId: version_id }); + assert.deepEqual(version_res.TagSet, tag_set2.TagSet); + + await s3_uid6.deleteObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key }); + res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key }); + assert.equal(res.TagSet.length, 0); + + await s3_uid6.deleteObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, VersionId: version_id }); + res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, VersionId: version_id }); + assert.equal(res.TagSet.length, 0); + }); + + mocha.it('content directory - object tagging - versioning suspended', async function() { + const dir_tagging_key = 'dir_tagging/'; + const tag_set1 = { TagSet: [{ Key: "key1", Value: "Value1" }] }; + const tag_set2 = { TagSet: [{ Key: "key2", Value: "Value2" }] }; + + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res_put = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, Body: body1 }); + await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, Body: body1 }); + const version_id = res_put.VersionId; + + await s3_uid6.putObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, Tagging: tag_set1 }); + let res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key }); + assert.deepEqual(res.TagSet, tag_set1.TagSet); + + await s3_uid6.putObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, + Tagging: tag_set2, VersionId: version_id }); + res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key }); + assert.notDeepEqual(res.TagSet, tag_set2); + + const version_res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, + Key: dir_tagging_key, VersionId: version_id }); + assert.deepEqual(version_res.TagSet, tag_set2.TagSet); + + await s3_uid6.deleteObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key }); + res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key }); + assert.equal(res.TagSet.length, 0); + + await s3_uid6.deleteObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, VersionId: version_id }); + res = await s3_uid6.getObjectTagging({ Bucket: content_dir_bucket_name, Key: dir_tagging_key, VersionId: version_id }); + assert.equal(res.TagSet.length, 0); + }); + + mocha.it('content directory - list object', async function() { + const res = await s3_uid6.listObjects({ Bucket: content_dir_bucket_name }); + assert.equal(res.Contents.length, 10); + }); + + mocha.it('content directory - list object versions - enabled mode', async function() { + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key = "list_key/"; + const version_ids = new Set(); + let push_res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: key, Body: body1 }); + version_ids.add(push_res.VersionId); + push_res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: key, Body: body1 }); + version_ids.add(push_res.VersionId); + push_res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: key, Body: body1 }); + version_ids.add(push_res.VersionId); + const list_res = await s3_uid6.listObjectVersions({ Bucket: content_dir_bucket_name }); + const key_list_versions = list_res.Versions.filter(v => v.Key === key); + assert.equal(key_list_versions.length, version_ids.size); + key_list_versions.forEach(v => { + assert(version_ids.has(v.VersionId)); + if (v.VersionId === push_res.VersionId) { + assert.equal(v.IsLatest, true); + assert(!v.Key.includes(config.NSFS_FOLDER_OBJECT_NAME)); + assert(!v.VersionId.includes(config.NSFS_FOLDER_OBJECT_NAME)); + } + }); + }); + + mocha.it('content directory - list object versions - delete marker', async function() { + const key = "list_key/"; + const delete_res = await s3_uid6.deleteObject({ Bucket: content_dir_bucket_name, Key: key }); + const list_res = await s3_uid6.listObjectVersions({ Bucket: content_dir_bucket_name }); + const key_list_versions = list_res.Versions.filter(v => v.Key === key); + assert.equal(key_list_versions.length, 3); + assert.equal(list_res.DeleteMarkers.length, 3); // 2 previous delete markers + 1 new delete marker + let is_delete_marker_present = false; + for (const delete_marker of list_res.DeleteMarkers) { + if (delete_marker.versionId === delete_res.created_version_id) { + is_delete_marker_present = true; + assert.equal(delete_marker.versionId, delete_res.created_version_id); + assert.equal(delete_marker.IsLatest, true); + assert(!delete_marker.Key.includes(config.NSFS_FOLDER_OBJECT_NAME)); + assert(!delete_marker.VersionId.includes(config.NSFS_FOLDER_OBJECT_NAME)); + } + } + assert(is_delete_marker_present); + }); + + mocha.it('content directory - list object versions - suspended mode', async function() { + const key = "list_key/"; + await s3_uid6.putBucketVersioning({ Bucket: content_dir_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const push_res = await s3_uid6.putObject({ Bucket: content_dir_bucket_name, Key: key, Body: body1 }); + const list_res = await s3_uid6.listObjectVersions({ Bucket: content_dir_bucket_name }); + const key_list_versions = list_res.Versions.filter(v => v.Key === key); + assert.equal(key_list_versions.length, 4); //3 from before + 1 new key + key_list_versions.forEach(v => { + if (v.VersionId === push_res.VersionId) { + assert.equal(v.VersionId, 'null'); + assert.equal(v.IsLatest, true); + assert(!v.key.includes(config.NSFS_FOLDER_OBJECT_NAME)); + assert(!v.VersionId.includes(config.NSFS_FOLDER_OBJECT_NAME)); + } + }); + }); + }); + + // The res of putBucketVersioning is different depends on the versioning state: + // * Enabled - Etag and VersionId; + // * Disabled and Suspended - only Etag. + // Hence, 'res.VersionId' would be undefined when versioning is Suspended. + // (We would still prefer using it as a parameter instead of explicitly send undefined to function compare_version_ids). + mocha.describe('put object - versioning suspended - simple cases', function() { + const is_enabled = false; + const key2 = 'water.txt'; + + // It was already set to Suspended in previous test (just to be on the safe side) + mocha.before('set bucket versioning - Suspended', async function() { + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + }); + + mocha.describe('put object', function() { + + mocha.it('put object 1st time - versioning suspended', async function() { + const res = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key2, Body: body1 }); + const comp_res = await compare_version_ids(suspended_full_path, key2, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + const exist = await fs_utils.file_not_exists(suspended_versions_path); + assert.ok(exist); + }); + + mocha.it('put object 2nd time - versioning suspended', async function() { + const res = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key2, Body: body2 }); + const comp_res = await compare_version_ids(suspended_full_path, key2, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + const exist = await fs_utils.file_not_exists(suspended_versions_path); + assert.ok(exist); + }); + + mocha.it('put object 2nd time - versioning suspended - disabled key', async function() { + const res = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: disabled_key, Body: body2 }); + const comp_res = await compare_version_ids(suspended_full_path, disabled_key, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + const exist = await fs_utils.file_not_exists(suspended_versions_path); + assert.ok(exist); + }); + + mocha.it('put object 1st time - versioning suspended - nested', async function() { + const res = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: nested_key1, Body: body1 }); + const exist = await fs_utils.file_not_exists(suspended_dir1_versions_path); + assert.ok(exist); + const comp_res = await compare_version_ids(suspended_full_path, nested_key1, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + }); + + mocha.it('put object 2nd time - versioning suspended - nested', async function() { + const res = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: nested_key1, Body: body2 }); + const version_path_nested = path.join(full_path, dir1, HIDDEN_VERSIONS_PATH); + const nested_versions_dir_exist = await fs_utils.file_exists(version_path_nested); + assert.ok(nested_versions_dir_exist); + const exist = await fs_utils.file_not_exists(suspended_dir1_versions_path); + assert.ok(exist); + const comp_res = await compare_version_ids(suspended_full_path, nested_key1, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + }); + }); + + // mpu = multipart upload + mocha.describe('mpu object', function() { + + mocha.it('mpu object 1st time - versioning suspended', async function() { + const mpu_res = await s3_uid6.createMultipartUpload({ Bucket: suspended_bucket_name, Key: mpu_key1 }); + const upload_id = mpu_res.UploadId; + const part1 = await s3_uid6.uploadPart({ + Bucket: suspended_bucket_name, Key: mpu_key1, Body: body1, UploadId: upload_id, PartNumber: 1 }); + const res = await s3_uid6.completeMultipartUpload({ + Bucket: suspended_bucket_name, + Key: mpu_key1, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }] + } + }); + const comp_res = await compare_version_ids(suspended_full_path, mpu_key1, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + }); + + mocha.it('mpu object 2nd time - versioning suspended', async function() { + const prev_version_id = await stat_and_get_version_id(suspended_full_path, mpu_key1); + const mpu_res = await s3_uid6.createMultipartUpload({ Bucket: suspended_bucket_name, Key: mpu_key1 }); + const upload_id = mpu_res.UploadId; + const part1 = await s3_uid6.uploadPart({ + Bucket: suspended_bucket_name, Key: mpu_key1, Body: body1, UploadId: upload_id, PartNumber: 1 }); + const part2 = await s3_uid6.uploadPart({ + Bucket: suspended_bucket_name, Key: mpu_key1, Body: body2, UploadId: upload_id, PartNumber: 2 }); + const res = await s3_uid6.completeMultipartUpload({ + Bucket: suspended_bucket_name, + Key: mpu_key1, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }, + { + ETag: part2.ETag, + PartNumber: 2 + }] + } + }); + const comp_res = await compare_version_ids(suspended_full_path, mpu_key1, res.VersionId, prev_version_id, is_enabled); + assert.ok(comp_res); + const exist = await version_file_must_not_exists(suspended_full_path, mpu_key1, '', prev_version_id); + assert.ok(exist); + }); + }); + }); + + mocha.describe('put object - versioning suspended - should have only one null version ID', function() { + + mocha.before('put object 2nd time - versioning enabled', async function() { + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: disabled_key2, Body: body2 }); + }); + + // Start with null version ID in .versions/ directory (this version that was created when versioning was disabled) + // and unique version ID as latest version (this version that was created when versioning was enabled). + mocha.it('put object 3rd time - versioning suspended', async function() { + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const is_enabled = false; + let exist = await version_file_exists(suspended_full_path, disabled_key2, '', 'null'); + assert.ok(exist); + const prev_version_id = await stat_and_get_version_id(suspended_full_path, disabled_key2); + const res = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: disabled_key2, Body: body3 }); + const comp_res = await compare_version_ids(suspended_full_path, disabled_key2, res.VersionId, prev_version_id, is_enabled); + assert.ok(comp_res); + // unique version ID should move to .versions/ directory + exist = await version_file_exists(suspended_full_path, disabled_key2, '', prev_version_id); + assert.ok(exist); + // latest version is null, we cannot have another null in .versions/ directory + exist = await version_file_must_not_exists(suspended_full_path, disabled_key2, '', 'null'); + assert.ok(exist); + }); + }); + + mocha.describe('put object - versioning suspended - unique version ID should move to .version/ directory', function() { + const key2 = 'soda.txt'; + + mocha.before('put object - versioning enabled', async function() { + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key2, Body: body1 }); + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + }); + + mocha.it('put object 2nd time - versioning suspended', async function() { + const is_enabled = false; + const prev_version_id = await stat_and_get_version_id(suspended_full_path, key2); + const comp_prev = check_enable_version_format(prev_version_id); + assert.ok(comp_prev); + const res = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key2, Body: body2 }); + const comp_res = await compare_version_ids(suspended_full_path, key2, res.VersionId, prev_version_id, is_enabled); + assert.ok(comp_res); + // unique version ID should move to .versions/ directory + const exist = await version_file_exists(suspended_full_path, key2, '', prev_version_id); + assert.ok(exist); + }); + + mocha.it('put object 3rd time - versioning suspended', async function() { + const is_enabled = false; + const prev_version_id = await stat_and_get_version_id(suspended_full_path, key2); + const comp_prev = check_null_version_id(prev_version_id); + assert.ok(comp_prev); + const res = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key2, Body: body3 }); + const comp_res = await compare_version_ids(suspended_full_path, key2, res.VersionId, prev_version_id, is_enabled); + assert.ok(comp_res); + // the new version is with a null version ID (the previous was overridden) + const exist = await version_file_must_not_exists(suspended_full_path, key2, '', prev_version_id); + assert.ok(exist); + }); + }); + + mocha.describe('copy object - versioning enabled - nsfs copy fallback flow', function() { + + mocha.it('copy object - target bucket versioning enabled - 1st', async function() { + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: copied_key1, CopySource: `${bucket_name}/${key1}` }); + const comp_res = await compare_version_ids(full_path, copied_key1, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('copy object - target bucket versioning enabled - 2nd', async function() { + const prev_version_id = await stat_and_get_version_id(full_path, copied_key1); + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: copied_key1, CopySource: `${bucket_name}/${key1}` }); + const comp_res = await compare_version_ids(full_path, copied_key1, res.VersionId, prev_version_id); + assert.ok(comp_res); + const exist = await version_file_exists(full_path, copied_key1, '', prev_version_id); + assert.ok(exist); + }); + + mocha.it('copy object latest - source bucket versioning enabled', async function() { + const key = 'copied_key2.txt'; + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: key, + CopySource: `${bucket_name}/${key1}`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, bucket_name, key, body2); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(full_path, key, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('copy object latest & versionId - source bucket versioning enabled - version does not exist', async function() { + const key = 'copied_key3.txt'; + const non_exist_version = 'mtime-dsbfjb-ino-sjfhjsa'; + try { + await s3_uid6.copyObject({ Bucket: bucket_name, Key: key, + CopySource: `${bucket_name}/${key1}?versionId=${non_exist_version}`}); + assert.fail('copy non existing version should have failed'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('copy object latest & versionId - source bucket versioning enabled', async function() { + const key = 'copied_key3.txt'; + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: key, + CopySource: `${bucket_name}/${key1}?versionId=${key1_cur_ver}`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, bucket_name, key, body2); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(full_path, key, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('copy object version id - source bucket versioning enabled', async function() { + const key = 'copied_key4.txt'; + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: key, + CopySource: `${bucket_name}/${key1}?versionId=${key1_ver1}`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, bucket_name, key, body1); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(full_path, key, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('copy object version null - version is in .versions/', async function() { + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: copied_key5, + CopySource: `${bucket_name}/${disabled_key}?versionId=null`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, bucket_name, copied_key5, body1); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(full_path, copied_key5, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('copy object version null - no version null - should fail', async function() { + try { + await s3_uid6.copyObject({ Bucket: bucket_name, Key: copied_key5, + CopySource: `${bucket_name}/${key1}?versionId=null`}); + assert.fail('should have failed'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('copy object - version does not exist - should fail', async function() { + try { + await s3_uid6.copyObject({ Bucket: bucket_name, Key: copied_key5, + CopySource: `${bucket_name}/${key1}?versionId=mtime-123-ino-123`}); + assert.fail('should have failed'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('copy object version null from disabled bucket - version is in parent', async function() { + const key = 'copied_key6.txt'; + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: key, + CopySource: `${disabled_bucket_name}/${disabled_key}?versionId=null`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, bucket_name, key, body1); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(full_path, key, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('set bucket versioning - Enabled', async function() { + await s3_uid6.putBucketVersioning({ Bucket: disabled_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const res = await s3_uid6.getBucketVersioning({ Bucket: disabled_bucket_name }); + assert.equal(res.Status, 'Enabled'); + }); + + mocha.it('copy object version null from enabled bucket - version is in parent', async function() { + const key = 'copied_key7.txt'; + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: key, + CopySource: `${disabled_bucket_name}/${disabled_key}?versionId=null`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, bucket_name, key, body1); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(full_path, key, res.VersionId); + assert.ok(comp_res); + }); + + mocha.it('delete object latest - non existing version', async function() { + const non_exist_version = 'mtime-dsad-ino-sdfasd'; + const res = await s3_uid6.deleteObject({ Bucket: bucket_name, Key: disabled_key, VersionId: non_exist_version }); + assert.equal(res.VersionId, non_exist_version); + }); + + mocha.it('delete object latest - create dm & move latest -> .versions/', async function() { + const prev_version_id = await stat_and_get_version_id(full_path, disabled_key); + const max_version1 = await find_max_version_past(full_path, disabled_key, ''); + const res = await s3_uid6.deleteObject({ Bucket: bucket_name, Key: disabled_key }); + assert.equal(res.DeleteMarker, true); + + await fs_utils.file_must_not_exist(path.join(full_path, disabled_key)); + const exist = await version_file_exists(full_path, disabled_key, '', prev_version_id); + assert.ok(exist); + const max_version2 = await find_max_version_past(full_path, disabled_key, ''); + assert.notEqual(max_version2, max_version1); + const is_dm = await is_delete_marker(full_path, '', disabled_key, max_version2); + assert.ok(is_dm); + assert.equal(res.VersionId, max_version2); + }); + + mocha.it('delete object - create dm & move latest -> .versions/ - 1st', async function() { + const prev_version_id = await stat_and_get_version_id(full_path, key1); + const max_version1 = await find_max_version_past(full_path, key1, ''); + const res = await s3_uid6.deleteObject({ Bucket: bucket_name, Key: key1 }); + assert.equal(res.DeleteMarker, true); + + await fs_utils.file_must_not_exist(path.join(full_path, key1)); + const exist = await version_file_exists(full_path, key1, '', prev_version_id); + assert.ok(exist); + const max_version2 = await find_max_version_past(full_path, key1, ''); + assert.notEqual(max_version2, max_version1); + const is_dm = await is_delete_marker(full_path, '', key1, max_version2); + assert.ok(is_dm); + assert.equal(res.VersionId, max_version2); + }); + + mocha.it('delete object - create dm & move latest -> .versions/ - 2nd time', async function() { + const max_version1 = await find_max_version_past(full_path, key1, ''); + const res = await s3_uid6.deleteObject({ Bucket: bucket_name, Key: key1 }); + assert.equal(res.DeleteMarker, true); + + await fs_utils.file_must_not_exist(path.join(full_path, key1)); + const exist = await version_file_exists(full_path, key1, '', max_version1); + assert.ok(exist); + const max_version2 = await find_max_version_past(full_path, key1, ''); + assert.notEqual(max_version2, max_version1); + assert.equal(max_version2, res.VersionId); + const is_dm = await is_delete_marker(full_path, '', key1, max_version2); + assert.ok(is_dm); + }); + + mocha.it('delete object - versioning enabled - nested key (more than 1 level)', async function() { + const res = await s3_uid6.deleteObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level3 }); + assert.equal(res.DeleteMarker, true); + const exist = await fs_utils.file_not_exists(path.join(nested_keys_full_path, nested_key_level3)); + assert.ok(exist); + const version_path_nested = path.join(nested_keys_full_path, dir_path_nested, HIDDEN_VERSIONS_PATH); + const exist2 = await fs_utils.file_exists(version_path_nested); + assert.ok(exist2); + }); + + mocha.it('delete object - versioning enabled - nested key (more than 1 level) - delete inside directory', async function() { + const res = await s3_uid6.deleteObject({ Bucket: nested_keys_bucket_name, Key: dir_path_nested }); + assert.equal(res.DeleteMarker, true); + const version_path_nested = path.join(nested_keys_full_path, dir_path_nested, HIDDEN_VERSIONS_PATH); + const exist2 = await fs_utils.file_exists(version_path_nested); + assert.ok(exist2); + }); + + mocha.it('copy object version id - version matches mtime and inode', async function() { + const key = 'copied_key5.txt'; + const res = await s3_uid6.copyObject({ Bucket: bucket_name, Key: key, + CopySource: `${bucket_name}/${key1}?versionId=${key1_ver1}`}); + const obj_path = path.join(full_path, key); + const stat = await nb_native().fs.stat(DEFAULT_FS_CONFIG, obj_path); + const expcted_version = 'mtime-' + stat.mtimeNsBigint.toString(36) + '-ino-' + stat.ino.toString(36); + assert.equal(expcted_version, res.VersionId); + }); + + mocha.it('delete object - versioning enabled - nested key (more than 1 level)- delete partial directory', async function() { + //TODO key, key/ can't coexist. fails to create key delete marker. re-enable once implemented + this.skip(); // eslint-disable-line no-invalid-this + const parital_nested_directory = dir_path_complete.slice(0, -1); // the directory without the last slash + const folder_path_nested = path.join(nested_keys_full_path, dir_path_complete, NSFS_FOLDER_OBJECT_NAME); + const body_of_copied_key = 'make the lemon lemonade'; + await s3_uid6.putObject({ Bucket: nested_keys_bucket_name, Key: dir_path_complete, Body: body_of_copied_key }); + await fs_utils.file_must_exist(folder_path_nested); + await s3_uid6.deleteObject({ Bucket: nested_keys_bucket_name, Key: parital_nested_directory }); + await fs_utils.file_must_exist(folder_path_nested); + }); + + mocha.it('delete object - versioning enabled - nested key (more than 1 level)- delete complete directory', async function() { + const res = await s3_uid6.deleteObject({ Bucket: nested_keys_bucket_name, Key: dir_path_complete }); + assert.equal(res.DeleteMarker, true); + const folder_path_nested = path.join(nested_keys_full_path, dir_path_complete, NSFS_FOLDER_OBJECT_NAME); + await fs_utils.file_must_not_exist(folder_path_nested); + await fs_utils.file_must_exist(path.join(nested_keys_full_path, dir_path_complete)); //delete marker exist + }); + }); + + mocha.describe('copy object (latest version) - versioning suspended - nsfs copy fallback flow', function() { + const is_enabled = false; + const key_to_copy = 'orange.txt'; + const body_of_copied_key = 'drink orange juice'; + let copied_key = 'copied_orange.txt'; + + // It was already set to Suspended in previous test (just to be on the safe side) + mocha.before('set bucket versioning - Suspended', async function() { + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key_to_copy, Body: body_of_copied_key }); + }); + + mocha.it('copy object - target bucket versioning suspended - 1st', async function() { + const res = await s3_uid6.copyObject({ Bucket: suspended_bucket_name, Key: copied_key, + CopySource: `${suspended_bucket_name}/${key_to_copy}` }); + const comp_res = await compare_version_ids(suspended_full_path, copied_key, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + }); + + mocha.it('copy object - target bucket versioning suspended - 2nd', async function() { + const prev_version_id = await stat_and_get_version_id(suspended_full_path, copied_key); + const res = await s3_uid6.copyObject({ Bucket: suspended_bucket_name, Key: copied_key, + CopySource: `${suspended_bucket_name}/${key_to_copy}` }); + const comp_res = await compare_version_ids(suspended_full_path, copied_key, res.VersionId, prev_version_id, is_enabled); + assert.ok(comp_res); + const exist = await version_file_must_not_exists(suspended_full_path, copied_key, '', prev_version_id); + assert.ok(exist); + }); + + mocha.it('copy object latest - source bucket versioning suspended', async function() { + copied_key = 'copied_orange2.txt'; + const res = await s3_uid6.copyObject({ Bucket: suspended_bucket_name, Key: copied_key, + CopySource: `${suspended_bucket_name}/${key_to_copy}`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, suspended_bucket_name, copied_key, body_of_copied_key); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(suspended_full_path, copied_key, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + }); + }); + + mocha.describe('copy object by version id - versioning suspended - nsfs copy fallback flow', function() { + const is_enabled = false; + const key_to_copy = 'lemon.txt'; + const key_to_copy2 = 'lime.txt'; + const key_to_copy3 = 'avocado.txt'; + const body_of_copied_key = 'make the lemon lemonade'; + const body_of_copied_key_latest = 'lemons are yellow'; + let copied_key; + let version_id_to_copy; + let version_id_to_copy_latest; + + mocha.before('set bucket versioning - Enabled and then Suspended', async function() { + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key_to_copy3, Body: body1 }); + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const res_put = await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key_to_copy, + Body: body_of_copied_key }); + version_id_to_copy = res_put.VersionId; + const res_put_latest = await s3_uid6.putObject({ Bucket: suspended_bucket_name, + Key: key_to_copy, Body: body_of_copied_key_latest }); + version_id_to_copy_latest = res_put_latest.VersionId; + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key_to_copy2, Body: body_of_copied_key }); + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key_to_copy3, Body: body2 }); + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + }); + + mocha.it('copy object latest & versionId - source bucket versioning suspended', async function() { + copied_key = 'copied_lemon.txt'; + const res = await s3_uid6.copyObject({ Bucket: suspended_bucket_name, Key: copied_key, + CopySource: `${suspended_bucket_name}/${key_to_copy}?versionId=${version_id_to_copy_latest}`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, suspended_bucket_name, copied_key, body_of_copied_key_latest); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(suspended_full_path, copied_key, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + }); + + mocha.it('copy object version id - source bucket versioning suspended', async function() { + copied_key = 'copied_lemon2.txt'; + const res = await s3_uid6.copyObject({ Bucket: suspended_bucket_name, Key: copied_key, + CopySource: `${suspended_bucket_name}/${key_to_copy}?versionId=${version_id_to_copy}`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, suspended_bucket_name, copied_key, body_of_copied_key); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(suspended_full_path, copied_key, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + }); + + mocha.it('copy object version null - versioning suspended - no version null - should fail', async function() { + copied_key = 'copied_lime.txt'; + try { + await s3_uid6.copyObject({ Bucket: suspended_bucket_name, Key: copied_key, + CopySource: `${suspended_bucket_name}/${key_to_copy2}?versionId=null`}); + assert.fail('should have failed'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('copy object - version does not exist - should fail', async function() { + copied_key = 'copied_lime2.txt'; + try { + await s3_uid6.copyObject({ Bucket: suspended_bucket_name, Key: copied_key, + CopySource: `${suspended_bucket_name}/${key_to_copy2}?versionId=mtime-123-ino-123`}); + assert.fail('should have failed'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('copy object version null - version is in .versions/ - versioning suspended', async function() { + copied_key = 'copied_avocado.txt'; + const res = await s3_uid6.copyObject({ Bucket: suspended_bucket_name, Key: copied_key, + CopySource: `${suspended_bucket_name}/${key_to_copy3}?versionId=null`}); + const body_comp_res = await get_obj_and_compare_data(s3_uid6, suspended_bucket_name, copied_key, body1); + assert.ok(body_comp_res); + const comp_res = await compare_version_ids(suspended_full_path, copied_key, res.VersionId, undefined, is_enabled); + assert.ok(comp_res); + }); + }); + + mocha.describe('object tagging', function() { + + const tagging_key = "key_tagging"; + const tag_set1 = {TagSet: [{Key: "key1", Value: "Value1"}]}; + const tag_set2 = {TagSet: [{Key: "key2", Value: "Value2"}]}; + const tag_set3 = { TagSet: [{ "Key": "designation", "Value": "confidential" }, + { "Key": "department", "Value": "finance" }, + { "Key": "team", "Value": "payroll" }] + }; + let version_id; + + mocha.before(async function() { + await s3_uid6.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const res_put = await s3_uid6.putObject({ Bucket: bucket_name, Key: tagging_key, Body: body1 }); + await s3_uid6.putObject({ Bucket: bucket_name, Key: tagging_key, Body: body1 }); + version_id = res_put.VersionId; + }); + + mocha.it("put object tagging - no versionId", async function() { + await s3_uid6.putObjectTagging({ Bucket: bucket_name, Key: tagging_key, Tagging: tag_set1}); + const res = await s3_uid6.getObjectTagging({Bucket: bucket_name, Key: tagging_key}); + assert.deepEqual(res.TagSet, tag_set1.TagSet); + }); + + mocha.it("put object tagging - specific versionId", async function() { + await s3_uid6.putObjectTagging({ Bucket: bucket_name, Key: tagging_key, Tagging: tag_set2, versionId: version_id}); + const res = await s3_uid6.getObjectTagging({Bucket: bucket_name, Key: tagging_key}); + assert.notDeepEqual(res.TagSet, tag_set2); + const version_res = await s3_uid6.getObjectTagging({Bucket: bucket_name, Key: tagging_key}); + assert.deepEqual(version_res.TagSet, tag_set2.TagSet); + }); + + mocha.it("head object with tagging - test header x-amz-tagging-count (1 tag)", async function() { + await s3_uid6.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + await s3_uid6.putObjectTagging({ Bucket: bucket_name, Key: tagging_key, Tagging: tag_set2, versionId: version_id}); + s3_uid6.middlewareStack.add( + next => async args => { + const result = await next(args); + result.output.$metadata.headers = result.response.headers; + return result; + } + ); + + const res = await s3_uid6.headObject({ Bucket: bucket_name, Key: tagging_key, versionId: version_id}); + assert.equal(res.$metadata.headers['x-amz-tagging-count'], tag_set2.TagSet.length); + }); + + mocha.it("head object with tagging - test header x-amz-tagging-count (3 tags)", async function() { + await s3_uid6.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + await s3_uid6.putObjectTagging({ Bucket: bucket_name, Key: tagging_key, Tagging: tag_set3, versionId: version_id}); + s3_uid6.middlewareStack.add( + next => async args => { + const result = await next(args); + result.output.$metadata.headers = result.response.headers; + return result; + } + ); + + const res = await s3_uid6.headObject({ Bucket: bucket_name, Key: tagging_key, versionId: version_id}); + assert.equal(res.$metadata.headers['x-amz-tagging-count'], tag_set3.TagSet.length); + }); + + mocha.it("delete object tagging - no versionId", async function() { + await s3_uid6.deleteObjectTagging({ Bucket: bucket_name, Key: tagging_key}); + const res = await s3_uid6.getObjectTagging({Bucket: bucket_name, Key: tagging_key}); + assert.equal(res.TagSet.length, 0); + }); + + mocha.it("delete object tagging - specific versionId", async function() { + await s3_uid6.deleteObjectTagging({ Bucket: bucket_name, Key: tagging_key, versionId: version_id}); + const res = await s3_uid6.getObjectTagging({Bucket: bucket_name, Key: tagging_key, versionId: version_id}); + assert.equal(res.TagSet.length, 0); + }); + }); + + mocha.describe('get object attributes', function() { + const key = 'my-key-att'; + let version_id; + + mocha.before(async function() { + await s3_uid6.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const res_put = await s3_uid6.putObject({ Bucket: bucket_name, Key: key, Body: body1 }); + await s3_uid6.putObject({ Bucket: bucket_name, Key: key, Body: body1 }); + version_id = res_put.VersionId; + }); + + mocha.it("get object attributes - attributes (only ETag)", async function() { + const res = await s3_uid6.getObjectAttributes({ + Bucket: bucket_name, + Key: key, + VersionId: version_id, + ObjectAttributes: ['ETag'], + }); + assert.ok(res.$metadata.httpStatusCode === 200); + assert.ok(res.ETag !== undefined); + assert.ok(res.VersionId === version_id); + assert.ok(res.LastModified !== undefined); + // was not send in ObjectAttributes + assert.ok(res.StorageClass === undefined); + assert.ok(res.ObjectSize === undefined); + }); + + mocha.it("get object attributes - attributes (only StorageClass)", async function() { + const res = await s3_uid6.getObjectAttributes({ + Bucket: bucket_name, + Key: key, + VersionId: version_id, + ObjectAttributes: ['StorageClass'], + }); + assert.ok(res.$metadata.httpStatusCode === 200); + assert.ok(res.StorageClass === 'STANDARD'); + assert.ok(res.VersionId === version_id); + assert.ok(res.LastModified !== undefined); + // was not send in ObjectAttributes + assert.ok(res.ETag === undefined); + assert.ok(res.ObjectSize === undefined); + }); + + mocha.it("get object attributes - attributes (only ObjectSize)", async function() { + const res = await s3_uid6.getObjectAttributes({ + Bucket: bucket_name, + Key: key, + VersionId: version_id, + ObjectAttributes: ['ObjectSize'], + }); + assert.ok(res.$metadata.httpStatusCode === 200); + assert.ok(res.ObjectSize === body1.length); + assert.ok(res.VersionId === version_id); + assert.ok(res.LastModified !== undefined); + // was not send in ObjectAttributes + assert.ok(res.ETag === undefined); + assert.ok(res.StorageClass === undefined); + }); + + mocha.it("get object attributes - attributes (all 3 supported attributes)", async function() { + const res = await s3_uid6.getObjectAttributes({ + Bucket: bucket_name, + Key: key, + VersionId: version_id, + ObjectAttributes: ['ETag', 'ObjectSize', 'StorageClass'], + }); + assert.ok(res.$metadata.httpStatusCode === 200); + assert.ok(res.ETag !== undefined); + assert.ok(res.ObjectSize === body1.length); + assert.ok(res.StorageClass === 'STANDARD'); + assert.ok(res.VersionId === version_id); + assert.ok(res.LastModified !== undefined); + }); + + mocha.it("get object attributes - should fail - with invalid attribute", async function() { + try { + await s3_uid6.getObjectAttributes({ + Bucket: bucket_name, + Key: key, + VersionId: version_id, + ObjectAttributes: ['non-existing-attribute'], + }); + } catch (err) { + assert.strictEqual(err.$metadata.httpStatusCode, 400); + assert.strictEqual(err.Code, 'InvalidArgument'); + } + }); + + mocha.it("get object attributes - check version-id header", async function() { + s3_uid6.middlewareStack.add( + next => async args => { + const result = await next(args); + result.output.$metadata.headers = result.response.headers; + return result; + } + ); + + const res = await s3_uid6.getObjectAttributes({ + Bucket: bucket_name, + Key: key, + VersionId: version_id, + ObjectAttributes: ['ETag', 'ObjectSize', 'StorageClass'], + }); + assert.ok(res.$metadata.httpStatusCode === 200); + assert.equal(res.$metadata.headers['x-amz-version-id'], version_id); + }); + }); + + + // dm = delete marker + mocha.describe('delete object latest - versioning suspended', function() { + const key_to_delete = 'mango.txt'; + const key_to_delete2 = 'plum.txt'; + const key_to_delete3 = 'kiwi.txt'; + + mocha.before('set bucket versioning - Enabled and then Suspended', async function() { + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key_to_delete, Body: body1 }); + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key_to_delete3, Body: body1 }); + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + await s3_uid6.putObject({ Bucket: suspended_bucket_name, Key: key_to_delete2, Body: body1 }); + }); + + mocha.it('delete object latest - create dm & move latest -> .versions/', async function() { + const prev_version_id = await stat_and_get_version_id(suspended_full_path, key_to_delete); + const max_version_before_delete = await find_max_version_past(suspended_full_path, key_to_delete, ''); + const res = await s3_uid6.deleteObject({ Bucket: suspended_bucket_name, Key: key_to_delete }); + assert.equal(res.DeleteMarker, true); + + let exist = await fs_utils.file_not_exists(path.join(suspended_full_path, key_to_delete)); + assert.ok(exist); + exist = await version_file_exists(suspended_full_path, key_to_delete, '', prev_version_id); + assert.ok(exist); + const max_version_after_delete = await find_max_version_past(suspended_full_path, key_to_delete, ''); + assert.notEqual(max_version_after_delete, max_version_before_delete); + const is_dm = await is_delete_marker(suspended_full_path, '', key_to_delete, max_version_after_delete); + assert.ok(is_dm); + assert.equal(res.VersionId, max_version_after_delete); + }); + + mocha.it('delete object - create dm & remove latest', async function() { + const prev_version_id = await stat_and_get_version_id(suspended_full_path, key_to_delete2); + const res = await s3_uid6.deleteObject({ Bucket: suspended_bucket_name, Key: key_to_delete2 }); + assert.equal(res.DeleteMarker, true); + + let exist = await fs_utils.file_not_exists(path.join(suspended_full_path, key_to_delete2)); + assert.ok(exist); + exist = await version_file_exists(suspended_full_path, key_to_delete2, '', prev_version_id); + assert.ok(exist); + const max_version_after_delete = await find_max_version_past(suspended_full_path, key_to_delete2, ''); + const is_dm = await is_delete_marker(suspended_full_path, '', key_to_delete2, max_version_after_delete); + assert.ok(is_dm); + assert.equal(res.VersionId, max_version_after_delete); + }); + + mocha.it('delete object latest - non existing key', async function() { + const non_existing_key = 'non_existing_key.txt'; + const res = await s3_uid6.deleteObject({ Bucket: suspended_bucket_name, Key: non_existing_key }); + assert.equal(res.DeleteMarker, true); + + const max_version_after_delete = await find_max_version_past(suspended_full_path, key_to_delete, ''); + const is_dm = await is_delete_marker(suspended_full_path, '', key_to_delete, max_version_after_delete); + assert.ok(is_dm); + assert.equal(res.VersionId, max_version_after_delete); + }); + + mocha.it('delete object when the latest is a dm with a unique version ID - should create another dm with null version ID', async function() { + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const prev_dm = await s3_uid6.deleteObject({ Bucket: suspended_bucket_name, Key: key_to_delete3 }); + assert.equal(prev_dm.DeleteMarker, true); + const comp_prev = check_enable_version_format(prev_dm.VersionId); + assert.ok(comp_prev); + + await s3_uid6.putBucketVersioning({ Bucket: suspended_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_uid6.deleteObject({ Bucket: suspended_bucket_name, Key: key_to_delete3 }); + assert.equal(res.DeleteMarker, true); + const comp_cur = check_null_version_id(res.VersionId); + assert.ok(comp_cur); + const latest_dm_version = await find_max_version_past(suspended_full_path, key_to_delete3, ''); + assert.equal(res.VersionId, latest_dm_version); + const is_dm = await is_delete_marker(suspended_full_path, '', key_to_delete3, latest_dm_version); + assert.ok(is_dm); + const version_path = path.join(suspended_full_path, '.versions', key_to_delete3 + '_' + latest_dm_version); + const version_info = await stat_and_get_all(version_path, ''); + assert.equal(version_info.xattr[XATTR_VERSION_ID], NULL_VERSION_ID); + // check second latest is still non current xattr + const second_latest_version_path = path.join(suspended_full_path, '.versions', key_to_delete3 + '_' + prev_dm.VersionId); + await check_non_current_xattr_exists(second_latest_version_path); + }); + }); + + mocha.describe('delete object version id - versioning enabled', function() { + const delete_object_test_bucket_reg = 'delete-object-test-bucket-reg'; + const delete_object_test_bucket_null = 'delete-object-test-bucket-null'; + const delete_object_test_bucket_dm = 'delete-object-test-bucket-dm'; + + const full_delete_path = path.join(tmp_fs_root, delete_object_test_bucket_reg); + const full_delete_path_null = path.join(tmp_fs_root, delete_object_test_bucket_null); + const full_delete_path_dm = path.join(tmp_fs_root, delete_object_test_bucket_dm); + + let account_with_access; + mocha.describe('delete object - versioning enabled', function() { + mocha.describe('delete object - regular version - versioning enabled', async function() { + mocha.before(async function() { + const res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path_param, { default_resource: nsr}); + account_with_access = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await account_with_access.createBucket({ Bucket: delete_object_test_bucket_reg }); + await put_allow_all_bucket_policy(account_with_access, delete_object_test_bucket_reg); + await account_with_access.createBucket({ Bucket: delete_object_test_bucket_null }); + await put_allow_all_bucket_policy(account_with_access, delete_object_test_bucket_null); + await account_with_access.createBucket({ Bucket: delete_object_test_bucket_dm }); + await put_allow_all_bucket_policy(account_with_access, delete_object_test_bucket_dm); + }); + + mocha.it('delete version id - fake id - nothing to remove', async function() { + const max_version1 = await find_max_version_past(full_path, key1, ''); + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, Key: key1, VersionId: 'mtime-123-ino-123'}); + const max_version2 = await find_max_version_past(full_path, key1, ''); + assert.equal(max_version1, max_version2); + }); + + mocha.it('delete object version id - latest - second latest is null version', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['null', 'regular']); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + const second_latest_version_path = path.join(full_delete_path, '.versions', key1 + '_null'); + await check_non_current_xattr_exists(second_latest_version_path); + + const delete_res = await account_with_access.deleteObject({ + Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[1].VersionId + }); + assert.equal(delete_res.VersionId, cur_version_id1); + + const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); + assert.notEqual(cur_version_id1, cur_version_id2); + assert.equal('null', cur_version_id2); + // check second latest current xattr removed + const latest_version_path = path.join(full_delete_path, key1); + await check_non_current_xattr_does_not_exist(latest_version_path); + await fs_utils.file_must_not_exist(path.join(full_delete_path, key1 + '_' + upload_res_arr[1].VersionId)); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + await delete_object_versions(full_delete_path, key1); + }); + + mocha.it('delete object version id - latest - second latest is delete marker version ', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['regular', 'delete_marker', 'regular']); + const max_version0 = await find_max_version_past(full_delete_path, key1, ''); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + assert.equal(upload_res_arr[2].VersionId, cur_version_id1); + const is_dm = await is_delete_marker(full_delete_path, '', key1, max_version0); + assert.ok(is_dm); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[2].VersionId }); + assert.equal(delete_res.VersionId, cur_version_id1); + await fs_utils.file_must_not_exist(path.join(full_delete_path, key1)); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, max_version0); + await delete_object_versions(full_delete_path, key1); + }); + + mocha.it('delete object version id - in .versions/', async function() { + const put_res = await account_with_access.putObject({ + Bucket: delete_object_test_bucket_reg, Key: key1, Body: body1 }); + await account_with_access.putObject({ Bucket: delete_object_test_bucket_reg, Key: key1, Body: body1 }); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: put_res.VersionId }); + assert.equal(put_res.VersionId, delete_res.VersionId); + const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); + assert.equal(cur_version_id1, cur_version_id2); + const exist = await version_file_must_not_exists(full_delete_path, key1, '', put_res.VersionId); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + }); + + mocha.it('delete object version id - latest - no second latest', async function() { + const cur_version_id = await stat_and_get_version_id(full_delete_path, key1); + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: cur_version_id }); + await fs_utils.file_must_not_exist(path.join(full_delete_path, key1 + '_' + cur_version_id)); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + }); + + mocha.it('delete object version id - in .versions/ 2 - latest exist and it\'s a regular version', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['regular', 'regular', 'regular']); + + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + assert.equal(cur_version_id1, upload_res_arr[2].VersionId); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[1].VersionId }); + assert.equal(upload_res_arr[1].VersionId, delete_res.VersionId); + const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); + assert.equal(cur_version_id1, cur_version_id2); + const exist = await version_file_must_not_exists(full_delete_path, key1, '', upload_res_arr[1].VersionId); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, upload_res_arr[0].VersionId); + await delete_object_versions(full_delete_path, key1); + }); + + mocha.it('delete object version id - in .versions/ 3 - latest exist and it\'s a delete marker', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['regular', 'regular', 'delete_marker']); + + await fs_utils.file_must_not_exist(path.join(full_delete_path, key1)); + const latest_dm_version_id1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(latest_dm_version_id1, upload_res_arr[2].VersionId); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[1].VersionId }); + assert.equal(upload_res_arr[1].VersionId, delete_res.VersionId); + + await fs_utils.file_must_not_exist(path.join(full_delete_path, key1)); + const latest_dm_version_id2 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(latest_dm_version_id1, latest_dm_version_id2); + const version_deleted = await version_file_must_not_exists(full_delete_path, key1, '', upload_res_arr[1].VersionId); + assert.ok(version_deleted); + await delete_object_versions(full_delete_path, key1); + }); + + mocha.it('delete object version id - latest - second latest is regular version ', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['regular', 'regular']); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[1].VersionId }); + + const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); + + assert.notEqual(cur_version_id1, cur_version_id2); + assert.equal(upload_res_arr[0].VersionId, cur_version_id2); + await fs_utils.file_must_not_exist(path.join(full_delete_path, key1 + '_' + upload_res_arr[1].VersionId)); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + await delete_object_versions(full_delete_path, key1); + }); + + mocha.it('delete object version null - latest, no second latest', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_null, key1, ['null']); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path_null, key1); + assert.equal(upload_res_arr[0].VersionId, undefined); + assert.equal(cur_version_id1, 'null'); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_null, + Key: key1, VersionId: 'null' }); + + await fs_utils.file_must_not_exist(path.join(full_delete_path_null, key1)); + const max_version1 = await find_max_version_past(full_delete_path_null, key1, ''); + assert.equal(max_version1, undefined); + await delete_object_versions(full_delete_path_null, key1); + }); + + mocha.it('delete object version null - version is in .versions/', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_null, key1, ['null', 'regular']); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path_null, key1); + assert.equal(upload_res_arr[0].VersionId, undefined); + assert.notEqual(cur_version_id1, 'null'); + assert.equal(cur_version_id1, upload_res_arr[1].VersionId); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_null, + Key: key1, VersionId: 'null' }); + + const max_version1 = await find_max_version_past(full_delete_path_null, key1, ''); + assert.equal(max_version1, undefined); + const cur_version_id2 = await stat_and_get_version_id(full_delete_path_null, key1); + assert.equal(cur_version_id1, cur_version_id2); + + await delete_object_versions(full_delete_path_null, key1); + }); + + mocha.it('delete object version delete marker - latest - second latest is a null version', async function() { + await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['null', 'delete_marker']); + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + assert.equal(delete_res.DeleteMarker, true); + assert.equal(delete_res.VersionId, max_version); + + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, undefined); + await fs_utils.file_must_exist(path.join(full_delete_path_dm, key1)); + await version_file_must_not_exists(full_delete_path_dm, key1, '', second_max_version1); + const new_latest_ver_id = await stat_and_get_version_id(full_delete_path_dm, key1); + assert.equal(new_latest_ver_id, 'null'); + + await delete_object_versions(full_delete_path_dm, key1); + }); + + mocha.it('delete object version delete marker - non latest', async function() { + await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'delete_marker', 'delete_marker']); + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: second_max_version1 }); + + assert.equal(delete_res.DeleteMarker, true); + assert.equal(delete_res.VersionId, second_max_version1); + + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + await version_file_must_not_exists(full_delete_path, key1, '', second_max_version1); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, max_version); + + await delete_object_versions(full_delete_path_dm, key1); + }); + + mocha.it('delete object version delete marker - latest - second latest is a delete marker', async function() { + await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'delete_marker', 'delete_marker']); + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + await version_file_exists(full_delete_path_dm, key1, '', second_max_version1); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, second_max_version1); + + await delete_object_versions(full_delete_path_dm, key1); + }); + + mocha.it('delete object version delete marker - latest - second latest is a regular version', async function() { + const put_res = await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'regular', 'delete_marker']); + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + await fs_utils.file_must_exist(path.join(full_delete_path_dm, key1)); + await version_file_must_not_exists(full_delete_path_dm, key1, '', second_max_version1); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.notEqual(max_version1, second_max_version1); + assert.equal(put_res[0].VersionId, max_version1); + + await delete_object_versions(full_delete_path_dm, key1); + }); + + mocha.it('delete object version delete marker - latest - no second latest', async function() { + const put_res = await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['delete_marker']); + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(put_res[0].VersionId, max_version); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + await version_file_must_not_exists(full_delete_path_dm, key1, '', max_version); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, undefined); + await delete_object_versions(full_delete_path_dm, key1); + }); + + mocha.it('delete object version delete marker - in .versions/ - latest exist', async function() { + const put_res = await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'delete_marker', 'regular']); + const ltst_version_id1 = await stat_and_get_version_id(full_delete_path_dm, key1); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + await fs_utils.file_must_exist(path.join(full_delete_path_dm, key1)); + const ltst_version_id2 = await stat_and_get_version_id(full_delete_path_dm, key1); + assert.equal(ltst_version_id1, ltst_version_id2); + await version_file_exists(full_delete_path_dm, key1, '', second_max_version1); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, second_max_version1); + assert.equal(put_res[0].VersionId, max_version1); + + await delete_object_versions(full_delete_path_dm, key1); + }); + + mocha.it('delete object version delete marker - in .versions/ - latest is a delete marker', async function() { + const put_res = await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'delete_marker', 'delete_marker']); + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + const latest_dm1 = await find_max_version_past(full_delete_path_dm, key1, ''); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: put_res[1].VersionId }); + + await fs_utils.file_must_not_exist(path.join(full_delete_path_dm, key1)); + const latest_dm2 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(latest_dm1, latest_dm2); + await delete_object_versions(full_delete_path_dm, key1); + }); + }); + }); + }); + + // Delete object version id is the same implementation to Enabled and Suspended. + // Thus, the tests are the same and their objective is only to validate that future code changes + // would not change the behavior of delete object version with suspended mode. + mocha.describe('delete object version id - versioning suspended', function() { + const delete_object_test_bucket_reg = 'delete-object-suspended-test-bucket-reg'; + const delete_object_test_bucket_null = 'delete-object-suspended-test-bucket-null'; + const delete_object_test_bucket_dm = 'delete-object-test-suspended-bucket-dm'; + + const full_delete_path = path.join(tmp_fs_root, delete_object_test_bucket_reg); + const full_delete_path_null = path.join(tmp_fs_root, delete_object_test_bucket_null); + const full_delete_path_dm = path.join(tmp_fs_root, delete_object_test_bucket_dm); + + let account_with_access; + mocha.before(async function() { + const res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path_param, { default_resource: nsr }); + account_with_access = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await account_with_access.createBucket({ Bucket: delete_object_test_bucket_reg }); + await put_allow_all_bucket_policy(account_with_access, delete_object_test_bucket_reg); + await account_with_access.createBucket({ Bucket: delete_object_test_bucket_null }); + await put_allow_all_bucket_policy(account_with_access, delete_object_test_bucket_null); + await account_with_access.createBucket({ Bucket: delete_object_test_bucket_dm }); + await put_allow_all_bucket_policy(account_with_access, delete_object_test_bucket_dm); + }); + + mocha.describe('delete object - regular version - versioning suspended', async function() { + + mocha.it('delete version id - fake id - nothing to remove - bucket will be suspended', async function() { + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, Key: key1, VersionId: 'mtime-123-ino-123'}); + const max_version2 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version2, undefined); + }); + + mocha.it('delete object version id - latest - second latest is null version - versioning suspended', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['null', 'regular']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[1].VersionId }); + assert.equal(delete_res.VersionId, cur_version_id1); + + const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); + assert.notEqual(cur_version_id1, cur_version_id2); + assert.equal('null', cur_version_id2); + const exist = await fs_utils.file_not_exists(path.join(full_delete_path, key1 + '_' + upload_res_arr[1].VersionId)); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + await delete_object_versions(full_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version id - latest - second latest is delete marker version - versioning suspended', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['regular', 'delete_marker', 'regular']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const max_version0 = await find_max_version_past(full_delete_path, key1, ''); + + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + assert.equal(upload_res_arr[2].VersionId, cur_version_id1); + const is_dm = await is_delete_marker(full_delete_path, '', key1, max_version0); + assert.ok(is_dm); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[2].VersionId }); + assert.equal(delete_res.VersionId, cur_version_id1); + const exist = await fs_utils.file_not_exists(path.join(full_delete_path, key1)); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, max_version0); + await delete_object_versions(full_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version id - in .versions/ - versioning suspended', async function() { + const put_res = await account_with_access.putObject({ + Bucket: delete_object_test_bucket_reg, Key: key1, Body: body1 }); + await account_with_access.putObject({ Bucket: delete_object_test_bucket_reg, Key: key1, Body: body1 }); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: put_res.VersionId }); + assert.equal(put_res.VersionId, delete_res.VersionId); + const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); + assert.equal(cur_version_id1, cur_version_id2); + const exist = await version_file_must_not_exists(full_delete_path, key1, '', put_res.VersionId); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + }); + + mocha.it('delete object version id - latest - no second latest - versioning suspended', async function() { + const cur_version_id = await stat_and_get_version_id(full_delete_path, key1); + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: cur_version_id }); + const exist = await fs_utils.file_not_exists(path.join(full_delete_path, key1 + '_' + cur_version_id)); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version id - in .versions/ 2 - latest exist and it\'s a regular version - versioning suspended', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['regular', 'regular', 'regular']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + assert.equal(cur_version_id1, upload_res_arr[2].VersionId); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[1].VersionId }); + assert.equal(upload_res_arr[1].VersionId, delete_res.VersionId); + const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); + assert.equal(cur_version_id1, cur_version_id2); + const exist = await version_file_must_not_exists(full_delete_path, key1, '', upload_res_arr[1].VersionId); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, upload_res_arr[0].VersionId); + await delete_object_versions(full_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version id - in .versions/ 3 - latest exist and it\'s a delete marker - versioning suspended', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['regular', 'regular', 'delete_marker']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + + let exist = await fs_utils.file_not_exists(path.join(full_delete_path, key1)); + assert.ok(exist); + const latest_dm_version_id1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(latest_dm_version_id1, upload_res_arr[2].VersionId); + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[1].VersionId }); + assert.equal(upload_res_arr[1].VersionId, delete_res.VersionId); + + exist = await fs_utils.file_not_exists(path.join(full_delete_path, key1)); + assert.ok(exist); + const latest_dm_version_id2 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(latest_dm_version_id1, latest_dm_version_id2); + const version_deleted = await version_file_must_not_exists(full_delete_path, key1, '', upload_res_arr[1].VersionId); + assert.ok(version_deleted); + await delete_object_versions(full_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version id - latest - second latest is regular version - versioning suspended', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['regular', 'regular']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, + Key: key1, VersionId: upload_res_arr[1].VersionId }); + + const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); + assert.notEqual(cur_version_id1, cur_version_id2); + assert.equal(upload_res_arr[0].VersionId, cur_version_id2); + const exist = await fs_utils.file_not_exists(path.join(full_delete_path, key1 + '_' + upload_res_arr[1].VersionId)); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path, key1, ''); + assert.equal(max_version1, undefined); + await delete_object_versions(full_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_reg, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + }); + + mocha.describe('delete object - null version - versioning suspended', async function() { + + mocha.it('delete object version null - latest, no second latest - versioning suspended', async function() { + const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_null, key1, ['null']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_null, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const cur_version_id1 = await stat_and_get_version_id(full_delete_path_null, key1); + assert.equal(upload_res_arr[0].VersionId, undefined); + assert.equal(cur_version_id1, 'null'); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_null, + Key: key1, VersionId: 'null' }); + + const exist = await fs_utils.file_not_exists(path.join(full_delete_path_null, key1)); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path_null, key1, ''); + assert.equal(max_version1, undefined); + await delete_object_versions(full_delete_path_null, key1); + }); + + mocha.it('delete object version null - version is in .versions/ - versioning suspended', async function() { + const upload_res_arr = []; // would contain ['null', 'regular'] put_res + const put_res_null = await account_with_access.putObject({ Bucket: delete_object_test_bucket_null, + Key: key1, Body: body1 }); + upload_res_arr.push(put_res_null); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_null, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const put_res_regular = await account_with_access.putObject({ Bucket: delete_object_test_bucket_null, + Key: key1, Body: body1 }); + upload_res_arr.push(put_res_regular); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_null, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + + const cur_version_id1 = await stat_and_get_version_id(full_delete_path_null, key1); + assert.equal(upload_res_arr[0].VersionId, undefined); + assert.notEqual(cur_version_id1, 'null'); + assert.equal(cur_version_id1, upload_res_arr[1].VersionId); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_null, + Key: key1, VersionId: 'null' }); + + const max_version1 = await find_max_version_past(full_delete_path_null, key1, ''); + assert.equal(max_version1, undefined); + const cur_version_id2 = await stat_and_get_version_id(full_delete_path_null, key1); + assert.equal(cur_version_id1, cur_version_id2); + + await delete_object_versions(full_delete_path_null, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_null, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + }); + + mocha.describe('delete object - delete marker version - versioning suspended', async function() { + + mocha.it('delete object version delete marker - latest - second latest is a null version - versioning suspended', async function() { + await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['null', 'delete_marker']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + + let exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + assert.equal(delete_res.DeleteMarker, true); + assert.equal(delete_res.VersionId, max_version); + + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, undefined); + exist = await fs_utils.file_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + exist = await version_file_must_not_exists(full_delete_path_dm, key1, '', second_max_version1); + assert.ok(exist); + const new_latest_ver_id = await stat_and_get_version_id(full_delete_path_dm, key1); + assert.equal(new_latest_ver_id, 'null'); + + await delete_object_versions(full_delete_path_dm, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version delete marker - non latest - versioning suspended', async function() { + await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'delete_marker', 'delete_marker']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + let exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + + const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: second_max_version1 }); + + assert.equal(delete_res.DeleteMarker, true); + assert.equal(delete_res.VersionId, second_max_version1); + + exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + exist = await version_file_must_not_exists(full_delete_path_dm, key1, '', second_max_version1); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, max_version); + + await delete_object_versions(full_delete_path_dm, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version delete marker - latest - second latest is a delete marker - versioning suspended', async function() { + await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'delete_marker', 'delete_marker']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + + let exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + exist = await version_file_exists(full_delete_path_dm, key1, '', second_max_version1); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, second_max_version1); + + await delete_object_versions(full_delete_path_dm, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version delete marker - latest - second latest is a regular version - versioning suspended', async function() { + const put_res = await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'regular', 'delete_marker']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + let exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + exist = await fs_utils.file_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + exist = await version_file_must_not_exists(full_delete_path_dm, key1, '', second_max_version1); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.notEqual(max_version1, second_max_version1); + assert.equal(put_res[0].VersionId, max_version1); + + await delete_object_versions(full_delete_path_dm, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version delete marker - latest - no second latest - versioning suspended', async function() { + const put_res = await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['delete_marker']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + let exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(put_res[0].VersionId, max_version); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + exist = await version_file_must_not_exists(full_delete_path_dm, key1, '', max_version); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, undefined); + await delete_object_versions(full_delete_path_dm, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version delete marker - in .versions/ - latest exist', async function() { + const put_res = await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'delete_marker', 'regular']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const ltst_version_id1 = await stat_and_get_version_id(full_delete_path_dm, key1); + const max_version = await find_max_version_past(full_delete_path_dm, key1, ''); + const second_max_version1 = await find_max_version_past(full_delete_path_dm, key1, '', [max_version]); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: max_version }); + + let exist = await fs_utils.file_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + const ltst_version_id2 = await stat_and_get_version_id(full_delete_path_dm, key1); + assert.equal(ltst_version_id1, ltst_version_id2); + exist = await version_file_exists(full_delete_path_dm, key1, '', second_max_version1); + assert.ok(exist); + const max_version1 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(max_version1, second_max_version1); + assert.equal(put_res[0].VersionId, max_version1); + + await delete_object_versions(full_delete_path_dm, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete object version delete marker - in .versions/ - latest is a delete marker', async function() { + const put_res = await upload_object_versions(account_with_access, delete_object_test_bucket_dm, key1, ['regular', 'delete_marker', 'delete_marker']); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + + let exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + const latest_dm1 = await find_max_version_past(full_delete_path_dm, key1, ''); + + await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_dm, + Key: key1, VersionId: put_res[1].VersionId }); + + exist = await fs_utils.file_not_exists(path.join(full_delete_path_dm, key1)); + assert.ok(exist); + const latest_dm2 = await find_max_version_past(full_delete_path_dm, key1, ''); + assert.equal(latest_dm1, latest_dm2); + await delete_object_versions(full_delete_path_dm, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_object_test_bucket_dm, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + }); + }); + + mocha.describe('delete multiple objects - versioning enabled', function() { + const delete_multi_object_test_bucket = 'delete-multi-object-test-bucket'; + const full_multi_delete_path = path.join(tmp_fs_root, delete_multi_object_test_bucket); + let account_with_access; + + mocha.before(async function() { + const res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path_param, { default_resource: nsr }); + account_with_access = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await account_with_access.createBucket({ Bucket: delete_multi_object_test_bucket }); + await put_allow_all_bucket_policy(account_with_access, delete_multi_object_test_bucket); + }); + + mocha.it('delete multiple objects - no version id - versioning disabled', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(80000); + const keys = []; + for (let i = 0; i < 50; i++) { + const random_key = (Math.random() + 1).toString(36).substring(7); + keys.push(random_key); + await upload_object_versions(account_with_access, delete_multi_object_test_bucket, random_key, ['null']); + } + const to_delete_arr = keys.map(key => ({ Key: key })); + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: to_delete_arr } }); + assert.equal(delete_res.Deleted.length, 50); + assert.deepStrictEqual(delete_res.Deleted, to_delete_arr); + for (const res of delete_res.Deleted) { + assert.equal(res.DeleteMarker, undefined); + assert.equal(res.VersionId, undefined); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + await fs_utils.file_must_not_exist(versions_dir); + const objects = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, full_multi_delete_path); + assert.equal(objects.length, 1); + assert.ok(objects[0].name.startsWith('.noobaa-nsfs_')); + + }); + + mocha.it('delete multiple objects - no version id', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + const versions_type_arr = ['null']; + for (let i = 0; i < 300; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + const arr = []; + for (let i = 0; i < 200; i++) { + arr.push({ Key: 'a' }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 1); + for (const res of delete_res.Deleted) { + assert.equal(res.DeleteMarker, true); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + assert.equal(versions.length, 302); // 1 null version + 30 dm and versions + 1 new dm + await delete_object_versions(full_multi_delete_path, key1); + await delete_object_versions(full_multi_delete_path, 'a'); + }); + + mocha.it('delete multiple objects - delete only delete markers', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(60000); + const versions_type_arr = []; + for (let i = 0; i < 300; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + const arr = []; + for (let i = 0; i < 300; i++) { + if (i % 2 === 1) arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 150); + for (const res of delete_res.Deleted) { + assert.equal(res.DeleteMarker, true); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + assert.equal(versions.length, 149); + await fs_utils.file_must_exist(path.join(full_multi_delete_path, key1)); + const latest_stat = await stat_and_get_all(full_multi_delete_path, key1); + assert.equal(latest_stat.xattr[XATTR_VERSION_ID], put_res[298].VersionId); + await delete_object_versions(full_multi_delete_path, key1); + }); + + mocha.it('delete multiple objects - delete only regular versions key1, delete delete markers key2', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + const key2 = 'key2'; + const versions_type_arr = []; + for (let i = 0; i < 300; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + const put_res2 = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key2, versions_type_arr); + const arr = []; + for (let i = 0; i < 300; i++) { + if (i % 2 === 0) arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + if (i % 2 === 1) arr.push({ Key: key2, VersionId: put_res2[i].VersionId }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + + assert.equal(delete_res.Deleted.length, 300); + for (const res of delete_res.Deleted.slice(0, 150)) { + assert.equal(res.DeleteMarker, undefined); + } + for (const res of delete_res.Deleted.slice(150)) { + assert.equal(res.DeleteMarker, true); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + // 150 of key1 and 149 of key2 (latest version of key2 is in the parent dir) + assert.equal(versions.length, 299); + await fs_utils.file_must_not_exist(path.join(full_multi_delete_path, key1)); + await fs_utils.file_must_exist(path.join(full_multi_delete_path, key2)); + const latest_dm_version = await find_max_version_past(full_multi_delete_path, key1); + const version_path = path.join(full_multi_delete_path, '.versions', key1 + '_' + latest_dm_version); + const version_info = await stat_and_get_all(version_path, ''); + assert.equal(version_info.xattr[XATTR_DELETE_MARKER], 'true'); + assert.equal(version_info.xattr[XATTR_VERSION_ID], put_res[299].VersionId); + await delete_object_versions(full_multi_delete_path, key1); + await delete_object_versions(full_multi_delete_path, key2); + }); + + mocha.it('delete multiple objects - delete regular versions & delete markers - new latest is dm', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + const versions_type_arr = []; + for (let i = 0; i < 300; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + const arr = []; + for (let i = 200; i < 300; i++) { + arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 100); + for (let i = 0; i < 100; i++) { + if (i % 2 === 1) assert.equal(delete_res.Deleted[i].DeleteMarker, true); + if (i % 2 === 0) assert.equal(delete_res.Deleted[i].DeleteMarker, undefined); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + assert.equal(versions.length, 200); + await fs_utils.file_must_not_exist(path.join(full_multi_delete_path, key1)); + const latest_dm_version = await find_max_version_past(full_multi_delete_path, key1); + const version_path = path.join(full_multi_delete_path, '.versions', key1 + '_' + latest_dm_version); + const version_info = await stat_and_get_all(version_path, ''); + assert.equal(version_info.xattr[XATTR_VERSION_ID], put_res[199].VersionId); + await delete_object_versions(full_multi_delete_path, key1); + }); + + mocha.it('delete multiple objects - delete regular versions & delete markers - new latest is regular version', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + const versions_type_arr = []; + for (let i = 0; i < 300; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + const arr = []; + for (let i = 100; i < 200; i++) { + arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + arr.push({ Key: key1, VersionId: put_res[299].VersionId }); + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 101); + for (let i = 0; i < 100; i++) { + if (i % 2 === 1) assert.equal(delete_res.Deleted[i].DeleteMarker, true); + if (i % 2 === 0) assert.equal(delete_res.Deleted[i].DeleteMarker, undefined); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + + assert.equal(versions.length, 198); + await fs_utils.file_must_exist(path.join(full_multi_delete_path, key1)); + const latest_stat = await stat_and_get_all(full_multi_delete_path, key1); + assert.equal(latest_stat.xattr[XATTR_VERSION_ID], put_res[298].VersionId); + await delete_object_versions(full_multi_delete_path, key1); + }); + + mocha.it('delete multiple objects - delete keys & regular versions & delete markers ', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + const versions_type_arr = []; + for (let i = 0; i < 300; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + const arr = []; + for (let i = 0; i < 50; i++) { + arr.push({ Key: key1 }); + } + for (let i = 100; i < 200; i++) { + arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 101); // 100 dm and versions + 1 new dm + for (let i = 0; i < 1; i++) { + assert.notEqual(delete_res.Deleted[i].DeleteMarkerVersionId, undefined); + assert.equal(delete_res.Deleted[i].DeleteMarker, true); + } + for (let i = 1; i < 101; i++) { + if (i % 2 === 0) assert.equal(delete_res.Deleted[i].DeleteMarker, true); + if (i % 2 === 1) assert.equal(delete_res.Deleted[i].DeleteMarker, undefined); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + + assert.equal(versions.length, 201); // 200 dm and versions + 1 new dm + await fs_utils.file_must_not_exist(path.join(full_multi_delete_path, key1)); + await delete_object_versions(full_multi_delete_path, key1); + }); + + + mocha.it('delete multiple objects - delete regular versions & delete markers & latest & keys- ', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + const versions_type_arr = []; + for (let i = 0; i < 300; i++) { + versions_type_arr.push(i % 2 === 1 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + const arr = []; + for (let i = 200; i < 300; i++) { + arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + + for (let i = 0; i < 50; i++) { + arr.push({ Key: key1 }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 101); + for (let i = 0; i < 100; i++) { + if (i % 2 === 1) assert.equal(delete_res.Deleted[i].DeleteMarker, undefined); + if (i % 2 === 0) assert.equal(delete_res.Deleted[i].DeleteMarker, true); + } + for (let i = 100; i < 101; i++) { + assert.notEqual(delete_res.Deleted[i].DeleteMarkerVersionId, undefined); + assert.equal(delete_res.Deleted[i].DeleteMarker, true); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + + assert.equal(versions.length, 201); // 200 dm and versions + dm + new dm + await fs_utils.file_must_not_exist(path.join(full_multi_delete_path, key1)); + await delete_object_versions(full_multi_delete_path, key1); + }); + + mocha.it('delete multiple objects - versioning enabled (more than 1 level)', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + const nested_key_level4 = '/company/department/team/2024/January/employee_123.txt'; + const arr_keys_only = []; + const arr_keys_and_version_id = []; + const num_of_versions = 5; + for (let i = 0; i < num_of_versions; i++) { + const body_to_add = `zzzzz-${i}`; + const res = await account_with_access.putObject({ + Bucket: delete_multi_object_test_bucket, Key: nested_key_level4, Body: body_to_add }); + arr_keys_only.push({ Key: nested_key_level4 }); + arr_keys_and_version_id.push({ Key: nested_key_level4, VersionId: res.VersionId }); + } + + // no version-id (latest) + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr_keys_only } }); + assert.equal(delete_res.Deleted.length, 1); + + // with version-id + const delete_res2 = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr_keys_and_version_id } }); + assert.equal(delete_res2.Deleted.length, num_of_versions); + + // delete the delete marker of latest version + await s3_uid6.deleteObject({ Bucket: delete_multi_object_test_bucket, + Key: delete_res.Deleted[0].Key, VersionId: delete_res.Deleted[0].DeleteMarkerVersionId}); + + // to validate that the object was deleted completely + const list_object_versions_res = await s3_uid6.listObjectVersions({Bucket: delete_multi_object_test_bucket}); + assert.ok(list_object_versions_res.Versions === undefined); + assert.ok(list_object_versions_res.DeleteMarkers === undefined); + }); + }); + + mocha.describe('delete multiple objects - versioning suspended', function() { + const delete_multi_object_test_bucket = 'delete-multi-object-test-bucket-suspended'; + const full_multi_delete_path = path.join(tmp_fs_root, delete_multi_object_test_bucket); + let account_with_access; + + mocha.before(async function() { + const res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path_param, { default_resource: nsr }); + account_with_access = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await account_with_access.createBucket({ Bucket: delete_multi_object_test_bucket }); + await put_allow_all_bucket_policy(account_with_access, delete_multi_object_test_bucket); + }); + + mocha.it('delete multiple objects - no version id - versioning disabled - will be suspended', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(80000); + const keys = []; + for (let i = 0; i < 50; i++) { + const random_key = (Math.random() + 1).toString(36).substring(7); + keys.push(random_key); + await upload_object_versions(account_with_access, delete_multi_object_test_bucket, random_key, ['null']); + } + const to_delete_arr = keys.map(key => ({ Key: key })); + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: to_delete_arr } }); + assert.equal(delete_res.Deleted.length, 50); + assert.deepStrictEqual(delete_res.Deleted, to_delete_arr); + for (const res of delete_res.Deleted) { + assert.equal(res.DeleteMarker, undefined); + assert.equal(res.VersionId, undefined); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const exist = await fs_utils.file_not_exists(versions_dir); + assert.ok(exist); + const objects = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, full_multi_delete_path); + assert.equal(objects.length, 1); + assert.ok(objects[0].name.startsWith('.noobaa-nsfs_')); + }); + + mocha.it('delete multiple objects - no version id - versioning suspended', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(60000); + const versions_type_arr = ['null']; + for (let i = 0; i < 3; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const arr = []; + for (let i = 0; i < 2; i++) { + arr.push({ Key: 'a' }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 1); + for (const res of delete_res.Deleted) { + assert.equal(res.DeleteMarker, true); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + assert.equal(versions.length, 4); //key1 versions: null, regular and dm (3 total) + a versions: dm (1 total) + await delete_object_versions(full_multi_delete_path, key1); + await delete_object_versions(full_multi_delete_path, 'a'); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete multiple objects - delete only delete markers - versioning suspended', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(60000); + const versions_type_arr = []; + for (let i = 0; i < 5; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const arr = []; + for (let i = 0; i < 5; i++) { + if (i % 2 === 1) arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 2); + for (const res of delete_res.Deleted) { + assert.equal(res.DeleteMarker, true); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + assert.equal(versions.length, 2); + const exist = await fs_utils.file_exists(path.join(full_multi_delete_path, key1)); + assert.ok(exist); + const latest_stat = await stat_and_get_all(full_multi_delete_path, key1); + assert.equal(latest_stat.xattr[XATTR_VERSION_ID], put_res[4].VersionId); + await delete_object_versions(full_multi_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete multiple objects - delete only regular versions key1, delete delete markers key2 - versioning suspended', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(60000); + const key2 = 'key2'; + const versions_type_arr = []; + for (let i = 0; i < 8; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + const put_res2 = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key2, versions_type_arr); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const arr = []; + for (let i = 0; i < 8; i++) { + if (i % 2 === 0) arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + if (i % 2 === 1) arr.push({ Key: key2, VersionId: put_res2[i].VersionId }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 8); + for (const res of delete_res.Deleted.slice(0, 4)) { + assert.equal(res.DeleteMarker, undefined); + } + for (const res of delete_res.Deleted.slice(4)) { + assert.equal(res.DeleteMarker, true); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + // 4 of key1 and 3 of key2 (latest version of key2 is in the parent dir) + assert.equal(versions.length, 7); + let exist = await fs_utils.file_not_exists(path.join(full_multi_delete_path, key1)); + assert.ok(exist); + exist = await fs_utils.file_exists(path.join(full_multi_delete_path, key2)); + assert.ok(exist); + const latest_dm_version = await find_max_version_past(full_multi_delete_path, key1); + const version_path = path.join(full_multi_delete_path, '.versions', key1 + '_' + latest_dm_version); + const version_info = await stat_and_get_all(version_path, ''); + assert.equal(version_info.xattr[XATTR_DELETE_MARKER], 'true'); + assert.equal(version_info.xattr[XATTR_VERSION_ID], put_res[7].VersionId); + await delete_object_versions(full_multi_delete_path, key1); + await delete_object_versions(full_multi_delete_path, key2); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete multiple objects - delete regular versions & delete markers - new latest is dm - versioning suspended', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(60000); + const versions_type_arr = []; + for (let i = 0; i < 5; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const arr = []; + for (let i = 2; i < 5; i++) { + arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 3); + for (let i = 0; i < 3; i++) { + if (i % 2 === 1) assert.equal(delete_res.Deleted[i].DeleteMarker, true); + if (i % 2 === 0) assert.equal(delete_res.Deleted[i].DeleteMarker, undefined); + } + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + assert.equal(versions.length, 2); + const exist = await fs_utils.file_not_exists(path.join(full_multi_delete_path, key1)); + assert.ok(exist); + const latest_dm_version = await find_max_version_past(full_multi_delete_path, key1); + const version_path = path.join(full_multi_delete_path, '.versions', key1 + '_' + latest_dm_version); + const version_info = await stat_and_get_all(version_path, ''); + assert.equal(version_info.xattr[XATTR_VERSION_ID], put_res[1].VersionId); + await delete_object_versions(full_multi_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete multiple objects - delete regular versions & delete markers - new latest is regular version - versioning suspended', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(60000); + const versions_type_arr = []; + for (let i = 0; i < 5; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const arr = []; + for (let i = 0; i < 4; i++) { + arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 4); + for (let i = 0; i < 4; i++) { + if (i % 2 === 0) assert.equal(delete_res.Deleted[i].DeleteMarker, undefined); + if (i % 2 === 1) assert.equal(delete_res.Deleted[i].DeleteMarker, true); + } + + const exist = await fs_utils.file_exists(path.join(full_multi_delete_path, key1)); + assert.ok(exist); + const latest_stat = await stat_and_get_all(full_multi_delete_path, key1); + assert.equal(latest_stat.xattr[XATTR_VERSION_ID], put_res[4].VersionId); + await delete_object_versions(full_multi_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete multiple objects - delete keys & regular versions & delete markers - versioning suspended', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(60000); + const versions_type_arr = []; + for (let i = 0; i < 30; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const arr = []; + for (let i = 0; i < 5; i++) { + arr.push({ Key: key1 }); + } + for (let i = 10; i < 20; i++) { + arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 11); + assert.equal(delete_res.Deleted[0].DeleteMarker, true); + assert.equal(delete_res.Deleted[0].DeleteMarkerVersionId, 'null'); + for (let i = 1; i < 11; i++) { + if (i % 2 === 0) assert.equal(delete_res.Deleted[i].DeleteMarker, true); + if (i % 2 === 1) assert.equal(delete_res.Deleted[i].DeleteMarker, undefined); + } + + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + assert.equal(versions.length, 21); // 30 - 10 (deleted id-versions) + 1 (null dm on the key delete) + const exist = await fs_utils.file_not_exists(path.join(full_multi_delete_path, key1)); + assert.ok(exist); + await delete_object_versions(full_multi_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + + mocha.it('delete multiple objects - delete regular versions & delete markers & latest & keys- - versioning suspended', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(60000); + const versions_type_arr = []; + for (let i = 0; i < 30; i++) { + versions_type_arr.push(i % 2 === 0 ? 'regular' : 'delete_marker'); + } + const put_res = await upload_object_versions(account_with_access, delete_multi_object_test_bucket, key1, versions_type_arr); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const arr = []; + for (let i = 20; i < 30; i++) { + arr.push({ Key: key1, VersionId: put_res[i].VersionId }); + } + for (let i = 0; i < 5; i++) { + arr.push({ Key: key1 }); + } + const delete_res = await account_with_access.deleteObjects({ + Bucket: delete_multi_object_test_bucket, Delete: { Objects: arr } }); + assert.equal(delete_res.Deleted.length, 11); + for (let i = 0; i < 10; i++) { + if (i % 2 === 0) assert.equal(delete_res.Deleted[i].DeleteMarker, undefined); + if (i % 2 === 1) assert.equal(delete_res.Deleted[i].DeleteMarker, true); + } + assert.equal(delete_res.Deleted[10].DeleteMarkerVersionId, NULL_VERSION_ID); + assert.equal(delete_res.Deleted[10].DeleteMarker, true); + + const versions_dir = path.join(full_multi_delete_path, '.versions'); + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + assert.equal(versions.length, 21); // 30 - 10 (deleted id-versions) + 1 (null dm on the key delete) + const exist = await fs_utils.file_not_exists(path.join(full_multi_delete_path, key1)); + assert.ok(exist); + await delete_object_versions(full_multi_delete_path, key1); + await s3_uid6.putBucketVersioning({ Bucket: delete_multi_object_test_bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + }); + }); + + mocha.describe('list object version - check null version', function() { + const list_object_versions_test_bucket = 'list-object-versions-test-bucket'; + // const full_path_list_object_versions_test_bucket = tmp_fs_root + '/' + list_object_versions_test_bucket; + let account_with_access; + const key_to_list1 = 'car.txt'; + const key_to_list2 = 'bike.txt'; + const key_to_list3 = 'ship.txt'; + + mocha.before(async function() { + const res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path_param, { default_resource: nsr }); + account_with_access = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await account_with_access.createBucket({ Bucket: list_object_versions_test_bucket }); + await put_allow_all_bucket_policy(account_with_access, list_object_versions_test_bucket); + }); + + mocha.it('list object versions - only null versions - versioning disabled', async function() { + await account_with_access.putObject({ Bucket: list_object_versions_test_bucket, + Key: key_to_list1, Body: body1 }); + await account_with_access.putObject({ Bucket: list_object_versions_test_bucket, + Key: key_to_list2, Body: body1 }); + const list_object_versions_res = await account_with_access.listObjectVersions({ + Bucket: list_object_versions_test_bucket}); + //bike.txt before car.txt (a-z sort) + assert.equal(list_object_versions_res.Versions[0].Key, key_to_list2); + assert.equal(list_object_versions_res.Versions[0].VersionId, NULL_VERSION_ID); + assert.equal(list_object_versions_res.Versions[1].Key, key_to_list1); + assert.equal(list_object_versions_res.Versions[1].VersionId, NULL_VERSION_ID); + }); + + mocha.it('list object versions - no null version id - with versioning enabled', async function() { + const versions_type_arr = []; + for (let i = 0; i < 3; i++) { + versions_type_arr.push('regular'); + } + const put_res = await upload_object_versions(account_with_access, list_object_versions_test_bucket, + key_to_list3, versions_type_arr); + const list_object_versions_res = await account_with_access.listObjectVersions( + {Bucket: list_object_versions_test_bucket, Prefix: key_to_list3}); + // the order is from latest to oldest + assert.equal(list_object_versions_res.Versions[0].Key, key_to_list3); + assert.equal(list_object_versions_res.Versions[0].VersionId, put_res[2].VersionId); + assert.equal(list_object_versions_res.Versions[1].Key, key_to_list3); + assert.equal(list_object_versions_res.Versions[1].VersionId, put_res[1].VersionId); + assert.equal(list_object_versions_res.Versions[2].Key, key_to_list3); + assert.equal(list_object_versions_res.Versions[2].VersionId, put_res[0].VersionId); + }); + + mocha.it('list object versions - latest version is null version id', async function() { + await account_with_access.putBucketVersioning({ Bucket: list_object_versions_test_bucket, + VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + await account_with_access.putObject({ Bucket: list_object_versions_test_bucket, + Key: key_to_list3, Body: body1 }); + const list_object_versions_res = await account_with_access.listObjectVersions( + {Bucket: list_object_versions_test_bucket, Prefix: key_to_list3}); + // latest version is null (the order is from latest to oldest), hence null is first + assert.equal(list_object_versions_res.Versions[0].Key, key_to_list3); + assert.equal(list_object_versions_res.Versions[0].VersionId, NULL_VERSION_ID); + }); + + mocha.it('list object versions - oldest version is null version id', async function() { + await account_with_access.putBucketVersioning({ Bucket: list_object_versions_test_bucket, + VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const versions_type_arr = []; + for (let i = 0; i < 3; i++) { + versions_type_arr.push('regular'); + } + await upload_object_versions(account_with_access, list_object_versions_test_bucket, + key_to_list2, versions_type_arr); + + + const list_object_versions_res = await account_with_access.listObjectVersions( + {Bucket: list_object_versions_test_bucket, Prefix: key_to_list2}); + for (let i = 0; i < 3; i++) { + assert.equal(list_object_versions_res.Versions[i].Key, key_to_list2); + assert.notEqual(list_object_versions_res.Versions[i].VersionId, NULL_VERSION_ID); + } + // oldest version is null (the order is from latest to oldest), hence null is last + assert.equal(list_object_versions_res.Versions[3].Key, key_to_list2); + assert.equal(list_object_versions_res.Versions[3].VersionId, NULL_VERSION_ID); + }); + }); +}); + +mocha.describe('bucketspace namespace_fs - versioning', function() { + mocha.describe('List-objects', function() { + const nsr = 'noobaa-nsr'; + const bucket_name = 'noobaa-bucket'; + const tmp_fs_root2 = path.join(TMP_PATH, 'test_namespace_fs_list_objects'); + const bucket_path = '/bucket'; + const full_path = tmp_fs_root2 + bucket_path; + const version_dir = '/.versions'; + const full_path_version_dir = full_path + `${version_dir}`; + const dir1 = full_path + '/dir1'; + const dir1_version_dir = dir1 + `${version_dir}`; + const dir2 = full_path + '/dir2'; + const dir2_version_dir = dir2 + `${version_dir}`; + let s3_client; + let s3_admin; + const accounts = []; + const key = 'key'; + const body = 'AAAA'; + const version_key = 'version_key'; + const version_body = 'A1A1A1A'; + + mocha.before(async function() { + this.timeout(0); // eslint-disable-line no-invalid-this + if (invalid_nsfs_root_permissions()) this.skip(); // eslint-disable-line no-invalid-this + // create paths + await fs_utils.create_fresh_path(tmp_fs_root2, 0o777); + await fs_utils.create_fresh_path(full_path, 0o770); + await fs_utils.file_must_exist(full_path); + await fs_utils.create_fresh_path(full_path_version_dir, 0o770); + await fs_utils.file_must_exist(full_path_version_dir); + await fs_utils.create_fresh_path(dir1, 0o770); + await fs_utils.file_must_exist(dir1); + await fs_utils.create_fresh_path(dir1_version_dir, 0o770); + await fs_utils.file_must_exist(dir1_version_dir); + await fs_utils.create_fresh_path(dir2, 0o770); + await fs_utils.file_must_exist(dir2); + await fs_utils.create_fresh_path(dir2_version_dir, 0o770); + await fs_utils.file_must_exist(dir2_version_dir); + const ls_new_buckets_path = get_new_buckets_path_by_test_env(tmp_fs_root2, '/'); + if (is_nc_coretest) { + const { uid, gid } = get_admin_mock_account_details(); + await set_path_permissions_and_owner(full_path, { uid, gid }, 0o700); + } + // export dir as a bucket + await rpc_client.pool.create_namespace_resource({ + name: nsr, + nsfs_config: { + fs_root_path: tmp_fs_root2, + } + }); + const obj_nsr = { resource: nsr, path: bucket_path }; + await rpc_client.bucket.create_bucket({ + name: bucket_name, + namespace: { + read_resources: [obj_nsr], + write_resource: obj_nsr + } + }); + const policy = { + Version: '2012-10-17', + Statement: [{ + Sid: 'id-1', + Effect: 'Allow', + Principal: { AWS: "*" }, + Action: ['s3:*'], + Resource: [`arn:aws:s3:::*`] + }, + ] + }; + // create accounts + let res = await generate_nsfs_account(rpc_client, EMAIL, ls_new_buckets_path, { admin: true }); + s3_admin = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await s3_admin.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(policy) + }); + // create nsfs account + res = await generate_nsfs_account(rpc_client, EMAIL, ls_new_buckets_path); + s3_client = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + accounts.push(res.email); + await s3_client.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const bucket_ver = await s3_client.getBucketVersioning({ Bucket: bucket_name }); + assert.equal(bucket_ver.Status, 'Enabled'); + await create_object(`${full_path}/${key}`, body, 'null'); + await create_object(`${full_path_version_dir}/${version_key}`, version_body, 'null'); + await create_object(`${dir1}/${key}`, body, 'null'); + await create_object(`${dir1_version_dir}/${version_key}`, version_body, 'null'); + await create_object(`${dir2}/${key}`, body, 'null'); + await create_object(`${dir1_version_dir}/${version_key}`, version_body, 'null'); + }); + + mocha.after(async () => { + this.timeout(0); // eslint-disable-line no-invalid-this + fs_utils.folder_delete(tmp_fs_root); + for (const email of accounts) { + await rpc_client.account.delete_account({ email }); + } + }); + + mocha.it('list objects - should return only latest object', async function() { + const res = await s3_client.listObjects({Bucket: bucket_name}); + res.Contents.forEach(val => { + if (val.Key.includes('version') === true) { + assert.fail('Not Expected: list objects returned contents fo .version dir'); + } + }); + }); + }); + + mocha.describe('Get/Head object', function() { + const nsr = 'get-head-versioned-nsr'; + const bucket_name = 'get-head-versioned-bucket'; + const disabled_bucket_name = 'get-head-disabled-bucket'; + const nested_keys_bucket_name = 'get-head-versioned-bucket2'; + const tmp_fs_root3 = path.join(TMP_PATH, 'test_namespace_fs_get_objects'); + + const bucket_path = '/get-head-bucket/'; + const nested_keys_bucket_path = '/bucket_with_nested_keys'; + const vesion_dir = '/.versions'; + const full_path = path.join(tmp_fs_root3, bucket_path); + const disabled_bucket_path = '/get-head-disabled_bucket'; + const disabled_full_path = path.join(tmp_fs_root3, disabled_bucket_path); + const nested_keys_full_path = path.join(tmp_fs_root3, nested_keys_bucket_path); + const version_dir_path = path.join(full_path, vesion_dir); + let file_pointer; + const versionID_1 = 'mtime-12a345b-ino-c123d45'; + const versionID_2 = 'mtime-e56789f-ino-h56g789'; + const versionID_3 = 'mtime-1i357k9-ino-13l57j9'; + let s3_client; + let s3_admin; + const accounts = []; + const dis_version_key = 'dis_version'; + const dis_version_body = 'AAAAA'; + const en_version_key = 'en_version'; + const en_version_body = 'BBBBB'; + const en_version_key_v1 = versionID_2; + const en_version_body_v1 = 'CCCCC'; + const key_version = en_version_key + '_' + en_version_key_v1; + mocha.before(async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(300000); + if (invalid_nsfs_root_permissions()) this.skip(); // eslint-disable-line no-invalid-this + // create paths + await fs_utils.create_fresh_path(tmp_fs_root3, 0o777); + await fs_utils.create_fresh_path(full_path, 0o770); + await fs_utils.file_must_exist(full_path); + await fs_utils.create_fresh_path(version_dir_path, 0o770); + await fs_utils.file_must_exist(version_dir_path); + await fs_utils.create_fresh_path(disabled_full_path, 0o770); + await fs_utils.file_must_exist(disabled_full_path); + await fs_utils.create_fresh_path(nested_keys_full_path, 0o770); + await fs_utils.file_must_exist(nested_keys_full_path); + const new_buckets_path3 = get_new_buckets_path_by_test_env(tmp_fs_root3, '/'); + if (is_nc_coretest) { + const { uid, gid } = get_admin_mock_account_details(); + await set_path_permissions_and_owner(full_path, { uid, gid }, 0o700); + await set_path_permissions_and_owner(disabled_full_path, { uid, gid }, 0o700); + await set_path_permissions_and_owner(nested_keys_full_path, { uid, gid }, 0o700); + } + // export dir as a bucket + await rpc_client.pool.create_namespace_resource({ + name: nsr, + nsfs_config: { + fs_root_path: tmp_fs_root3, + } + }); + const obj_nsr = { resource: nsr, path: bucket_path }; + await rpc_client.bucket.create_bucket({ + name: bucket_name, + namespace: { + read_resources: [obj_nsr], + write_resource: obj_nsr + } + }); + + const disabled_nsr = { resource: nsr, path: disabled_bucket_path }; + await rpc_client.bucket.create_bucket({ + name: disabled_bucket_name, + namespace: { + read_resources: [disabled_nsr], + write_resource: disabled_nsr + } + }); + + const nested_keys_nsr = { resource: nsr, path: nested_keys_bucket_path }; + await rpc_client.bucket.create_bucket({ + name: nested_keys_bucket_name, + namespace: { + read_resources: [nested_keys_nsr], + write_resource: nested_keys_nsr + } + }); + + const policy = { + Version: '2012-10-17', + Statement: [{ + Sid: 'id-1', + Effect: 'Allow', + Principal: { AWS: "*" }, + Action: ['s3:*'], + Resource: [`arn:aws:s3:::*`] + }, + ] + }; + // create accounts + let res = await generate_nsfs_account(rpc_client, EMAIL, new_buckets_path3, { admin: true }); + s3_admin = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await s3_admin.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(policy) + }); + await s3_admin.putBucketPolicy({ + Bucket: disabled_bucket_name, + Policy: JSON.stringify(policy) + }); + await s3_admin.putBucketPolicy({ + Bucket: nested_keys_bucket_name, + Policy: JSON.stringify(policy) + }); + // create nsfs account + res = await generate_nsfs_account(rpc_client, EMAIL, new_buckets_path3); + s3_client = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + accounts.push(res.email); + // create a file in version disabled bucket + await create_object(`${disabled_full_path}/${dis_version_key}`, dis_version_body, 'null'); + // create a file after when versioning not enabled + await create_object(`${full_path}/${dis_version_key}`, dis_version_body, 'null'); + await s3_client.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + await s3_client.putBucketVersioning({ Bucket: nested_keys_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const bucket_ver = await s3_client.getBucketVersioning({ Bucket: bucket_name }); + assert.equal(bucket_ver.Status, 'Enabled'); + // create base file after versioning enabled + file_pointer = await create_object(`${full_path}/${en_version_key}`, en_version_body, versionID_1, true); + }); + + mocha.after(async () => { + if (file_pointer) await file_pointer.close(DEFAULT_FS_CONFIG); + fs_utils.folder_delete(tmp_fs_root); + for (const email of accounts) { + await rpc_client.account.delete_account({ email }); + } + }); + + mocha.it('get object, versioning not enabled - should return latest object', async function() { + const res = await s3_client.getObject({Bucket: disabled_bucket_name, Key: dis_version_key}); + const body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, dis_version_body); + }); + + mocha.it('get object, versioning not enabled, version id specified - should return NoSuchKey', async function() { + try { + await s3_client.getObject({Bucket: disabled_bucket_name, Key: dis_version_key, VersionId: versionID_1}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('get object, file created before the version is enabled - should return latest object', async function() { + const res = await s3_client.getObject({Bucket: bucket_name, Key: dis_version_key}); + const body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, dis_version_body); + }); + + mocha.it('get object, file created before the version is enabled, version id specified - should return ENOENT', async function() { + try { + await s3_client.getObject({Bucket: bucket_name, Key: dis_version_key, VersionId: versionID_1}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('get object, with version enabled, no version id specified - should return latest object', async function() { + const res = await s3_client.getObject({Bucket: bucket_name, Key: en_version_key}); + const body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, en_version_body); + }); + + mocha.it('get object, with version enabled, with version id specified and matching with main file - should return latest object', async function() { + const res = await s3_client.getObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_1}); + const body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, en_version_body); + }); + + mocha.it('get object, with version enabled, with version id specified and not matching with main file - should return $VersionId object', async function() { + await create_object(`${version_dir_path}/${key_version}`, en_version_body_v1, versionID_2); + const res = await s3_client.getObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_2}); + const body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, en_version_body_v1); + }); + + mocha.it('get object, with version enabled, with version id specified and versioned object not present - should return NoSuchKey', async function() { + try { + await s3_client.getObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_3}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('get object, with version enabled, with wrong version id specified - should return Bad Request', async function() { + try { + await s3_client.getObject({Bucket: bucket_name, Key: en_version_key, VersionId: 'ctime-12-ino-12a'}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.Code, 'BadRequest'); + } + }); + + mocha.it('get object - versioning enabled - nested key (more than 1 level)', async function() { + const body1 = 'A'.repeat(5); + const body2 = body1 + 'B'.repeat(3); + const body3 = body2 + 'C'.repeat(7); + const dir_path_nested = 'documents/finance/2024/October/'; + const nested_key_level4 = path.join(dir_path_nested, 'report.pdf'); + await s3_client.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level4, Body: body1 }); + // only after the second PUT object we expect to have the .versions under the parent directory of the file + const put_res2 = await s3_client.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level4, Body: body2}); + await s3_client.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level4, Body: body3}); + + // latest + const get_res = await s3_client.getObject({Bucket: nested_keys_bucket_name, Key: nested_key_level4}); + const body_as_string = await get_res.Body.transformToString(); + assert.equal(body_as_string, body3); + + // by version-id + const get_res2 = await s3_client.getObject({ + Bucket: nested_keys_bucket_name, Key: nested_key_level4, VersionId: put_res2.VersionId}); + const body_as_string2 = await get_res2.Body.transformToString(); + assert.equal(body_as_string2, body2); + + // partial path of (directory without the key) + try { + await s3_client.getObject({Bucket: nested_keys_bucket_name, Key: dir_path_nested}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.name, 'NoSuchKey'); + } + }); + + mocha.it('head object, version not enabled - should return latest object md', async function() { + try { + await s3_client.headObject({Bucket: disabled_bucket_name, Key: dis_version_key}); + assert.ok('Expected latest head object returned'); + } catch (err) { + assert.fail('Latest head object not found'); + } + }); + + mocha.it('head object, version not enabled, version specified - should return object NotFound', async function() { + try { + await s3_client.headObject({Bucket: disabled_bucket_name, Key: dis_version_key, VersionId: versionID_1}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.name, 'NotFound'); + } + }); + + mocha.it('head object, file created before the version is enabled - should return latest object md', async function() { + try { + await s3_client.headObject({Bucket: bucket_name, Key: dis_version_key}); + assert.ok('Expected latest head object returned'); + } catch (err) { + assert.fail('Latest head object not found'); + } + }); + + mocha.it('head object, file created before the version is enabled, version specified - should return NotFound', async function() { + try { + await s3_client.headObject({Bucket: bucket_name, Key: dis_version_key, VersionId: versionID_1}); + assert.ok('Expected latest head object returned'); + } catch (err) { + assert.equal(err.name, 'NotFound'); + } + }); + + mocha.it('head object, with version enabled, no version id specified - should return latest object md', async function() { + try { + await s3_client.headObject({Bucket: bucket_name, Key: en_version_key}); + assert.ok('Expected latest head object returned'); + } catch (err) { + assert.fail(`Failed with an error: ${err.Code}`); + } + }); + + mocha.it('head object, with version enabled, with version id specified and matching with main file - should return latest object md', async function() { + try { + await s3_client.headObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_1}); + assert.ok('Expected latest head object returned'); + } catch (err) { + assert.fail(`Failed with an error: ${err.Code}`); + } + }); + + mocha.it('head object, with version enabled, with version id specified and not matching with main file - should return $VersionId object md', async function() { + await create_object(`${version_dir_path}/${key_version}`, en_version_body_v1, versionID_2); + try { + await s3_client.headObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_2}); + assert.ok('Expected versioned object returned'); + } catch (err) { + assert.fail(`Failed with an error: ${err.Code}`); + } + }); + + mocha.it('head object, with version enabled, versioned object not created - should return NotFound', async function() { + try { + await s3_client.headObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_3}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.name, 'NotFound'); + } + }); + + mocha.it('head object, with version enabled, wrong version id specified - should return NotFound', async function() { + try { + await s3_client.headObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_3}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.name, 'NotFound'); + } + }); + + mocha.it('head object - versioning enabled - nested key (more than 1 level)', async function() { + const body1 = 'A'.repeat(5); + const body2 = body1 + 'B'.repeat(3); + const body3 = body2 + 'C'.repeat(7); + const dir_path_nested = 'documents/finance/2024/October/'; + const nested_key_level4 = path.join(dir_path_nested, 'report.pdf'); + await s3_client.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level4, Body: body1 }); + // only after the second PUT object we expect to have the .versions under the parent directory of the file + const put_res2 = await s3_client.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level4, Body: body2}); + await s3_client.putObject({ Bucket: nested_keys_bucket_name, Key: nested_key_level4, Body: body3}); + + // latest + const head_res = await s3_client.headObject({Bucket: nested_keys_bucket_name, Key: nested_key_level4}); + assert.equal(head_res.ContentLength, body3.length); + + // by version-id + const head_res2 = await s3_client.headObject({ + Bucket: nested_keys_bucket_name, Key: nested_key_level4, VersionId: put_res2.VersionId}); + assert.equal(head_res2.ContentLength, body2.length); + + // partial path of (directory without the key) + try { + await s3_client.headObject({Bucket: nested_keys_bucket_name, Key: dir_path_nested}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.name, 'NotFound'); + } + }); + + mocha.it('Put object when version disbled and do put on the same object when version enabled - get object should return versioned object', async function() { + let res = await s3_client.getObject({Bucket: disabled_bucket_name, Key: dis_version_key}); + let body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, dis_version_body); + const version_obj_path = disabled_full_path + '/.versions'; + const version_key = dis_version_key + '_null'; + const version_body = 'DDDDD'; + const version_body_new_version = 'EEEEE'; + await s3_client.putBucketVersioning({ Bucket: disabled_bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const bucket_ver = await s3_client.getBucketVersioning({ Bucket: disabled_bucket_name }); + assert.equal(bucket_ver.Status, 'Enabled'); + // May be below 3 steps can be replaced with put_object() to create versioned object + await fs_utils.create_fresh_path(version_obj_path, 0o770); + await fs_utils.file_must_exist(version_obj_path); + await create_object(`${version_obj_path}/${version_key}`, version_body, 'null'); + await create_object(`${disabled_full_path}/${dis_version_key}`, version_body_new_version, 'mtime-jhdfbkjsd-ino-bnsdf7f'); + res = await s3_client.getObject({Bucket: disabled_bucket_name, Key: dis_version_key, VersionId: 'null'}); + body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, version_body); + }); + + mocha.it('get object, with version enabled, delete marker placed on latest object - should return NoSuchKey', async function() { + const xattr_delete_marker = { [XATTR_DELETE_MARKER]: 'true' }; + await file_pointer.replacexattr(DEFAULT_FS_CONFIG, xattr_delete_marker); + try { + await s3_client.getObject({Bucket: bucket_name, Key: en_version_key}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.Code, 'NoSuchKey'); + } + }); + + mocha.it('get object, with version enabled, delete marker placed on latest object, version id specified - should return the version object', async function() { + const xattr_delete_marker = { [XATTR_DELETE_MARKER]: 'true' }; + await file_pointer.replacexattr(DEFAULT_FS_CONFIG, xattr_delete_marker); + const res = await s3_client.getObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_2}); + const body_as_string = await res.Body.transformToString(); + assert.equal(body_as_string, en_version_body_v1); + }); + + mocha.it('head object, with version enabled, delete marked xattr placed on latest object - should return ENOENT', async function() { + const xattr_delete_marker = { [XATTR_DELETE_MARKER]: 'true' }; + await file_pointer.replacexattr(DEFAULT_FS_CONFIG, xattr_delete_marker); + try { + await s3_client.headObject({Bucket: bucket_name, Key: en_version_key}); + assert.fail('Should fail'); + } catch (err) { + assert.equal(err.name, 'NotFound'); + } + }); + + mocha.it('head object, with version enabled, delete marked xattr placed on latest object, version id specified - should return the version object', async function() { + const xattr_delete_marker = { [XATTR_DELETE_MARKER]: 'true' }; + await file_pointer.replacexattr(DEFAULT_FS_CONFIG, xattr_delete_marker); + try { + await s3_client.headObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_2}); + assert.ok('Expected versioned object returned'); + } catch (err) { + assert.fail(`Failed with an error: ${err.Code}`); + } + }); + + mocha.it('head object, with version enabled, version id specified delete marker - should throw error with code 405', async function() { + try { + await s3_client.headObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_1}); + assert.fail('Should fail'); + } catch (err) { + assert.strictEqual(err.$metadata.httpStatusCode, 405); + assert.ok(err.$response.headers['last-modified'] !== undefined, 'Should have last-modified header'); + assert.ok(err.$response.headers['x-amz-delete-marker'] === 'true', 'Should have x-amz-delete-marker header with value true'); + // In headObject the AWS SDK doesn't return the err.Code + // In AWS CLI it looks: + // An error occurred (405) when calling the HeadObject operation: Method Not Allowed + // in the docs: https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html + // if the HEAD request generates an error, it returns a generic code, such as ... + // 405 Method Not Allowed, ... It's not possible to retrieve the exact exception of these error codes. + } + }); + + mocha.it('get object, with version enabled, version id specified delete marker - should throw error with code 405', async function() { + try { + await s3_client.getObject({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_1}); + assert.fail('Should fail'); + } catch (err) { + assert.strictEqual(err.$metadata.httpStatusCode, 405); + assert.strictEqual(err.Code, 'MethodNotAllowed'); + assert.ok(err.$response.headers['last-modified'] !== undefined, 'Should have last-modified header'); + assert.ok(err.$response.headers['x-amz-delete-marker'] === 'true', 'Should have x-amz-delete-marker header with value true'); + // In AWS CLI it looks: + // An error occurred (MethodNotAllowed) when calling the GetObject operation: The specified method is not allowed against this resource. + } + }); + + mocha.it('get object attributes, with version enabled, version id specified delete marker - should throw error with code 405', async function() { + try { + await s3_client.getObjectAttributes({Bucket: bucket_name, Key: en_version_key, VersionId: versionID_1, ObjectAttributes: ['ETag']}); + assert.fail('Should fail'); + } catch (err) { + assert.strictEqual(err.$metadata.httpStatusCode, 405); + assert.strictEqual(err.Code, 'MethodNotAllowed'); + assert.ok(err.$response.headers['last-modified'] !== undefined, 'Should have last-modified header'); + assert.ok(err.$response.headers['x-amz-delete-marker'] === 'true', 'Should have x-amz-delete-marker header with value true'); + // In AWS CLI it looks: + // An error occurred (MethodNotAllowed) when calling the GetObjectAttributes operation: The specified method is not allowed against this resource. + } + }); + }); +}); + + + +///// UTILS /////// + +async function delete_object_versions(bucket_path, key) { + // delete past versions + const versions_dir = path.join(bucket_path, '.versions'); + try { + const versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + + for (const entry of versions) { + if (entry.name.startsWith(key)) { + await fs_utils.file_delete(path.join(versions_dir, entry.name)); + } + } + } catch (err) { + console.log('find_max_version_past: .versions is missing'); + } + // delete latest version + await fs_utils.file_delete(path.join(bucket_path, key)); +} + +// upload_object_versions +// object_types_arr: +// 1. Would contain strings 'regular', 'null', and 'delete_marker'. +// 2. If array contains n elements then: +// at index 0 is the version that was put first (oldest version) +// at index n-1 is the version that was put last (newest version) +async function upload_object_versions(s3_client, bucket, key, object_types_arr) { + const res = []; + const versioning_status = await s3_client.getBucketVersioning({ Bucket: bucket }); + for (const obj_type of object_types_arr) { + if (obj_type === 'regular' || obj_type === 'null') { + if (!versioning_status.Status && obj_type === 'regular') { + await s3_client.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + } + const random_body = (Math.random() + 1).toString(36).substring(7); + const put_res = await s3_client.putObject({ Bucket: bucket, Key: key, Body: random_body }); + res.push(put_res); + } else if (obj_type === 'delete_marker') { + if (!versioning_status.Status) { + await s3_client.putBucketVersioning({ Bucket: bucket, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + } + const delete_res = await s3_client.deleteObject({ Bucket: bucket, Key: key }); + res.push(delete_res); + } + } + return res; +} +// add the prev xattr optimization +async function find_max_version_past(full_path, key, dir, skip_list) { + const versions_dir = path.join(full_path, dir || '', '.versions'); + try { + //let versions = await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir); + let max_mtime_nsec = 0; + let max_path; + const versions = (await nb_native().fs.readdir(DEFAULT_FS_CONFIG, versions_dir)).filter(entry => { + const index = entry.name.endsWith('_null') ? entry.name.lastIndexOf('_null') : + entry.name.lastIndexOf('_mtime-'); + // don't fail if version entry name is invalid, just keep searching + return index > 0 && entry.name.slice(0, index) === key; + }); + for (const entry of versions) { + if (skip_list ? !skip_list.includes(entry.name.slice(key.length + 1)) : true) { + const version_str = entry.name.slice(key.length + 1); + const { mtimeNsBigint } = _extract_version_info_from_xattr(version_str) || + (await nb_native().fs.stat(DEFAULT_FS_CONFIG, path.join(versions_dir, entry.name))); + + if (mtimeNsBigint > max_mtime_nsec) { + max_mtime_nsec = mtimeNsBigint; + max_path = entry.name; + } + } + } + return max_path && max_path.slice(key.length + 1); + } catch (err) { + console.log('find_max_version_past: .versions is missing', err); + } +} + +function _extract_version_info_from_xattr(version_id_str) { + if (version_id_str === 'null') return; + const arr = version_id_str.split('mtime-').join('').split('-ino-'); + if (arr.length < 2) throw new Error('Invalid version_id_string, cannot extact attributes'); + return { mtimeNsBigint: size_utils.string_to_bigint(arr[0], 36), ino: parseInt(arr[1], 36) }; +} + +/** + * version_file_exists returns path of version in .versions + * @param {String} full_path + * @param {String} key + * @param {String} dir + * @param {String} version_id + * @returns {Promise} + */ +async function version_file_exists(full_path, key, dir, version_id) { + const version_path = path.join(full_path, dir, '.versions', key + '_' + version_id); + await fs_utils.file_must_exist(version_path); + await check_non_current_xattr_exists(version_path, ''); + return true; +} + +async function version_file_must_not_exists(full_path, key, dir, version_id) { + const version_path = path.join(full_path, dir, '.versions', key + '_' + version_id); + await fs_utils.file_must_not_exist(version_path); + return true; +} + +async function get_obj_and_compare_data(s3, bucket_name, key, expected_body) { + const get_res = await s3.getObject({ Bucket: bucket_name, Key: key }); + const body_as_string = await get_res.Body.transformToString(); + assert.equal(body_as_string, expected_body); + return true; +} + +async function is_delete_marker(full_path, dir, key, version, check_non_current_version = true) { + const version_path = path.join(full_path, dir, '.versions', key + '_' + version); + const stat = await nb_native().fs.stat(DEFAULT_FS_CONFIG, version_path); + return stat && stat.xattr[XATTR_DELETE_MARKER] && (check_non_current_version ? stat.xattr[XATTR_NON_CURRENT_TIMESTASMP] : true); +} + +async function stat_and_get_version_id(full_path, key) { + const key_path = path.join(full_path, key); + const stat = await nb_native().fs.stat(DEFAULT_FS_CONFIG, key_path); + return get_version_id_by_xattr(stat); +} + +async function stat_and_get_all(full_path, key) { + const key_path = path.join(full_path, key); + const stat = await nb_native().fs.stat(DEFAULT_FS_CONFIG, key_path); + return stat; +} + +async function compare_version_ids(full_path, key, put_result_version_id, prev_version_id, is_enabled = true) { + const key_path = path.join(full_path, key); + const stat = await nb_native().fs.stat(DEFAULT_FS_CONFIG, key_path); + const new_version_id = is_enabled ? get_version_id_by_stat(stat) : NULL_VERSION_ID; + const xattr_version_id = get_version_id_by_xattr(stat); + if (is_enabled) { + assert.equal(new_version_id, put_result_version_id); + } else { + // When versioning is Suspended or Disabled the response of put object will include only Etag. + // Hence, put_result_version_id should be undefined. + assert.notEqual(new_version_id, put_result_version_id); + } + assert.equal(new_version_id, xattr_version_id); + if (prev_version_id) { + if (is_enabled) { + // When versioning is Enabled the version IDs are unique. + // Hence, the new version ID must be different than the previous one. + assert.notEqual(new_version_id, prev_version_id); + } + } + return true; +} + +function get_version_id_by_stat(stat) { + return 'mtime-' + stat.mtimeNsBigint.toString(36) + '-ino-' + stat.ino.toString(36); +} + +function get_version_id_by_xattr(stat) { + return (stat && stat.xattr[XATTR_VERSION_ID]) || 'null'; +} + +function check_enable_version_format(version_id) { + const v_parts = version_id.split('-'); + if (v_parts[0] !== 'mtime' || v_parts[2] !== 'ino') { + return false; + } + + const version_format = /^[a-z0-9]+$/; + if (!version_format.test(v_parts[1]) || !version_format.test(v_parts[3])) { + return false; + } + return true; +} + +function check_null_version_id(version_id) { + return version_id === NULL_VERSION_ID; +} + +async function check_no_user_attributes(full_path, key) { + const stat = await stat_and_get_all(full_path, key); + const user_xattr = _.pickBy(stat?.xattr, (val, name) => name?.startsWith(XATTR_USER_PREFIX)); + return Object.keys(user_xattr).length === 0; +} + +async function create_empty_content_dir(fs_context, bucket_path, key) { + const full_path = path.join(bucket_path, key); + await nb_native().fs.mkdir(fs_context, full_path); + const fd = await nb_native().fs.open(fs_context, full_path, 'r'); + await fd.replacexattr(fs_context, {[XATTR_DIR_CONTENT]: '0'}); + fd.close(fs_context); + +} + +/** + * check_non_current_xattr_exists checks that the XATTR_NON_CURRENT_TIMESTASMP xattr exists + * @param {String} full_path + * @param {String} [key] + * @returns {Promise} + */ +async function check_non_current_xattr_exists(full_path, key = '') { + const stat = await stat_and_get_all(full_path, key); + assert.ok(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP]); +} + +/** + * check_non_current_xattr_does_not_exist checks that the XATTR_NON_CURRENT_TIMESTASMP xattr does not exist + * @param {String} full_path + * @param {String} [key] + * @returns {Promise} + */ +async function check_non_current_xattr_does_not_exist(full_path, key = '') { + const stat = await stat_and_get_all(full_path, key); + assert.equal(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP], undefined); +} + +async function put_allow_all_bucket_policy(s3_client, bucket) { + const policy = { + Version: '2012-10-17', + Statement: [{ + Sid: 'id-1', + Effect: 'Allow', + Principal: { AWS: "*" }, + Action: ['s3:*'], + Resource: [`arn:aws:s3:::*`] + } + ] + }; + // create accounts + await s3_client.putBucketPolicy({ + Bucket: bucket, + Policy: JSON.stringify(policy) + }); +} + +mocha.describe('List-objects', function() { + const DEFAULT_MAX_KEYS = 1000; + const nsr = 'noobaa-nsr-object-vesions'; + const bucket_name = 'noobaa-bucket-object-vesions'; + const bucket_name2 = 'noobaa-bucket-object-versions-2'; + const tmp_fs_root4 = path.join(TMP_PATH, 'test_bucketspace_list_object_versions'); + const bucket_path = '/bucket'; + const bucket_path2 = '/bucket2'; + const full_path = tmp_fs_root4 + bucket_path; + const full_path2 = tmp_fs_root4 + bucket_path2; + const version_dir = '/.versions'; + const full_path_version_dir = full_path + `${version_dir}`; + const dir1 = full_path + '/dir1'; + const dir1_version_dir = dir1 + `${version_dir}`; + + let s3_client; + let s3_admin; + const accounts = []; + const key = 'search_key'; + const body = 'AAAA'; + const version_key_1 = 'search_key_mtime-crh3783sxk3k-ino-guty'; + const version_key_2 = 'search_key_mtime-crkfjum9883k-ino-guu7'; + const version_key_3 = 'search_key_mtime-crkfjx1hui2o-ino-guu9'; + const version_key_4 = 'search_key_mtime-crkfjx1hui2o-ino-guuh'; + const key_version = 'mtime-gffdt785k3k-ino-fgy'; + + const dir_key = 'dir1/delete_marker_key'; + const dir_version_key_1 = 'delete_marker_key_mtime-crkfjknr7xmo-ino-guu4'; + const dir_version_key_2 = 'delete_marker_key_mtime-crkfjr98uiv4-ino-guu6'; + const version_body = 'A1A1A1A'; + + let file_pointer; + + mocha.before(async function() { + this.timeout(0); // eslint-disable-line no-invalid-this + if (process.getgid() !== 0 || process.getuid() !== 0) { + console.log('No Root permissions found in env. Skipping test'); + this.skip(); // eslint-disable-line no-invalid-this + } + // create paths + await fs_utils.create_fresh_path(tmp_fs_root4, 0o777); + await fs_utils.create_fresh_path(full_path, 0o770); + await fs_utils.file_must_exist(full_path); + await fs_utils.create_fresh_path(full_path2, 0o777); + await fs_utils.file_must_exist(full_path2); + await fs_utils.create_fresh_path(full_path_version_dir, 0o770); + await fs_utils.file_must_exist(full_path_version_dir); + await fs_utils.create_fresh_path(dir1, 0o770); + await fs_utils.file_must_exist(dir1); + await fs_utils.create_fresh_path(dir1_version_dir, 0o770); + await fs_utils.file_must_exist(dir1_version_dir); + const new_bucket_path4 = get_new_buckets_path_by_test_env(tmp_fs_root4, '/'); + if (is_nc_coretest) { + const { uid, gid } = get_admin_mock_account_details(); + await set_path_permissions_and_owner(full_path, { uid, gid }, 0o700); + } + // export dir as a bucket + await rpc_client.pool.create_namespace_resource({ + name: nsr, + nsfs_config: { + fs_root_path: tmp_fs_root4, + } + }); + const obj_nsr = { resource: nsr, path: bucket_path }; + const obj_nsr2 = { resource: nsr, path: bucket_path2 }; + await rpc_client.bucket.create_bucket({ + name: bucket_name, + namespace: { + read_resources: [obj_nsr], + write_resource: obj_nsr + } + }); + await rpc_client.bucket.create_bucket({ + name: bucket_name2, + namespace: { + read_resources: [obj_nsr2], + write_resource: obj_nsr2 + } + }); + const policy = { + Version: '2012-10-17', + Statement: [{ + Sid: 'id-1', + Effect: 'Allow', + Principal: { AWS: "*" }, + Action: ['s3:*'], + Resource: [`arn:aws:s3:::*`] + }, + ] + }; + // create accounts + let res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path4, { admin: true }); + s3_admin = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + await s3_admin.putBucketPolicy({ + Bucket: bucket_name, + Policy: JSON.stringify(policy) + }); + await s3_admin.putBucketPolicy({ + Bucket: bucket_name2, + Policy: JSON.stringify(policy) + }); + // create nsfs account + res = await generate_nsfs_account(rpc_client, EMAIL, new_bucket_path4); + s3_client = generate_s3_client(res.access_key, res.secret_key, CORETEST_ENDPOINT); + accounts.push(res.email); + await s3_client.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const bucket_ver = await s3_client.getBucketVersioning({ Bucket: bucket_name }); + assert.equal(bucket_ver.Status, 'Enabled'); + await create_object(`${full_path}/${key}`, body, key_version); + await create_object(`${full_path_version_dir}/${version_key_1}`, version_body, 'mtime-crh3783sxk3k-ino-guty'); + await create_object(`${full_path_version_dir}/${version_key_2}`, version_body, 'mtime-crkfjum9883k-ino-guu7'); + await create_object(`${full_path_version_dir}/${version_key_3}`, version_body, 'mtime-crkfjx1hui2o-ino-guu9'); + file_pointer = await create_object(`${full_path}/${dir_key}`, version_body, 'null', true); + await create_object(`${dir1_version_dir}/${dir_version_key_1}`, version_body, 'mtime-crkfjknr7xmo-ino-guu4'); + await create_object(`${dir1_version_dir}/${dir_version_key_2}`, version_body, 'mtime-crkfjr98uiv4-ino-guu6'); + }); + + mocha.after(async () => { + this.timeout(0); // eslint-disable-line no-invalid-this + if (file_pointer) await file_pointer.close(DEFAULT_FS_CONFIG); + fs_utils.folder_delete(tmp_fs_root); + for (const email of accounts) { + await rpc_client.account.delete_account({ email }); + } + }); + + mocha.beforeEach(async () => { + this.timeout(0); // eslint-disable-line no-invalid-this + await fs_utils.create_fresh_path(full_path2, 0o777); + await P.delay(100); // sometime we saw that the check failed although the path is created a line before + const file_exists = await fs_utils.file_exists(full_path2); + assert.ok(file_exists); + }); + + mocha.it('list objects - should return only latest object', async function() { + const res = await s3_client.listObjects({Bucket: bucket_name}); + let count = 0; + res.Contents.forEach(val => { + if (val.Key === key) { + count += 1; + } + }); + assert.equal(count, 1); + }); + + mocha.it('list object versions - should return all versions of all the object', async function() { + const res = await s3_client.listObjectVersions({Bucket: bucket_name}); + let count = 0; + res.Versions.forEach(val => { + count += 1; + }); + assert.equal(count, 7); + }); + + mocha.it('list object versions - should by pass the dir cache and load the objects from the disk', async function() { + // Creation of version under HIDDER_VERSION_PATH will trigger load() call + await create_object(`${full_path_version_dir}/${version_key_4}`, version_body, 'null'); + let res = await s3_client.listObjectVersions({Bucket: bucket_name}); + let count = 0; + res.Versions.forEach(val => { + count += 1; + }); + assert.equal(count, 8); + // Deletion of version under HIDDER_VERSION_PATH will trigger load() call + await fs_utils.file_delete(path.join(full_path_version_dir, version_key_4)); + res = await s3_client.listObjectVersions({Bucket: bucket_name}); + count = 0; + res.Versions.forEach(val => { + count += 1; + }); + assert.equal(count, 7); + }); + + mocha.it('list object versions - should return all the versions of the requested object', async function() { + const res = await s3_client.listObjectVersions({Bucket: bucket_name, KeyMarker: key}); + let count = 0; + res.Versions.forEach(val => { + count += 1; + }); + assert.equal(count, 3); + }); + + mocha.it('list object versions - should return only max_key number of elements', async function() { + const res = await s3_client.listObjectVersions({Bucket: bucket_name, MaxKeys: 2}); + let count = 0; + res.Versions.forEach(val => { + count += 1; + }); + assert.equal(count, 2); + }); + + mocha.it('list object versions - should return only max keys of the versions of the requested object and nextKeyMarker should point to the next version', async function() { + const res = await s3_client.listObjectVersions({Bucket: bucket_name, KeyMarker: key, MaxKeys: 2}); + let count = 0; + assert(res.KeyMarker, key); + assert(res.NextKeyMarker, key); + assert(res.NextVersionIdMarker, version_key_2); + res.Versions.forEach(val => { + count += 1; + }); + assert.equal(count, 2); + }); + + mocha.it('list object versions - should return only max keys of the versions of the requested object from the version id marker', async function() { + const res = await s3_client.listObjectVersions({Bucket: bucket_name, + KeyMarker: key, MaxKeys: 2, + VersionIdMarker: version_key_3}); + let count = 0; + res.Versions.forEach(val => { + count += 1; + }); + assert.equal(count, 2); + }); + + mocha.it('list object versions - should fail because of missing key_marker', async function() { + try { + await s3_client.listObjectVersions({Bucket: bucket_name, + MaxKeys: 2, VersionIdMarker: version_key_3}); + assert.fail(`list object versions passed though key marker is missing`); + } catch (err) { + assert.equal(err.Code, 'InvalidArgument'); + } + }); + + mocha.it('list objects - deleted object should not be listed', async function() { + const xattr_delete_marker = { [XATTR_DELETE_MARKER]: 'true' }; + file_pointer.replacexattr(DEFAULT_FS_CONFIG, xattr_delete_marker); + const res = await s3_client.listObjects({Bucket: bucket_name}); + let count = 0; + res.Contents.forEach(val => { + if (val.Key === dir_key) { + count += 1; + } + }); + assert.equal(count, 0); + }); + + mocha.it('list object versions - All versions of the object should be listed', async function() { + const xattr_delete_marker = { [XATTR_DELETE_MARKER]: 'true' }; + file_pointer.replacexattr(DEFAULT_FS_CONFIG, xattr_delete_marker); + const res = await s3_client.listObjectVersions({Bucket: bucket_name}); + let count = 0; + res.Versions.forEach(val => { + if (val.Key === dir_key) { + count += 1; + } + }); + assert.equal(count, 2); + }); + + mocha.it('list object versions - Check whether the deleted object is the latest and should be present under delete markers as well', async function() { + const xattr_delete_marker = { [XATTR_DELETE_MARKER]: 'true' }; + file_pointer.replacexattr(DEFAULT_FS_CONFIG, xattr_delete_marker); + const res = await s3_client.listObjectVersions({Bucket: bucket_name}); + + res.DeleteMarkers.forEach(val => { + if (val.Key === dir_key) { + assert.equal(val.IsLatest, true); + assert.equal(res.DeleteMarkers[0].IsLatest, true); + assert.equal(res.DeleteMarkers[0].Key, dir_key); + } + }); + }); + + mocha.it('list object versions - should not list .versions folder', async function() { + const res = await s3_client.listObjectVersions({Bucket: bucket_name, Delimiter: "/"}); + res.CommonPrefixes?.forEach(obj => { + assert.notEqual(obj.Prefix, ".versions/"); + }); + }); + + mocha.it('list object versions - should show isLatest flag only for latest objects', async function() { + const res = await s3_client.listObjectVersions({Bucket: bucket_name}); + let count = 0; + res.Versions.forEach(val => { + if (val.IsLatest) { + if (val.Key === key) { + assert.equal(val.VersionId, key_version); + } + count += 1; + } + }); + res.DeleteMarkers.forEach(val => { + if (val.IsLatest) { + assert.equal(val.VersionId, 'null'); + count += 1; + } + }); + assert.equal(count, 2); + }); + + mocha.it('list object versions - should show version-id in suspention mode', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({Bucket: bucket_name}); + let count = 0; + res.Versions.forEach(val => { + if (val.VersionId !== 'null') { + count += 1; + } + }); + assert.equal(count, 6); + }); + + mocha.it('list object versions - no objects in the bucket (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.ok(res.Versions === undefined); // no versions yet + }); + + mocha.it('list object versions - no objects in the bucket (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.ok(res.Versions === undefined); // no versions yet + }); + + mocha.it('list object versions - 1 nested object in the bucket with unique version id (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'New/Year/Happy.txt'; + const put_res = await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, 1); + assert.equal(res.Versions[0].VersionId, put_res.VersionId); + }); + + mocha.it('list object versions - 1 nested object in the bucket with version id null (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const key1 = 'New/Year/Happy.txt'; + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, 1); + assert.equal(res.Versions[0].VersionId, NULL_VERSION_ID); + }); + + mocha.it('list object versions - 1 delete marker only (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'Moo/Koo/Loo.txt'; + const put_res = await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + const delete_res = await s3_client.deleteObject({ Bucket: bucket_name2, Key: key1 }); // create delete-marker + await s3_client.deleteObject({ Bucket: bucket_name2, Key: key1, VersionId: put_res.VersionId }); // delete the version + + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.ok(res.Versions === undefined); + assert.equal(res.DeleteMarkers.length, 1); + assert.equal(res.DeleteMarkers[0].VersionId, delete_res.VersionId); + assert.equal(res.DeleteMarkers[0].IsLatest, true); + }); + + mocha.it('list object versions - 1 delete marker only (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'Moo/Koo/Loo.txt'; + const put_res = await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + const delete_res = await s3_client.deleteObject({ Bucket: bucket_name2, Key: key1 }); // create delete-marker + await s3_client.deleteObject({ Bucket: bucket_name2, Key: key1, VersionId: put_res.VersionId }); // delete the version + + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.ok(res.Versions === undefined); + assert.equal(res.DeleteMarkers.length, 1); + assert.equal(res.DeleteMarkers[0].VersionId, delete_res.VersionId); + assert.equal(res.DeleteMarkers[0].IsLatest, true); + }); + + mocha.it('list object versions - 3 objects (1 of them with 2 versions) - ' + + 'check the sorting (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'zoom.txt'; + const key2 = 'all.txt'; + const key3 = 'big.txt'; + + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key2, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key3, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); // another version + + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, 4); + // expect: all.txt, big.txt, zoom.txt (latest) and then another zoom.txt + assert.equal(res.Versions[0].Key, key2); + assert.equal(res.Versions[0].IsLatest, true); + assert.equal(res.Versions[1].Key, key3); + assert.equal(res.Versions[1].IsLatest, true); + assert.equal(res.Versions[2].Key, key1); + assert.equal(res.Versions[2].IsLatest, true); + assert.equal(res.Versions[3].Key, key1); + assert.equal(res.Versions[3].IsLatest, false); + }); + + mocha.it('list object versions - 3 objects (1 of them with 2 versions) - ' + + 'check the sorting (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'zoom.txt'; + const key2 = 'all.txt'; + const key3 = 'big.txt'; + + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key2, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key3, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); // another version + + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, 4); + // expect: all.txt, big.txt, zoom.txt (latest) and then another zoom.txt + assert.equal(res.Versions[0].Key, key2); + assert.equal(res.Versions[0].IsLatest, true); + assert.equal(res.Versions[1].Key, key3); + assert.equal(res.Versions[1].IsLatest, true); + assert.equal(res.Versions[2].Key, key1); + assert.equal(res.Versions[2].IsLatest, true); + assert.equal(res.Versions[3].Key, key1); + assert.equal(res.Versions[3].IsLatest, false); + }); + + mocha.it('list object versions - 4 objects (3 of them deleted) - ' + + 'check the sorting (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'aa.txt'; + const key2 = 'af.txt'; + const key3 = 'am.txt'; + const key4 = 'ay.txt'; + + await s3_client.putObject({ Bucket: bucket_name2, Key: key3, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key4, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key2, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + + await s3_client.deleteObject({ Bucket: bucket_name2, Key: key1}); + await s3_client.deleteObject({ Bucket: bucket_name2, Key: key2}); + await s3_client.deleteObject({ Bucket: bucket_name2, Key: key3}); + + + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, 4); + assert.equal(res.DeleteMarkers.length, 3); + // expect: versions: aa.txt, af.txt, am.txt (not latest) ay.txt (only latest), delete markers: aa.txt, af.txt, am.txt + assert.equal(res.Versions[3].Key, key4); + assert.equal(res.Versions[3].IsLatest, true); + const arr_to_compare = res.Versions.slice(0, -1); //without the last item + const comp_res = arr_to_compare.every(item => item.IsLatest === false); + assert.ok(comp_res); + assert.equal(res.DeleteMarkers[0].Key, key1); + assert.equal(res.DeleteMarkers[1].Key, key2); + assert.equal(res.DeleteMarkers[2].Key, key3); + const comp_res2 = res.DeleteMarkers.every(item => item.IsLatest === true); + assert.ok(comp_res2); + }); + + mocha.it('list object versions - 4 objects (3 of them deleted) - ' + + 'check the sorting (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'aa.txt'; + const key2 = 'af.txt'; + const key3 = 'am.txt'; + const key4 = 'ay.txt'; + + await s3_client.putObject({ Bucket: bucket_name2, Key: key3, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key4, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key2, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + + await s3_client.deleteObject({ Bucket: bucket_name2, Key: key1}); + await s3_client.deleteObject({ Bucket: bucket_name2, Key: key2}); + await s3_client.deleteObject({ Bucket: bucket_name2, Key: key3}); + + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, 4); + assert.equal(res.DeleteMarkers.length, 3); + // expect: versions: aa.txt, af.txt, am.txt (not latest) ay.txt (only latest), delete markers: aa.txt, af.txt, am.txt + assert.equal(res.Versions[3].Key, key4); + assert.equal(res.Versions[3].IsLatest, true); + const arr_to_compare = res.Versions.slice(0, -1); //without the last item + const comp_res = arr_to_compare.every(item => item.IsLatest === false); + assert.ok(comp_res); + assert.equal(res.DeleteMarkers[0].Key, key1); + assert.equal(res.DeleteMarkers[1].Key, key2); + assert.equal(res.DeleteMarkers[2].Key, key3); + const comp_res2 = res.DeleteMarkers.every(item => item.IsLatest === true); + assert.ok(comp_res2); + }); + + mocha.it('list object versions - 10 versions of the same key in the bucket (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'NiceDay.txt'; + const number_of_versions = 10; + const versions_arr = await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, number_of_versions); + const latest_versions = res.Versions.filter(version => version.IsLatest); + assert.equal(latest_versions.length, 1); + assert.equal(latest_versions[0].VersionId, versions_arr[number_of_versions - 1]); + }); + + mocha.it('list object versions - 10 uploads of the same key in the bucket (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const key1 = 'NiceDay.txt'; + const number_of_versions = 10; + await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, 1); + const latest_versions = res.Versions.filter(version => version.IsLatest); + assert.equal(latest_versions.length, 1); + assert.equal(res.Versions[0].VersionId, NULL_VERSION_ID); + }); + + mocha.it('list object versions - 10 versions of the same key in the bucket (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'NiceDay.txt'; + const number_of_versions = 10; + const versions_arr = await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, number_of_versions); + const latest_versions = res.Versions.filter(version => version.IsLatest); + assert.equal(latest_versions.length, 1); + assert.equal(latest_versions[0].VersionId, versions_arr[number_of_versions - 1]); + }); + + mocha.it('list object versions - 1001 versions of the same key in the bucket - ' + + 'use NextKeyMarker and NextVersionIdMarker for listing over 1,000 versions (versioning enabled)', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'NiceDay.txt'; + const number_of_versions = 1001; + await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + // list the first 1,000 keys + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, true); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, DEFAULT_MAX_KEYS); + const latest_versions = res.Versions.filter(version => version.IsLatest); + assert.equal(latest_versions.length, 1); + // list the next keys (1) + const res2 = await s3_client.listObjectVersions({ Bucket: bucket_name2, + KeyMarker: res.NextKeyMarker, VersionIdMarker: res.NextVersionIdMarker}); + assert.equal(res2.IsTruncated, false); + assert.equal(res2.Versions.length, number_of_versions - DEFAULT_MAX_KEYS); + }); + + mocha.it('list object versions - 1001 versions of the same key in the bucket - ' + + 'use NextKeyMarker and NextVersionIdMarker for listing over 1,000 versions (versioning suspended)', async function() { + const self = this; // eslint-disable-line no-invalid-this + self.timeout(150000); + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'NiceDay.txt'; + const number_of_versions = 1001; + await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + // list the first 1,000 keys + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, true); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, DEFAULT_MAX_KEYS); + const latest_versions = res.Versions.filter(version => version.IsLatest); + assert.equal(latest_versions.length, 1); + // list the next keys (1) + const res2 = await s3_client.listObjectVersions({ Bucket: bucket_name2, + KeyMarker: res.NextKeyMarker, VersionIdMarker: res.NextVersionIdMarker}); + assert.equal(res2.IsTruncated, false); + assert.equal(res2.Versions.length, number_of_versions - DEFAULT_MAX_KEYS); + }); + + mocha.it('list object versions - 5 versions of each key out of 3 keys in the bucket - ' + + 'use KeyMarkerKey and MaxKeys (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'Day1.txt'; + const key2 = 'Day2.txt'; + const key3 = 'Day3.txt'; + const max_keys = 3; + const number_of_versions = 5; + const versions_arr1 = await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + await _upload_versions(bucket_name2, key2, number_of_versions, s3_client); + await _upload_versions(bucket_name2, key3, number_of_versions, s3_client); + + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2, + KeyMarkerKey: 'Day1', MaxKeys: max_keys}); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, true); + assert.equal(res.MaxKeys, max_keys); + assert.equal(res.Versions.length, max_keys); + const res2 = await s3_client.listObjectVersions({ Bucket: bucket_name2, + KeyMarker: 'Day1', VersionIdMarker: res.NextVersionIdMarker, MaxKeys: number_of_versions - max_keys}); + assert.equal(res2.$metadata.httpStatusCode, 200); + assert.equal(res2.IsTruncated, true); + assert.equal(res2.MaxKeys, number_of_versions - max_keys); + assert.equal(res2.Versions.length, number_of_versions - max_keys); + const arr_day1_versions = [...res.Versions, ...res2.Versions]; + const latest_versions = arr_day1_versions.filter(version => version.IsLatest); + assert.equal(latest_versions.length, 1); + assert.equal(latest_versions[0].VersionId, versions_arr1[number_of_versions - 1]); + }); + + mocha.it('list object versions - 5 versions of each key out of 3 keys in the bucket - ' + + 'use KeyMarkerKey and MaxKeys (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'Day1.txt'; + const key2 = 'Day2.txt'; + const key3 = 'Day3.txt'; + const max_keys = 3; + const number_of_versions = 5; + const versions_arr1 = await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + await _upload_versions(bucket_name2, key2, number_of_versions, s3_client); + await _upload_versions(bucket_name2, key3, number_of_versions, s3_client); + + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2, + KeyMarkerKey: 'Day1', MaxKeys: max_keys}); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, true); + assert.equal(res.MaxKeys, max_keys); + assert.equal(res.Versions.length, max_keys); + const res2 = await s3_client.listObjectVersions({ Bucket: bucket_name2, + KeyMarker: 'Day1', VersionIdMarker: res.NextVersionIdMarker, MaxKeys: number_of_versions - max_keys}); + assert.equal(res2.$metadata.httpStatusCode, 200); + assert.equal(res2.IsTruncated, true); + assert.equal(res2.MaxKeys, number_of_versions - max_keys); + assert.equal(res2.Versions.length, number_of_versions - max_keys); + const arr_day1_versions = [...res.Versions, ...res2.Versions]; + const latest_versions = arr_day1_versions.filter(version => version.IsLatest); + assert.equal(latest_versions.length, 1); + assert.equal(latest_versions[0].VersionId, versions_arr1[number_of_versions - 1]); + }); + + mocha.it('list object versions - using Delimiter and Prefix parameters (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'photos/2006/January/sample.jpg'; + const key2 = 'photos/2006/February/sample.jpg'; + const key3 = 'photos/2006/March/sample.jpg'; + const key4 = 'videos/2006/March/sample.wmv'; + const key5 = 'sample.jpg'; + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key2, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key3, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key4, Body: body }); + const put_res = await s3_client.putObject({ Bucket: bucket_name2, Key: key5, Body: body }); + + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2, Delimiter: '/'}); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.CommonPrefixes.length, 2); + const comp_res = res.CommonPrefixes.every(item => ['photos/', 'videos/'].includes(item.Prefix)); + assert.ok(comp_res); + assert.equal(res.Versions.length, 1); + assert.equal(res.Versions[0].Key, key5); + assert.equal(res.Versions[0].VersionId, put_res.VersionId); + assert.equal(res.Versions[0].IsLatest, true); + + const res2 = await s3_client.listObjectVersions({ Bucket: bucket_name2, Delimiter: '/', Prefix: 'photos/2006/'}); + assert.equal(res2.$metadata.httpStatusCode, 200); + assert.equal(res2.IsTruncated, false); + assert.equal(res2.CommonPrefixes.length, 3); + const comp_res2 = res2.CommonPrefixes.every(item => ['photos/2006/January/', 'photos/2006/February/', 'photos/2006/March/'].includes(item.Prefix)); + assert.ok(comp_res2); + assert.ok(res2.Versions === undefined); + }); + + mocha.it('list object versions - using Delimiter and Prefix parameters (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'photos/2006/January/sample.jpg'; + const key2 = 'photos/2006/February/sample.jpg'; + const key3 = 'photos/2006/March/sample.jpg'; + const key4 = 'videos/2006/March/sample.wmv'; + const key5 = 'sample.jpg'; + await s3_client.putObject({ Bucket: bucket_name2, Key: key1, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key2, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key3, Body: body }); + await s3_client.putObject({ Bucket: bucket_name2, Key: key4, Body: body }); + const put_res = await s3_client.putObject({ Bucket: bucket_name2, Key: key5, Body: body }); + + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2, Delimiter: '/'}); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.CommonPrefixes.length, 2); + const comp_res = res.CommonPrefixes.every(item => ['photos/', 'videos/'].includes(item.Prefix)); + assert.ok(comp_res); + assert.equal(res.Versions.length, 1); + assert.equal(res.Versions[0].Key, key5); + assert.equal(res.Versions[0].VersionId, put_res.VersionId); + assert.equal(res.Versions[0].IsLatest, true); + + const res2 = await s3_client.listObjectVersions({ Bucket: bucket_name2, Delimiter: '/', Prefix: 'photos/2006/'}); + assert.equal(res2.$metadata.httpStatusCode, 200); + assert.equal(res2.IsTruncated, false); + assert.equal(res2.CommonPrefixes.length, 3); + const comp_res2 = res2.CommonPrefixes.every(item => ['photos/2006/January/', 'photos/2006/February/', 'photos/2006/March/'].includes(item.Prefix)); + assert.ok(comp_res2); + assert.ok(res2.Versions === undefined); + }); + + mocha.it('list object versions - after multipart upload 1 part (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const mpu_key1 = 'mpu_key1.txt'; + const res_mpu = await s3_client.createMultipartUpload({ Bucket: bucket_name2, Key: mpu_key1 }); + const upload_id = res_mpu.UploadId; + const body1 = 'AAAAABBBBBCCCCC'; + const part1 = await s3_client.uploadPart({ + Bucket: bucket_name2, Key: mpu_key1, Body: body1, UploadId: upload_id, PartNumber: 1 }); + const res_cmpu = await s3_client.completeMultipartUpload({ + Bucket: bucket_name2, + Key: mpu_key1, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }] + } + }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2}); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.Versions[0].VersionId, res_cmpu.VersionId); + }); + + mocha.it('list object versions - after multipart upload 2 parts (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const mpu_key1 = 'mpu_key1.txt'; + const res_mpu = await s3_client.createMultipartUpload({ Bucket: bucket_name2, Key: mpu_key1 }); + const upload_id = res_mpu.UploadId; + const body1 = 'AAAAABBBBBCCCCC'; + const body2 = 'DDDDDEEEEEEFFFF'; + const part1 = await s3_client.uploadPart({ + Bucket: bucket_name2, Key: mpu_key1, Body: body1, UploadId: upload_id, PartNumber: 1 }); + const part2 = await s3_client.uploadPart({ + Bucket: bucket_name2, Key: mpu_key1, Body: body2, UploadId: upload_id, PartNumber: 2 }); + const res_cmpu = await s3_client.completeMultipartUpload({ + Bucket: bucket_name2, + Key: mpu_key1, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }, + { + ETag: part2.ETag, + PartNumber: 2 + }] + } + }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2}); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.Versions[0].VersionId, res_cmpu.VersionId); + }); + + mocha.it('list object versions - after multipart upload 1 part (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const mpu_key1 = 'mpu_key1.txt'; + const res_mpu = await s3_client.createMultipartUpload({ Bucket: bucket_name2, Key: mpu_key1 }); + const upload_id = res_mpu.UploadId; + const body1 = 'AAAAABBBBBCCCCC'; + const part1 = await s3_client.uploadPart({ + Bucket: bucket_name2, Key: mpu_key1, Body: body1, UploadId: upload_id, PartNumber: 1 }); + await s3_client.completeMultipartUpload({ + Bucket: bucket_name2, + Key: mpu_key1, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }] + } + }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2}); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.Versions[0].VersionId, NULL_VERSION_ID); + }); + + mocha.it('list object versions - after multipart upload 2 parts (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const mpu_key1 = 'mpu_key1.txt'; + const res_mpu = await s3_client.createMultipartUpload({ Bucket: bucket_name2, Key: mpu_key1 }); + const upload_id = res_mpu.UploadId; + const body1 = 'AAAAABBBBBCCCCC'; + const body2 = 'DDDDDEEEEEEFFFF'; + const part1 = await s3_client.uploadPart({ + Bucket: bucket_name2, Key: mpu_key1, Body: body1, UploadId: upload_id, PartNumber: 1 }); + const part2 = await s3_client.uploadPart({ + Bucket: bucket_name2, Key: mpu_key1, Body: body2, UploadId: upload_id, PartNumber: 2 }); + await s3_client.completeMultipartUpload({ + Bucket: bucket_name2, + Key: mpu_key1, + UploadId: upload_id, + MultipartUpload: { + Parts: [{ + ETag: part1.ETag, + PartNumber: 1 + }, + { + ETag: part2.ETag, + PartNumber: 2 + }] + } + }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2}); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.Versions[0].VersionId, NULL_VERSION_ID); + }); + + mocha.it('list object versions - after 3 deletes (non-existing keys) (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'NiceMonth.txt'; + const delete_arr = []; + const res_delete1 = await s3_client.deleteObject({Bucket: bucket_name2, Key: key1}); + delete_arr.push(res_delete1.VersionId); + const res_delete2 = await s3_client.deleteObject({Bucket: bucket_name2, Key: key1}); + delete_arr.push(res_delete2.VersionId); + const res_delete3 = await s3_client.deleteObject({Bucket: bucket_name2, Key: key1}); + delete_arr.push(res_delete3.VersionId); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions, undefined); + assert.equal(res.DeleteMarkers.length, 3); + const comp_res = res.DeleteMarkers.every(item => delete_arr.includes(item.VersionId)); + assert.ok(comp_res); + }); + + mocha.it('list object versions - after 3 deletes (non-existing keys) (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'NiceMonth.txt'; + const delete_arr = []; + const res_delete1 = await s3_client.deleteObject({Bucket: bucket_name2, Key: key1}); + delete_arr.push(res_delete1.VersionId); + const res_delete2 = await s3_client.deleteObject({Bucket: bucket_name2, Key: key1}); + delete_arr.push(res_delete2.VersionId); + const res_delete3 = await s3_client.deleteObject({Bucket: bucket_name2, Key: key1}); + delete_arr.push(res_delete3.VersionId); + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions, undefined); + assert.equal(res.DeleteMarkers.length, 3); + const comp_res = res.DeleteMarkers.every(item => delete_arr.includes(item.VersionId)); + assert.ok(comp_res); + }); + + mocha.it('list object versions - 2 versions of the same key, 1 delete latest (versioning enabled)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'NiceWeek.txt'; + const number_of_versions = 2; + const versions_arr = await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + const res_delete = await s3_client.deleteObject({Bucket: bucket_name2, Key: key1}); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, number_of_versions); + const comp_res = res.Versions.every(item => versions_arr.includes(item.VersionId)); + assert.ok(comp_res); + assert.equal(res.DeleteMarkers.length, 1); + assert.equal(res.DeleteMarkers[0].VersionId, res_delete.VersionId); + assert.equal(res.DeleteMarkers[0].IsLatest, true); + }); + + mocha.it('list object versions - 2 versions of the same key, 1 delete latest (versioning suspended)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const key1 = 'NiceWeek.txt'; + const number_of_versions = 2; + const versions_arr = await _upload_versions(bucket_name2, key1, number_of_versions, s3_client); + const res_delete = await s3_client.deleteObject({Bucket: bucket_name2, Key: key1}); + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Suspended' } }); + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2 }); + assert.equal(res.$metadata.httpStatusCode, 200); + assert.equal(res.IsTruncated, false); + assert.equal(res.MaxKeys, DEFAULT_MAX_KEYS); + assert.equal(res.Versions.length, number_of_versions); + const comp_res = res.Versions.every(item => versions_arr.includes(item.VersionId)); + assert.ok(comp_res); + assert.equal(res.DeleteMarkers.length, 1); + assert.equal(res.DeleteMarkers[0].VersionId, res_delete.VersionId); + assert.equal(res.DeleteMarkers[0].IsLatest, true); + }); + + mocha.it('list object versions - versioning enabled - nested key (more than 1 level)', async function() { + await s3_client.putBucketVersioning({ Bucket: bucket_name2, VersioningConfiguration: { MFADelete: 'Disabled', Status: 'Enabled' } }); + const dir_path_nested = 'user/u1/photos/2006/January/'; + const nested_key_level5 = path.join(dir_path_nested, 'sample.jpg'); + const number_of_versions = 5; + for (let i = 0; i < number_of_versions; i++) { + const body_to_add = `YYY-${i}`; + await s3_client.putObject({ Bucket: bucket_name2, Key: nested_key_level5, Body: body_to_add }); + } + + const res = await s3_client.listObjectVersions({ Bucket: bucket_name2}); + assert.equal(res.Versions.length, number_of_versions); + const comp_res = res.Versions.every(item => item.Key === nested_key_level5); + assert.ok(comp_res); + }); + +}); + +async function create_object(object_path, data, version_id, return_fd) { + const target_file = await nb_native().fs.open(DEFAULT_FS_CONFIG, object_path, 'w+'); + await fs.promises.writeFile(object_path, data); + if (version_id !== 'null') { + const xattr_version_id = { [XATTR_VERSION_ID]: `${version_id}` }; + await target_file.replacexattr(DEFAULT_FS_CONFIG, xattr_version_id); + } + if (return_fd) return target_file; + await target_file.close(DEFAULT_FS_CONFIG); +} + +/** + * _upload_versions uploads number_of_versions of key in bucket with a body of random data + * note: this function is not concurrent, it's a helper function for preparing a bucket with a couple of versions + * @param {string} bucket + * @param {string} key + * @param {number} number_of_versions + * @param {object} s3_client + */ +async function _upload_versions(bucket, key, number_of_versions, s3_client) { + const versions_arr = []; + for (let i = 0; i < number_of_versions; i++) { + const body = `some-data-${i}-` + 'A'.repeat(i); + const res = await s3_client.putObject({ Bucket: bucket, Key: key, Body: body }); + versions_arr.push(res.VersionId); + } + return versions_arr; +} diff --git a/src/test/unit_tests/jest_tests/test_nsfs_concurrency.test.js b/src/test/integration_tests/nsfs/test_nsfs_concurrency.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_nsfs_concurrency.test.js rename to src/test/integration_tests/nsfs/test_nsfs_concurrency.test.js diff --git a/src/test/unit_tests/test_nsfs_integration.js b/src/test/integration_tests/nsfs/test_nsfs_integration.js similarity index 98% rename from src/test/unit_tests/test_nsfs_integration.js rename to src/test/integration_tests/nsfs/test_nsfs_integration.js index 7a46c7ff5d..9038669ef6 100644 --- a/src/test/unit_tests/test_nsfs_integration.js +++ b/src/test/integration_tests/nsfs/test_nsfs_integration.js @@ -12,32 +12,31 @@ const util = require('util'); const http = require('http'); const mocha = require('mocha'); const assert = require('assert'); -const http_utils = require('../../util/http_utils'); -const config = require('../../../config'); -const fs_utils = require('../../util/fs_utils'); +const http_utils = require('../../../util/http_utils'); +const config = require('../../../../config'); +const fs_utils = require('../../../util/fs_utils'); const fetch = require('node-fetch'); -const P = require('../../util/promise'); -const cloud_utils = require('../../util/cloud_utils'); -const SensitiveString = require('../../util/sensitive_string'); -const S3Error = require('../../endpoint/s3/s3_errors').S3Error; -const test_utils = require('../system_tests/test_utils'); -const { stat, open } = require('../../util/nb_native')().fs; -const { get_process_fs_context } = require('../../util/native_fs_utils'); -const { TYPES } = require('../../manage_nsfs/manage_nsfs_constants'); -const ManageCLIError = require('../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; -const { TMP_PATH, IS_GPFS, is_nc_coretest, get_coretest_path, invalid_nsfs_root_permissions, +const P = require('../../../util/promise'); +const cloud_utils = require('../../../util/cloud_utils'); +const SensitiveString = require('../../../util/sensitive_string'); +const S3Error = require('../../../endpoint/s3/s3_errors').S3Error; +const test_utils = require('../../system_tests/test_utils'); +const { stat, open } = require('../../../util/nb_native')().fs; +const { get_process_fs_context } = require('../../../util/native_fs_utils'); +const { TYPES } = require('../../../manage_nsfs/manage_nsfs_constants'); +const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; +const { TMP_PATH, IS_GPFS, is_nc_coretest, require_coretest, invalid_nsfs_root_permissions, generate_s3_policy, create_fs_user_by_platform, delete_fs_user_by_platform, get_new_buckets_path_by_test_env, - generate_s3_client, exec_manage_cli, generate_anon_s3_client, generate_nsfs_account } = require('../system_tests/test_utils'); -const nc_mkm = require('../../manage_nsfs/nc_master_key_manager').get_instance(); + generate_s3_client, exec_manage_cli, generate_anon_s3_client, generate_nsfs_account } = require('../../system_tests/test_utils'); +const nc_mkm = require('../../../manage_nsfs/nc_master_key_manager').get_instance(); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); -const native_fs_utils = require('../../util/native_fs_utils'); -const mongo_utils = require('../../util/mongo_utils'); +const native_fs_utils = require('../../../util/native_fs_utils'); +const mongo_utils = require('../../../util/mongo_utils'); -const coretest_path = get_coretest_path(); -const coretest = require(coretest_path); +const coretest = require_coretest(); const { rpc_client, EMAIL, PASSWORD, SYSTEM, get_admin_mock_account_details, NC_CORETEST_CONFIG_FS } = coretest; coretest.setup({}); let CORETEST_ENDPOINT; diff --git a/src/test/unit_tests/jest_tests/test_versioning_concurrency.test.js b/src/test/integration_tests/nsfs/test_versioning_concurrency.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_versioning_concurrency.test.js rename to src/test/integration_tests/nsfs/test_versioning_concurrency.test.js diff --git a/src/test/system_tests/test_md_aggregator.js b/src/test/system_tests/test_md_aggregator.js index 8ef37659c8..348853826d 100644 --- a/src/test/system_tests/test_md_aggregator.js +++ b/src/test/system_tests/test_md_aggregator.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const util = require('util'); const argv = require('minimist')(process.argv); -const http_utils = require('../../../src/util/http_utils'); +const http_utils = require('../../util/http_utils'); const dotenv = require('../../util/dotenv'); dotenv.load(); diff --git a/src/test/system_tests/test_utils.js b/src/test/system_tests/test_utils.js index 1d813f3776..d0760abfb5 100644 --- a/src/test/system_tests/test_utils.js +++ b/src/test/system_tests/test_utils.js @@ -223,7 +223,7 @@ function generate_s3_policy(principal, bucket, action) { Version: '2012-10-17', Statement: [{ Effect: 'Allow', - Principal: { AWS: [principal] }, + Principal: { AWS: _.flatten([principal]) }, Action: action, Resource: [ `arn:aws:s3:::${bucket}/*`, @@ -248,11 +248,11 @@ function invalid_nsfs_root_permissions() { } /** - * get_coretest_path returns coretest path according to process.env.NC_CORETEST value - * @returns {string} + * require_coretest returns coretest path according to process.env.NC_CORETEST value + * @returns {*} */ -function get_coretest_path() { - return process.env.NC_CORETEST ? './nc_coretest' : './coretest'; +function require_coretest() { + return process.env.NC_CORETEST ? require('../utils/coretest/nc_coretest') : require('../utils/coretest/coretest'); } /** @@ -919,7 +919,7 @@ exports.generate_s3_policy = generate_s3_policy; exports.generate_s3_client = generate_s3_client; exports.generate_iam_client = generate_iam_client; exports.invalid_nsfs_root_permissions = invalid_nsfs_root_permissions; -exports.get_coretest_path = get_coretest_path; +exports.require_coretest = require_coretest; exports.exec_manage_cli = exec_manage_cli; exports.create_fs_user_by_platform = create_fs_user_by_platform; exports.delete_fs_user_by_platform = delete_fs_user_by_platform; diff --git a/src/test/unit_tests/jest_tests/test_iam_ops_input_validation.test.js b/src/test/unit_tests/api/iam/test_iam_ops_input_validation.test.js similarity index 95% rename from src/test/unit_tests/jest_tests/test_iam_ops_input_validation.test.js rename to src/test/unit_tests/api/iam/test_iam_ops_input_validation.test.js index dd5acff194..4cdd3b89fd 100644 --- a/src/test/unit_tests/jest_tests/test_iam_ops_input_validation.test.js +++ b/src/test/unit_tests/api/iam/test_iam_ops_input_validation.test.js @@ -1,17 +1,17 @@ /* Copyright (C) 2024 NooBaa */ 'use strict'; -const { IamError } = require('../../../endpoint/iam/iam_errors'); -const iam_create_user_op = require('../../../endpoint/iam/ops/iam_create_user'); -const iam_get_user_op = require('../../../endpoint/iam/ops/iam_get_user'); -const iam_update_user_op = require('../../../endpoint/iam/ops/iam_update_user'); -const iam_delete_user_op = require('../../../endpoint/iam/ops/iam_delete_user'); -const iam_list_users_op = require('../../../endpoint/iam/ops/iam_list_users'); -const iam_create_access_key_op = require('../../../endpoint/iam/ops/iam_create_access_key'); -const iam_get_access_key_last_used_op = require('../../../endpoint/iam/ops/iam_get_access_key_last_used'); -const iam_update_access_key_op = require('../../../endpoint/iam/ops/iam_update_access_key'); -const iam_delete_access_key_op = require('../../../endpoint/iam/ops/iam_delete_access_key'); -const iam_list_access_keys_op = require('../../../endpoint/iam/ops/iam_list_access_keys'); +const { IamError } = require('../../../../endpoint/iam/iam_errors'); +const iam_create_user_op = require('../../../../endpoint/iam/ops/iam_create_user'); +const iam_get_user_op = require('../../../../endpoint/iam/ops/iam_get_user'); +const iam_update_user_op = require('../../../../endpoint/iam/ops/iam_update_user'); +const iam_delete_user_op = require('../../../../endpoint/iam/ops/iam_delete_user'); +const iam_list_users_op = require('../../../../endpoint/iam/ops/iam_list_users'); +const iam_create_access_key_op = require('../../../../endpoint/iam/ops/iam_create_access_key'); +const iam_get_access_key_last_used_op = require('../../../../endpoint/iam/ops/iam_get_access_key_last_used'); +const iam_update_access_key_op = require('../../../../endpoint/iam/ops/iam_update_access_key'); +const iam_delete_access_key_op = require('../../../../endpoint/iam/ops/iam_delete_access_key'); +const iam_list_access_keys_op = require('../../../../endpoint/iam/ops/iam_list_access_keys'); class NoErrorThrownError extends Error {} diff --git a/src/test/unit_tests/jest_tests/test_list_object.test.js b/src/test/unit_tests/api/s3/test_list_object.test.js similarity index 96% rename from src/test/unit_tests/jest_tests/test_list_object.test.js rename to src/test/unit_tests/api/s3/test_list_object.test.js index ea1fabdee8..0b2c41e5f7 100644 --- a/src/test/unit_tests/jest_tests/test_list_object.test.js +++ b/src/test/unit_tests/api/s3/test_list_object.test.js @@ -4,10 +4,10 @@ const fs = require('fs'); const path = require('path'); -const fs_utils = require('../../../util/fs_utils'); -const nb_native = require('../../../util/nb_native'); -const {TMP_PATH} = require('../../system_tests/test_utils'); -const { get_process_fs_context } = require('../../../util/native_fs_utils'); +const fs_utils = require('../../../../util/fs_utils'); +const nb_native = require('../../../../util/nb_native'); +const {TMP_PATH} = require('../../../system_tests/test_utils'); +const { get_process_fs_context } = require('../../../../util/native_fs_utils'); const tmp_fs_path = path.join(TMP_PATH, 'test_list_object'); const DEFAULT_FS_CONFIG = get_process_fs_context(); diff --git a/src/test/unit_tests/test_ns_list_objects.js b/src/test/unit_tests/api/s3/test_ns_list_objects.js similarity index 99% rename from src/test/unit_tests/test_ns_list_objects.js rename to src/test/unit_tests/api/s3/test_ns_list_objects.js index 6a1be3dc4e..09173e5b79 100644 --- a/src/test/unit_tests/test_ns_list_objects.js +++ b/src/test/unit_tests/api/s3/test_ns_list_objects.js @@ -3,10 +3,10 @@ const mocha = require('mocha'); const assert = require('assert'); -const config = require('../../../config'); +const config = require('../../../../../config'); -const buffer_utils = require('../../util/buffer_utils'); -const fs_utils = require('../../util/fs_utils'); +const buffer_utils = require('../../../../util/buffer_utils'); +const fs_utils = require('../../../../util/fs_utils'); // eslint-disable-next-line max-lines-per-function function test_ns_list_objects(ns, object_sdk, bucket) { diff --git a/src/test/unit_tests/test_s3select.js b/src/test/unit_tests/api/s3/test_s3select.js similarity index 98% rename from src/test/unit_tests/test_s3select.js rename to src/test/unit_tests/api/s3/test_s3select.js index e4e3e634f2..e960b7b4ee 100644 --- a/src/test/unit_tests/test_s3select.js +++ b/src/test/unit_tests/api/s3/test_s3select.js @@ -3,11 +3,11 @@ const mocha = require('mocha'); const assert = require('assert'); -const s3select_utils = require('../../util/s3select'); +const s3select_utils = require('../../../../util/s3select'); const stream = require('stream'); -const nb_native = require('../../util/nb_native')(); +const nb_native = require('../../../../util/nb_native')(); const { Transform } = require('readable-stream'); -const stream_utils = require('../../util/stream_utils'); +const stream_utils = require('../../../../util/stream_utils'); const fs = require('fs'); const { tmpdir } = require('os'); const { sep } = require('path'); diff --git a/src/test/unit_tests/test_migration_to_postgres.js b/src/test/unit_tests/db/test_migration_to_postgres.js similarity index 97% rename from src/test/unit_tests/test_migration_to_postgres.js rename to src/test/unit_tests/db/test_migration_to_postgres.js index 44a5bcf51d..15e08e21b1 100644 --- a/src/test/unit_tests/test_migration_to_postgres.js +++ b/src/test/unit_tests/db/test_migration_to_postgres.js @@ -4,7 +4,7 @@ // TODO: Should implement the test // const { PostgresClient } = require('../../util/postgres_client'); -const mongo_client = require('../../util/mongo_client'); +const mongo_client = require('../../../util/mongo_client'); const { Pool } = require('pg'); const _ = require('lodash'); diff --git a/src/test/unit_tests/test_postgres_client.js b/src/test/unit_tests/db/test_postgres_client.js similarity index 98% rename from src/test/unit_tests/test_postgres_client.js rename to src/test/unit_tests/db/test_postgres_client.js index 73b23f11e3..1ad4de7cdd 100644 --- a/src/test/unit_tests/test_postgres_client.js +++ b/src/test/unit_tests/db/test_postgres_client.js @@ -4,12 +4,12 @@ // TODO: Should implement the test const mocha = require('mocha'); -const { PostgresClient } = require('../../util/postgres_client'); -const SensitiveString = require('../../util/sensitive_string'); +const { PostgresClient } = require('../../../util/postgres_client'); +const SensitiveString = require('../../../util/sensitive_string'); const assert = require('assert'); const _ = require('lodash'); const wtf = require('wtfnode'); -const P = require('../../util/promise'); +const P = require('../../../util/promise'); const test_schema = { $id: 'test_postgres_client_schema', diff --git a/src/test/unit_tests/test_schema_keywords.js b/src/test/unit_tests/db/test_schema_keywords.js similarity index 97% rename from src/test/unit_tests/test_schema_keywords.js rename to src/test/unit_tests/db/test_schema_keywords.js index 34bd5a864a..2c546fad34 100644 --- a/src/test/unit_tests/test_schema_keywords.js +++ b/src/test/unit_tests/db/test_schema_keywords.js @@ -3,8 +3,8 @@ const mocha = require('mocha'); const { default: Ajv } = require('ajv'); -const schema_keywords = require('../../util/schema_keywords'); -const SensitiveString = require('../../util/sensitive_string'); +const schema_keywords = require('../../../util/schema_keywords'); +const SensitiveString = require('../../../util/sensitive_string'); const mongodb = require('mongodb'); const assert = require('assert'); diff --git a/src/test/unit_tests/index.js b/src/test/unit_tests/index.js deleted file mode 100644 index d4e947c6b5..0000000000 --- a/src/test/unit_tests/index.js +++ /dev/null @@ -1,109 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - - -const coretest = require('./coretest'); -coretest.setup({ incomplete_rpc_coverage: 'show' }); - -// --------------------------------------- -// Tests that does not require hosts pools -// --------------------------------------- - -//JSON SCHEMA -require('./test_schema_keywords'); - -// UTILS -require('./test_linked_list'); -require('./test_keys_lock'); -require('./test_lru'); -require('./test_lru_cache'); -require('./test_prefetch'); -require('./test_promise_utils'); -require('./test_rpc'); -require('./test_semaphore'); -require('./test_delayed_trigger'); -require('./test_fs_utils'); -require('./test_signature_utils'); -require('./test_http_utils'); -require('./test_v8_optimizations'); -require('./test_ssl_utils'); -require('./test_zip_utils'); -require('./test_wait_queue'); -require('./test_kmeans'); -require('./test_sensitive_wrapper'); -// require('./test_debug_module'); -require('./test_range_stream'); -require('./test_buffer_pool'); - -// // STORES -require('./test_md_store'); -require('./test_nodes_store'); -require('./test_system_store'); - - -// // CORE -// require('./test_mapper'); -require('./test_map_client'); -//require('./test_map_reader'); -require('./test_map_deleter'); -require('./test_chunk_coder'); -require('./test_chunk_splitter'); -require('./test_chunk_config_utils'); -//require('./test_md_aggregator_unit'); -require('./test_agent_blocks_verifier'); -require('./test_s3_list_objects'); -require('./test_nb_native_b64'); -require('./test_bucket_chunks_builder'); -require('./test_mirror_writer'); -require('./test_namespace_fs'); -require('./test_ns_list_objects'); -require('./test_namespace_fs_mpu'); -require('./test_nb_native_fs'); -require('./test_s3select'); -require('./test_nsfs_glacier_backend'); - -// // SERVERS - -// ------------------------------------ -// A test that initialize the pool list -// ------------------------------------ -require('./test_system_servers'); - -// ------------------------------ -// Tests that require hosts pools -// ------------------------------ - -// SERVERS -require('./test_host_server'); -require('./test_node_server'); - -// CORE -require('./test_map_builder'); // Requires pools -require('./test_map_reader'); ///////////// -require('./test_object_io'); -require('./test_agent_blocks_reclaimer'); -require('./test_s3_ops'); -require('./test_s3_encryption'); -require('./test_s3_bucket_policy'); -// require('./test_node_allocator'); -require('./test_namespace_cache'); -require('./test_namespace_auth'); -require('./test_encryption'); -require('./test_bucket_replication'); -require('./test_sts'); -require('./test_cloud_utils'); -require('./test_upgrade_scripts.js'); -require('./test_tiering_ttl_worker'); -// require('./test_tiering_upload'); -//require('./test_s3_worm'); -require('./test_bucket_logging'); -require('./test_notifications'); - -// UPGRADE -// require('./test_postgres_upgrade'); // TODO currently working with mongo -> once changing to postgres - need to uncomment - -// Lifecycle -require('./test_lifecycle'); - -// MD Sequence -require('./test_mdsequence'); diff --git a/src/test/unit_tests/test_bucket_chunks_builder.js b/src/test/unit_tests/internal/test_bucket_chunks_builder.js similarity index 98% rename from src/test/unit_tests/test_bucket_chunks_builder.js rename to src/test/unit_tests/internal/test_bucket_chunks_builder.js index dc4d963e9c..d94d3cf7df 100644 --- a/src/test/unit_tests/test_bucket_chunks_builder.js +++ b/src/test/unit_tests/internal/test_bucket_chunks_builder.js @@ -7,8 +7,8 @@ const assert = require('assert'); const sinon = require('sinon'); const argv = require('minimist')(process.argv); -const { BucketChunksBuilder } = require('../../server/bg_services/bucket_chunks_builder'); -const dbg = require('../../util/debug_module')(__filename); +const { BucketChunksBuilder } = require('../../../server/bg_services/bucket_chunks_builder'); +const dbg = require('../../../util/debug_module')(__filename); // argv.verbose = true; if (argv.verbose) { diff --git a/src/test/unit_tests/jest_tests/test_bucket_diff.test.js b/src/test/unit_tests/internal/test_bucket_diff.test.js similarity index 99% rename from src/test/unit_tests/jest_tests/test_bucket_diff.test.js rename to src/test/unit_tests/internal/test_bucket_diff.test.js index fede84de81..49bb0c8c0c 100644 --- a/src/test/unit_tests/jest_tests/test_bucket_diff.test.js +++ b/src/test/unit_tests/internal/test_bucket_diff.test.js @@ -5,7 +5,7 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const { BucketDiff } = require('../../../server/utils/bucket_diff.js'); -const replication_utils = require('../../../server/utils/replication_utils'); +const replication_utils = require('../../../server/utils/replication_utils.js'); // @ts-ignore const mock_fn = jest.fn(); diff --git a/src/test/unit_tests/jest_tests/test_bucket_log_based_replication.test.js b/src/test/unit_tests/internal/test_bucket_log_based_replication.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_bucket_log_based_replication.test.js rename to src/test/unit_tests/internal/test_bucket_log_based_replication.test.js diff --git a/src/test/unit_tests/test_chunk_coder.js b/src/test/unit_tests/internal/test_chunk_coder.js similarity index 96% rename from src/test/unit_tests/test_chunk_coder.js rename to src/test/unit_tests/internal/test_chunk_coder.js index 3162b5d8b9..4a026c1bb6 100644 --- a/src/test/unit_tests/test_chunk_coder.js +++ b/src/test/unit_tests/internal/test_chunk_coder.js @@ -8,15 +8,15 @@ const crypto = require('crypto'); const Chance = require('chance'); const assert = require('assert'); -const config = require('../../../config'); -const nb_native = require('../../util/nb_native'); -const RandStream = require('../../util/rand_stream'); -const ChunkCoder = require('../../util/chunk_coder'); -const ChunkEraser = require('../../util/chunk_eraser'); -const Speedometer = require('../../util/speedometer'); -const stream_utils = require('../../util/stream_utils'); -const FlattenStream = require('../../util/flatten_stream'); -const ChunkSplitter = require('../../util/chunk_splitter'); +const config = require('../../../../config'); +const nb_native = require('../../../util/nb_native'); +const RandStream = require('../../../util/rand_stream'); +const ChunkCoder = require('../../../util/chunk_coder'); +const ChunkEraser = require('../../../util/chunk_eraser'); +const Speedometer = require('../../../util/speedometer'); +const stream_utils = require('../../../util/stream_utils'); +const FlattenStream = require('../../../util/flatten_stream'); +const ChunkSplitter = require('../../../util/chunk_splitter'); const chance = new Chance(); diff --git a/src/test/unit_tests/test_chunk_config_utils.js b/src/test/unit_tests/internal/test_chunk_config_utils.js similarity index 96% rename from src/test/unit_tests/test_chunk_config_utils.js rename to src/test/unit_tests/internal/test_chunk_config_utils.js index 02a330dddb..c68217ba63 100644 --- a/src/test/unit_tests/test_chunk_config_utils.js +++ b/src/test/unit_tests/internal/test_chunk_config_utils.js @@ -5,8 +5,8 @@ const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const config = require('../../../config'); -const chunk_config_utils = require('../../server/utils/chunk_config_utils'); +const config = require('../../../../config'); +const chunk_config_utils = require('../../../server/utils/chunk_config_utils'); mocha.describe('chunk_config_utils', function() { diff --git a/src/test/unit_tests/test_chunk_splitter.js b/src/test/unit_tests/internal/test_chunk_splitter.js similarity index 95% rename from src/test/unit_tests/test_chunk_splitter.js rename to src/test/unit_tests/internal/test_chunk_splitter.js index 5b01c5d84b..6ce117da6c 100644 --- a/src/test/unit_tests/test_chunk_splitter.js +++ b/src/test/unit_tests/internal/test_chunk_splitter.js @@ -6,9 +6,9 @@ const mocha = require('mocha'); const crypto = require('crypto'); const assert = require('assert'); -const P = require('../../util/promise'); -const RandStream = require('../../util/rand_stream'); -const ChunkSplitter = require('../../util/chunk_splitter'); +const P = require('../../../util/promise'); +const RandStream = require('../../../util/rand_stream'); +const ChunkSplitter = require('../../../util/chunk_splitter'); function log(...args) { if (process.env.SUPPRESS_LOGS) return; diff --git a/src/test/unit_tests/jest_tests/test_chunked_content_decoder.test.js b/src/test/unit_tests/internal/test_chunked_content_decoder.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_chunked_content_decoder.test.js rename to src/test/unit_tests/internal/test_chunked_content_decoder.test.js diff --git a/src/test/unit_tests/test_debug_module.js b/src/test/unit_tests/internal/test_debug_module.js similarity index 97% rename from src/test/unit_tests/test_debug_module.js rename to src/test/unit_tests/internal/test_debug_module.js index e9444d9008..8d5419a638 100644 --- a/src/test/unit_tests/test_debug_module.js +++ b/src/test/unit_tests/internal/test_debug_module.js @@ -2,12 +2,12 @@ 'use strict'; const _ = require('lodash'); -const P = require('../../util/promise'); +const P = require('../../../util/promise'); const mocha = require('mocha'); const assert = require('assert'); const fs = require('fs'); -const DebugModule = require('../../util/debug_module'); -const os_utils = require('../../util/os_utils'); +const DebugModule = require('../../../util/debug_module'); +const os_utils = require('../../../util/os_utils'); // File Content Verifier according to given expected result (positive/negative) function file_content_verify(flag, expected) { diff --git a/src/test/unit_tests/jest_tests/test_file_writer.test.js b/src/test/unit_tests/internal/test_file_writer.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_file_writer.test.js rename to src/test/unit_tests/internal/test_file_writer.test.js diff --git a/src/test/unit_tests/test_keys_semaphore.js b/src/test/unit_tests/internal/test_keys_semaphore.js similarity index 93% rename from src/test/unit_tests/test_keys_semaphore.js rename to src/test/unit_tests/internal/test_keys_semaphore.js index d2a261ee7a..fe1bcf4aef 100644 --- a/src/test/unit_tests/test_keys_semaphore.js +++ b/src/test/unit_tests/internal/test_keys_semaphore.js @@ -4,8 +4,8 @@ const mocha = require('mocha'); const assert = require('assert'); -const P = require('../../util/promise'); -const KeysSemaphore = require('../../util/keys_semaphore'); +const P = require('../../../util/promise'); +const KeysSemaphore = require('../../../util/keys_semaphore'); mocha.describe('keys_semaphore', function() { diff --git a/src/test/unit_tests/test_mapper.js b/src/test/unit_tests/internal/test_mapper.js similarity index 99% rename from src/test/unit_tests/test_mapper.js rename to src/test/unit_tests/internal/test_mapper.js index 9581a41468..633928ee03 100644 --- a/src/test/unit_tests/test_mapper.js +++ b/src/test/unit_tests/internal/test_mapper.js @@ -3,7 +3,7 @@ 'use strict'; // setup coretest first to prepare the env -const coretest = require('./coretest'); +const coretest = require('../../utils/coretest/coretest'); coretest.no_setup(); const _ = require('lodash'); @@ -12,8 +12,8 @@ const mocha = require('mocha'); const assert = require('assert'); const mongodb = require('mongodb'); -const config = require('../../../config.js'); -const mapper = require('../../server/object_services/mapper'); +const config = require('../../../../config.js'); +const mapper = require('../../../server/object_services/mapper'); coretest.describe_mapper_test_case({ name: 'mapper', diff --git a/src/test/unit_tests/test_mirror_writer.js b/src/test/unit_tests/internal/test_mirror_writer.js similarity index 96% rename from src/test/unit_tests/test_mirror_writer.js rename to src/test/unit_tests/internal/test_mirror_writer.js index 8a6f351f6a..453c5cc164 100644 --- a/src/test/unit_tests/test_mirror_writer.js +++ b/src/test/unit_tests/internal/test_mirror_writer.js @@ -9,11 +9,11 @@ const sinon = require('sinon'); const _ = require('lodash'); const { ObjectId } = require('mongodb'); -const { MirrorWriter } = require('../../server/bg_services/mirror_writer'); -const config = require('../../../config'); +const { MirrorWriter } = require('../../../server/bg_services/mirror_writer'); +const config = require('../../../../config'); const argv = require('minimist')(process.argv); -const dbg = require('../../util/debug_module')(__filename); +const dbg = require('../../../util/debug_module')(__filename); // argv.verbose = true; if (argv.verbose) { dbg.set_module_level(5, 'core'); diff --git a/src/test/unit_tests/test_namespace_cache.js b/src/test/unit_tests/internal/test_namespace_cache.js similarity index 99% rename from src/test/unit_tests/test_namespace_cache.js rename to src/test/unit_tests/internal/test_namespace_cache.js index e03f41adee..4af8c3c4fc 100644 --- a/src/test/unit_tests/test_namespace_cache.js +++ b/src/test/unit_tests/internal/test_namespace_cache.js @@ -6,13 +6,13 @@ const mocha = require('mocha'); const assert = require('assert'); const crypto = require('crypto'); -const P = require('../../util/promise'); -const metric_utils = require('../utils/metrics'); -const { RpcError } = require('../../rpc'); -const buffer_utils = require('../../util/buffer_utils'); -const NamespaceCache = require('../../sdk/namespace_cache'); -const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector'); -const cache_config = require('../../../config').NAMESPACE_CACHING; +const P = require('../../../util/promise'); +const metric_utils = require('../../utils/metrics'); +const { RpcError } = require('../../../rpc'); +const buffer_utils = require('../../../util/buffer_utils'); +const NamespaceCache = require('../../../sdk/namespace_cache'); +const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); +const cache_config = require('../../../../config').NAMESPACE_CACHING; const EVENT_UPLOAD_FINISH = 'event_upload_finish'; const EVENT_DELETE_OBJ = 'event_delete_object'; diff --git a/src/test/unit_tests/jest_tests/test_newline_reader.test.js b/src/test/unit_tests/internal/test_newline_reader.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_newline_reader.test.js rename to src/test/unit_tests/internal/test_newline_reader.test.js diff --git a/src/test/unit_tests/jest_tests/test_noobaa_s3_client.test.js b/src/test/unit_tests/internal/test_noobaa_s3_client.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_noobaa_s3_client.test.js rename to src/test/unit_tests/internal/test_noobaa_s3_client.test.js diff --git a/src/test/unit_tests/jest_tests/test_fs_napi_concurrency.test.js b/src/test/unit_tests/native/test_fs_napi_concurrency.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_fs_napi_concurrency.test.js rename to src/test/unit_tests/native/test_fs_napi_concurrency.test.js diff --git a/src/test/unit_tests/test_nb_native_b64.js b/src/test/unit_tests/native/test_nb_native_b64.js similarity index 96% rename from src/test/unit_tests/test_nb_native_b64.js rename to src/test/unit_tests/native/test_nb_native_b64.js index baa7646012..4f13b010a1 100644 --- a/src/test/unit_tests/test_nb_native_b64.js +++ b/src/test/unit_tests/native/test_nb_native_b64.js @@ -4,7 +4,7 @@ const mocha = require('mocha'); const assert = require('assert'); const crypto = require('crypto'); -const nb_native = require('../../util/nb_native'); +const nb_native = require('../../../util/nb_native'); mocha.describe('nb_native b64', function() { diff --git a/src/test/unit_tests/test_nb_native_fs.js b/src/test/unit_tests/native/test_nb_native_fs.js similarity index 98% rename from src/test/unit_tests/test_nb_native_fs.js rename to src/test/unit_tests/native/test_nb_native_fs.js index bb67ca3f62..5629f235aa 100644 --- a/src/test/unit_tests/test_nb_native_fs.js +++ b/src/test/unit_tests/native/test_nb_native_fs.js @@ -6,10 +6,10 @@ const _ = require('lodash'); const fs = require('fs'); const mocha = require('mocha'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); -const os_utils = require('../../util/os_utils'); -const nb_native = require('../../util/nb_native'); -const { get_process_fs_context } = require('../../util/native_fs_utils'); +const fs_utils = require('../../../util/fs_utils'); +const os_utils = require('../../../util/os_utils'); +const nb_native = require('../../../util/nb_native'); +const { get_process_fs_context } = require('../../../util/native_fs_utils'); const DEFAULT_FS_CONFIG = get_process_fs_context(); diff --git a/src/test/unit_tests/test_nb_native_gpfs.js b/src/test/unit_tests/native/test_nb_native_gpfs.js similarity index 96% rename from src/test/unit_tests/test_nb_native_gpfs.js rename to src/test/unit_tests/native/test_nb_native_gpfs.js index 2d3ff113c1..5fa6b676f7 100644 --- a/src/test/unit_tests/test_nb_native_gpfs.js +++ b/src/test/unit_tests/native/test_nb_native_gpfs.js @@ -4,9 +4,9 @@ const fs = require('fs'); const mocha = require('mocha'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); -const nb_native = require('../../util/nb_native'); -const { get_process_fs_context } = require('../../util/native_fs_utils'); +const fs_utils = require('../../../util/fs_utils'); +const nb_native = require('../../../util/nb_native'); +const { get_process_fs_context } = require('../../../util/native_fs_utils'); const DEFAULT_FS_CONFIG = get_process_fs_context('GPFS'); diff --git a/src/test/unit_tests/test_nb_native_hashes.js b/src/test/unit_tests/native/test_nb_native_hashes.js similarity index 97% rename from src/test/unit_tests/test_nb_native_hashes.js rename to src/test/unit_tests/native/test_nb_native_hashes.js index df60a5ddcd..c750a7e3ab 100644 --- a/src/test/unit_tests/test_nb_native_hashes.js +++ b/src/test/unit_tests/native/test_nb_native_hashes.js @@ -4,7 +4,7 @@ const mocha = require('mocha'); const assert = require('assert'); const crypto = require('crypto'); -const nb_native = require('../../util/nb_native'); +const nb_native = require('../../../util/nb_native'); mocha.describe('nb_native hashes', async function() { function md5(input) { diff --git a/src/test/unit_tests/jest_tests/test_config_dir_restructure_upgrade_script.test.js b/src/test/unit_tests/nc/configuration/test_config_dir_restructure_upgrade_script.test.js similarity index 98% rename from src/test/unit_tests/jest_tests/test_config_dir_restructure_upgrade_script.test.js rename to src/test/unit_tests/nc/configuration/test_config_dir_restructure_upgrade_script.test.js index 150f2e54c1..500507b44a 100644 --- a/src/test/unit_tests/jest_tests/test_config_dir_restructure_upgrade_script.test.js +++ b/src/test/unit_tests/nc/configuration/test_config_dir_restructure_upgrade_script.test.js @@ -3,19 +3,19 @@ const _ = require('lodash'); const path = require('path'); -const P = require('../../../util/promise'); -const config = require('../../../../config'); -const nb_native = require('../../../util/nb_native'); -const { ConfigFS, CONFIG_TYPES } = require('../../../sdk/config_fs'); -const dbg = require('../../../util/debug_module')(__filename); +const P = require('../../../../util/promise'); +const config = require('../../../../../config'); +const nb_native = require('../../../../util/nb_native'); +const { ConfigFS, CONFIG_TYPES } = require('../../../../sdk/config_fs'); +const dbg = require('../../../../util/debug_module')(__filename); const { create_config_dir, create_identity_dir_if_missing, clean_config_dir, fail_test_if_default_config_dir_exists, TEST_TIMEOUT, write_manual_config_file, write_manual_old_account_config_file, create_file, - symlink_account_name, symlink_account_access_keys } = require('../../system_tests/test_utils'); -const { get_process_fs_context, is_path_exists } = require('../../../util/native_fs_utils'); + symlink_account_name, symlink_account_access_keys } = require('../../../system_tests/test_utils'); +const { get_process_fs_context, is_path_exists } = require('../../../../util/native_fs_utils'); const { move_old_accounts_dir, create_account_access_keys_index_if_missing, create_account_name_index_if_missing, - create_identity_if_missing, prepare_account_upgrade_params, upgrade_account_config_file, upgrade_accounts_config_files, run } = require('../../../upgrade/nc_upgrade_scripts/1.0.0/config_dir_restructure'); -const { create_fresh_path } = require('../../../util/fs_utils'); -const native_fs_utils = require('../../../util/native_fs_utils'); + create_identity_if_missing, prepare_account_upgrade_params, upgrade_account_config_file, upgrade_accounts_config_files, run } = require('../../../../upgrade/nc_upgrade_scripts/1.0.0/config_dir_restructure'); +const { create_fresh_path } = require('../../../../util/fs_utils'); +const native_fs_utils = require('../../../../util/native_fs_utils'); const mock_access_key = 'Zzto3OwtGflQrqD41h3SEXAMPLE'; const mock_secret_key = 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsDy3oEXAMPLE'; diff --git a/src/test/unit_tests/jest_tests/test_config_dir_structure.test.js b/src/test/unit_tests/nc/configuration/test_config_dir_structure.test.js similarity index 96% rename from src/test/unit_tests/jest_tests/test_config_dir_structure.test.js rename to src/test/unit_tests/nc/configuration/test_config_dir_structure.test.js index 2abc14edc0..ed7e6674d0 100644 --- a/src/test/unit_tests/jest_tests/test_config_dir_structure.test.js +++ b/src/test/unit_tests/nc/configuration/test_config_dir_structure.test.js @@ -2,11 +2,11 @@ 'use strict'; const path = require('path'); -const config = require('../../../../config'); -const { ConfigFS } = require('../../../sdk/config_fs'); +const config = require('../../../../../config'); +const { ConfigFS } = require('../../../../sdk/config_fs'); const { TMP_PATH, create_redirect_file, create_config_dir, clean_config_dir, - fail_test_if_default_config_dir_exists, TEST_TIMEOUT } = require('../../system_tests/test_utils'); -const { get_process_fs_context, is_path_exists } = require('../../../util/native_fs_utils'); + fail_test_if_default_config_dir_exists, TEST_TIMEOUT } = require('../../../system_tests/test_utils'); +const { get_process_fs_context, is_path_exists } = require('../../../../util/native_fs_utils'); const tmp_fs_path = path.join(TMP_PATH, 'test_config_dir_restructure'); const config_root = path.join(tmp_fs_path, 'config_root'); diff --git a/src/test/unit_tests/jest_tests/test_config_fs.test.js b/src/test/unit_tests/nc/configuration/test_config_fs.test.js similarity index 95% rename from src/test/unit_tests/jest_tests/test_config_fs.test.js rename to src/test/unit_tests/nc/configuration/test_config_fs.test.js index 21d66ab7d2..598ab9ae1a 100644 --- a/src/test/unit_tests/jest_tests/test_config_fs.test.js +++ b/src/test/unit_tests/nc/configuration/test_config_fs.test.js @@ -4,11 +4,11 @@ const _ = require('lodash'); const os = require('os'); const path = require('path'); -const config = require('../../../../config'); -const pkg = require('../../../../package.json'); -const { TMP_PATH } = require('../../system_tests/test_utils'); -const { get_process_fs_context } = require('../../../util/native_fs_utils'); -const { ConfigFS } = require('../../../sdk/config_fs'); +const config = require('../../../../../config'); +const pkg = require('../../../../../package.json'); +const { TMP_PATH } = require('../../../system_tests/test_utils'); +const { get_process_fs_context } = require('../../../../util/native_fs_utils'); +const { ConfigFS } = require('../../../../sdk/config_fs'); const tmp_fs_path = path.join(TMP_PATH, 'test_config_fs'); const config_root = path.join(tmp_fs_path, 'config_root'); diff --git a/src/test/unit_tests/jest_tests/test_config_fs_backward_compatibility.test.js b/src/test/unit_tests/nc/configuration/test_config_fs_backward_compatibility.test.js similarity index 97% rename from src/test/unit_tests/jest_tests/test_config_fs_backward_compatibility.test.js rename to src/test/unit_tests/nc/configuration/test_config_fs_backward_compatibility.test.js index 8d4743ae15..3d7390e288 100644 --- a/src/test/unit_tests/jest_tests/test_config_fs_backward_compatibility.test.js +++ b/src/test/unit_tests/nc/configuration/test_config_fs_backward_compatibility.test.js @@ -2,9 +2,9 @@ 'use strict'; const path = require('path'); -const fs_utils = require('../../../util/fs_utils'); -const { ConfigFS, CONFIG_TYPES } = require('../../../sdk/config_fs'); -const { TMP_PATH, TEST_TIMEOUT, write_manual_config_file, write_manual_old_account_config_file } = require('../../system_tests/test_utils'); +const fs_utils = require('../../../../util/fs_utils'); +const { ConfigFS, CONFIG_TYPES } = require('../../../../sdk/config_fs'); +const { TMP_PATH, TEST_TIMEOUT, write_manual_config_file, write_manual_old_account_config_file } = require('../../../system_tests/test_utils'); const tmp_fs_path = path.join(TMP_PATH, 'test_config_fs_backward_compatibility'); diff --git a/src/test/unit_tests/jest_tests/test_nc_account_invalid_mkm_integration.test.js b/src/test/unit_tests/nc/configuration/test_nc_account_invalid_mkm_integration.test.js similarity index 96% rename from src/test/unit_tests/jest_tests/test_nc_account_invalid_mkm_integration.test.js rename to src/test/unit_tests/nc/configuration/test_nc_account_invalid_mkm_integration.test.js index 8e580cf157..9be3dd3536 100644 --- a/src/test/unit_tests/jest_tests/test_nc_account_invalid_mkm_integration.test.js +++ b/src/test/unit_tests/nc/configuration/test_nc_account_invalid_mkm_integration.test.js @@ -6,13 +6,13 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const fs = require('fs'); const path = require('path'); -const fs_utils = require('../../../util/fs_utils'); -const { exec_manage_cli, set_path_permissions_and_owner, TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); -const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); -const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; -const ManageCLIResponse = require('../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; -const { get_process_fs_context } = require('../../../util/native_fs_utils'); -const nb_native = require('../../../util/nb_native'); +const fs_utils = require('../../../../util/fs_utils'); +const { exec_manage_cli, set_path_permissions_and_owner, TMP_PATH, set_nc_config_dir_in_config } = require('../../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const ManageCLIError = require('../../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; +const ManageCLIResponse = require('../../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; +const { get_process_fs_context } = require('../../../../util/native_fs_utils'); +const nb_native = require('../../../../util/nb_native'); const tmp_fs_path = path.join(TMP_PATH, 'test_nc_invalid_mkm_integration'); const config_root = path.join(tmp_fs_path, 'config_root_account_mkm_integration'); diff --git a/src/test/unit_tests/jest_tests/test_nc_master_keys.test.js b/src/test/unit_tests/nc/configuration/test_nc_master_keys.test.js similarity index 95% rename from src/test/unit_tests/jest_tests/test_nc_master_keys.test.js rename to src/test/unit_tests/nc/configuration/test_nc_master_keys.test.js index bdc9b59658..ffb267295a 100644 --- a/src/test/unit_tests/jest_tests/test_nc_master_keys.test.js +++ b/src/test/unit_tests/nc/configuration/test_nc_master_keys.test.js @@ -4,14 +4,14 @@ const fs = require('fs'); const path = require('path'); -const config = require('../../../../config'); -const fs_utils = require('../../../util/fs_utils'); -const nb_native = require('../../../util/nb_native'); -const cloud_utils = require('../../../util/cloud_utils'); -const nc_mkm = require('../../../manage_nsfs/nc_master_key_manager'); -const { get_process_fs_context } = require('../../../util/native_fs_utils'); -const nsfs_schema_utils = require('../../../manage_nsfs/nsfs_schema_utils'); -const { fail_test_if_default_config_dir_exists } = require('../../../test/system_tests/test_utils'); +const config = require('../../../../../config'); +const fs_utils = require('../../../../util/fs_utils'); +const nb_native = require('../../../../util/nb_native'); +const cloud_utils = require('../../../../util/cloud_utils'); +const nc_mkm = require('../../../../manage_nsfs/nc_master_key_manager'); +const { get_process_fs_context } = require('../../../../util/native_fs_utils'); +const nsfs_schema_utils = require('../../../../manage_nsfs/nsfs_schema_utils'); +const { fail_test_if_default_config_dir_exists } = require('../../../system_tests/test_utils'); const DEFAULT_FS_CONFIG = get_process_fs_context(); const MASTER_KEYS_JSON_PATH = path.join(config.NSFS_NC_DEFAULT_CONF_DIR, 'master_keys.json'); diff --git a/src/test/unit_tests/jest_tests/test_nc_master_keys_exec.test.js b/src/test/unit_tests/nc/configuration/test_nc_master_keys_exec.test.js similarity index 93% rename from src/test/unit_tests/jest_tests/test_nc_master_keys_exec.test.js rename to src/test/unit_tests/nc/configuration/test_nc_master_keys_exec.test.js index 9dde0c8b4a..6f7aaa959d 100644 --- a/src/test/unit_tests/jest_tests/test_nc_master_keys_exec.test.js +++ b/src/test/unit_tests/nc/configuration/test_nc_master_keys_exec.test.js @@ -5,11 +5,11 @@ const fs = require('fs'); const path = require('path'); const crypto = require('crypto'); -const config = require('../../../../config'); -const fs_utils = require('../../../util/fs_utils'); -const cloud_utils = require('../../../util/cloud_utils'); -const db_client = require('../../../util/db_client').instance(); -const { TMP_PATH, fail_test_if_default_config_dir_exists } = require('../../system_tests/test_utils'); +const config = require('../../../../../config'); +const fs_utils = require('../../../../util/fs_utils'); +const cloud_utils = require('../../../../util/cloud_utils'); +const db_client = require('../../../../util/db_client').instance(); +const { TMP_PATH, fail_test_if_default_config_dir_exists } = require('../../../system_tests/test_utils'); const MKM_GET_EXEC_PATH = path.join(config.NSFS_NC_DEFAULT_CONF_DIR, 'file_get'); const MKM_PUT_EXEC_PATH = path.join(config.NSFS_NC_DEFAULT_CONF_DIR, 'file_put'); @@ -90,7 +90,7 @@ describe('NC master key manager tests - exec store type', () => { let initial_master_key; let initial_master_keys_by_id; - const nc_mkm = require('../../../manage_nsfs/nc_master_key_manager'); + const nc_mkm = require('../../../../manage_nsfs/nc_master_key_manager'); const nc_mkm_instance = nc_mkm.get_instance(); @@ -185,8 +185,8 @@ function compare_master_keys_json_and_active_root_key(active_master_key, master_ * @returns {Promise} */ async function read_master_keys_json(master_keys_json_path) { - const nb_native = require('../../../util/nb_native'); - const { get_process_fs_context } = require('../../../util/native_fs_utils'); + const nb_native = require('../../../../util/nb_native'); + const { get_process_fs_context } = require('../../../../util/native_fs_utils'); const DEFAULT_FS_CONFIG = get_process_fs_context(); const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, master_keys_json_path); const master_keys = JSON.parse(data.toString()); diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js b/src/test/unit_tests/nc/configuration/test_nc_nsfs_bucket_schema_validation.test.js similarity index 98% rename from src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js rename to src/test/unit_tests/nc/configuration/test_nc_nsfs_bucket_schema_validation.test.js index 457239f4d4..e628cfdc50 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js +++ b/src/test/unit_tests/nc/configuration/test_nc_nsfs_bucket_schema_validation.test.js @@ -3,10 +3,10 @@ 'use strict'; -const nsfs_schema_utils = require('../../../manage_nsfs/nsfs_schema_utils'); -const RpcError = require('../../../rpc/rpc_error'); -const test_utils = require('../../system_tests/test_utils'); -const config = require('../../../../config'); +const nsfs_schema_utils = require('../../../../manage_nsfs/nsfs_schema_utils'); +const RpcError = require('../../../../rpc/rpc_error'); +const test_utils = require('../../../system_tests/test_utils'); +const config = require('../../../../../config'); describe('schema validation NC NSFS bucket', () => { const versioning_enabled = 'ENABLED'; diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_config_schema_validation.test.js b/src/test/unit_tests/nc/configuration/test_nc_nsfs_config_schema_validation.test.js similarity index 99% rename from src/test/unit_tests/jest_tests/test_nc_nsfs_config_schema_validation.test.js rename to src/test/unit_tests/nc/configuration/test_nc_nsfs_config_schema_validation.test.js index 6be08c2214..2887ba7d59 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_config_schema_validation.test.js +++ b/src/test/unit_tests/nc/configuration/test_nc_nsfs_config_schema_validation.test.js @@ -3,9 +3,9 @@ 'use strict'; -const nsfs_schema_utils = require('../../../manage_nsfs/nsfs_schema_utils'); -const RpcError = require('../../../rpc/rpc_error'); -const config = require('../../../../config'); +const nsfs_schema_utils = require('../../../../manage_nsfs/nsfs_schema_utils'); +const RpcError = require('../../../../rpc/rpc_error'); +const config = require('../../../../../config'); describe('schema validation NC NSFS config', () => { diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_new_buckets_path_validation.test.js b/src/test/unit_tests/nc/configuration/test_nc_nsfs_new_buckets_path_validation.test.js similarity index 98% rename from src/test/unit_tests/jest_tests/test_nc_nsfs_new_buckets_path_validation.test.js rename to src/test/unit_tests/nc/configuration/test_nc_nsfs_new_buckets_path_validation.test.js index 01013beb45..f7e78c0a4d 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_new_buckets_path_validation.test.js +++ b/src/test/unit_tests/nc/configuration/test_nc_nsfs_new_buckets_path_validation.test.js @@ -6,10 +6,10 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const fs = require('fs'); const path = require('path'); -const config = require('../../../../config'); -const fs_utils = require('../../../util/fs_utils'); -const test_utils = require('../../system_tests/test_utils'); -const { get_fs_context, is_dir_accessible } = require('../../../util/native_fs_utils'); +const config = require('../../../../../config'); +const fs_utils = require('../../../../util/fs_utils'); +const test_utils = require('../../../system_tests/test_utils'); +const { get_fs_context, is_dir_accessible } = require('../../../../util/native_fs_utils'); const MAC_PLATFORM = 'darwin'; let tmp_fs_path = '/tmp/test_nc_nsfs_new_buckets_path_validation'; diff --git a/src/test/unit_tests/jest_tests/test_nc_upgrade_manager.test.js b/src/test/unit_tests/nc/configuration/test_nc_upgrade_manager.test.js similarity index 99% rename from src/test/unit_tests/jest_tests/test_nc_upgrade_manager.test.js rename to src/test/unit_tests/nc/configuration/test_nc_upgrade_manager.test.js index aa28ce053d..242fdca5ce 100644 --- a/src/test/unit_tests/jest_tests/test_nc_upgrade_manager.test.js +++ b/src/test/unit_tests/nc/configuration/test_nc_upgrade_manager.test.js @@ -10,14 +10,14 @@ const os = require('os'); const util = require('util'); const _ = require('lodash'); const path = require('path'); -const fs_utils = require('../../../util/fs_utils'); -const config = require('../../../../config'); -const pkg = require('../../../../package.json'); +const fs_utils = require('../../../../util/fs_utils'); +const config = require('../../../../../config'); +const pkg = require('../../../../../package.json'); const { NCUpgradeManager, DEFAULT_NC_UPGRADE_SCRIPTS_DIR, OLD_DEFAULT_PACKAGE_VERSION, - OLD_DEFAULT_CONFIG_DIR_VERSION } = require('../../../upgrade/nc_upgrade_manager'); -const { ConfigFS, CONFIG_DIR_PHASES } = require('../../../sdk/config_fs'); + OLD_DEFAULT_CONFIG_DIR_VERSION } = require('../../../../upgrade/nc_upgrade_manager'); +const { ConfigFS, CONFIG_DIR_PHASES } = require('../../../../sdk/config_fs'); const { TMP_PATH, create_redirect_file, create_config_dir, - fail_test_if_default_config_dir_exists, clean_config_dir, TEST_TIMEOUT } = require('../../system_tests/test_utils'); + fail_test_if_default_config_dir_exists, clean_config_dir, TEST_TIMEOUT } = require('../../../system_tests/test_utils'); const config_root = path.join(TMP_PATH, 'config_root_nc_upgrade_manager_test'); const config_fs = new ConfigFS(config_root); diff --git a/src/test/unit_tests/jest_tests/test_nc_lifecycle.test.js b/src/test/unit_tests/nc/lifecycle/test_nc_lifecycle.test.js similarity index 98% rename from src/test/unit_tests/jest_tests/test_nc_lifecycle.test.js rename to src/test/unit_tests/nc/lifecycle/test_nc_lifecycle.test.js index a61c36e77f..a483e912c0 100644 --- a/src/test/unit_tests/jest_tests/test_nc_lifecycle.test.js +++ b/src/test/unit_tests/nc/lifecycle/test_nc_lifecycle.test.js @@ -7,13 +7,13 @@ process.env.DISABLE_INIT_RANDOM_SEED = 'true'; const path = require('path'); const crypto = require('crypto'); -const config = require('../../../../config'); -const fs_utils = require('../../../util/fs_utils'); -const NamespaceFS = require('../../../sdk/namespace_fs'); -const buffer_utils = require('../../../util/buffer_utils'); -const lifecycle_utils = require('../../../../src/util/lifecycle_utils'); -const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); -const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT } = require('../../system_tests/test_utils'); +const config = require('../../../../../config'); +const fs_utils = require('../../../../util/fs_utils'); +const NamespaceFS = require('../../../../sdk/namespace_fs'); +const buffer_utils = require('../../../../util/buffer_utils'); +const lifecycle_utils = require('../../../../util/lifecycle_utils'); +const endpoint_stats_collector = require('../../../../sdk/endpoint_stats_collector'); +const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT } = require('../../../system_tests/test_utils'); const config_root = path.join(TMP_PATH, 'config_root_nc_lifecycle'); diff --git a/src/test/unit_tests/jest_tests/test_nc_lifecycle_gpfs_ilm_integration.test.js b/src/test/unit_tests/nc/lifecycle/test_nc_lifecycle_gpfs_ilm_integration.test.js similarity index 98% rename from src/test/unit_tests/jest_tests/test_nc_lifecycle_gpfs_ilm_integration.test.js rename to src/test/unit_tests/nc/lifecycle/test_nc_lifecycle_gpfs_ilm_integration.test.js index a6d94b0b77..110e73b6e1 100644 --- a/src/test/unit_tests/jest_tests/test_nc_lifecycle_gpfs_ilm_integration.test.js +++ b/src/test/unit_tests/nc/lifecycle/test_nc_lifecycle_gpfs_ilm_integration.test.js @@ -6,12 +6,12 @@ process.env.DISABLE_INIT_RANDOM_SEED = 'true'; const fs = require('fs'); const path = require('path'); -const config = require('../../../../config'); -const { ConfigFS } = require('../../../sdk/config_fs'); -const { file_delete, create_fresh_path } = require('../../../util/fs_utils'); -const { read_file } = require('../../../util/native_fs_utils'); -const { run_or_skip_test, TMP_PATH, create_file, IS_GPFS } = require('../../system_tests/test_utils'); -const { NCLifecycle, ILM_POLICIES_TMP_DIR, ILM_CANDIDATES_TMP_DIR } = require('../../../manage_nsfs/nc_lifecycle'); +const config = require('../../../../../config'); +const { ConfigFS } = require('../../../../sdk/config_fs'); +const { file_delete, create_fresh_path } = require('../../../../util/fs_utils'); +const { read_file } = require('../../../../util/native_fs_utils'); +const { run_or_skip_test, TMP_PATH, create_file, IS_GPFS } = require('../../../system_tests/test_utils'); +const { NCLifecycle, ILM_POLICIES_TMP_DIR, ILM_CANDIDATES_TMP_DIR } = require('../../../../manage_nsfs/nc_lifecycle'); const config_root = path.join(TMP_PATH, 'config_root_nc_lifecycle'); const config_fs = new ConfigFS(config_root); diff --git a/src/test/unit_tests/jest_tests/test_nc_lifecycle_posix_integration.test.js b/src/test/unit_tests/nc/lifecycle/test_nc_lifecycle_posix_integration.test.js similarity index 99% rename from src/test/unit_tests/jest_tests/test_nc_lifecycle_posix_integration.test.js rename to src/test/unit_tests/nc/lifecycle/test_nc_lifecycle_posix_integration.test.js index fe578e0d07..0b55795386 100644 --- a/src/test/unit_tests/jest_tests/test_nc_lifecycle_posix_integration.test.js +++ b/src/test/unit_tests/nc/lifecycle/test_nc_lifecycle_posix_integration.test.js @@ -8,20 +8,20 @@ process.env.DISABLE_INIT_RANDOM_SEED = 'true'; const path = require('path'); const fs = require('fs'); -const config = require('../../../../config'); -const fs_utils = require('../../../util/fs_utils'); -const { ConfigFS } = require('../../../sdk/config_fs'); -const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT, exec_manage_cli, create_system_json, update_file_mtime } = require('../../system_tests/test_utils'); -const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); -const NamespaceFS = require('../../../sdk/namespace_fs'); -const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); -const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses'); -const { ManageCLIError } = require('../../../manage_nsfs/manage_nsfs_cli_errors'); -const buffer_utils = require('../../../util/buffer_utils'); +const config = require('../../../../../config'); +const fs_utils = require('../../../../util/fs_utils'); +const { ConfigFS } = require('../../../../sdk/config_fs'); +const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT, exec_manage_cli, create_system_json, update_file_mtime } = require('../../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../../manage_nsfs/manage_nsfs_constants'); +const NamespaceFS = require('../../../../sdk/namespace_fs'); +const endpoint_stats_collector = require('../../../../sdk/endpoint_stats_collector'); +const { ManageCLIResponse } = require('../../../../manage_nsfs/manage_nsfs_cli_responses'); +const { ManageCLIError } = require('../../../../manage_nsfs/manage_nsfs_cli_errors'); +const buffer_utils = require('../../../../util/buffer_utils'); const crypto = require('crypto'); -const NsfsObjectSDK = require('../../../sdk/nsfs_object_sdk'); -const nb_native = require('../../../util/nb_native'); -const native_fs_utils = require('../../../util/native_fs_utils'); +const NsfsObjectSDK = require('../../../../sdk/nsfs_object_sdk'); +const nb_native = require('../../../../util/nb_native'); +const native_fs_utils = require('../../../../util/native_fs_utils'); const LIFECYCLE_RULE_STATUS_ENUM = Object.freeze({ ENABLED: 'Enabled', diff --git a/src/test/unit_tests/nc_index.js b/src/test/unit_tests/nc_index.js deleted file mode 100644 index d11bd050bb..0000000000 --- a/src/test/unit_tests/nc_index.js +++ /dev/null @@ -1,35 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - - -const coretest = require('./nc_coretest'); -coretest.setup(); - -require('./test_namespace_fs'); -require('./test_ns_list_objects'); -require('./test_namespace_fs_mpu'); -require('./test_nb_native_fs'); -require('./test_nc_cli'); -require('./test_nc_health'); -require('./test_nsfs_access'); -require('./test_nsfs_integration'); -require('./test_bucketspace_fs'); -require('./test_nsfs_glacier_backend'); -require('./test_s3_bucket_policy'); -require('./test_nsfs_versioning'); -require('./test_bucketspace_versioning'); -require('./test_nc_bucket_logging'); -require('./test_nc_online_upgrade_s3_integrations'); -require('./test_public_access_block'); -require('./test_nc_lifecycle_expiration'); - -// running with iam port -require('./test_nc_iam_basic_integration.js'); // please notice that we use a different setup -// running with a couple of forks - please notice and add only relevant tests here -require('./test_nc_with_a_couple_of_forks.js'); // please notice that we use a different setup - -// TODO: uncomment when supported -//require('./test_s3_ops'); -//require('./test_s3_list_objects'); -//require('./test_s3_encryption'); -// require('./test_s3select'); diff --git a/src/test/unit_tests/jest_tests/test_accountspace_fs.test.js b/src/test/unit_tests/nsfs/test_accountspace_fs.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_accountspace_fs.test.js rename to src/test/unit_tests/nsfs/test_accountspace_fs.test.js diff --git a/src/test/unit_tests/test_bucketspace_fs.js b/src/test/unit_tests/nsfs/test_bucketspace_fs.js similarity index 98% rename from src/test/unit_tests/test_bucketspace_fs.js rename to src/test/unit_tests/nsfs/test_bucketspace_fs.js index 0bccb0e85f..d785a65f6a 100644 --- a/src/test/unit_tests/test_bucketspace_fs.js +++ b/src/test/unit_tests/nsfs/test_bucketspace_fs.js @@ -8,23 +8,23 @@ const _ = require('lodash'); const path = require('path'); const mocha = require('mocha'); const assert = require('assert'); -const P = require('../../util/promise'); -const config = require('../../../config'); -const fs_utils = require('../../util/fs_utils'); +const P = require('../../../util/promise'); +const config = require('../../../../config'); +const fs_utils = require('../../../util/fs_utils'); const { get_process_fs_context, read_file, get_user_by_distinguished_name, get_bucket_tmpdir_name, update_config_file -} = require('../../util/native_fs_utils'); -const nb_native = require('../../util/nb_native'); -const SensitiveString = require('../../util/sensitive_string'); -const NamespaceFS = require('../../sdk/namespace_fs'); -const BucketSpaceFS = require('../../sdk/bucketspace_fs'); -const { TMP_PATH, generate_s3_policy } = require('../system_tests/test_utils'); -const { CONFIG_SUBDIRS, JSON_SUFFIX } = require('../../sdk/config_fs'); -const nc_mkm = require('../../manage_nsfs/nc_master_key_manager').get_instance(); +} = require('../../../util/native_fs_utils'); +const nb_native = require('../../../util/nb_native'); +const SensitiveString = require('../../../util/sensitive_string'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const BucketSpaceFS = require('../../../sdk/bucketspace_fs'); +const { TMP_PATH, generate_s3_policy } = require('../../system_tests/test_utils'); +const { CONFIG_SUBDIRS, JSON_SUFFIX } = require('../../../sdk/config_fs'); +const nc_mkm = require('../../../manage_nsfs/nc_master_key_manager').get_instance(); const XATTR_INTERNAL_NOOBAA_PREFIX = 'user.noobaa.'; const XATTR_VERSION_ID = XATTR_INTERNAL_NOOBAA_PREFIX + 'version_id'; diff --git a/src/test/unit_tests/test_namespace_fs.js b/src/test/unit_tests/nsfs/test_namespace_fs.js similarity index 99% rename from src/test/unit_tests/test_namespace_fs.js rename to src/test/unit_tests/nsfs/test_namespace_fs.js index 661c00f3fd..ace8e28c6d 100644 --- a/src/test/unit_tests/test_namespace_fs.js +++ b/src/test/unit_tests/nsfs/test_namespace_fs.js @@ -11,20 +11,20 @@ const mocha = require('mocha'); const crypto = require('crypto'); const assert = require('assert'); const os = require('os'); -const P = require('../../util/promise'); -const config = require('../../../config'); -const os_utils = require('../../util/os_utils'); -const fs_utils = require('../../util/fs_utils'); -const nb_native = require('../../util/nb_native'); -const NamespaceFS = require('../../sdk/namespace_fs'); -const s3_utils = require('../../endpoint/s3/s3_utils'); -const buffer_utils = require('../../util/buffer_utils'); -const { S3Error } = require('../../endpoint/s3/s3_errors'); -const test_ns_list_objects = require('./test_ns_list_objects'); -const { TMP_PATH } = require('../system_tests/test_utils'); -const { get_process_fs_context } = require('../../util/native_fs_utils'); -const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector'); -const SensitiveString = require('../../util/sensitive_string'); +const P = require('../../../util/promise'); +const config = require('../../../../config'); +const os_utils = require('../../../util/os_utils'); +const fs_utils = require('../../../util/fs_utils'); +const nb_native = require('../../../util/nb_native'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const s3_utils = require('../../../endpoint/s3/s3_utils'); +const buffer_utils = require('../../../util/buffer_utils'); +const { S3Error } = require('../../../endpoint/s3/s3_errors'); +const test_ns_list_objects = require('../api/s3/test_ns_list_objects'); +const { TMP_PATH } = require('../../system_tests/test_utils'); +const { get_process_fs_context } = require('../../../util/native_fs_utils'); +const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); +const SensitiveString = require('../../../util/sensitive_string'); const inspect = (x, max_arr = 5) => util.inspect(x, { colors: true, depth: null, maxArrayLength: max_arr }); @@ -73,7 +73,7 @@ mocha.describe('namespace_fs', function() { const upload_bkt = 'test_ns_uploads_object'; const mpu_bkt = 'test_ns_multipart_upload'; - const src_key = 'test/unit_tests/test_namespace_fs.js'; + const src_key = 'test/unit_tests/nsfs/test_namespace_fs.js'; const tmp_fs_path = path.join(TMP_PATH, 'test_namespace_fs'); const dummy_object_sdk = make_dummy_object_sdk(tmp_fs_path); const ns_src_bucket_path = `./${src_bkt}`; diff --git a/src/test/unit_tests/test_namespace_fs_mpu.js b/src/test/unit_tests/nsfs/test_namespace_fs_mpu.js similarity index 97% rename from src/test/unit_tests/test_namespace_fs_mpu.js rename to src/test/unit_tests/nsfs/test_namespace_fs_mpu.js index 9f11327de9..b52be5e6fe 100644 --- a/src/test/unit_tests/test_namespace_fs_mpu.js +++ b/src/test/unit_tests/nsfs/test_namespace_fs_mpu.js @@ -10,15 +10,15 @@ const util = require('util'); const mocha = require('mocha'); const crypto = require('crypto'); const assert = require('assert'); -const P = require('../../util/promise'); -const config = require('../../../config'); -const fs_utils = require('../../util/fs_utils'); -const time_utils = require('../../util/time_utils'); -const NamespaceFS = require('../../sdk/namespace_fs'); -const s3_utils = require('../../endpoint/s3/s3_utils'); -const buffer_utils = require('../../util/buffer_utils'); -const { TMP_PATH } = require('../system_tests/test_utils'); -const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector'); +const P = require('../../../util/promise'); +const config = require('../../../../config'); +const fs_utils = require('../../../util/fs_utils'); +const time_utils = require('../../../util/time_utils'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const s3_utils = require('../../../endpoint/s3/s3_utils'); +const buffer_utils = require('../../../util/buffer_utils'); +const { TMP_PATH } = require('../../system_tests/test_utils'); +const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); const inspect = (x, max_arr = 5) => util.inspect(x, { colors: true, depth: null, maxArrayLength: max_arr }); diff --git a/src/test/unit_tests/test_nsfs_access.js b/src/test/unit_tests/nsfs/test_nsfs_access.js similarity index 96% rename from src/test/unit_tests/test_nsfs_access.js rename to src/test/unit_tests/nsfs/test_nsfs_access.js index a97582292b..aef8fc203a 100644 --- a/src/test/unit_tests/test_nsfs_access.js +++ b/src/test/unit_tests/nsfs/test_nsfs_access.js @@ -5,15 +5,15 @@ const path = require('path'); const mocha = require('mocha'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); -const nb_native = require('../../util/nb_native'); -const test_utils = require('../system_tests/test_utils'); +const fs_utils = require('../../../util/fs_utils'); +const nb_native = require('../../../util/nb_native'); +const test_utils = require('../../system_tests/test_utils'); const fs = require('fs'); -const { TMP_PATH } = require('../system_tests/test_utils'); -const NamespaceFS = require('../../sdk/namespace_fs'); -const buffer_utils = require('../../util/buffer_utils'); -const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector'); -const SensitiveString = require('../../util/sensitive_string'); +const { TMP_PATH } = require('../../system_tests/test_utils'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const buffer_utils = require('../../../util/buffer_utils'); +const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); +const SensitiveString = require('../../../util/sensitive_string'); const new_umask = process.env.NOOBAA_ENDPOINT_UMASK || 0o000; const old_umask = process.umask(new_umask); diff --git a/src/test/unit_tests/test_nsfs_glacier_backend.js b/src/test/unit_tests/nsfs/test_nsfs_glacier_backend.js similarity index 96% rename from src/test/unit_tests/test_nsfs_glacier_backend.js rename to src/test/unit_tests/nsfs/test_nsfs_glacier_backend.js index c273b28425..7acd171b7c 100644 --- a/src/test/unit_tests/test_nsfs_glacier_backend.js +++ b/src/test/unit_tests/nsfs/test_nsfs_glacier_backend.js @@ -8,18 +8,18 @@ const mocha = require('mocha'); const crypto = require('crypto'); const assert = require('assert'); const os = require('os'); -const config = require('../../../config'); -const NamespaceFS = require('../../sdk/namespace_fs'); -const s3_utils = require('../../endpoint/s3/s3_utils'); -const buffer_utils = require('../../util/buffer_utils'); -const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector'); -const { NewlineReader } = require('../../util/file_reader'); -const { TapeCloudGlacier, TapeCloudUtils } = require('../../sdk/glacier_tapecloud'); -const { PersistentLogger } = require('../../util/persistent_logger'); -const { Glacier } = require('../../sdk/glacier'); -const { Semaphore } = require('../../util/semaphore'); -const nb_native = require('../../util/nb_native'); -const { handler: s3_get_bucket } = require('../../endpoint/s3/ops/s3_get_bucket'); +const config = require('../../../../config'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const s3_utils = require('../../../endpoint/s3/s3_utils'); +const buffer_utils = require('../../../util/buffer_utils'); +const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); +const { NewlineReader } = require('../../../util/file_reader'); +const { TapeCloudGlacier, TapeCloudUtils } = require('../../../sdk/glacier_tapecloud'); +const { PersistentLogger } = require('../../../util/persistent_logger'); +const { Glacier } = require('../../../sdk/glacier'); +const { Semaphore } = require('../../../util/semaphore'); +const nb_native = require('../../../util/nb_native'); +const { handler: s3_get_bucket } = require('../../../endpoint/s3/ops/s3_get_bucket'); const inspect = (x, max_arr = 5) => util.inspect(x, { colors: true, depth: null, maxArrayLength: max_arr }); diff --git a/src/test/unit_tests/test_nsfs_versioning.js b/src/test/unit_tests/nsfs/test_nsfs_versioning.js similarity index 94% rename from src/test/unit_tests/test_nsfs_versioning.js rename to src/test/unit_tests/nsfs/test_nsfs_versioning.js index e79a46ecff..fa0248ce4f 100644 --- a/src/test/unit_tests/test_nsfs_versioning.js +++ b/src/test/unit_tests/nsfs/test_nsfs_versioning.js @@ -8,11 +8,11 @@ const path = require('path'); const mocha = require('mocha'); const assert = require('assert'); const crypto = require('crypto'); -const fs_utils = require('../../util/fs_utils'); -const NamespaceFS = require('../../sdk/namespace_fs'); -const buffer_utils = require('../../util/buffer_utils'); -const native_fs_utils = require('../../util/native_fs_utils'); -const { TMP_PATH, invalid_nsfs_root_permissions } = require('../system_tests/test_utils'); +const fs_utils = require('../../../util/fs_utils'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const buffer_utils = require('../../../util/buffer_utils'); +const native_fs_utils = require('../../../util/native_fs_utils'); +const { TMP_PATH, invalid_nsfs_root_permissions } = require('../../system_tests/test_utils'); function make_dummy_object_sdk(nsfs_config, uid, gid) { diff --git a/src/test/unit_tests/sudo_index.js b/src/test/unit_tests/sudo_index.js deleted file mode 100644 index a96b19f437..0000000000 --- a/src/test/unit_tests/sudo_index.js +++ /dev/null @@ -1,25 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - - -const coretest = require('./coretest'); -const test_utils = require('../system_tests/test_utils'); -coretest.setup({ incomplete_rpc_coverage: 'show' }); - -// --------------------------------------- -// Tests that does not require hosts pools -// --------------------------------------- - -if (test_utils.invalid_nsfs_root_permissions()) { - throw new Error(`Insufficient uid gid: pgid: ${process.getgid()}, puid: ${process.getuid()}`); -} - -// // CORE -require('./test_nsfs_access'); -require('./test_nsfs_integration'); -require('./test_bucketspace_versioning'); -require('./test_bucketspace_fs'); -require('./test_nsfs_versioning'); -require('./test_nc_cli'); -require('./test_nc_health'); - diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/LICENSE b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/LICENSE similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/LICENSE rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/LICENSE diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/NOTICE b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/NOTICE similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/NOTICE rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/NOTICE diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-key-duplicate/get-header-key-duplicate.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-multiline/get-header-value-multiline.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-order/get-header-value-order.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-header-value-trim/get-header-value-trim.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-unreserved/get-unreserved.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-utf8/get-utf8.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-empty-query-key/get-vanilla-empty-query-key.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-order-key-case/get-vanilla-query-order-key-case.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query-unreserved/get-vanilla-query-unreserved.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-query/get-vanilla-query.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla-utf8-query/get-vanilla-utf8-query.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/get-vanilla/get-vanilla.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative-relative/get-relative-relative.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-relative/get-relative.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-dot-slash/get-slash-dot-slash.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash-pointless-dot/get-slash-pointless-dot.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slash/get-slash.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-slashes/get-slashes.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/get-space/get-space.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/normalize-path.txt b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/normalize-path.txt similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/normalize-path/normalize-path.txt rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/normalize-path/normalize-path.txt diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-case/post-header-key-case.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-key-sort/post-header-key-sort.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-header-value-case/post-header-value-case.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-after/post-sts-header-after.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/post-sts-header-before/post-sts-header-before.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/readme.txt b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/readme.txt similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-sts-token/readme.txt rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-sts-token/readme.txt diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-empty-query-value/post-vanilla-empty-query-value.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-nonunreserved/post-vanilla-query-nonunreserved.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query-space/post-vanilla-query-space.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla-query/post-vanilla-query.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-vanilla/post-vanilla.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded-parameters/post-x-www-form-urlencoded-parameters.sts diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.creq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.creq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.creq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.creq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sreq b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sreq rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sreq diff --git a/src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sts b/src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sts similarity index 100% rename from src/test/unit_tests/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sts rename to src/test/unit_tests/utils/signature_test_suite/aws4_testsuite/post-x-www-form-urlencoded/post-x-www-form-urlencoded.sts diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_GET_ixt43o0n.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_GET_ixt43o0n.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_GET_ixt43o0n.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_GET_ixt43o0n.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_GET_j4dv016i.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_GET_j4dv016i.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_GET_j4dv016i.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_GET_j4dv016i.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_GET_j4dviqlb.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_GET_j4dviqlb.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_GET_j4dviqlb.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_GET_j4dviqlb.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfg5hub.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfg5hub.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfg5hub.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfg5hub.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsgw.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsgw.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsgw.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsgw.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsjt.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsjt.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsjt.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsjt.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsk1.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsk1.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsk1.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsk1.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfskj.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfskj.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfskj.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfskj.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsl1.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsl1.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsl1.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsl1.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsl4.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsl4.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgfsl4.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgfsl4.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgpe7c.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgpe7c.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgpe7c.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgpe7c.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgtw54.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgtw54.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwfgtw54.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwfgtw54.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirgh.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirgh.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirgh.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirgh.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirjj.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirjj.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirjj.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirjj.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirjp.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirjp.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirjp.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirjp.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirjt.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirjt.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirjt.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirjt.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirjv.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirjv.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirjv.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirjv.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirk8.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirk8.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirk8.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirk8.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirkc.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirkc.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirkc.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirkc.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirkk.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirkk.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirkk.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirkk.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirl2.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirl2.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirl2.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirl2.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirl7.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirl7.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirl7.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirl7.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirl9.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirl9.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirl9.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirl9.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirla.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirla.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirla.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirla.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirll.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirll.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirll.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirll.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirlu.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirlu.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirlu.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirlu.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirlw.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirlw.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirlw.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirlw.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirm2.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirm2.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirm2.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirm2.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirma.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirma.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdirma.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdirma.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdis8b.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdis8b.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdis8b.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdis8b.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdis8d.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdis8d.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdis8d.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdis8d.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdis9x.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdis9x.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdis9x.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdis9x.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisc8.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisc8.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisc8.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisc8.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiscl.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiscl.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiscl.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiscl.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiscn.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiscn.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiscn.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiscn.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisgt.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisgt.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisob.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisob.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisob.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisob.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisoh.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisoh.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisoh.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisoh.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdissw.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdissw.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdissw.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdissw.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisug.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisug.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdisug.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdisug.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiszd.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiszd.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiszd.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiszd.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiszf.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiszf.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiszf.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiszf.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiszi.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiszi.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdiszi.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdiszi.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit02.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit02.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit02.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit02.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit12.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit12.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit12.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit12.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit1y.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit1y.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit1y.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit1y.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit3a.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit3a.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit3a.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit3a.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit5n.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit5n.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit5n.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit5n.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit9b.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit9b.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit9b.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit9b.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit9c.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit9c.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdit9c.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdit9c.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdita2.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdita2.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdita2.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdita2.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgditi0.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgditi0.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgditi0.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgditi0.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgditnm.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgditnm.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgditnm.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgditnm.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdito6.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdito6.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgdito6.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgdito6.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgditoz.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgditoz.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgditoz.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgditoz.sreq diff --git a/src/test/unit_tests/signature_test_suite/awscli/awscli_iwgditp9.sreq b/src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgditp9.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awscli/awscli_iwgditp9.sreq rename to src/test/unit_tests/utils/signature_test_suite/awscli/awscli_iwgditp9.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsumhdig_namewith=.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsumhdig_namewith=.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsumhdig_namewith=.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsumhdig_namewith=.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsun15mj_namewith%253D.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsun15mj_namewith%253D.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsun15mj_namewith%253D.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsun15mj_namewith%253D.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsun46rc_namewith$.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsun46rc_namewith$.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsun46rc_namewith$.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsun46rc_namewith$.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsun7xep_namewith&.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsun7xep_namewith&.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsun7xep_namewith&.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsun7xep_namewith&.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsundqfu_namewith,.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsundqfu_namewith,.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsundqfu_namewith,.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsundqfu_namewith,.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsunew5a_namewith:.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsunew5a_namewith:.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsunew5a_namewith:.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsunew5a_namewith:.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsung2nb_namewith@.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsung2nb_namewith@.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkcpp/awssdkcpp_lsung2nb_namewith@.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkcpp/awssdkcpp_lsung2nb_namewith@.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjava/awssdkjava_PUT_j3y3x79m.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjava/awssdkjava_PUT_j3y3x79m.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjava/awssdkjava_PUT_j3y3x79m.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjava/awssdkjava_PUT_j3y3x79m.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwfras1h.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwfras1h.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwfras1h.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwfras1h.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwfrazqq.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwfrazqq.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwfrazqq.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwfrazqq.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwfrd85t.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwfrd85t.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwfrd85t.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwfrd85t.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4z7vq.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4z7vq.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4z7vq.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4z7vq.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4z8rp.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4z8rp.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4z8rp.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4z8rp.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4zat7.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4zat7.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4zat7.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4zat7.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4zavj.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4zavj.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4zavj.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4zavj.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4zgyj.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4zgyj.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg4zgyj.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg4zgyj.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5embj.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5embj.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5embj.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5embj.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5en9c.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5en9c.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5en9c.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5en9c.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5enz6.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5enz6.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5enz6.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5enz6.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5eo3v.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5eo3v.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5eo3v.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5eo3v.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5eo67.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5eo67.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5eo67.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5eo67.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5eo8w.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5eo8w.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5eo8w.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5eo8w.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f124.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f124.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f124.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f124.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f12h.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f12h.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f12h.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f12h.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f131.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f131.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f131.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f131.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f13m.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f13m.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f13m.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f13m.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f16a.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f16a.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f16a.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f16a.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f16b.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f16b.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwg5f16b.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwg5f16b.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwgdhgq4.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwgdhgq4.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwgdhgq4.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwgdhgq4.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwgdhl00.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwgdhl00.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwgdhl00.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwgdhl00.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwgdhm2o.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwgdhm2o.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwgdhm2o.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwgdhm2o.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwgdhm3y.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwgdhm3y.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkjs/mozilla_iwgdhm3y.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkjs/mozilla_iwgdhm3y.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjrwfj.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjrwfj.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjrwfj.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjrwfj.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjscft.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjscft.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjscft.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjscft.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjsior.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjsior.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjsior.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjsior.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjukzm.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjukzm.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjukzm.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjukzm.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjunsr.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjunsr.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjunsr.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjunsr.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjuq0e.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjuq0e.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfjuq0e.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfjuq0e.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk13hp.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk13hp.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk13hp.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk13hp.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk33s8.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk33s8.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk33s8.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk33s8.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk36zu.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk36zu.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk36zu.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk36zu.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk3r71.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk3r71.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk3r71.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk3r71.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk3vfs.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk3vfs.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfk3vfs.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfk3vfs.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfqyfi9.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfqyfi9.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfqyfi9.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfqyfi9.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfqyqjo.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfqyqjo.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_iwfqyqjo.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_iwfqyqjo.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_lsunnm4a_namewith%3D.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_lsunnm4a_namewith%3D.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_lsunnm4a_namewith%3D.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_lsunnm4a_namewith%3D.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_lsunp5sx_namewith%40.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_lsunp5sx_namewith%40.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_lsunp5sx_namewith%40.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_lsunp5sx_namewith%40.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_lsunppst_namewith%24.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_lsunppst_namewith%24.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_lsunppst_namewith%24.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_lsunppst_namewith%24.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_lsunqlri_namewith%253D.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_lsunqlri_namewith%253D.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdknodejs/awssdknodejs_lsunqlri_namewith%253D.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdknodejs/awssdknodejs_lsunqlri_namewith%253D.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5j8rpgr.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5j8rpgr.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5j8rpgr.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5j8rpgr.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5j9wb4d.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5j9wb4d.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5j9wb4d.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5j9wb4d.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5ja2mvl.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5ja2mvl.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5ja2mvl.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkruby2/awssdkruby2_GET_j5ja2mvl.sreq diff --git a/src/test/unit_tests/signature_test_suite/awssdkruby2/awssdkruby2_PUT_j5jajnre.sreq b/src/test/unit_tests/utils/signature_test_suite/awssdkruby2/awssdkruby2_PUT_j5jajnre.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/awssdkruby2/awssdkruby2_PUT_j5jajnre.sreq rename to src/test/unit_tests/utils/signature_test_suite/awssdkruby2/awssdkruby2_PUT_j5jajnre.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_GET_ixt3qlyg.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_GET_ixt3qlyg.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_GET_ixt3qlyg.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_GET_ixt3qlyg.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_GET_ixt3xo48.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_GET_ixt3xo48.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_GET_ixt3xo48.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_GET_ixt3xo48.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdegdm.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdegdm.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdegdm.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdegdm.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdegnh.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdegnh.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdegnh.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdegnh.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfcmd.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfcmd.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfcmd.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfcmd.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfcqg.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfcqg.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfcqg.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfcqg.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfcrc.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfcrc.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfcrc.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfcrc.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfcvj.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfcvj.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfcvj.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfcvj.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdffdd.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdffdd.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdffdd.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdffdd.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfkde.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfkde.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfkde.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfkde.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfkmo.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfkmo.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfkmo.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfkmo.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfmbq.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfmbq.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfmbq.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfmbq.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfme4.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfme4.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfme4.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfme4.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfmmm.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfmmm.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfmmm.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfmmm.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfmp9.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfmp9.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfmp9.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfmp9.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfmyw.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfmyw.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfmyw.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfmyw.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfnpb.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfnpb.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfnpb.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfnpb.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfnrp.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfnrp.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfnrp.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfnrp.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfnx4.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfnx4.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfnx4.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfnx4.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfnz2.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfnz2.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfnz2.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfnz2.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfo3w.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfo3w.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfo3w.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfo3w.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfpgw.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfpgw.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfpgw.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfpgw.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfpzn.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfpzn.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfpzn.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfpzn.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfq23.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfq23.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfq23.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfq23.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqcm.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqcm.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqcm.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqcm.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqey.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqey.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqey.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqey.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqlr.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqlr.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqlr.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqlr.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqob.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqob.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqob.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqob.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqp9.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqp9.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqp9.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqp9.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqtg.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqtg.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqtg.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqtg.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqz6.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqz6.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfqz6.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfqz6.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfr39.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfr39.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfr39.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfr39.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfrbs.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfrbs.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfrbs.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfrbs.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfrea.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfrea.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfrea.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfrea.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfrum.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfrum.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfrum.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfrum.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfs7m.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfs7m.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfs7m.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfs7m.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfsft.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfsft.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfsft.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfsft.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfsi6.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfsi6.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfsi6.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfsi6.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfsiq.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfsiq.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfsiq.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfsiq.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdftul.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdftul.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdftul.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdftul.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdftx3.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdftx3.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdftx3.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdftx3.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfugt.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfugt.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfugt.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfugt.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfuk5.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfuk5.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfuk5.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfuk5.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfura.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfura.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfura.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfura.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfutd.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfutd.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfutd.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfutd.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfuwf.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfuwf.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfuwf.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfuwf.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfv5d.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfv5d.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfv5d.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfv5d.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfv7e.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfv7e.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfv7e.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfv7e.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfvrf.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfvrf.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfvrf.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfvrf.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfvtr.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfvtr.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfvtr.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfvtr.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwcr.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwcr.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwcr.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwcr.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwf9.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwf9.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwf9.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwf9.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwmy.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwmy.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwmy.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwmy.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwqc.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwqc.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwqc.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwqc.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwvi.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwvi.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwvi.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwvi.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwyh.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwyh.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfwyh.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfwyh.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfy9s.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfy9s.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfy9s.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfy9s.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfycg.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfycg.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfycg.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfycg.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfz3j.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfz3j.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfz3j.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfz3j.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfz70.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfz70.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfz70.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfz70.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzpw.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzpw.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzpw.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzpw.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzrm.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzrm.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzrm.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzrm.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzua.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzua.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzua.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzua.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzwj.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzwj.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzwj.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzwj.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzwu.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzwu.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzwu.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzwu.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzz0.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzz0.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdfzz0.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdfzz0.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdg017.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdg017.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdg017.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdg017.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdg02o.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdg02o.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdg02o.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdg02o.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdg035.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdg035.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdg035.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdg035.sreq diff --git a/src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdg04o.sreq b/src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdg04o.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/cyberduck/cyberduck_iwgdg04o.sreq rename to src/test/unit_tests/utils/signature_test_suite/cyberduck/cyberduck_iwgdg04o.sreq diff --git a/src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2l4dq.sreq b/src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2l4dq.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2l4dq.sreq rename to src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2l4dq.sreq diff --git a/src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2m390.sreq b/src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2m390.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2m390.sreq rename to src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2m390.sreq diff --git a/src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2p70i.sreq b/src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2p70i.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2p70i.sreq rename to src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2p70i.sreq diff --git a/src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2pmp6.sreq b/src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2pmp6.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2pmp6.sreq rename to src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2pmp6.sreq diff --git a/src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2qe9y.sreq b/src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2qe9y.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/postman/postmanruntime_iwf2qe9y.sreq rename to src/test/unit_tests/utils/signature_test_suite/postman/postmanruntime_iwf2qe9y.sreq diff --git a/src/test/unit_tests/signature_test_suite/presigned/curl_iwfgwr8w.sreq b/src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfgwr8w.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/presigned/curl_iwfgwr8w.sreq rename to src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfgwr8w.sreq diff --git a/src/test/unit_tests/signature_test_suite/presigned/curl_iwfk6hmm.sreq b/src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk6hmm.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/presigned/curl_iwfk6hmm.sreq rename to src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk6hmm.sreq diff --git a/src/test/unit_tests/signature_test_suite/presigned/curl_iwfk6t1i.sreq b/src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk6t1i.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/presigned/curl_iwfk6t1i.sreq rename to src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk6t1i.sreq diff --git a/src/test/unit_tests/signature_test_suite/presigned/curl_iwfk7a6w.sreq b/src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk7a6w.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/presigned/curl_iwfk7a6w.sreq rename to src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk7a6w.sreq diff --git a/src/test/unit_tests/signature_test_suite/presigned/curl_iwfk833f.sreq b/src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk833f.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/presigned/curl_iwfk833f.sreq rename to src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk833f.sreq diff --git a/src/test/unit_tests/signature_test_suite/presigned/curl_iwfk8xcq.sreq b/src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk8xcq.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/presigned/curl_iwfk8xcq.sreq rename to src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk8xcq.sreq diff --git a/src/test/unit_tests/signature_test_suite/presigned/curl_iwfk9ovx.sreq b/src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk9ovx.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/presigned/curl_iwfk9ovx.sreq rename to src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfk9ovx.sreq diff --git a/src/test/unit_tests/signature_test_suite/presigned/curl_iwfkuaka.sreq b/src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfkuaka.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/presigned/curl_iwfkuaka.sreq rename to src/test/unit_tests/utils/signature_test_suite/presigned/curl_iwfkuaka.sreq diff --git a/src/test/unit_tests/signature_test_suite/rgw/rgw_HEAD_BZ.sreq b/src/test/unit_tests/utils/signature_test_suite/rgw/rgw_HEAD_BZ.sreq similarity index 100% rename from src/test/unit_tests/signature_test_suite/rgw/rgw_HEAD_BZ.sreq rename to src/test/unit_tests/utils/signature_test_suite/rgw/rgw_HEAD_BZ.sreq diff --git a/src/test/unit_tests/test_buffer_pool.js b/src/test/unit_tests/utils/test_buffer_pool.js similarity index 98% rename from src/test/unit_tests/test_buffer_pool.js rename to src/test/unit_tests/utils/test_buffer_pool.js index 3ccaf6c08e..6535ed642b 100644 --- a/src/test/unit_tests/test_buffer_pool.js +++ b/src/test/unit_tests/utils/test_buffer_pool.js @@ -2,8 +2,8 @@ 'use strict'; const mocha = require('mocha'); const assert = require('assert'); -const buffer_utils = require('../../util/buffer_utils'); -const semaphore = require('../../util/semaphore'); +const buffer_utils = require('../../../util/buffer_utils'); +const semaphore = require('../../../util/semaphore'); mocha.describe('Test buffers pool', function() { diff --git a/src/test/unit_tests/test_cloud_utils.js b/src/test/unit_tests/utils/test_cloud_utils.js similarity index 95% rename from src/test/unit_tests/test_cloud_utils.js rename to src/test/unit_tests/utils/test_cloud_utils.js index 7e069516bc..67d5a179fc 100644 --- a/src/test/unit_tests/test_cloud_utils.js +++ b/src/test/unit_tests/utils/test_cloud_utils.js @@ -5,8 +5,8 @@ const mocha = require('mocha'); const assert = require('assert'); const sinon = require('sinon'); const AWS = require('aws-sdk'); -const cloud_utils = require('../../util/cloud_utils'); -const dbg = require('../../util/debug_module')(__filename); +const cloud_utils = require('../../../util/cloud_utils'); +const dbg = require('../../../util/debug_module')(__filename); const fs = require("fs"); const projectedServiceAccountToken = "/var/run/secrets/openshift/serviceaccount/token"; const fakeAccessKeyId = "fakeAccessKeyId"; diff --git a/src/test/unit_tests/test_delayed_trigger.js b/src/test/unit_tests/utils/test_delayed_trigger.js similarity index 96% rename from src/test/unit_tests/test_delayed_trigger.js rename to src/test/unit_tests/utils/test_delayed_trigger.js index f57f52a992..9794c1297b 100644 --- a/src/test/unit_tests/test_delayed_trigger.js +++ b/src/test/unit_tests/utils/test_delayed_trigger.js @@ -5,9 +5,9 @@ const mocha = require('mocha'); const assert = require('assert'); const { setTimeout: delay, setImmediate: immediate } = require('node:timers/promises'); -const error_utils = require('../../util/error_utils'); -const DelayedTrigger = require('../../util/delayed_trigger'); -const WaitQueue = require('../../util/wait_queue'); +const error_utils = require('../../../util/error_utils'); +const DelayedTrigger = require('../../../util/delayed_trigger'); +const WaitQueue = require('../../../util/wait_queue'); // class to test the expectations from DelayedTrigger class TriggerTester { diff --git a/src/test/unit_tests/jest_tests/test_entropy_utils.test.js b/src/test/unit_tests/utils/test_entropy_utils.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_entropy_utils.test.js rename to src/test/unit_tests/utils/test_entropy_utils.test.js diff --git a/src/test/unit_tests/test_fs_utils.js b/src/test/unit_tests/utils/test_fs_utils.js similarity index 97% rename from src/test/unit_tests/test_fs_utils.js rename to src/test/unit_tests/utils/test_fs_utils.js index 09eb3542dc..5ac05d9372 100644 --- a/src/test/unit_tests/test_fs_utils.js +++ b/src/test/unit_tests/utils/test_fs_utils.js @@ -5,7 +5,7 @@ const mocha = require('mocha'); const assert = require('assert'); -const fs_utils = require('../../util/fs_utils'); +const fs_utils = require('../../../util/fs_utils'); function log(...args) { if (process.env.SUPPRESS_LOGS) return; diff --git a/src/test/unit_tests/test_http_utils.js b/src/test/unit_tests/utils/test_http_utils.js similarity index 96% rename from src/test/unit_tests/test_http_utils.js rename to src/test/unit_tests/utils/test_http_utils.js index e0b0e656aa..1dee5b01b1 100644 --- a/src/test/unit_tests/test_http_utils.js +++ b/src/test/unit_tests/utils/test_http_utils.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const http_utils = require('../../util/http_utils'); +const http_utils = require('../../../util/http_utils'); mocha.describe('http_utils', function() { diff --git a/src/test/unit_tests/jest_tests/test_iam_utils.test.js b/src/test/unit_tests/utils/test_iam_utils.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_iam_utils.test.js rename to src/test/unit_tests/utils/test_iam_utils.test.js diff --git a/src/test/unit_tests/test_keys_lock.js b/src/test/unit_tests/utils/test_keys_lock.js similarity index 99% rename from src/test/unit_tests/test_keys_lock.js rename to src/test/unit_tests/utils/test_keys_lock.js index 8e44dcff32..e04e5be2bb 100644 --- a/src/test/unit_tests/test_keys_lock.js +++ b/src/test/unit_tests/utils/test_keys_lock.js @@ -3,7 +3,7 @@ const mocha = require('mocha'); const assert = require('assert'); -const KeysLock = require('../../util/keys_lock'); +const KeysLock = require('../../../util/keys_lock'); mocha.describe('keys_lock', function() { diff --git a/src/test/unit_tests/test_kmeans.js b/src/test/unit_tests/utils/test_kmeans.js similarity index 98% rename from src/test/unit_tests/test_kmeans.js rename to src/test/unit_tests/utils/test_kmeans.js index 59489ef522..4ad492e64f 100644 --- a/src/test/unit_tests/test_kmeans.js +++ b/src/test/unit_tests/utils/test_kmeans.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const kmeans = require('../../util/kmeans'); +const kmeans = require('../../../util/kmeans'); mocha.describe('kmeans errors', function() { diff --git a/src/test/unit_tests/test_linked_list.js b/src/test/unit_tests/utils/test_linked_list.js similarity index 96% rename from src/test/unit_tests/test_linked_list.js rename to src/test/unit_tests/utils/test_linked_list.js index 851d23ce3c..6d6810f567 100644 --- a/src/test/unit_tests/test_linked_list.js +++ b/src/test/unit_tests/utils/test_linked_list.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const LinkedList = require('../../util/linked_list'); +const LinkedList = require('../../../util/linked_list'); mocha.describe('linked_list', function() { diff --git a/src/test/unit_tests/test_lru.js b/src/test/unit_tests/utils/test_lru.js similarity index 92% rename from src/test/unit_tests/test_lru.js rename to src/test/unit_tests/utils/test_lru.js index 77d6e3341c..37fe5c7fe5 100644 --- a/src/test/unit_tests/test_lru.js +++ b/src/test/unit_tests/utils/test_lru.js @@ -4,8 +4,8 @@ const mocha = require('mocha'); const assert = require('assert'); -const P = require('../../util/promise'); -const LRU = require('../../util/lru'); +const P = require('../../../util/promise'); +const LRU = require('../../../util/lru'); mocha.describe('lru', function() { @@ -35,14 +35,13 @@ mocha.describe('lru', function() { }); lru._sanity(); - let item = lru.find_or_add_item(1); - item.foo = 'bar'; + lru.find_or_add_item(1); lru._sanity(); - item = lru.find_or_add_item(2); + lru.find_or_add_item(2); lru._sanity(); - item = lru.find_or_add_item(1); + const item = lru.find_or_add_item(1); assert.strictEqual(item.foo, undefined); lru._sanity(); }); diff --git a/src/test/unit_tests/test_lru_cache.js b/src/test/unit_tests/utils/test_lru_cache.js similarity index 98% rename from src/test/unit_tests/test_lru_cache.js rename to src/test/unit_tests/utils/test_lru_cache.js index 2a44b65a56..ccf2bb47bf 100644 --- a/src/test/unit_tests/test_lru_cache.js +++ b/src/test/unit_tests/utils/test_lru_cache.js @@ -3,7 +3,7 @@ const mocha = require('mocha'); const assert = require('assert'); -const LRUCache = require('../../util/lru_cache'); +const LRUCache = require('../../../util/lru_cache'); mocha.describe('lru_cache', function() { diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js b/src/test/unit_tests/utils/test_nc_nsfs_account_schema_validation.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js rename to src/test/unit_tests/utils/test_nc_nsfs_account_schema_validation.test.js diff --git a/src/test/unit_tests/jest_tests/test_net_utils.test.js b/src/test/unit_tests/utils/test_net_utils.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_net_utils.test.js rename to src/test/unit_tests/utils/test_net_utils.test.js diff --git a/src/test/unit_tests/jest_tests/test_os_utils.test.js b/src/test/unit_tests/utils/test_os_utils.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_os_utils.test.js rename to src/test/unit_tests/utils/test_os_utils.test.js diff --git a/src/test/unit_tests/test_prefetch.js b/src/test/unit_tests/utils/test_prefetch.js similarity index 93% rename from src/test/unit_tests/test_prefetch.js rename to src/test/unit_tests/utils/test_prefetch.js index 13c4895c0f..556107bf77 100644 --- a/src/test/unit_tests/test_prefetch.js +++ b/src/test/unit_tests/utils/test_prefetch.js @@ -2,10 +2,10 @@ 'use strict'; const _ = require('lodash'); -const P = require('../../util/promise'); +const P = require('../../../util/promise'); const mocha = require('mocha'); // var assert = require('assert'); -const Prefetch = require('../../util/prefetch'); +const Prefetch = require('../../../util/prefetch'); function log(...args) { if (process.env.SUPPRESS_LOGS) return; diff --git a/src/test/unit_tests/test_promise_utils.js b/src/test/unit_tests/utils/test_promise_utils.js similarity index 97% rename from src/test/unit_tests/test_promise_utils.js rename to src/test/unit_tests/utils/test_promise_utils.js index b05473cd53..f7141358c7 100644 --- a/src/test/unit_tests/test_promise_utils.js +++ b/src/test/unit_tests/utils/test_promise_utils.js @@ -1,8 +1,8 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const P = require('../../util/promise'); -const Defer = require('../../util/defer'); +const P = require('../../../util/promise'); +const Defer = require('../../../util/defer'); const mocha = require('mocha'); const assert = require('assert'); diff --git a/src/test/unit_tests/test_range_stream.js b/src/test/unit_tests/utils/test_range_stream.js similarity index 98% rename from src/test/unit_tests/test_range_stream.js rename to src/test/unit_tests/utils/test_range_stream.js index d1021c5642..752dcba9d3 100644 --- a/src/test/unit_tests/test_range_stream.js +++ b/src/test/unit_tests/utils/test_range_stream.js @@ -4,7 +4,7 @@ const mocha = require('mocha'); const assert = require('assert'); -const RangeStream = require('../../util/range_stream'); +const RangeStream = require('../../../util/range_stream'); mocha.describe('range_stream', function() { diff --git a/src/test/unit_tests/test_rpc.js b/src/test/unit_tests/utils/test_rpc.js similarity index 99% rename from src/test/unit_tests/test_rpc.js rename to src/test/unit_tests/utils/test_rpc.js index 28f23d401d..a5e79d7276 100644 --- a/src/test/unit_tests/test_rpc.js +++ b/src/test/unit_tests/utils/test_rpc.js @@ -8,8 +8,8 @@ const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const ssl_utils = require('../../util/ssl_utils'); -const { RPC, RpcError, RpcSchema, RPC_BUFFERS } = require('../../rpc'); +const ssl_utils = require('../../../util/ssl_utils'); +const { RPC, RpcError, RpcSchema, RPC_BUFFERS } = require('../../../rpc'); function log(...args) { if (process.env.SUPPRESS_LOGS) return; diff --git a/src/test/unit_tests/jest_tests/test_s3_utils.test.js b/src/test/unit_tests/utils/test_s3_utils.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_s3_utils.test.js rename to src/test/unit_tests/utils/test_s3_utils.test.js diff --git a/src/test/unit_tests/test_semaphore.js b/src/test/unit_tests/utils/test_semaphore.js similarity index 98% rename from src/test/unit_tests/test_semaphore.js rename to src/test/unit_tests/utils/test_semaphore.js index 12e593c65f..cd8859001e 100644 --- a/src/test/unit_tests/test_semaphore.js +++ b/src/test/unit_tests/utils/test_semaphore.js @@ -4,7 +4,7 @@ const util = require('util'); const mocha = require('mocha'); const assert = require('assert'); -const semaphore = require('../../util/semaphore'); +const semaphore = require('../../../util/semaphore'); const setImmediateAsync = util.promisify(setImmediate); diff --git a/src/test/unit_tests/test_sensitive_wrapper.js b/src/test/unit_tests/utils/test_sensitive_wrapper.js similarity index 96% rename from src/test/unit_tests/test_sensitive_wrapper.js rename to src/test/unit_tests/utils/test_sensitive_wrapper.js index cdbfd9e867..f386dec4c5 100644 --- a/src/test/unit_tests/test_sensitive_wrapper.js +++ b/src/test/unit_tests/utils/test_sensitive_wrapper.js @@ -7,8 +7,8 @@ const { default: Ajv } = require('ajv'); const util = require('util'); const BSON = require('bson'); const assert = require('assert'); -const { KEYWORDS } = require('../../util/schema_keywords'); -const SensitiveString = require('../../util/sensitive_string'); +const { KEYWORDS } = require('../../../util/schema_keywords'); +const SensitiveString = require('../../../util/sensitive_string'); mocha.describe('SensitiveString', function() { const ajv = new Ajv({ verbose: true, allErrors: true }); diff --git a/src/test/unit_tests/test_signature_utils.js b/src/test/unit_tests/utils/test_signature_utils.js similarity index 99% rename from src/test/unit_tests/test_signature_utils.js rename to src/test/unit_tests/utils/test_signature_utils.js index ae18a118e4..a4d19f4f7e 100644 --- a/src/test/unit_tests/test_signature_utils.js +++ b/src/test/unit_tests/utils/test_signature_utils.js @@ -10,7 +10,7 @@ const http = require('http'); const mocha = require('mocha'); const crypto = require('crypto'); -const signature_utils = require('../../util/signature_utils'); +const signature_utils = require('../../../util/signature_utils'); function log(...args) { if (process.env.SUPPRESS_LOGS) return; diff --git a/src/test/unit_tests/test_ssl_utils.js b/src/test/unit_tests/utils/test_ssl_utils.js similarity index 98% rename from src/test/unit_tests/test_ssl_utils.js rename to src/test/unit_tests/utils/test_ssl_utils.js index 8e2b45cbb4..ea7f50ae73 100644 --- a/src/test/unit_tests/test_ssl_utils.js +++ b/src/test/unit_tests/utils/test_ssl_utils.js @@ -8,8 +8,8 @@ const assert = require('assert'); const https = require('https'); // const P = require('../../util/promise'); -const ssl_utils = require('../../util/ssl_utils'); -const nb_native = require('../../util/nb_native'); +const ssl_utils = require('../../../util/ssl_utils'); +const nb_native = require('../../../util/nb_native'); mocha.describe('ssl_utils', function() { diff --git a/src/test/unit_tests/jest_tests/test_url_utils.test.js b/src/test/unit_tests/utils/test_url_utils.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_url_utils.test.js rename to src/test/unit_tests/utils/test_url_utils.test.js diff --git a/src/test/unit_tests/test_v8_optimizations.js b/src/test/unit_tests/utils/test_v8_optimizations.js similarity index 96% rename from src/test/unit_tests/test_v8_optimizations.js rename to src/test/unit_tests/utils/test_v8_optimizations.js index e364d54bc4..77059fcca7 100644 --- a/src/test/unit_tests/test_v8_optimizations.js +++ b/src/test/unit_tests/utils/test_v8_optimizations.js @@ -11,9 +11,9 @@ const mocha = require('mocha'); const assert = require('assert'); const stream = require('stream'); -const P = require('../../util/promise'); -const FrameStream = require('../../util/frame_stream'); -const buffer_utils = require('../../util/buffer_utils'); +const P = require('../../../util/promise'); +const FrameStream = require('../../../util/frame_stream'); +const buffer_utils = require('../../../util/buffer_utils'); const OPTIMIZED = 'OPTIMIZED'; const NOT_OPTIMIZED = 'NOT_OPTIMIZED'; diff --git a/src/test/unit_tests/jest_tests/test_validation_utils.test.js b/src/test/unit_tests/utils/test_validation_utils.test.js similarity index 100% rename from src/test/unit_tests/jest_tests/test_validation_utils.test.js rename to src/test/unit_tests/utils/test_validation_utils.test.js diff --git a/src/test/unit_tests/test_wait_queue.js b/src/test/unit_tests/utils/test_wait_queue.js similarity index 96% rename from src/test/unit_tests/test_wait_queue.js rename to src/test/unit_tests/utils/test_wait_queue.js index f0dd995c6e..3c70281dff 100644 --- a/src/test/unit_tests/test_wait_queue.js +++ b/src/test/unit_tests/utils/test_wait_queue.js @@ -5,8 +5,8 @@ const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); -const P = require('../../util/promise'); -const WaitQueue = require('../../util/wait_queue'); +const P = require('../../../util/promise'); +const WaitQueue = require('../../../util/wait_queue'); mocha.describe('wait_queue', function() { diff --git a/src/test/unit_tests/test_zip_utils.js b/src/test/unit_tests/utils/test_zip_utils.js similarity index 97% rename from src/test/unit_tests/test_zip_utils.js rename to src/test/unit_tests/utils/test_zip_utils.js index c27fc5ce71..659db81f35 100644 --- a/src/test/unit_tests/test_zip_utils.js +++ b/src/test/unit_tests/utils/test_zip_utils.js @@ -9,8 +9,8 @@ const assert = require('assert'); const crypto = require('crypto'); const chance = require('chance')(); -const zip_utils = require('../../util/zip_utils'); -const fs_utils = require('../../util/fs_utils'); +const zip_utils = require('../../../util/zip_utils'); +const fs_utils = require('../../../util/fs_utils'); mocha.describe('zip_utils', function() { @@ -125,7 +125,7 @@ function random_file_name(len) { } /** - * @param {string} range + * @param {string} range * @returns {string} */ function charset_range(range) { diff --git a/src/test/unit_tests/coretest.js b/src/test/utils/coretest/coretest.js similarity index 92% rename from src/test/unit_tests/coretest.js rename to src/test/utils/coretest/coretest.js index d47d32828c..f308e7a4ae 100644 --- a/src/test/unit_tests/coretest.js +++ b/src/test/utils/coretest/coretest.js @@ -18,11 +18,11 @@ process.env.JWT_SECRET = CORETEST; const root_secret = crypto.randomBytes(32).toString('base64'); process.env.ACCOUNTS_CACHE_EXPIRY = '1'; -require('../../util/dotenv').load(); -require('../../util/panic'); -require('../../util/fips'); +require('../../../util/dotenv').load(); +require('../../../util/panic'); +require('../../../util/fips'); -const config = require('../../../config.js'); +const config = require('../../../../config.js'); config.test_mode = true; config.NODES_FREE_SPACE_RESERVE = 10 * 1024 * 1024; config.NSFS_VERSIONING_ENABLED = true; @@ -30,29 +30,29 @@ config.OBJECT_SDK_BUCKET_CACHE_EXPIRY_MS = 1; config.ROOT_KEY_MOUNT = '/tmp/noobaa-server/root_keys'; -const dbg = require('../../util/debug_module')(__filename); +const dbg = require('../../../util/debug_module')(__filename); const dbg_level = (process.env.SUPPRESS_LOGS && -5) || (argv.verbose && 5) || 0; dbg.set_module_level(dbg_level, 'core'); -const P = require('../../util/promise'); -const ObjectIO = require('../../sdk/object_io'); -const endpoint = require('../../endpoint/endpoint'); -const db_client = require('../../util/db_client'); -const server_rpc = require('../../server/server_rpc'); -const auth_server = require('../../server/common_services/auth_server'); -const node_server = require('../../server/node_services/node_server'); -const account_server = require('../../server/system_services/account_server'); -const system_server = require('../../server/system_services/system_server'); -const system_store = require('../../server/system_services/system_store').get_instance(); -const MDStore = require('../../server/object_services/md_store').MDStore; -const pool_server = require('../../server/system_services/pool_server'); -const pool_ctrls = require('../../server/system_services/pool_controllers'); -const { PersistentLogger } = require('../../util/persistent_logger'); +const P = require('../../../util/promise'); +const ObjectIO = require('../../../sdk/object_io'); +const endpoint = require('../../../endpoint/endpoint'); +const db_client = require('../../../util/db_client'); +const server_rpc = require('../../../server/server_rpc'); +const auth_server = require('../../../server/common_services/auth_server'); +const node_server = require('../../../server/node_services/node_server'); +const account_server = require('../../../server/system_services/account_server'); +const system_server = require('../../../server/system_services/system_server'); +const system_store = require('../../../server/system_services/system_store').get_instance(); +const MDStore = require('../../../server/object_services/md_store').MDStore; +const pool_server = require('../../../server/system_services/pool_server'); +const pool_ctrls = require('../../../server/system_services/pool_controllers'); +const { PersistentLogger } = require('../../../util/persistent_logger'); const os = require('os'); -const { TMP_PATH } = require('../system_tests/test_utils'); +const { TMP_PATH } = require('../../system_tests/test_utils'); // Set the pools server pool controller factory to create pools with // backed by in process agents. @@ -97,14 +97,14 @@ function new_rpc_client() { function init_all_collections() { /* eslint-disable global-require */ - require('../../server/notifications/alerts_log_store').instance(); - require('../../server/analytic_services/activity_log_store').ActivityLogStore.instance(); - require('../../server/analytic_services/io_stats_store').IoStatsStore.instance(); - require('../../server/analytic_services/bucket_stats_store').BucketStatsStore.instance(); - require('../../server/analytic_services/history_data_store').HistoryDataStore.instance(); - require('../../server/analytic_services/activity_log_store').ActivityLogStore.instance(); + require('../../../server/notifications/alerts_log_store').instance(); + require('../../../server/analytic_services/activity_log_store').ActivityLogStore.instance(); + require('../../../server/analytic_services/io_stats_store').IoStatsStore.instance(); + require('../../../server/analytic_services/bucket_stats_store').BucketStatsStore.instance(); + require('../../../server/analytic_services/history_data_store').HistoryDataStore.instance(); + require('../../../server/analytic_services/activity_log_store').ActivityLogStore.instance(); // eslint-disable-next-line no-unused-expressions - require('../../server/analytic_services/endpoint_stats_store').EndpointStatsStore.instance; + require('../../../server/analytic_services/endpoint_stats_store').EndpointStatsStore.instance; } function setup(options = {}) { diff --git a/src/test/unit_tests/nc_coretest.js b/src/test/utils/coretest/nc_coretest.js similarity index 97% rename from src/test/unit_tests/nc_coretest.js rename to src/test/utils/coretest/nc_coretest.js index 19179f0222..a2a7137e1d 100644 --- a/src/test/unit_tests/nc_coretest.js +++ b/src/test/utils/coretest/nc_coretest.js @@ -7,11 +7,11 @@ const _ = require('lodash'); const mocha = require('mocha'); const child_process = require('child_process'); const argv = require('minimist')(process.argv); -const SensitiveString = require('../../util/sensitive_string'); -const { exec_manage_cli, TMP_PATH, create_redirect_file, delete_redirect_file } = require('../system_tests/test_utils'); -const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); -const { ConfigFS } = require('../../sdk/config_fs'); -const os_utils = require('../../util/os_utils'); +const SensitiveString = require('../../../util/sensitive_string'); +const { exec_manage_cli, TMP_PATH, create_redirect_file, delete_redirect_file } = require('../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); +const { ConfigFS } = require('../../../sdk/config_fs'); +const os_utils = require('../../../util/os_utils'); // keep me first - this is setting envs that should be set before other modules are required. const NC_CORETEST = 'nc_coretest'; @@ -22,25 +22,25 @@ process.env.DEBUG_MODE = 'true'; // process.env.ACCOUNTS_CACHE_EXPIRY = '1'; In NC we check if the config file was changed as validation process.env.NC_CORETEST = 'true'; -require('../../util/dotenv').load(); -require('../../util/panic'); -require('../../util/fips'); +require('../../../util/dotenv').load(); +require('../../../util/panic'); +require('../../../util/fips'); -const config = require('../../../config.js'); +const config = require('../../../../config.js'); config.test_mode = true; config.NC_MASTER_KEYS_FILE_LOCATION = master_key_location; const new_umask = process.env.NOOBAA_ENDPOINT_UMASK || 0o000; const old_umask = process.umask(new_umask); console.log('test_nsfs_access: replacing old umask: ', old_umask.toString(8), 'with new umask: ', new_umask.toString(8)); -const dbg = require('../../util/debug_module')(__filename); +const dbg = require('../../../util/debug_module')(__filename); const dbg_level = (process.env.SUPPRESS_LOGS && -5) || (argv.verbose && 5) || 0; dbg.set_module_level(dbg_level, 'core'); -const P = require('../../util/promise'); +const P = require('../../../util/promise'); let _setup = false; let _nsfs_process = false; diff --git a/src/test/utils/index/index.js b/src/test/utils/index/index.js new file mode 100644 index 0000000000..851d5c1dea --- /dev/null +++ b/src/test/utils/index/index.js @@ -0,0 +1,110 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; + + +const coretest = require('../coretest/coretest'); +coretest.setup({ incomplete_rpc_coverage: 'show' }); + +// --------------------------------------- +// Tests that does not require hosts pools +// --------------------------------------- + +//JSON SCHEMA +require('../../unit_tests/db/test_schema_keywords'); + +// UTILS +require('../../unit_tests/utils/test_linked_list'); +require('../../unit_tests/utils/test_keys_lock'); +require('../../unit_tests/utils/test_lru'); +require('../../unit_tests/utils/test_lru_cache'); +require('../../unit_tests/utils/test_prefetch'); +require('../../unit_tests/utils/test_promise_utils'); +require('../../unit_tests/utils/test_rpc'); +require('../../unit_tests/utils/test_semaphore'); +require('../../unit_tests/utils/test_delayed_trigger'); +require('../../unit_tests/utils/test_fs_utils'); +require('../../unit_tests/utils/test_signature_utils'); +require('../../unit_tests/utils/test_http_utils'); +require('../../unit_tests/utils/test_v8_optimizations'); +require('../../unit_tests/utils/test_ssl_utils'); +require('../../unit_tests/utils/test_zip_utils'); +require('../../unit_tests/utils/test_wait_queue'); +require('../../unit_tests/utils/test_kmeans'); +require('../../unit_tests/utils/test_sensitive_wrapper'); +// require('./test_debug_module'); +require('../../unit_tests/utils/test_range_stream'); +require('../../unit_tests/utils/test_buffer_pool'); + +// // STORES +require('../../integration_tests/db/test_md_store'); +require('../../integration_tests/db/test_nodes_store'); +require('../../integration_tests/db/test_system_store'); + + +// // CORE +// require('./test_mapper'); +require('../../integration_tests/internal/test_map_client'); +//require('./test_map_reader'); +require('../../integration_tests/internal/test_map_deleter'); +require('../../unit_tests/internal/test_chunk_coder'); +require('../../unit_tests/internal/test_chunk_splitter'); +require('../../unit_tests/internal/test_chunk_config_utils'); +//require('./test_md_aggregator_unit'); +require('../../integration_tests/internal/test_agent_blocks_verifier'); +require('../../integration_tests/api/s3/test_s3_list_objects'); +require('../../unit_tests/native/test_nb_native_b64'); +require('../../unit_tests/internal/test_bucket_chunks_builder'); +require('../../unit_tests/internal/test_mirror_writer'); +require('../../unit_tests/nsfs/test_namespace_fs'); +require('../../unit_tests/api/s3/test_ns_list_objects'); +require('../../unit_tests/nsfs/test_namespace_fs_mpu'); +require('../../unit_tests/native/test_nb_native_fs'); +require('../../unit_tests/api/s3/test_s3select'); +require('../../unit_tests/nsfs/test_nsfs_glacier_backend'); + +// // SERVERS + +// ------------------------------------ +// A test that initialize the pool list +// ------------------------------------ +require('../../integration_tests/internal/test_system_servers'); + +// ------------------------------ +// Tests that require hosts pools +// ------------------------------ + +// SERVERS +require('../../integration_tests/internal/test_host_server'); +require('../../integration_tests/internal/test_node_server'); + +// CORE +require('../../integration_tests/internal/test_map_builder'); // Requires pools +require('../../integration_tests/internal/test_map_reader'); ///////////// +require('../../integration_tests/internal/test_object_io'); +require('../../integration_tests/internal/test_agent_blocks_reclaimer'); +require('../../integration_tests/api/s3/test_s3_ops'); +require('../../integration_tests/api/s3/test_s3_encryption'); +require('../../integration_tests/api/s3/test_s3_bucket_policy'); +// require('./test_node_allocator'); +require('../../unit_tests/internal/test_namespace_cache'); +require('../../integration_tests/api/s3/test_namespace_auth'); +require('../../integration_tests/internal/test_encryption'); +require('../../integration_tests/api/s3/test_bucket_replication'); +require('../../integration_tests/api/sts/test_sts'); +require('../../unit_tests/utils/test_cloud_utils'); +require('../../integration_tests/internal/test_upgrade_scripts.js'); +require('../../integration_tests/internal/test_tiering_ttl_worker'); +// require('./test_tiering_upload'); +//require('./test_s3_worm'); +require('../../integration_tests/api/s3/test_bucket_logging'); +require('../../integration_tests/api/s3/test_notifications'); + +// UPGRADE +// require('./test_postgres_upgrade'); // TODO currently working with mongo -> once changing to postgres - need to uncomment + +// Lifecycle +require('../../integration_tests/api/s3/test_lifecycle'); + +// MD Sequence +require('../../integration_tests/db/test_mdsequence'); + diff --git a/src/test/utils/index/nc_index.js b/src/test/utils/index/nc_index.js new file mode 100644 index 0000000000..45f0a03772 --- /dev/null +++ b/src/test/utils/index/nc_index.js @@ -0,0 +1,35 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; + + +const coretest = require('../coretest/nc_coretest'); +coretest.setup(); + +require('../../unit_tests/nsfs/test_namespace_fs'); +require('../../unit_tests/api/s3/test_ns_list_objects'); +require('../../unit_tests/nsfs/test_namespace_fs_mpu'); +require('../../unit_tests/native/test_nb_native_fs'); +require('../../integration_tests/nc/cli/test_nc_cli'); +require('../../integration_tests/nc/tools/test_nc_health'); +require('../../unit_tests/nsfs/test_nsfs_access'); +require('../../integration_tests/nsfs/test_nsfs_integration'); +require('../../unit_tests/nsfs/test_bucketspace_fs'); +require('../../unit_tests/nsfs/test_nsfs_glacier_backend'); +require('../../integration_tests/api/s3/test_s3_bucket_policy'); +require('../../unit_tests/nsfs/test_nsfs_versioning'); +require('../../integration_tests/nsfs/test_bucketspace_versioning'); +require('../../integration_tests/nc/tools/test_nc_bucket_logging'); +require('../../integration_tests/nc/tools/test_nc_online_upgrade_s3_integrations'); +require('../../integration_tests/api/s3/test_public_access_block'); +require('../../integration_tests/nc/lifecycle/test_nc_lifecycle_expiration'); + +// running with iam port +require('../../integration_tests/api/iam/test_nc_iam_basic_integration.js'); // please notice that we use a different setup +// running with a couple of forks - please notice and add only relevant tests here +require('../../integration_tests/nc/test_nc_with_a_couple_of_forks.js'); // please notice that we use a different setup + +// TODO: uncomment when supported +//require('./test_s3_ops'); +//require('./test_s3_list_objects'); +//require('./test_s3_encryption'); +// require('./test_s3select'); diff --git a/src/test/utils/index/sudo_index.js b/src/test/utils/index/sudo_index.js new file mode 100644 index 0000000000..8d18a4ae03 --- /dev/null +++ b/src/test/utils/index/sudo_index.js @@ -0,0 +1,25 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; + + +const coretest = require('../coretest/coretest'); +const test_utils = require('../../system_tests/test_utils'); +coretest.setup({ incomplete_rpc_coverage: 'show' }); + +// --------------------------------------- +// Tests that does not require hosts pools +// --------------------------------------- + +if (test_utils.invalid_nsfs_root_permissions()) { + throw new Error(`Insufficient uid gid: pgid: ${process.getgid()}, puid: ${process.getuid()}`); +} + +// // CORE +require('../../unit_tests/nsfs/test_nsfs_access'); +require('../../integration_tests/nsfs/test_nsfs_integration'); +require('../../integration_tests/nsfs/test_bucketspace_versioning'); +require('../../unit_tests/nsfs/test_bucketspace_fs'); +require('../../unit_tests/nsfs/test_nsfs_versioning'); +require('../../integration_tests/nc/cli/test_nc_cli'); +require('../../integration_tests/nc/tools/test_nc_health'); + diff --git a/src/test/system_tests/nc_test_utils.js b/src/test/utils/nc_test_utils.js similarity index 96% rename from src/test/system_tests/nc_test_utils.js rename to src/test/utils/nc_test_utils.js index 78a4572bb6..599af91179 100644 --- a/src/test/system_tests/nc_test_utils.js +++ b/src/test/utils/nc_test_utils.js @@ -1,7 +1,7 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const test_utils = require('./test_utils'); +const test_utils = require('../system_tests/test_utils'); const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); /**