Merge pull request #8729 from jackyalbo/jacky-cors-fixes

Adding upgrade script to bucket CORS - new bucket default will be the same as in 5.18

Author: jackyalbo

config.js

@@ -158,15 +158,16 @@ config.ENDPOINT_HTTP_MAX_REQUESTS_PER_SOCKET = 0; // 0 = no limit
 // note that browsers do not really allow origin=* with credentials,
 // but we just allow both from our side for simplicity.
 config.S3_CORS_ENABLED = true;
-config.S3_CORS_ALLOW_ORIGIN = '*';
+config.S3_CORS_DEFAULTS_ENABLED = true;
+config.S3_CORS_ALLOW_ORIGIN = ['*'];
 config.S3_CORS_ALLOW_CREDENTIAL = 'true';
 config.S3_CORS_ALLOW_METHODS = [
     'GET',
     'POST',
     'PUT',
     'DELETE',
     'OPTIONS'
-].join(',');
+];
 config.S3_CORS_ALLOW_HEADERS = [
     'Content-Type',
     'Content-MD5',
@@ -177,11 +178,11 @@ config.S3_CORS_ALLOW_HEADERS = [
     'X-Amz-Content-Sha256',
     'amz-sdk-invocation-id',
     'amz-sdk-request',
-].join(',');
+];
 config.S3_CORS_EXPOSE_HEADERS = [
     'ETag',
     'X-Amz-Version-Id'
-].join(',');
+];
 config.STS_CORS_EXPOSE_HEADERS = 'ETag';
 
 config.DENY_UPLOAD_TO_STORAGE_CLASS_STANDARD = false;

src/sdk/bucketspace_fs.js

@@ -350,7 +350,14 @@ class BucketSpaceFS extends BucketSpaceSimpleFS {
             force_md5_etag: force_md5_etag,
             path: bucket_storage_path,
             should_create_underlying_storage: create_uls,
-            fs_backend: account.nsfs_account_config.fs_backend
+            fs_backend: account.nsfs_account_config.fs_backend,
+            cors_configuration_rules: config.S3_CORS_DEFAULTS_ENABLED ? [{
+                allowed_origins: config.S3_CORS_ALLOW_ORIGIN,
+                allowed_methods: config.S3_CORS_ALLOW_METHODS,
+                allowed_headers: config.S3_CORS_ALLOW_HEADERS,
+                expose_headers: config.S3_CORS_EXPOSE_HEADERS,
+
+            }] : undefined,
         };
     }
 

src/server/system_services/bucket_server.js

@@ -74,6 +74,12 @@ function new_bucket_defaults(name, system_id, tiering_policy_id, owner_account_i
         object_lock_configuration: config.WORM_ENABLED ? {
             object_lock_enabled: lock_enabled ? 'Enabled' : 'Disabled',
         } : undefined,
+        cors_configuration_rules: config.S3_CORS_DEFAULTS_ENABLED ? [{
+            allowed_origins: config.S3_CORS_ALLOW_ORIGIN,
+            allowed_methods: config.S3_CORS_ALLOW_METHODS,
+            allowed_headers: config.S3_CORS_ALLOW_HEADERS,
+            expose_headers: config.S3_CORS_EXPOSE_HEADERS,
+        }] : undefined,
     };
 }
 

src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh

@@ -25,6 +25,7 @@ export CONFIG_DIR=/etc/noobaa.conf.d/
 export FS_ROOT_1=/tmp/nsfs_root1/
 export FS_ROOT_2=/tmp/nsfs_root2/
 export CONFIG_JS_allow_anonymous_access_in_test=true # Needed for allowing anon access for tests using ACL='public-read-write'
+export CONFIG_JS_S3_CORS_DEFAULTS_ENABLED=false # Needed for disabling cors defaults for ceph cors test
 
 # ====================================================================================
 

src/test/system_tests/ceph_s3_tests/run_ceph_test_on_test_container.sh

@@ -38,6 +38,7 @@ export CEPH_TEST_LOGS_DIR=/logs/ceph-test-logs
 
 export CONFIG_JS_OBJECT_SDK_BUCKET_CACHE_EXPIRY_MS=0 # Needed for disabling cache for ceph cors test and maybe some more
 export CONFIG_JS_allow_anonymous_access_in_test=true # Needed for allowing anon access for tests using ACL='public-read-write'
+export CONFIG_JS_S3_CORS_DEFAULTS_ENABLED=false # Needed for disabling cors defaults for ceph cors test
 
 # ====================================================================================
 

src/test/unit_tests/test_upgrade_scripts.js

@@ -10,20 +10,38 @@ const { NodeHttpHandler } = require("@smithy/node-http-handler");
 const http = require('http');
 const system_store = require('../../server/system_services/system_store').get_instance();
 const upgrade_bucket_policy = require('../../upgrade/upgrade_scripts/5.15.6/upgrade_bucket_policy');
+const upgrade_bucket_cors = require('../../upgrade/upgrade_scripts/5.19.0/upgrade_bucket_cors');
 const dbg = require('../../util/debug_module')(__filename);
 const assert = require('assert');
 const mocha = require('mocha');
 const config = require('../../../config');
 
 const BKT = 'test-bucket';
+/** @type {S3} */
 let s3;
 
 async function _clean_all_bucket_policies() {
-    for (const bucket of system_store.data.buckets) {
-        if (bucket.s3_policy) {
-            await s3.deleteBucketPolicy({ Bucket: bucket.name.unwrap() });
+    const buckets = system_store.data.buckets.map(bucket => ({
+        _id: bucket._id,
+        $unset: { s3_policy: 1 }
+    }));
+    await system_store.make_changes({
+        update: {
+            buckets
         }
-    }
+    });
+}
+
+async function _clean_all_bucket_cors() {
+    const buckets = system_store.data.buckets.map(bucket => ({
+        _id: bucket._id,
+        $unset: { cors_configuration_rules: 1 }
+    }));
+    await system_store.make_changes({
+        update: {
+            buckets
+        }
+    });
 }
 
 mocha.describe('test upgrade scripts', async function() {
@@ -93,6 +111,24 @@ mocha.describe('test upgrade scripts', async function() {
         assert.strictEqual(new_policy.Statement[0].Resource[0], old_policy.statement[0].resource[0]);
     });
 
+    mocha.it('test upgrade bucket cors to version 5.19.0', async function() {
+
+        // clean all leftover bucket CORS configurations
+        await _clean_all_bucket_cors();
+
+        await upgrade_bucket_cors.run({ dbg, system_store });
+        const cors = await s3.getBucketCors({
+            Bucket: BKT,
+        });
+
+        dbg.log0('cors:', cors);
+
+        assert.deepEqual(cors.CORSRules[0].AllowedHeaders, config.S3_CORS_ALLOW_HEADERS);
+        assert.deepEqual(cors.CORSRules[0].AllowedMethods, config.S3_CORS_ALLOW_METHODS);
+        assert.deepEqual(cors.CORSRules[0].AllowedOrigins, config.S3_CORS_ALLOW_ORIGIN);
+        assert.deepEqual(cors.CORSRules[0].ExposeHeaders, config.S3_CORS_EXPOSE_HEADERS);
+    });
+
     mocha.after(async function() {
         await s3.deleteBucket({ Bucket: BKT });
     });

src/upgrade/upgrade_scripts/5.19.0/upgrade_bucket_cors.js

@@ -0,0 +1,39 @@
+/* Copyright (C) 2023 NooBaa */
+"use strict";
+
+const util = require('util');
+const config = require('../../../../config.js');
+
+async function run({ dbg, system_store }) {
+
+    try {
+        dbg.log0('Starting bucket CORS upgrade...');
+        const buckets = system_store.data.buckets
+            .map(bucket => ({
+                _id: bucket._id,
+                cors_configuration_rules: [{
+                    allowed_origins: config.S3_CORS_ALLOW_ORIGIN,
+                    allowed_methods: config.S3_CORS_ALLOW_METHODS,
+                    allowed_headers: config.S3_CORS_ALLOW_HEADERS,
+                    expose_headers: config.S3_CORS_EXPOSE_HEADERS,
+                }],
+            }));
+
+        if (buckets.length > 0) {
+            dbg.log0(`Adding default bucket CORS configuration to: ${buckets.map(bucket => util.inspect(bucket)).join(', ')}`);
+            await system_store.make_changes({ update: { buckets } });
+        } else {
+            dbg.log0('Upgrading buckets CORS configuration: no upgrade needed...');
+        }
+
+    } catch (err) {
+        dbg.error('Got error while upgrading buckets CORS configuration:', err);
+        throw err;
+    }
+}
+
+
+module.exports = {
+    run,
+    description: 'Update default CORS configuration for all buckets',
+};

src/util/http_utils.js

@@ -692,10 +692,10 @@ function set_cors_headers_s3(req, res, cors_rules) {
 function set_cors_headers_sts(req, res) {
     if (config.S3_CORS_ENABLED) {
         set_cors_headers(req, res, {
-            allow_origin: config.S3_CORS_ALLOW_ORIGIN,
+            allow_origin: config.S3_CORS_ALLOW_ORIGIN[0],
             allow_credentials: config.S3_CORS_ALLOW_CREDENTIAL,
-            allow_methods: config.S3_CORS_ALLOW_METHODS,
-            allow_headers: config.S3_CORS_ALLOW_HEADERS,
+            allow_methods: config.S3_CORS_ALLOW_METHODS.join(','),
+            allow_headers: config.S3_CORS_ALLOW_HEADERS.join(','),
             expose_headers: config.STS_CORS_EXPOSE_HEADERS,
         });
     }
@@ -756,7 +756,7 @@ function http_get(uri, options) {
  * @param {number} https_port
  * @param {('S3'|'IAM'|'STS'|'METRICS')} server_type 
  * @param {Object} request_handler 
-*/
+ */
 async function start_https_server(https_port, server_type, request_handler, nsfs_config_root) {
     const ssl_cert_info = await ssl_utils.get_ssl_cert_info(server_type, nsfs_config_root);
     const https_server = await ssl_utils.create_https_server(ssl_cert_info, true, request_handler);
@@ -775,7 +775,7 @@ async function start_https_server(https_port, server_type, request_handler, nsfs
  * @param {number} http_port
  * @param {('S3'|'IAM'|'STS'|'METRICS')} server_type 
  * @param {Object} request_handler 
-*/
+ */
 async function start_http_server(http_port, server_type, request_handler) {
     const http_server = http.createServer(request_handler);
     if (http_port > 0) {
@@ -790,7 +790,7 @@ async function start_http_server(http_port, server_type, request_handler) {
  * @param {number} port
  * @param {http.Server} server
  * @param {('S3'|'IAM'|'STS'|'METRICS')} server_type 
-*/
+ */
 function listen_port(port, server, server_type) {
     return new Promise((resolve, reject) => {
         if (server_type !== 'METRICS') {
@@ -810,7 +810,7 @@ function listen_port(port, server, server_type) {
 /**
  * Setup endpoint socket and server, Setup is not used for non-endpoint servers.
  * @param {http.Server} server
-*/
+ */
 function setup_endpoint_server(server) {
     // Handle 'Expect' header different than 100-continue to conform with AWS.
     // Consider any expect value as if the client is expecting 100-continue.