Merge pull request #8812 from romayalon/romy-response-headers-additions

S3 | GET/HEAD Object | Add Response Headers Support

Author: romayalon

src/endpoint/s3/ops/s3_get_object.js

@@ -48,7 +48,7 @@ async function get_object(req, res) {
             throw new S3Error(S3Error.InvalidObjectState);
         }
     }
-
+    http_utils.set_response_headers_from_request(req, res);
     const obj_size = object_md.size;
     const params = {
         object_md,

src/endpoint/s3/ops/s3_head_object.js

@@ -28,6 +28,7 @@ async function head_object(req, res) {
 
     s3_utils.set_response_object_md(res, object_md);
     s3_utils.set_encryption_response_headers(req, res, object_md.encryption);
+    http_utils.set_response_headers_from_request(req, res);
 }
 
 module.exports = {

src/test/unit_tests/test_nsfs_integration.js

@@ -60,6 +60,14 @@ const tmp_fs_root = path.join(TMP_PATH, 'test_bucket_namespace_fs');
 // on Containerized - new_buckets_path is the directory
 const new_bucket_path_param = get_new_buckets_path_by_test_env(tmp_fs_root, '/');
 
+// response headers
+const response_content_disposition = 'attachment';
+const response_content_language = 'hebrew';
+const response_content_type = 'application/json';
+const response_cache_control = 'no-cache';
+const response_expires = new Date();
+response_expires.setMilliseconds(0);
+
 // currently will pass only when running locally
 mocha.describe('bucket operations - namespace_fs', function() {
     const nsr = 'nsr';
@@ -2242,6 +2250,115 @@ mocha.describe('Presigned URL tests', function() {
         const expected_err = new S3Error(S3Error.AuthorizationQueryParametersError);
         await assert_throws_async(fetchData(invalid_expiry_presigned_url), expected_err.message);
     });
+
+    it('get-object - fetch valid presigned URL - 604800 seconds - epoch expiry - should return object data + return response headers', async () => {
+        const response_queries = {
+            ResponseContentDisposition: response_content_disposition,
+            ResponseContentLanguage: response_content_language,
+            ResponseContentType: response_content_type,
+            ResponseCacheControl: response_cache_control,
+            ResponseExpires: response_expires
+        };
+        const presigned_url_params_with_response_headers = { ...presigned_url_params, response_queries };
+        const url_with_response_headers = cloud_utils.get_signed_url(presigned_url_params_with_response_headers, 604800);
+        const headers = await fetchHeaders(url_with_response_headers);
+        assert.equal(headers.get('content-disposition'), response_content_disposition);
+        assert.equal(headers.get('content-language'), response_content_language);
+        assert.equal(headers.get('content-type'), response_content_type);
+        assert.equal(headers.get('cache-control'), response_cache_control);
+        assert.deepStrictEqual(headers.get('expires'), response_expires.toUTCString());
+    });
+
+    it('head-object - fetch valid presigned URL - 604800 seconds - epoch expiry - should return object data + return response headers', async () => {
+        const response_queries = {
+            ResponseContentDisposition: response_content_disposition,
+            ResponseContentLanguage: response_content_language,
+            ResponseContentType: response_content_type,
+            ResponseCacheControl: response_cache_control,
+            ResponseExpires: response_expires
+        };
+        const presigned_url_params_with_response_headers = { ...presigned_url_params, response_queries };
+        const url_with_response_headers = cloud_utils.get_signed_url(presigned_url_params_with_response_headers, 604800, 'headObject');
+        const headers = await fetchHeaders(url_with_response_headers, { method: 'HEAD' });
+        assert.equal(headers.get('content-disposition'), response_content_disposition);
+        assert.equal(headers.get('content-language'), response_content_language);
+        assert.equal(headers.get('content-type'), response_content_type);
+        assert.equal(headers.get('cache-control'), response_cache_control);
+        assert.deepStrictEqual(headers.get('expires'), response_expires.toUTCString());
+    });
+});
+
+mocha.describe('response headers test - regular request', function() {
+    this.timeout(50000); // eslint-disable-line no-invalid-this
+    const nsr = 'response_headers_nsr';
+    const account_name = 'response_header_account';
+    const fs_path = path.join(TMP_PATH, 'response_header_tests/');
+    const response_header_bucket = 'response-headerbucket';
+    const response_header_object = 'response-header-object.txt';
+    const response_header_body = 'response_header_body';
+    let s3_client;
+    let access_key;
+    let secret_key;
+    CORETEST_ENDPOINT = coretest.get_http_address();
+
+    mocha.before(async function() {
+        await fs_utils.create_fresh_path(fs_path);
+        await rpc_client.pool.create_namespace_resource({ name: nsr, nsfs_config: { fs_root_path: fs_path } });
+        const new_buckets_path = is_nc_coretest ? fs_path : '/';
+        const nsfs_account_config = {
+            uid: process.getuid(), gid: process.getgid(), new_buckets_path, nsfs_only: true
+        };
+        const account_params = { ...new_account_params, email: `${account_name}@noobaa.io`, name: account_name, default_resource: nsr, nsfs_account_config };
+        const res = await rpc_client.account.create_account(account_params);
+        access_key = res.access_keys[0].access_key;
+        secret_key = res.access_keys[0].secret_key;
+        s3_client = generate_s3_client(access_key.unwrap(), secret_key.unwrap(), CORETEST_ENDPOINT);
+        await s3_client.createBucket({ Bucket: response_header_bucket });
+        await s3_client.putObject({ Bucket: response_header_bucket, Key: response_header_object, Body: response_header_body });
+    });
+
+    mocha.after(async function() {
+        if (!is_nc_coretest) return;
+        await s3_client.deleteObject({ Bucket: response_header_bucket, Key: response_header_object });
+        await s3_client.deleteBucket({ Bucket: response_header_bucket });
+        await rpc_client.account.delete_account({ email: `${account_name}@noobaa.io` });
+        await fs_utils.folder_delete(fs_path);
+    });
+
+    it('get-object - response headers', async () => {
+        const res = await s3_client.getObject({
+            Bucket: response_header_bucket,
+            Key: response_header_object,
+            ResponseContentDisposition: response_content_disposition,
+            ResponseContentLanguage: response_content_language,
+            ResponseContentType: response_content_type,
+            ResponseCacheControl: response_cache_control,
+            ResponseExpires: response_expires,
+        });
+        assert.equal(res.ContentDisposition, response_content_disposition);
+        assert.equal(res.ContentLanguage, response_content_language);
+        assert.equal(res.ContentType, response_content_type);
+        assert.equal(res.CacheControl, response_cache_control);
+        assert.deepStrictEqual(res.Expires, response_expires);
+    });
+
+    it('head-object - response headers', async () => {
+        const res = await s3_client.headObject({
+            Bucket: response_header_bucket,
+            Key: response_header_object,
+            ResponseContentDisposition: response_content_disposition,
+            ResponseContentLanguage: response_content_language,
+            ResponseContentType: response_content_type,
+            ResponseCacheControl: response_cache_control,
+            ResponseExpires: response_expires,
+        });
+        assert.equal(res.ContentDisposition, response_content_disposition);
+        assert.equal(res.ContentLanguage, response_content_language);
+        assert.equal(res.ContentType, response_content_type);
+        assert.equal(res.CacheControl, response_cache_control);
+        assert.deepStrictEqual(res.Expires, response_expires);
+    });
+
 });
 
 async function fetchData(presigned_url) {
@@ -2257,3 +2374,17 @@ async function fetchData(presigned_url) {
     data = await response.text();
     return data.trim();
 }
+
+async function fetchHeaders(presigned_url, options) {
+    const response = await fetch(presigned_url, { ...options, agent: new http.Agent({ keepAlive: false }) });
+    let data;
+    if (!response.ok) {
+        data = (await response.text()).trim();
+        const err_json = (await http_utils.parse_xml_to_js(data)).Error;
+        const err = new Error(err_json.Message);
+        err.code = err_json.Code;
+        throw err;
+    }
+    return response.headers;
+}
+

src/util/cloud_utils.js

@@ -58,7 +58,8 @@ async function generate_aws_sts_creds(params, roleSessionName) {
     );
 }
 
-function get_signed_url(params, expiry = 604800) {
+function get_signed_url(params, expiry = 604800, custom_operation = 'getObject') {
+    const op = custom_operation;
     const s3 = new AWS.S3({
         endpoint: params.endpoint,
         credentials: {
@@ -76,12 +77,14 @@ function get_signed_url(params, expiry = 604800) {
             agent: http_utils.get_unsecured_agent(params.endpoint)
         }
     });
+    const response_queries = params.response_queries || {};
     return s3.getSignedUrl(
-        'getObject', {
+        op, {
             Bucket: params.bucket.unwrap(),
             Key: params.key,
             VersionId: params.version_id,
-            Expires: expiry
+            Expires: expiry,
+            ...response_queries
         }
     );
 }

src/util/http_utils.js

@@ -3,6 +3,7 @@
 /* eslint-disable no-control-regex */
 
 const _ = require('lodash');
+const util = require('util');
 const ip_module = require('ip');
 const net = require('net');
 const url = require('url');
@@ -908,6 +909,21 @@ function handle_server_error(err) {
     process.exit(1);
 }
 
+/**
+ * set_response_headers_from_request sets the response headers based on the request headers
+ * gap - response-content-encoding needs to be added with a more complex logic
+ * @param {http.IncomingMessage} req 
+ * @param {http.ServerResponse} res 
+ */
+function set_response_headers_from_request(req, res) {
+    dbg.log2(`set_response_headers_from_request req.query ${util.inspect(req.query)}`);
+    if (req.query['response-cache-control']) res.setHeader('Cache-Control', req.query['response-cache-control']);
+    if (req.query['response-content-disposition']) res.setHeader('Content-Disposition', req.query['response-content-disposition']);
+    if (req.query['response-content-language']) res.setHeader('Content-Language', req.query['response-content-language']);
+    if (req.query['response-content-type']) res.setHeader('Content-Type', req.query['response-content-type']);
+    if (req.query['response-expires']) res.setHeader('Expires', req.query['response-expires']);
+}
+
 exports.parse_url_query = parse_url_query;
 exports.parse_client_ip = parse_client_ip;
 exports.get_md_conditions = get_md_conditions;
@@ -944,3 +960,4 @@ exports.CONTENT_TYPE_APP_OCTET_STREAM = CONTENT_TYPE_APP_OCTET_STREAM;
 exports.CONTENT_TYPE_APP_JSON = CONTENT_TYPE_APP_JSON;
 exports.CONTENT_TYPE_APP_XML = CONTENT_TYPE_APP_XML;
 exports.CONTENT_TYPE_APP_FORM_URLENCODED = CONTENT_TYPE_APP_FORM_URLENCODED;
+exports.set_response_headers_from_request = set_response_headers_from_request;