Added unit tests for nc

Signed-off-by: Aayush <aayush@li-2392f9cc-287a-11b2-a85c-9fcc04b05da6.ibm.com>

Author: Aayush

src/test/system_tests/test_utils.js

@@ -790,6 +790,80 @@ const run_or_skip_test = cond => {
     } else return it.skip;
 };
 
+/**
+ * update_file_mtime updates the mtime of the target path
+ * Warnings:
+ *  - This operation would change the mtime of the file to 5 days ago - which means that it changes the etag / obj_id of the object
+ *  - Please do not use on versioned objects (version_id will not be changed, but the mtime will be changed) - might cause issues.
+ * @param {String} target_path
+ * @returns {Promise<Void>}
+ */
+async function update_file_mtime(target_path) {
+    const update_file_mtime_cmp = os_utils.IS_MAC ? `touch -t $(date -v -5d +"%Y%m%d%H%M.%S") ${target_path}` : `touch -d "5 days ago" ${target_path}`;
+    await os_utils.exec(update_file_mtime_cmp, { return_stdout: true });
+}
+
+/////////////////////////////////
+//////  LIFECYCLE UTILS   ///////
+/////////////////////////////////
+
+/**
+ * generate_lifecycle_rule generate an S3 lifecycle rule with optional filters and expiration currently (can be extend to support more lifecycle rule params)
+ *
+ * @param {number} expiration_days
+ * @param {string} id
+ * @param {string} [prefix]
+ * @param {Array<Object>} [tags]
+ * @param {number} [size_gt]
+ * @param {number} [size_lt]
+ * @returns {Object}
+ */
+function generate_lifecycle_rule(expiration_days, id, prefix, tags, size_gt, size_lt) {
+    const filters = {};
+    if (prefix) filters.Prefix = prefix;
+    if (Array.isArray(tags) && tags.length) filters.Tags = tags;
+    if (size_gt !== undefined) filters.ObjectSizeGreaterThan = size_gt;
+    if (size_lt !== undefined) filters.ObjectSizeLessThan = size_lt;
+
+    const filter = Object.keys(filters).length > 1 ? { And: filters } : filters;
+
+    return {
+        ID: id,
+        Status: 'Enabled',
+        Filter: filter,
+        Expiration: { Days: expiration_days },
+    };
+}
+
+/**
+ * validate_expiration_header validates the `x-amz-expiration` header against the object creation time, expected rule ID and expiration days
+ *
+ * The header is expected to have the format:
+ *   expiry-date="YYYY-MM-DDTHH:MM:SS.SSSZ", rule-id="RULE_ID"
+ *
+ * @param {string} expiration_header - expiration header value
+ * @param {string|Date} start_time - start/create time (string or Date) of the object
+ * @param {string} expected_rule_id - expected rule ID to match in the header
+ * @param {number} delta_days - expected number of days between start_time and expiry-date
+ * @returns {boolean} true if the header is valid and matches the expected_rule_id and delta_days otherwise false
+ */
+function validate_expiration_header(expiration_header, start_time, expected_rule_id, delta_days) {
+    const match = expiration_header.match(/expiry-date="(.+)", rule-id="(.+)"/);
+    if (!match) return false;
+
+    const [, expiry_str, rule_id] = match;
+    const expiration = new Date(expiry_str);
+    const start = new Date(start_time);
+    start.setUTCHours(0, 0, 0, 0); // adjusting to midnight UTC otherwise the tests will fail - fix for ceph-s3 tests
+
+    const days_diff = Math.floor((expiration.getTime() - start.getTime()) / (24 * 60 * 60 * 1000));
+
+    return days_diff === delta_days && rule_id === expected_rule_id;
+}
+
+exports.update_file_mtime = update_file_mtime;
+exports.generate_lifecycle_rule = generate_lifecycle_rule;
+exports.validate_expiration_header = validate_expiration_header;
 exports.run_or_skip_test = run_or_skip_test;
 exports.blocks_exist_on_cloud = blocks_exist_on_cloud;
 exports.create_hosts_pool = create_hosts_pool;

src/test/unit_tests/jest_tests/test_nc_lifecycle_posix_integration.test.js

@@ -11,11 +11,10 @@ const fs = require('fs');
 const config = require('../../../../config');
 const fs_utils = require('../../../util/fs_utils');
 const { ConfigFS } = require('../../../sdk/config_fs');
-const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT, exec_manage_cli, create_system_json } = require('../../system_tests/test_utils');
+const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT, exec_manage_cli, create_system_json, update_file_mtime } = require('../../system_tests/test_utils');
 const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants');
 const NamespaceFS = require('../../../sdk/namespace_fs');
 const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector');
-const os_utils = require('../../../util/os_utils');
 const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses');
 const { ManageCLIError } = require('../../../manage_nsfs/manage_nsfs_cli_errors');
 const buffer_utils = require('../../../util/buffer_utils');
@@ -2187,19 +2186,6 @@ async function create_object(object_sdk, bucket, key, size, is_old, tagging) {
     return res;
 }
 
-/**
- * update_file_mtime updates the mtime of the target path
- * Warnings:
- *  - This operation would change the mtime of the file to 5 days ago - which means that it changes the etag / obj_id of the object
- *  - Please do not use on versioned objects (version_id will not be changed, but the mtime will be changed) - might cause issues.
- * @param {String} target_path
- * @returns {Promise<Void>}
- */
-async function update_file_mtime(target_path) {
-    const update_file_mtime_cmp = os_utils.IS_MAC ? `touch -t $(date -v -5d +"%Y%m%d%H%M.%S") ${target_path}` : `touch -d "5 days ago" ${target_path}`;
-    await os_utils.exec(update_file_mtime_cmp, { return_stdout: true });
-}
-
 /**
  * updates the number of noncurrent days xattr of target path to be 5 days older. use only on noncurrent objects.
  * is use this function on latest object the xattr will be changed when the object turns noncurrent

src/test/unit_tests/nc_index.js

@@ -21,6 +21,7 @@ require('./test_bucketspace_versioning');
 require('./test_nc_bucket_logging');
 require('./test_nc_online_upgrade_s3_integrations');
 require('./test_public_access_block');
+require('./test_nc_bucket_lifecycle');
 
 // running with iam port
 require('./test_nc_iam_basic_integration.js'); // please notice that we use a different setup

src/test/unit_tests/test_lifecycle.js

@@ -22,6 +22,7 @@ const MDStore = require('../../server/object_services/md_store').MDStore;
 const coretest = require('./coretest');
 const lifecycle = require('../../server/bg_services/lifecycle');
 const http_utils = require('../../util/http_utils');
+const test_utils = require('../../../src/test/system_tests/test_utils');
 const commonTests = require('../lifecycle/common');
 const seed = crypto.randomBytes(16);
 const generator = crypto.createCipheriv('aes-128-gcm', seed, Buffer.alloc(12));
@@ -824,46 +825,14 @@ mocha.describe('lifecycle', () => {
             res = await s3.headObject({ Bucket: bucket, Key: key });
             assert.ok(res.Expiration, 'expiration header missing in headObject response');
 
-            const valid = validate_expiration_header(res.Expiration, start_time, expected_id, expected_days);
+            const valid = test_utils.validate_expiration_header(res.Expiration, start_time, expected_id, expected_days);
             assert.ok(valid, `expected rule ${expected_id} to match`);
         };
 
-        function generate_rule(id, prefix, tags, size_gt, size_lt, expiration_days) {
-            const filters = {};
-            if (prefix) filters.Prefix = prefix;
-            if (Array.isArray(tags) && tags.length) filters.Tags = tags;
-            if (size_gt !== undefined) filters.ObjectSizeGreaterThan = size_gt;
-            if (size_lt !== undefined) filters.ObjectSizeLessThan = size_lt;
-
-            const filter = Object.keys(filters).length > 1 ? { And: filters } : filters;
-
-            return {
-                ID: id,
-                Status: 'Enabled',
-                Filter: filter,
-                Expiration: { Days: expiration_days },
-            };
-        }
-
-        function validate_expiration_header(expiration_header, start_time, expected_rule_id, delta_days) {
-            const match = expiration_header.match(/expiry-date="(.+)", rule-id="(.+)"/);
-            if (!match) return false;
-            console.log("match: ", match);
-
-            const [, expiry_str, rule_id] = match;
-            const expiration = new Date(expiry_str);
-            const start = new Date(start_time);
-            start.setUTCHours(0, 0, 0, 0); // adjusting to midnight UTC otherwise the tests will fail - fix for ceph-s3 tests
-
-            const days_diff = Math.floor((expiration.getTime() - start.getTime()) / (24 * 60 * 60 * 1000));
-
-            return days_diff === delta_days && rule_id === expected_rule_id;
-        }
-
         mocha.it('should select rule with longest prefix', async () => {
             const rules = [
-                generate_rule('short-prefix', 'test1/', [], undefined, undefined, 10),
-                generate_rule('long-prefix', 'test1/logs/', [], undefined, undefined, 17),
+                test_utils.generate_lifecycle_rule(10, 'short-prefix', 'test1/', [], undefined, undefined),
+                test_utils.generate_lifecycle_rule(17, 'long-prefix', 'test1/logs/', [], undefined, undefined),
             ];
             await run_expiration_test({
                 rules,
@@ -875,11 +844,11 @@ mocha.describe('lifecycle', () => {
 
         mocha.it('should select rule with more tags when prefix is same', async () => {
             const rules = [
-                generate_rule('one-tag', 'test2/', [{ Key: 'env', Value: 'prod' }], undefined, undefined, 5),
-                generate_rule('two-tags', 'test2/', [
+                test_utils.generate_lifecycle_rule(5, 'one-tag', 'test2/', [{ Key: 'env', Value: 'prod' }], undefined, undefined),
+                test_utils.generate_lifecycle_rule(9, 'two-tags', 'test2/', [
                     { Key: 'env', Value: 'prod' },
                     { Key: 'team', Value: 'backend' }
-                ], undefined, undefined, 9),
+                ], undefined, undefined),
             ];
             await run_expiration_test({
                 rules,
@@ -892,8 +861,8 @@ mocha.describe('lifecycle', () => {
 
         mocha.it('should select rule with narrower size span when prefix and tags are matching', async () => {
             const rules = [
-                generate_rule('wide-range', 'test3/', [], 100, 10000, 4),
-                generate_rule('narrow-range', 'test3/', [], 1000, 5000, 6),
+                test_utils.generate_lifecycle_rule(4, 'wide-range', 'test3/', [], 100, 10000),
+                test_utils.generate_lifecycle_rule(6, 'narrow-range', 'test3/', [], 1000, 5000),
             ];
             await run_expiration_test({
                 rules,
@@ -906,8 +875,8 @@ mocha.describe('lifecycle', () => {
 
         mocha.it('should fallback to first matching rule if all filters are equal', async () => {
             const rules = [
-                generate_rule('rule-a', 'test4/', [], 0, 10000, 7),
-                generate_rule('rule-b', 'test4/', [], 0, 10000, 11),
+                test_utils.generate_lifecycle_rule(7, 'rule-a', 'test4/', [], 0, 10000),
+                test_utils.generate_lifecycle_rule(11, 'rule-b', 'test4/', [], 0, 10000),
             ];
             await run_expiration_test({
                 rules,

src/test/unit_tests/test_nc_bucket_lifecycle.js

@@ -0,0 +1,150 @@
+/* Copyright (C) 2024 NooBaa */
+'use strict';
+
+const path = require('path');
+const mocha = require('mocha');
+const assert = require('assert');
+const fs_utils = require('../../util/fs_utils');
+const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants');
+const { TMP_PATH, set_path_permissions_and_owner, invalid_nsfs_root_permissions, generate_s3_client, get_coretest_path,
+    generate_lifecycle_rule, validate_expiration_header, update_file_mtime, exec_manage_cli } = require('../system_tests/test_utils');
+
+const coretest_path = get_coretest_path();
+const coretest = require(coretest_path);
+const { rpc_client, EMAIL, get_admin_mock_account_details } = coretest;
+coretest.setup({});
+
+let s3_admin;
+
+const tmp_fs_root = path.join(TMP_PATH, 'test_nc_bucket_lifecycle/');
+
+/**
+ * is_nc_coretest returns true when the test runs on NC env
+ */
+const is_nc_coretest = process.env.NC_CORETEST === 'true';
+
+mocha.describe('nc lifecycle - check expiration header', async function() {
+    const bucket_path = path.join(tmp_fs_root, 'test-bucket/');
+    const bucket_name = 'test-bucket';
+
+    mocha.before(async function() {
+        this.timeout(0); // eslint-disable-line no-invalid-this
+        if (invalid_nsfs_root_permissions()) this.skip(); // eslint-disable-line no-invalid-this
+        // create paths
+        await fs_utils.create_fresh_path(tmp_fs_root, 0o777);
+        await fs_utils.create_fresh_path(bucket_path, 0o770);
+        await fs_utils.file_must_exist(bucket_path);
+
+        // set permissions
+        if (is_nc_coretest) {
+            const { uid, gid } = get_admin_mock_account_details();
+            await set_path_permissions_and_owner(bucket_path, { uid, gid }, 0o700);
+        }
+
+        // create s3_admin client
+        const admin = (await rpc_client.account.read_account({ email: EMAIL, }));
+        const admin_keys = admin.access_keys;
+        s3_admin = generate_s3_client(admin_keys[0].access_key.unwrap(),
+                        admin_keys[0].secret_key.unwrap(),
+                        coretest.get_http_address());
+
+        // create test bucket
+        const cli_bucket_options = {
+            name: bucket_name,
+            owner: admin.name,
+            path: bucket_path,
+        };
+        await exec_manage_cli(TYPES.BUCKET, ACTIONS.ADD, cli_bucket_options);
+    });
+
+    mocha.after(async function() {
+        this.timeout(0); // eslint-disable-line no-invalid-this
+        fs_utils.folder_delete(tmp_fs_root);
+    });
+
+    const run_expiration_test = async ({ rules, expected_id, expected_days, key, tagging = undefined, size = 1000}) => {
+        const putLifecycleParams = {
+            Bucket: bucket_name,
+            LifecycleConfiguration: { Rules: rules }
+        };
+        await s3_admin.putBucketLifecycleConfiguration(putLifecycleParams);
+
+        const putObjectParams = {
+            Bucket: bucket_name,
+            Key: key,
+            Body: 'x'.repeat(size) // default 1KB if size not specified
+        };
+        if (tagging) {
+            putObjectParams.Tagging = tagging;
+        }
+        const start_time = new Date();
+        let res = await s3_admin.putObject(putObjectParams);
+        assert.ok(res.Expiration, 'expiration header missing in putObject response');
+
+        // update file mtime to simulate a 5-days old object
+        await update_file_mtime(path.join(bucket_path, key));
+
+        res = await s3_admin.headObject({ Bucket: bucket_name, Key: key });
+        assert.ok(res.Expiration, 'expiration header missing in headObject response');
+
+        const valid = validate_expiration_header(res.Expiration, start_time, expected_id, expected_days - 5);
+        assert.ok(valid, `expected rule ${expected_id} to match`);
+    };
+
+    mocha.it('should select rule with longest prefix', async () => {
+        const rules = [
+            generate_lifecycle_rule(10, 'short-prefix', 'lifecycle-test1/', [], undefined, undefined),
+            generate_lifecycle_rule(17, 'long-prefix', 'lifecycle-test1/logs/', [], undefined, undefined),
+        ];
+        await run_expiration_test({
+            rules,
+            key: 'lifecycle-test1/logs//file.txt',
+            expected_id: 'long-prefix',
+            expected_days: 17
+        });
+    });
+
+    mocha.it('should select rule with more tags when prefix is same', async () => {
+        const rules = [
+            generate_lifecycle_rule(5, 'one-tag', 'lifecycle-test2/', [{ Key: 'env', Value: 'prod' }], undefined, undefined),
+            generate_lifecycle_rule(9, 'two-tags', 'lifecycle-test2/', [
+                { Key: 'env', Value: 'prod' },
+                { Key: 'team', Value: 'backend' }
+            ], undefined, undefined),
+        ];
+        await run_expiration_test({
+            rules,
+            key: 'lifecycle-test2/file2.txt',
+            tagging: 'env=prod&team=backend',
+            expected_id: 'two-tags',
+            expected_days: 9
+        });
+    });
+
+    mocha.it('should select rule with narrower size span when prefix and tags are matching', async () => {
+        const rules = [
+            generate_lifecycle_rule(4, 'wide-range', 'lifecycle-test3/', [], 100, 10000),
+            generate_lifecycle_rule(6, 'narrow-range', 'lifecycle-test3/', [], 1000, 5000),
+        ];
+        await run_expiration_test({
+            rules,
+            key: 'lifecycle-test3/file3.txt',
+            size: 1500,
+            expected_id: 'narrow-range',
+            expected_days: 6
+        });
+    });
+
+    mocha.it('should fallback to first matching rule if all filters are equal', async () => {
+        const rules = [
+            generate_lifecycle_rule(7, 'rule-a', 'lifecycle/test4/', [], 0, 10000),
+            generate_lifecycle_rule(11, 'rule-b', 'lifecycle/test4/', [], 0, 10000),
+        ];
+        await run_expiration_test({
+            rules,
+            key: 'lifecycle/test4/file4.txt',
+            expected_id: 'rule-a',
+            expected_days: 7
+        });
+    });
+});