Merge pull request #8975 from romayalon/romy-non-current-timestamp

romayalon · web-flow · commit 6db97335f120 · 2025-04-23T16:38:09.000+03:00
NC | Add non current timestamp xattr support
diff --git a/src/sdk/namespace_fs.js b/src/sdk/namespace_fs.js
@@ -62,6 +62,7 @@ const XATTR_PART_ETAG = XATTR_NOOBAA_INTERNAL_PREFIX + 'part_etag';
 const XATTR_VERSION_ID = XATTR_NOOBAA_INTERNAL_PREFIX + 'version_id';
 const XATTR_DELETE_MARKER = XATTR_NOOBAA_INTERNAL_PREFIX + 'delete_marker';
 const XATTR_DIR_CONTENT = XATTR_NOOBAA_INTERNAL_PREFIX + 'dir_content';
+const XATTR_NON_CURRENT_TIMESTASMP = XATTR_NOOBAA_INTERNAL_PREFIX + 'non_current_timestamp';
 const XATTR_TAG = XATTR_NOOBAA_INTERNAL_PREFIX + 'tag.';
 const HIDDEN_VERSIONS_PATH = '.versions';
 const NULL_VERSION_ID = 'null';
@@ -1399,10 +1400,10 @@ class NamespaceFS {
             fs_xattr = this._assign_md5_to_fs_xattr(digest, fs_xattr);
         }
         if (part_upload) {
-            fs_xattr = await this._assign_part_props_to_fs_xattr(fs_context, params.size, digest, offset, fs_xattr);
+            fs_xattr = this._assign_part_props_to_fs_xattr(params.size, digest, offset, fs_xattr);
         }
         if (!part_upload && (this._is_versioning_enabled() || this._is_versioning_suspended())) {
-            fs_xattr = await this._assign_versions_to_fs_xattr(stat, fs_xattr, undefined);
+            fs_xattr = this._assign_versions_to_fs_xattr(stat, fs_xattr, undefined);
         }
         if (!part_upload && params.storage_class) {
             fs_xattr = Object.assign(fs_xattr || {}, {
@@ -1556,6 +1557,7 @@ class NamespaceFS {
                     await native_fs_utils._make_path_dirs(versioned_path, fs_context);
                     await native_fs_utils.safe_move(fs_context, latest_ver_path, versioned_path, latest_ver_info,
                         gpfs_options?.move_to_versions, bucket_tmp_dir_path);
+                    await this._set_non_current_timestamp_on_past_version(fs_context, versioned_path);
                 }
                 if (is_dir_content) {
                     await this._move_directory_content_xattr_to_versioned_file(fs_context, key, versioned_path, latest_ver_path);
@@ -1597,17 +1599,19 @@ class NamespaceFS {
         const directory_stat = await native_fs_utils.stat_ignore_enoent(fs_context, latest_version_dir_path);
         const is_disabled_dir_content = directory_stat && directory_stat.xattr && directory_stat.xattr[XATTR_DIR_CONTENT];
         if (is_disabled_dir_content) {
+            let xattr = filter_fs_xattr(directory_stat.xattr);
+            xattr = this._assign_non_current_timestamp_xattr(xattr);
             if (this._is_versioning_enabled()) {
                 dbg.log1('NamespaceFS._move_to_dest_version latest object is a directory object with attributes on the directory. move the xattr to the new .version file');
                 if (versioned_path) {
-                    await this.set_fs_xattr_op(fs_context, versioned_path, filter_fs_xattr(directory_stat.xattr), undefined);
+                    await this.set_fs_xattr_op(fs_context, versioned_path, xattr, undefined);
                 } else {
                     //if no versioned_path, then we have empty content dir. need to create new .folder file
                     //this scenario happens only after moving from disabled to enabled mode or after upgrade. version-id is always null
                     versioned_path = this._get_version_path(key, NULL_VERSION_ID, true);
                     await native_fs_utils._make_path_dirs(versioned_path, fs_context);
                     //in case of empty directory object .folder of the latest doesn't exist. use 'w' to create it if its missing
-                    await this.set_fs_xattr_op(fs_context, versioned_path, filter_fs_xattr(directory_stat.xattr), undefined, "w");
+                    await this.set_fs_xattr_op(fs_context, versioned_path, xattr, undefined, 'w');
                 }
             }
             await this._clear_user_xattr(fs_context, latest_version_dir_path, XATTR_USER_PREFIX);
@@ -2384,17 +2388,30 @@ class NamespaceFS {
         return fs_xattr;
     }
 
-    async _assign_versions_to_fs_xattr(new_ver_stat, fs_xattr, delete_marker) {
+    /**
+     * _assign_versions_to_fs_xattr assigns version related xattrs to the file 
+     * 1. assign version_id xattr
+     * 2. if delete_marker - 
+     * 2.1. assigns delete_marker xattr
+     * 2.2. assigns non_current_timestamp xattr - on the current structure - delete marker is under .versions/
+     * @param {nb.NativeFSStats} new_ver_stat 
+     * @param {nb.NativeFSXattr} fs_xattr 
+     * @param {Boolean} [delete_marker]
+     * @returns {nb.NativeFSXattr}
+     */
+    _assign_versions_to_fs_xattr(new_ver_stat, fs_xattr, delete_marker = undefined) {
         fs_xattr = Object.assign(fs_xattr || {}, {
             [XATTR_VERSION_ID]: this._get_version_id_by_mode(new_ver_stat)
         });
 
-        if (delete_marker) fs_xattr[XATTR_DELETE_MARKER] = delete_marker;
-
+        if (delete_marker) {
+            fs_xattr[XATTR_DELETE_MARKER] = String(delete_marker);
+            fs_xattr = this._assign_non_current_timestamp_xattr(fs_xattr);
+        }
         return fs_xattr;
     }
 
-    async _assign_part_props_to_fs_xattr(fs_context, size, digest, offset, fs_xattr) {
+    _assign_part_props_to_fs_xattr(size, digest, offset, fs_xattr) {
         fs_xattr = Object.assign(fs_xattr || {}, {
             [XATTR_PART_SIZE]: size,
             [XATTR_PART_OFFSET]: offset,
@@ -2404,6 +2421,39 @@ class NamespaceFS {
         return fs_xattr;
     }
 
+    /**
+     * _assign_non_current_timestamp_xattr assigns non current timestamp xattr to file xattr
+     * @param {nb.NativeFSXattr} fs_xattr 
+     * @returns {nb.NativeFSXattr}
+     */
+    _assign_non_current_timestamp_xattr(fs_xattr = {}) {
+        fs_xattr = Object.assign(fs_xattr, {
+            [XATTR_NON_CURRENT_TIMESTASMP]: String(Date.now())
+        });
+        return fs_xattr;
+    }
+
+    /**
+     * _set_non_current_timestamp_on_past_version sets non current timestamp on past version - used as a hint for lifecycle process
+     * @param {nb.NativeFSContext} fs_context 
+     * @param {String} versioned_path 
+     * @returns {Promise<Void>}
+     */
+    async _set_non_current_timestamp_on_past_version(fs_context, versioned_path) {
+        const xattr = this._assign_non_current_timestamp_xattr();
+        await this.set_fs_xattr_op(fs_context, versioned_path, xattr);
+    }
+
+    /**
+     * _unset_non_current_timestamp_on_past_version unsets non current timestamp on past version - used as a hint for lifecycle process
+     * @param {nb.NativeFSContext} fs_context 
+     * @param {String} versioned_path 
+     * @returns {Promise<Void>}
+     */
+    async _unset_non_current_timestamp_on_past_version(fs_context, versioned_path) {
+        await this._clear_user_xattr(fs_context, versioned_path, XATTR_NON_CURRENT_TIMESTASMP);
+    }
+
     /**
      *
      * @param {*} fs_context - fs context object
@@ -3191,6 +3241,8 @@ class NamespaceFS {
                 const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
                 await native_fs_utils.safe_move_posix(fs_context, max_past_ver_info.path, latest_ver_path,
                     max_past_ver_info, bucket_tmp_dir_path);
+                // TODO - catch error if no such xattr
+                await this._unset_non_current_timestamp_on_past_version(fs_context, latest_ver_path);
                 break;
             } catch (err) {
                 dbg.warn(`NamespaceFS: _promote_version_to_latest failed error: retries=${retries}`, err);
@@ -3249,8 +3301,9 @@ class NamespaceFS {
                     const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
                     if (this._is_versioning_enabled() || suspended_and_latest_is_not_null) {
                         await native_fs_utils._make_path_dirs(versioned_path, fs_context);
-                         await native_fs_utils.safe_move_posix(fs_context, latest_ver_path, versioned_path, latest_ver_info,
+                        await native_fs_utils.safe_move_posix(fs_context, latest_ver_path, versioned_path, latest_ver_info,
                             bucket_tmp_dir_path);
+                        await this._set_non_current_timestamp_on_past_version(fs_context, versioned_path);
                         if (suspended_and_latest_is_not_null) {
                             // remove a version (or delete marker) with null version ID from .versions/ (if exists)
                             await this._delete_null_version_from_versions_directory(params.key, fs_context);
@@ -3333,7 +3386,7 @@ class NamespaceFS {
                 }
                 const file_path = this._get_version_path(params.key, delete_marker_version_id, is_dir);
 
-                const fs_xattr = await this._assign_versions_to_fs_xattr(stat, undefined, true);
+                const fs_xattr = this._assign_versions_to_fs_xattr(stat, undefined, true);
                 if (fs_xattr) await upload_params.target_file.replacexattr(fs_context, fs_xattr);
                 // create .version in case we don't have it yet
                 await native_fs_utils._make_path_dirs(file_path, fs_context);
diff --git a/src/test/unit_tests/test_bucketspace_versioning.js b/src/test/unit_tests/test_bucketspace_versioning.js
@@ -25,7 +25,8 @@ coretest.setup({});
 const XATTR_INTERNAL_NOOBAA_PREFIX = 'user.noobaa.';
 const XATTR_VERSION_ID = XATTR_INTERNAL_NOOBAA_PREFIX + 'version_id';
 const XATTR_DELETE_MARKER = XATTR_INTERNAL_NOOBAA_PREFIX + 'delete_marker';
-const XATTR_DIR_CONTENT = XATTR_INTERNAL_NOOBAA_PREFIX + "dir_content";
+const XATTR_NON_CURRENT_TIMESTASMP = XATTR_INTERNAL_NOOBAA_PREFIX + 'non_current_timestamp';
+const XATTR_DIR_CONTENT = XATTR_INTERNAL_NOOBAA_PREFIX + 'dir_content';
 const XATTR_USER_PREFIX = 'user.';
 const NULL_VERSION_ID = 'null';
 const HIDDEN_VERSIONS_PATH = '.versions';
@@ -1414,6 +1415,9 @@ mocha.describe('bucketspace namespace_fs - versioning', function() {
             const version_path = path.join(suspended_full_path, '.versions', key_to_delete3 + '_' + latest_dm_version);
             const version_info = await stat_and_get_all(version_path, '');
             assert.equal(version_info.xattr[XATTR_VERSION_ID], NULL_VERSION_ID);
+            // check second latest is still non current xattr
+            const second_latest_version_path = path.join(suspended_full_path, '.versions', key_to_delete3 + '_' + prev_dm.VersionId);
+            await check_non_current_xattr_exists(second_latest_version_path);
         });
     });
 
@@ -1450,6 +1454,8 @@ mocha.describe('bucketspace namespace_fs - versioning', function() {
             mocha.it('delete object version id - latest - second latest is null version', async function() {
                 const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['null', 'regular']);
                 const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1);
+                const second_latest_version_path = path.join(full_delete_path, '.versions', key1 + '_null');
+                await check_non_current_xattr_exists(second_latest_version_path);
 
                 const delete_res = await account_with_access.deleteObject({
                     Bucket: delete_object_test_bucket_reg,
@@ -1460,6 +1466,9 @@ mocha.describe('bucketspace namespace_fs - versioning', function() {
                 const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1);
                 assert.notEqual(cur_version_id1, cur_version_id2);
                 assert.equal('null', cur_version_id2);
+                // check second latest current xattr removed
+                const latest_version_path = path.join(full_delete_path, key1);
+                await check_non_current_xattr_does_not_exist(latest_version_path);
                 await fs_utils.file_must_not_exist(path.join(full_delete_path, key1 + '_' + upload_res_arr[1].VersionId));
                 const max_version1 = await find_max_version_past(full_delete_path, key1, '');
                 assert.equal(max_version1, undefined);
@@ -3408,9 +3417,18 @@ function _extract_version_info_from_xattr(version_id_str) {
     return { mtimeNsBigint: size_utils.string_to_bigint(arr[0], 36), ino: parseInt(arr[1], 36) };
 }
 
+/**
+ * version_file_exists returns path of version in .versions
+ * @param {String} full_path 
+ * @param {String} key 
+ * @param {String} dir 
+ * @param {String} version_id 
+ * @returns {Promise<Boolean>}
+ */
 async function version_file_exists(full_path, key, dir, version_id) {
     const version_path = path.join(full_path, dir, '.versions', key + '_' + version_id);
     await fs_utils.file_must_exist(version_path);
+    await check_non_current_xattr_exists(version_path, '');
     return true;
 }
 
@@ -3427,10 +3445,10 @@ async function get_obj_and_compare_data(s3, bucket_name, key, expected_body) {
     return true;
 }
 
-async function is_delete_marker(full_path, dir, key, version) {
+async function is_delete_marker(full_path, dir, key, version, check_non_current_version = true) {
     const version_path = path.join(full_path, dir, '.versions', key + '_' + version);
     const stat = await nb_native().fs.stat(DEFAULT_FS_CONFIG, version_path);
-    return stat && stat.xattr[XATTR_DELETE_MARKER];
+    return stat && stat.xattr[XATTR_DELETE_MARKER] && (check_non_current_version ? stat.xattr[XATTR_NON_CURRENT_TIMESTASMP] : true);
 }
 
 async function stat_and_get_version_id(full_path, key) {
@@ -3508,6 +3526,28 @@ async function create_empty_content_dir(fs_context, bucket_path, key) {
 
 }
 
+/**
+ * check_non_current_xattr_exists checks that the XATTR_NON_CURRENT_TIMESTASMP xattr exists
+ * @param {String} full_path 
+ * @param {String} [key] 
+ * @returns {Promise<Void>}
+ */
+async function check_non_current_xattr_exists(full_path, key = '') {
+    const stat = await stat_and_get_all(full_path, key);
+    assert.ok(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP]);
+}
+
+/**
+ * check_non_current_xattr_does_not_exist checks that the XATTR_NON_CURRENT_TIMESTASMP xattr does not exist
+ * @param {String} full_path 
+ * @param {String} [key] 
+ * @returns {Promise<Void>}
+ */
+async function check_non_current_xattr_does_not_exist(full_path, key = '') {
+    const stat = await stat_and_get_all(full_path, key);
+    assert.equal(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP], undefined);
+}
+
 async function put_allow_all_bucket_policy(s3_client, bucket) {
     const policy = {
         Version: '2012-10-17',
