Skip to content

Commit df85b02

Browse files
romayalonromayalon
committed
NC | Add non current timestamp xattr support
Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> (cherry picked from commit a0df317)
1 parent 7303929 commit df85b02

File tree

2 files changed

+101
-10
lines changed

2 files changed

+101
-10
lines changed

‎src/sdk/namespace_fs.js

Lines changed: 59 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ const XATTR_PART_ETAG = XATTR_NOOBAA_INTERNAL_PREFIX + 'part_etag';
6161
const XATTR_VERSION_ID = XATTR_NOOBAA_INTERNAL_PREFIX + 'version_id';
6262
const XATTR_DELETE_MARKER = XATTR_NOOBAA_INTERNAL_PREFIX + 'delete_marker';
6363
const XATTR_DIR_CONTENT = XATTR_NOOBAA_INTERNAL_PREFIX + 'dir_content';
64+
const XATTR_NON_CURRENT_TIMESTASMP = XATTR_NOOBAA_INTERNAL_PREFIX + 'non_current_timestamp';
6465
const XATTR_TAG = XATTR_NOOBAA_INTERNAL_PREFIX + 'tag.';
6566
const HIDDEN_VERSIONS_PATH = '.versions';
6667
const NULL_VERSION_ID = 'null';
@@ -1385,10 +1386,10 @@ class NamespaceFS {
13851386
fs_xattr = this._assign_md5_to_fs_xattr(digest, fs_xattr);
13861387
}
13871388
if (part_upload) {
1388-
fs_xattr = await this._assign_part_props_to_fs_xattr(fs_context, params.size, digest, offset, fs_xattr);
1389+
fs_xattr = this._assign_part_props_to_fs_xattr(params.size, digest, offset, fs_xattr);
13891390
}
13901391
if (!part_upload && (this._is_versioning_enabled() || this._is_versioning_suspended())) {
1391-
fs_xattr = await this._assign_versions_to_fs_xattr(stat, fs_xattr, undefined);
1392+
fs_xattr = this._assign_versions_to_fs_xattr(stat, fs_xattr, undefined);
13921393
}
13931394
if (!part_upload && params.storage_class) {
13941395
fs_xattr = Object.assign(fs_xattr || {}, {
@@ -1540,6 +1541,7 @@ class NamespaceFS {
15401541
await native_fs_utils._make_path_dirs(versioned_path, fs_context);
15411542
await native_fs_utils.safe_move(fs_context, latest_ver_path, versioned_path, latest_ver_info,
15421543
gpfs_options?.move_to_versions, bucket_tmp_dir_path);
1544+
await this._set_non_current_timestamp_on_past_version(fs_context, versioned_path);
15431545
}
15441546
try {
15451547
// move new version to latest_ver_path (key path)
@@ -2312,17 +2314,30 @@ class NamespaceFS {
23122314
return fs_xattr;
23132315
}
23142316

2315-
async _assign_versions_to_fs_xattr(new_ver_stat, fs_xattr, delete_marker) {
2317+
/**
2318+
* _assign_versions_to_fs_xattr assigns version related xattrs to the file
2319+
* 1. assign version_id xattr
2320+
* 2. if delete_marker -
2321+
* 2.1. assigns delete_marker xattr
2322+
* 2.2. assigns non_current_timestamp xattr - on the current structure - delete marker is under .versions/
2323+
* @param {nb.NativeFSStats} new_ver_stat
2324+
* @param {nb.NativeFSXattr} fs_xattr
2325+
* @param {Boolean} [delete_marker]
2326+
* @returns {nb.NativeFSXattr}
2327+
*/
2328+
_assign_versions_to_fs_xattr(new_ver_stat, fs_xattr, delete_marker = undefined) {
23162329
fs_xattr = Object.assign(fs_xattr || {}, {
23172330
[XATTR_VERSION_ID]: this._get_version_id_by_mode(new_ver_stat)
23182331
});
23192332

2320-
if (delete_marker) fs_xattr[XATTR_DELETE_MARKER] = delete_marker;
2321-
2333+
if (delete_marker) {
2334+
fs_xattr[XATTR_DELETE_MARKER] = String(delete_marker);
2335+
fs_xattr = this._assign_non_current_timestamp_xattr(fs_xattr);
2336+
}
23222337
return fs_xattr;
23232338
}
23242339

2325-
async _assign_part_props_to_fs_xattr(fs_context, size, digest, offset, fs_xattr) {
2340+
_assign_part_props_to_fs_xattr(size, digest, offset, fs_xattr) {
23262341
fs_xattr = Object.assign(fs_xattr || {}, {
23272342
[XATTR_PART_SIZE]: size,
23282343
[XATTR_PART_OFFSET]: offset,
@@ -2332,6 +2347,39 @@ class NamespaceFS {
23322347
return fs_xattr;
23332348
}
23342349

2350+
/**
2351+
* _assign_non_current_timestamp_xattr assigns non current timestamp xattr to file xattr
2352+
* @param {nb.NativeFSXattr} fs_xattr
2353+
* @returns {nb.NativeFSXattr}
2354+
*/
2355+
_assign_non_current_timestamp_xattr(fs_xattr = {}) {
2356+
fs_xattr = Object.assign(fs_xattr, {
2357+
[XATTR_NON_CURRENT_TIMESTASMP]: String(Date.now())
2358+
});
2359+
return fs_xattr;
2360+
}
2361+
2362+
/**
2363+
* _set_non_current_timestamp_on_past_version sets non current timestamp on past version - used as a hint for lifecycle process
2364+
* @param {nb.NativeFSContext} fs_context
2365+
* @param {String} versioned_path
2366+
* @returns {Promise<Void>}
2367+
*/
2368+
async _set_non_current_timestamp_on_past_version(fs_context, versioned_path) {
2369+
const xattr = this._assign_non_current_timestamp_xattr();
2370+
await this.set_fs_xattr_op(fs_context, versioned_path, xattr);
2371+
}
2372+
2373+
/**
2374+
* _unset_non_current_timestamp_on_past_version unsets non current timestamp on past version - used as a hint for lifecycle process
2375+
* @param {nb.NativeFSContext} fs_context
2376+
* @param {String} versioned_path
2377+
* @returns {Promise<Void>}
2378+
*/
2379+
async _unset_non_current_timestamp_on_past_version(fs_context, versioned_path) {
2380+
await this._clear_user_xattr(fs_context, versioned_path, XATTR_NON_CURRENT_TIMESTASMP);
2381+
}
2382+
23352383
/**
23362384
*
23372385
* @param {*} fs_context - fs context object
@@ -3088,6 +3136,8 @@ class NamespaceFS {
30883136
const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
30893137
await native_fs_utils.safe_move_posix(fs_context, max_past_ver_info.path, latest_ver_path,
30903138
max_past_ver_info, bucket_tmp_dir_path);
3139+
// TODO - catch error if no such xattr
3140+
await this._unset_non_current_timestamp_on_past_version(fs_context, latest_ver_path);
30913141
break;
30923142
} catch (err) {
30933143
dbg.warn(`NamespaceFS: _promote_version_to_latest failed error: retries=${retries}`, err);
@@ -3144,8 +3194,9 @@ class NamespaceFS {
31443194
const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path();
31453195
if (this._is_versioning_enabled() || suspended_and_latest_is_not_null) {
31463196
await native_fs_utils._make_path_dirs(versioned_path, fs_context);
3147-
await native_fs_utils.safe_move_posix(fs_context, latest_ver_path, versioned_path, latest_ver_info,
3197+
await native_fs_utils.safe_move_posix(fs_context, latest_ver_path, versioned_path, latest_ver_info,
31483198
bucket_tmp_dir_path);
3199+
await this._set_non_current_timestamp_on_past_version(fs_context, versioned_path);
31493200
if (suspended_and_latest_is_not_null) {
31503201
// remove a version (or delete marker) with null version ID from .versions/ (if exists)
31513202
await this._delete_null_version_from_versions_directory(params.key, fs_context);
@@ -3225,7 +3276,7 @@ class NamespaceFS {
32253276
}
32263277
const file_path = this._get_version_path(params.key, delete_marker_version_id);
32273278

3228-
const fs_xattr = await this._assign_versions_to_fs_xattr(stat, undefined, true);
3279+
const fs_xattr = this._assign_versions_to_fs_xattr(stat, undefined, true);
32293280
if (fs_xattr) await upload_params.target_file.replacexattr(fs_context, fs_xattr);
32303281
// create .version in case we don't have it yet
32313282
await native_fs_utils._make_path_dirs(file_path, fs_context);

‎src/test/unit_tests/test_bucketspace_versioning.js

Lines changed: 42 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ coretest.setup({});
2424
const XATTR_INTERNAL_NOOBAA_PREFIX = 'user.noobaa.';
2525
const XATTR_VERSION_ID = XATTR_INTERNAL_NOOBAA_PREFIX + 'version_id';
2626
const XATTR_DELETE_MARKER = XATTR_INTERNAL_NOOBAA_PREFIX + 'delete_marker';
27+
const XATTR_NON_CURRENT_TIMESTASMP = XATTR_INTERNAL_NOOBAA_PREFIX + 'non_current_timestamp';
2728
const NULL_VERSION_ID = 'null';
2829
const HIDDEN_VERSIONS_PATH = '.versions';
2930
const NSFS_FOLDER_OBJECT_NAME = '.folder';
@@ -1181,6 +1182,9 @@ mocha.describe('bucketspace namespace_fs - versioning', function() {
11811182
const version_path = path.join(suspended_full_path, '.versions', key_to_delete3 + '_' + latest_dm_version);
11821183
const version_info = await stat_and_get_all(version_path, '');
11831184
assert.equal(version_info.xattr[XATTR_VERSION_ID], NULL_VERSION_ID);
1185+
// check second latest is still non current xattr
1186+
const second_latest_version_path = path.join(suspended_full_path, '.versions', key_to_delete3 + '_' + prev_dm.VersionId);
1187+
await check_non_current_xattr_exists(second_latest_version_path);
11841188
});
11851189
});
11861190

@@ -1217,6 +1221,8 @@ mocha.describe('bucketspace namespace_fs - versioning', function() {
12171221
mocha.it('delete object version id - latest - second latest is null version', async function() {
12181222
const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['null', 'regular']);
12191223
const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1);
1224+
const second_latest_version_path = path.join(full_delete_path, '.versions', key1 + '_null');
1225+
await check_non_current_xattr_exists(second_latest_version_path);
12201226

12211227
const delete_res = await account_with_access.deleteObject({
12221228
Bucket: delete_object_test_bucket_reg,
@@ -1227,6 +1233,9 @@ mocha.describe('bucketspace namespace_fs - versioning', function() {
12271233
const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1);
12281234
assert.notEqual(cur_version_id1, cur_version_id2);
12291235
assert.equal('null', cur_version_id2);
1236+
// check second latest current xattr removed
1237+
const latest_version_path = path.join(full_delete_path, key1);
1238+
await check_non_current_xattr_does_not_exist(latest_version_path);
12301239
await fs_utils.file_must_not_exist(path.join(full_delete_path, key1 + '_' + upload_res_arr[1].VersionId));
12311240
const max_version1 = await find_max_version_past(full_delete_path, key1, '');
12321241
assert.equal(max_version1, undefined);
@@ -3175,9 +3184,18 @@ function _extract_version_info_from_xattr(version_id_str) {
31753184
return { mtimeNsBigint: size_utils.string_to_bigint(arr[0], 36), ino: parseInt(arr[1], 36) };
31763185
}
31773186

3187+
/**
3188+
* version_file_exists returns path of version in .versions
3189+
* @param {String} full_path
3190+
* @param {String} key
3191+
* @param {String} dir
3192+
* @param {String} version_id
3193+
* @returns {Promise<Boolean>}
3194+
*/
31783195
async function version_file_exists(full_path, key, dir, version_id) {
31793196
const version_path = path.join(full_path, dir, '.versions', key + '_' + version_id);
31803197
await fs_utils.file_must_exist(version_path);
3198+
await check_non_current_xattr_exists(version_path, '');
31813199
return true;
31823200
}
31833201

@@ -3194,10 +3212,10 @@ async function get_obj_and_compare_data(s3, bucket_name, key, expected_body) {
31943212
return true;
31953213
}
31963214

3197-
async function is_delete_marker(full_path, dir, key, version) {
3215+
async function is_delete_marker(full_path, dir, key, version, check_non_current_version = true) {
31983216
const version_path = path.join(full_path, dir, '.versions', key + '_' + version);
31993217
const stat = await nb_native().fs.stat(DEFAULT_FS_CONFIG, version_path);
3200-
return stat && stat.xattr[XATTR_DELETE_MARKER];
3218+
return stat && stat.xattr[XATTR_DELETE_MARKER] && (check_non_current_version ? stat.xattr[XATTR_NON_CURRENT_TIMESTASMP] : true);
32013219
}
32023220

32033221
async function stat_and_get_version_id(full_path, key) {
@@ -3260,6 +3278,28 @@ function check_null_version_id(version_id) {
32603278
return version_id === NULL_VERSION_ID;
32613279
}
32623280

3281+
/**
3282+
* check_non_current_xattr_exists checks that the XATTR_NON_CURRENT_TIMESTASMP xattr exists
3283+
* @param {String} full_path
3284+
* @param {String} [key]
3285+
* @returns {Promise<Void>}
3286+
*/
3287+
async function check_non_current_xattr_exists(full_path, key = '') {
3288+
const stat = await stat_and_get_all(full_path, key);
3289+
assert.ok(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP]);
3290+
}
3291+
3292+
/**
3293+
* check_non_current_xattr_does_not_exist checks that the XATTR_NON_CURRENT_TIMESTASMP xattr does not exist
3294+
* @param {String} full_path
3295+
* @param {String} [key]
3296+
* @returns {Promise<Void>}
3297+
*/
3298+
async function check_non_current_xattr_does_not_exist(full_path, key = '') {
3299+
const stat = await stat_and_get_all(full_path, key);
3300+
assert.equal(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP], undefined);
3301+
}
3302+
32633303
async function put_allow_all_bucket_policy(s3_client, bucket) {
32643304
const policy = {
32653305
Version: '2012-10-17',

0 commit comments

Comments
 (0)