NC | Versioning | Concurrent put of same key

Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com>

Author: romayalon

config.js

@@ -764,7 +764,7 @@ config.NSFS_REMOVE_PARTS_ON_COMPLETE = true;
 config.NSFS_BUF_POOL_WARNING_TIMEOUT = 2 * 60 * 1000;
 config.NSFS_SEM_WARNING_TIMEOUT = 10 * 60 * 1000;
 // number of rename retries in case of deleted destination directory
-config.NSFS_RENAME_RETRIES = 3;
+config.NSFS_RENAME_RETRIES = 10;
 
 config.NSFS_VERSIONING_ENABLED = true;
 config.NSFS_UPDATE_ISSUES_REPORT_ENABLED = true;

src/sdk/namespace_fs.js

@@ -1473,9 +1473,10 @@ class NamespaceFS {
                 break;
             } catch (err) {
                 retries -= 1;
-                if (retries <= 0 || !native_fs_utils.should_retry_link_unlink(is_gpfs, err)) throw err;
-                dbg.warn(`NamespaceFS._move_to_dest_version retrying retries=${retries}` +
+                const should_retry = native_fs_utils.should_retry_link_unlink(is_gpfs, err);
+                dbg.warn(`NamespaceFS._move_to_dest_version retrying retries=${retries} should_retry=${should_retry}` +
                     ` new_ver_tmp_path=${new_ver_tmp_path} latest_ver_path=${latest_ver_path}`, err);
+                if (!should_retry || retries <= 0) throw err;
             } finally {
                 if (gpfs_options) await this._close_files_gpfs(fs_context, gpfs_options.move_to_dst, open_mode);
             }

src/test/unit_tests/jest_tests/test_versioning_concurrency.test.js

@@ -0,0 +1,62 @@
+/* Copyright (C) 2016 NooBaa */
+'use strict';
+
+const path = require('path');
+const P = require('../../../util/promise');
+const fs_utils = require('../../../util/fs_utils');
+const NamespaceFS = require('../../../sdk/namespace_fs');
+const buffer_utils = require('../../../util/buffer_utils');
+const { TMP_PATH } = require('../../system_tests/test_utils');
+const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector');
+
+function make_dummy_object_sdk(nsfs_config, uid, gid) {
+    return {
+        requesting_account: {
+            nsfs_account_config: nsfs_config && {
+                uid: uid || process.getuid(),
+                gid: gid || process.getgid(),
+                backend: '',
+            }
+        },
+        abort_controller: new AbortController(),
+        throw_if_aborted() {
+            if (this.abort_controller.signal.aborted) throw new Error('request aborted signal');
+        }
+    };
+}
+
+const DUMMY_OBJECT_SDK = make_dummy_object_sdk(true);
+describe('test versioning concurrency', () => {
+    const tmp_fs_path = path.join(TMP_PATH, 'test_versioning_concurrency');
+
+    const nsfs = new NamespaceFS({
+        bucket_path: tmp_fs_path,
+        bucket_id: '1',
+        namespace_resource_id: undefined,
+        access_mode: undefined,
+        versioning: 'ENABLED',
+        force_md5_etag: false,
+        stats: endpoint_stats_collector.instance(),
+    });
+
+    beforeEach(async () => {
+        await fs_utils.create_fresh_path(tmp_fs_path);
+    });
+
+    afterEach(async () => {
+        await fs_utils.folder_delete(tmp_fs_path);
+    });
+
+    it('multiple puts of the same key', async () => {
+        const bucket = 'bucket1';
+        const key = 'key1';
+        for (let i = 0; i < 5; i++) {
+            const random_data = Buffer.from(String(i));
+            const body = buffer_utils.buffer_to_read_stream(random_data);
+            nsfs.upload_object({ bucket: bucket, key: key, source_stream: body }, DUMMY_OBJECT_SDK).catch(err => console.log('multiple puts of the same key error - ', err));
+        }
+        await P.delay(1000);
+        const versions = await nsfs.list_object_versions({ bucket: bucket }, DUMMY_OBJECT_SDK);
+        expect(versions.objects.length).toBe(5);
+    });
+});

src/util/native_fs_utils.js

@@ -239,7 +239,8 @@ async function safe_unlink_gpfs(fs_context, to_delete_path, to_delete_file, dir_
 function should_retry_link_unlink(is_gpfs, err) {
     return is_gpfs ?
         [gpfs_link_unlink_retry_err, gpfs_unlink_retry_catch].includes(err.code) :
-        [posix_link_retry_err, posix_unlink_retry_err].includes(err.message);
+        [posix_link_retry_err, posix_unlink_retry_err].includes(err.message) ||
+        ['EEXIST'].includes(err.code);
 }
 
 ////////////////////////