Skip to content

Commit 68abc12

Browse files
naveenpaul1naveenpaul1
authored
Merge pull request #8417 from naveenpaul1/tagging_issue
NSFS | Tagging bugs
2 parents 405905b + 5a9c83a commit 68abc12

File tree

2 files changed

+275
-6
lines changed

2 files changed

+275
-6
lines changed

‎src/sdk/namespace_fs.js

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1165,7 +1165,6 @@ class NamespaceFS {
11651165
await this._throw_if_storage_class_not_supported(params.storage_class);
11661166

11671167
upload_params = await this._start_upload(fs_context, object_sdk, file_path, params, open_mode);
1168-
11691168
if (!params.copy_source || upload_params.copy_res === copy_status_enum.FALLBACK) {
11701169
// We are taking the buffer size closest to the sized upload
11711170
const bp = multi_buffer_pool.get_buffers_pool(params.size);
@@ -1332,6 +1331,13 @@ class NamespaceFS {
13321331
await this.append_to_migrate_wal(file_path);
13331332
}
13341333
}
1334+
if (params.tagging) {
1335+
for (const { key, value } of params.tagging) {
1336+
fs_xattr = Object.assign(fs_xattr || {}, {
1337+
[XATTR_TAG + key]: value
1338+
});
1339+
}
1340+
}
13351341
if (fs_xattr && !is_dir_content && should_replace_xattr) await target_file.replacexattr(fs_context, fs_xattr);
13361342
// fsync
13371343
if (config.NSFS_TRIGGER_FSYNC) await target_file.fsync(fs_context);
@@ -1952,15 +1958,17 @@ class NamespaceFS {
19521958
async get_object_tagging(params, object_sdk) {
19531959
const tag_set = [];
19541960
let file_path;
1955-
if (params.version_id && this._is_versioning_enabled()) {
1956-
file_path = this._get_version_path(params.key, params.version_id);
1961+
let file;
1962+
const fs_context = this.prepare_fs_context(object_sdk);
1963+
// Version specific tag will return even if versioning suspended.
1964+
if (params.version_id) {
1965+
file_path = await this._find_version_path(fs_context, params, true);
19571966
} else {
1958-
file_path = this._get_file_path(params);
1967+
file_path = this._get_file_md_path(params);
19591968
}
19601969
try {
1961-
const fs_context = this.prepare_fs_context(object_sdk);
19621970
dbg.log0('NamespaceFS.get_object_tagging: param ', params, 'file_path :', file_path);
1963-
const file = await nb_native().fs.open(fs_context, file_path);
1971+
file = await nb_native().fs.open(fs_context, file_path);
19641972
const stat = await file.stat(fs_context);
19651973
if (stat.xattr) {
19661974
for (const [xattr_key, xattr_value] of Object.entries(stat.xattr)) {
@@ -1975,6 +1983,8 @@ class NamespaceFS {
19751983
} catch (err) {
19761984
dbg.error(`NamespaceFS.get_object_tagging: failed in dir ${file_path} with error: `, err);
19771985
throw native_fs_utils.translate_error_codes(err, native_fs_utils.entity_enum.OBJECT);
1986+
} finally {
1987+
if (file) await file.close(fs_context);
19781988
}
19791989
dbg.log0('NamespaceFS.get_object_tagging: return tagging ', tag_set, 'file_path :', file_path);
19801990
return { tagging: tag_set };

‎src/test/unit_tests/test_namespace_fs.js

Lines changed: 259 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
/* Copyright (C) 2020 NooBaa */
22
/*eslint max-lines-per-function: ["error", 900]*/
3+
/*eslint max-lines: ["error", 2100]*/
34
'use strict';
45

56
const _ = require('lodash');
@@ -1764,8 +1765,266 @@ mocha.describe('namespace_fs copy object', function() {
17641765
});
17651766
});
17661767

1768+
mocha.describe('namespace_fs upload object with tagging', function() {
1769+
1770+
const upload_bkt_tagging = 'test_ns_uploads_object_tagging';
1771+
const tmp_fs_path_tagging = path.join(TMP_PATH, 'test_namespace_fs_tagging');
1772+
const ns_tmp_bucket_path_tagging = `${tmp_fs_path_tagging}/${src_bkt}`;
1773+
const ns_tmp_tagging = new NamespaceFS({ bucket_path: ns_tmp_bucket_path_tagging, bucket_id: '3', namespace_resource_id: undefined });
1774+
mocha.before(async () => {
1775+
await P.all(_.map([src_bkt, upload_bkt_tagging], async buck =>
1776+
fs_utils.create_fresh_path(`${tmp_fs_path_tagging}/${buck}`)));
1777+
});
1778+
mocha.after(async () => {
1779+
await P.all(_.map([src_bkt, upload_bkt_tagging], async buck =>
1780+
fs_utils.folder_delete(`${tmp_fs_path_tagging}/${buck}`)));
1781+
await fs_utils.folder_delete(tmp_fs_path_tagging);
1782+
});
1783+
mocha.describe('upload_object (tagging)', function() {
1784+
const upload_key = 'upload_key_1';
1785+
const upload_folder_key = 'upload_folder_key1/';
1786+
const upload_folder_key_without_slash = 'upload_folder_key';
1787+
const data = crypto.randomBytes(100);
1788+
const tagging_1 = [{ key: 'tag1', value: 'val1' }];
1789+
const xattr = { key: 'key1', key2: 'value1' };
1790+
1791+
mocha.it('Upload object with tagging', async function() {
1792+
const params = {
1793+
bucket: upload_bkt_tagging,
1794+
key: upload_key,
1795+
xattr,
1796+
source_stream: buffer_utils.buffer_to_read_stream(data),
1797+
tagging: tagging_1,
1798+
};
1799+
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
1800+
console.log('upload_object (tag) response', inspect(upload_res));
1801+
const tag_res = await ns_tmp_tagging.get_object_tagging({
1802+
bucket: upload_bkt_tagging,
1803+
key: upload_key,
1804+
}, dummy_object_sdk);
1805+
console.log('get_object_tagging response', inspect(tag_res));
1806+
await validate_tagging(tag_res.tagging, tagging_1);
1807+
});
1808+
1809+
mocha.it('Upload object with multiple tagging', async function() {
1810+
const tagging_multi = [{ key: 'tag1', value: 'val1' },
1811+
{ key: 'tag2', value: 'val2' },
1812+
{ key: 'tag3', value: 'val3' }
1813+
];
1814+
const params = {
1815+
bucket: upload_bkt_tagging,
1816+
key: upload_key,
1817+
xattr,
1818+
source_stream: buffer_utils.buffer_to_read_stream(data),
1819+
tagging: tagging_multi,
1820+
};
1821+
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
1822+
console.log('upload_object (tag) response', inspect(upload_res));
1823+
const tag_res = await ns_tmp_tagging.get_object_tagging({
1824+
bucket: upload_bkt_tagging,
1825+
key: upload_key,
1826+
}, dummy_object_sdk);
1827+
console.log('get_object_tagging response', inspect(tag_res));
1828+
await validate_tagging(tag_res.tagging, tagging_multi);
1829+
});
1830+
1831+
mocha.it('Upload object with tagging, update tag', async function() {
1832+
const tagging_2 = [{ key: 'tag2', value: 'val2' }];
1833+
const params = {
1834+
bucket: upload_bkt_tagging,
1835+
key: upload_key,
1836+
xattr,
1837+
source_stream: buffer_utils.buffer_to_read_stream(data),
1838+
tagging: tagging_1,
1839+
};
1840+
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
1841+
console.log('upload_object (tag) response', inspect(upload_res));
1842+
const tag_res = await ns_tmp_tagging.get_object_tagging({
1843+
bucket: upload_bkt_tagging,
1844+
key: upload_key,
1845+
}, dummy_object_sdk);
1846+
console.log('get_object_tagging response', inspect(tag_res));
1847+
await validate_tagging(tag_res.tagging, tagging_1);
1848+
1849+
const update_params = {
1850+
bucket: upload_bkt_tagging,
1851+
key: upload_key,
1852+
xattr,
1853+
source_stream: buffer_utils.buffer_to_read_stream(data),
1854+
tagging: tagging_2,
1855+
};
1856+
const update_upload_res = await ns_tmp_tagging.upload_object(update_params, dummy_object_sdk);
1857+
console.log('upload_object (tag) response after update', inspect(update_upload_res));
1858+
const update_tag_res = await ns_tmp_tagging.get_object_tagging({
1859+
bucket: upload_bkt_tagging,
1860+
key: upload_key,
1861+
}, dummy_object_sdk);
1862+
console.log('get_object_tagging response after update', inspect(tag_res));
1863+
await validate_tagging(update_tag_res.tagging, tagging_2);
1864+
});
1865+
1866+
mocha.it('Upload folder object with tagging', async function() {
1867+
const params = {
1868+
bucket: upload_bkt_tagging,
1869+
key: upload_folder_key,
1870+
source_stream: buffer_utils.buffer_to_read_stream(data),
1871+
tagging: tagging_1,
1872+
};
1873+
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
1874+
console.log('upload_object (tag) response', inspect(upload_res));
1875+
const tag_res = await ns_tmp_tagging.get_object_tagging({
1876+
bucket: upload_bkt_tagging,
1877+
key: upload_folder_key,
1878+
}, dummy_object_sdk);
1879+
console.log('get_object_tagging response', inspect(tag_res));
1880+
await validate_tagging(tag_res.tagging, tagging_1);
1881+
});
1882+
1883+
mocha.it('Upload folder object with tagging, get object tagging key without slash', async function() {
1884+
const tagging_2 = [{ key: 'tag2', value: 'val2' }];
1885+
const params = {
1886+
bucket: upload_bkt_tagging,
1887+
key: upload_folder_key_without_slash,
1888+
xattr,
1889+
source_stream: buffer_utils.buffer_to_read_stream(data),
1890+
tagging: tagging_2,
1891+
};
1892+
const upload_res = await ns_tmp_tagging.upload_object(params, dummy_object_sdk);
1893+
console.log('upload_object (tag) response', inspect(upload_res));
1894+
const tag_res = await ns_tmp_tagging.get_object_tagging({
1895+
bucket: upload_bkt_tagging,
1896+
key: upload_folder_key_without_slash,
1897+
}, dummy_object_sdk);
1898+
console.log('get_object_tagging response', inspect(tag_res));
1899+
await validate_tagging(tag_res.tagging, tagging_2);
1900+
});
1901+
1902+
mocha.it('Upload folder object with tagging and version enabled', async function() {
1903+
ns_tmp_tagging.set_bucket_versioning('ENABLED', dummy_object_sdk);
1904+
const tagging_2 = [{ key: 'tag2', value: 'val2' }];
1905+
const tagging_3 = [{ key: 'tag3', value: 'val3' }];
1906+
const upload_key_2 = 'tagging_upload_key_2';
1907+
const params1 = {
1908+
bucket: upload_bkt_tagging,
1909+
key: upload_key_2,
1910+
source_stream: buffer_utils.buffer_to_read_stream(data),
1911+
tagging: tagging_1,
1912+
};
1913+
const upload_res = await ns_tmp_tagging.upload_object(params1, dummy_object_sdk);
1914+
console.log('upload_object (tag) response', inspect(upload_res));
1915+
const params2 = {
1916+
bucket: upload_bkt_tagging,
1917+
key: upload_key_2,
1918+
source_stream: buffer_utils.buffer_to_read_stream(data),
1919+
tagging: tagging_2,
1920+
};
1921+
const upload_res1 = await ns_tmp_tagging.upload_object(params2, dummy_object_sdk);
1922+
console.log('upload_object (tag) response', inspect(upload_res1));
1923+
1924+
//get tag for first version
1925+
const tag_res = await ns_tmp_tagging.get_object_tagging({
1926+
bucket: upload_bkt_tagging,
1927+
key: upload_key_2,
1928+
version_id: upload_res.version_id,
1929+
}, dummy_object_sdk);
1930+
console.log('get_object_tagging response', inspect(tag_res));
1931+
await validate_tagging(tag_res.tagging, tagging_1);
1932+
1933+
// suspend versioning, verion specific tag should retun.
1934+
ns_tmp_tagging.set_bucket_versioning('SUSPENDED', dummy_object_sdk);
1935+
const params3 = {
1936+
bucket: upload_bkt_tagging,
1937+
key: upload_key_2,
1938+
source_stream: buffer_utils.buffer_to_read_stream(data),
1939+
tagging: tagging_3,
1940+
};
1941+
const upload_res2 = await ns_tmp_tagging.upload_object(params3, dummy_object_sdk);
1942+
console.log('upload_object (tag) response', inspect(upload_res2));
1943+
1944+
//get tag for second version, after disabling version
1945+
const tag_res1 = await ns_tmp_tagging.get_object_tagging({
1946+
bucket: upload_bkt_tagging,
1947+
key: upload_key_2,
1948+
version_id: upload_res1.version_id,
1949+
}, dummy_object_sdk);
1950+
console.log('get_object_tagging response', inspect(tag_res1));
1951+
await validate_tagging(tag_res1.tagging, tagging_2);
1952+
1953+
//get tag for last item without version id
1954+
const tag_res2 = await ns_tmp_tagging.get_object_tagging({
1955+
bucket: upload_bkt_tagging,
1956+
key: upload_key_2,
1957+
}, dummy_object_sdk);
1958+
console.log('get_object_tagging response', inspect(tag_res2));
1959+
await validate_tagging(tag_res2.tagging, tagging_3);
1960+
});
1961+
1962+
mocha.it('get tag with last version', async function() {
1963+
ns_tmp_tagging.set_bucket_versioning('ENABLED', dummy_object_sdk);
1964+
const tagging_2 = [{ key: 'tag2', value: 'val2' }];
1965+
const upload_key_2 = 'tagging_upload_key_2';
1966+
const params1 = {
1967+
bucket: upload_bkt_tagging,
1968+
key: upload_key_2,
1969+
source_stream: buffer_utils.buffer_to_read_stream(data),
1970+
tagging: tagging_1,
1971+
};
1972+
const upload_res = await ns_tmp_tagging.upload_object(params1, dummy_object_sdk);
1973+
console.log('upload_object (tag) response@@@', inspect(upload_res));
1974+
//get tag for first version, with version id
1975+
const tag_res = await ns_tmp_tagging.get_object_tagging({
1976+
bucket: upload_bkt_tagging,
1977+
key: upload_key_2,
1978+
version_id: upload_res.version_id,
1979+
}, dummy_object_sdk);
1980+
await validate_tagging(tag_res.tagging, tagging_1);
1981+
//get tag for first version, without version id
1982+
const tag_res_1 = await ns_tmp_tagging.get_object_tagging({
1983+
bucket: upload_bkt_tagging,
1984+
key: upload_key_2,
1985+
}, dummy_object_sdk);
1986+
await validate_tagging(tag_res_1.tagging, tagging_1);
1987+
const params2 = {
1988+
bucket: upload_bkt_tagging,
1989+
key: upload_key_2,
1990+
source_stream: buffer_utils.buffer_to_read_stream(data),
1991+
tagging: tagging_2,
1992+
};
1993+
const upload_res1 = await ns_tmp_tagging.upload_object(params2, dummy_object_sdk);
1994+
console.log('upload_object (tag) response', inspect(upload_res1));
1995+
1996+
//get tag for lates version, with version id
1997+
const tag_res1 = await ns_tmp_tagging.get_object_tagging({
1998+
bucket: upload_bkt_tagging,
1999+
key: upload_key_2,
2000+
version_id: upload_res1.version_id,
2001+
}, dummy_object_sdk);
2002+
await validate_tagging(tag_res1.tagging, tagging_2);
2003+
//get tag for lates version, without version id
2004+
const tag_res1_1 = await ns_tmp_tagging.get_object_tagging({
2005+
bucket: upload_bkt_tagging,
2006+
key: upload_key_2,
2007+
}, dummy_object_sdk);
2008+
await validate_tagging(tag_res1_1.tagging, tagging_2);
2009+
2010+
});
2011+
2012+
});
2013+
});
17672014
});
17682015

2016+
async function validate_tagging(tag_res, tag_req) {
2017+
let count = 0;
2018+
if (tag_res.length === 0) {
2019+
assert.fail('object tag should not be empty');
2020+
}
2021+
for (const tagging of tag_res) {
2022+
assert.deepStrictEqual(tagging.key, tag_req[count].key);
2023+
assert.deepStrictEqual(tagging.value, tag_req[count].value);
2024+
count += 1;
2025+
}
2026+
}
2027+
17692028
//simulates object_sdk.fix_copy_source_params filtering of source xattr for copy object tests
17702029
async function _get_source_copy_xattr(copy_source, source_ns, object_sdk) {
17712030
const read_md_res = await source_ns.read_object_md({

0 commit comments

Comments
 (0)