Skip to content

Commit 2b84c69

Browse files
tangledbytestangledbytes
committed
encode copy-source in replication scanner
Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com> fix failing tests Signed-off-by: Utkarsh Srivastava <srivastavautkarsh8097@gmail.com>
1 parent c21797d commit 2b84c69

File tree

2 files changed

+44
-17
lines changed

2 files changed

+44
-17
lines changed

‎src/server/bg_services/replication_server.js

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ async function copy_objects_mixed_types(req) {
7979
if (keys_diff_map[key].length === 1) {
8080
const params = {
8181
Bucket: dst_bucket_name.unwrap(),
82-
CopySource: `/${src_bucket_name.unwrap()}/${key}`, // encodeURI for special chars is needed
82+
CopySource: encodeURI(`/${src_bucket_name.unwrap()}/${key}`),
8383
Key: key
8484
};
8585
try {
@@ -96,7 +96,7 @@ async function copy_objects_mixed_types(req) {
9696
const version = keys_diff_map[key][i].VersionId;
9797
const params = {
9898
Bucket: dst_bucket_name.unwrap(),
99-
CopySource: `/${src_bucket_name.unwrap()}/${key}?versionId=${version}`, // encodeURI for special chars is needed
99+
CopySource: encodeURI(`/${src_bucket_name.unwrap()}/${key}?versionId=${version}`),
100100
Key: key
101101
};
102102
try {

‎src/test/unit_tests/test_bucket_replication.js

Lines changed: 42 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,8 @@ mocha.describe('replication configuration bg worker tests', function() {
271271
const bucket_for_replications = 'bucket5-br-bg';
272272
const bucket_to_delete = 'bucket-to-delete';
273273
const buckets = [bucket1, bucket2, bucket_to_delete, bucket3, bucket4, bucket_for_replications];
274+
let uploaded_objects_count = 0;
275+
let uploaded_prefix_objects_count = 0;
274276
//const namespace_buckets = [];
275277
let s3_owner;
276278
let scanner;
@@ -282,6 +284,13 @@ mocha.describe('replication configuration bg worker tests', function() {
282284
region: 'us-east-1',
283285
httpOptions: { agent: new http.Agent({ keepAlive: false }) },
284286
};
287+
288+
// Special character items to ensure encoding of URI works OK in the replication scanner
289+
const special_character_items = [
290+
'key1273-2__#$!@%!#__BED-END-1-Carton-13.jpeg',
291+
'key1278-1__4267%2524__BED-END-1-Carton-13.jpeg'
292+
];
293+
285294
mocha.before('init scanner & populate buckets', async function() {
286295
// create buckets
287296
await P.all(_.map(buckets, async bucket_name => {
@@ -298,9 +307,23 @@ mocha.describe('replication configuration bg worker tests', function() {
298307
// populate buckets
299308
for (let i = 0; i < 10; i++) {
300309
let key = `key${i}`;
301-
if (i % 2 === 0) key = 'pref' + key;
310+
if (i % 2 === 0) {
311+
key = 'pref' + key;
312+
uploaded_prefix_objects_count += 1;
313+
}
302314
await put_object(s3_owner, bucket1, key);
315+
uploaded_objects_count += 1;
303316
}
317+
318+
// Add special characters items with prefix to the bucket
319+
await Promise.all(special_character_items.map(item => put_object(s3_owner, bucket1, 'pref' + item)));
320+
uploaded_objects_count += special_character_items.length;
321+
322+
// Add special characters items without prefix to the bucket
323+
await Promise.all(special_character_items.map(item => put_object(s3_owner, bucket1, item)));
324+
uploaded_objects_count += special_character_items.length;
325+
uploaded_prefix_objects_count += special_character_items.length;
326+
304327
cloud_utils.set_noobaa_s3_connection = () => {
305328
console.log('setting connection to coretest endpoint and access key');
306329
return s3_owner;
@@ -320,11 +343,13 @@ mocha.describe('replication configuration bg worker tests', function() {
320343
if (i % 2 === 0) key = 'pref' + key;
321344
await delete_object(s3_owner, bucket_name, key);
322345
}
346+
await Promise.all(special_character_items.map(item => delete_object(s3_owner, bucket_name, 'pref' + item)));
347+
await Promise.all(special_character_items.map(item => delete_object(s3_owner, bucket_name, item)));
323348
await rpc_client.bucket.delete_bucket({ name: bucket_name });
324349
}));
325350
});
326351

327-
mocha.it('run replication scanner and wait - no replication - nothing to upload', async function() {
352+
mocha.it('run replication scanner and wait - no replication rule - nothing to upload', async function() {
328353
const res1 = await scanner.run_batch();
329354
console.log('waiting for replication objects no objects to upload', res1);
330355
await _list_objects_and_wait(s3_owner, bucket_for_replications, 0);
@@ -345,16 +370,16 @@ mocha.describe('replication configuration bg worker tests', function() {
345370
[{ rule_id: 'rule-1', destination_bucket: bucket_for_replications, sync_versions: false, filter: { prefix: 'pref' } }], false);
346371
let res1 = await scanner.run_batch();
347372
console.log('waiting for replication objects - one rule one prefix', res1);
348-
let contents = await _list_objects_and_wait(s3_owner, bucket_for_replications, 5); //Check that the 5 objects were replicated
373+
let contents = await _list_objects_and_wait(s3_owner, bucket_for_replications, uploaded_prefix_objects_count); //Check that the desired objects were replicated
349374
console.log('contents', contents);
350375

351376
// delete object from dst
352377
await s3_owner.deleteObject({ Bucket: bucket_for_replications, Key: contents[0].Key }).promise();
353-
await _list_objects_and_wait(s3_owner, bucket_for_replications, 4); //Verify that the object was deleted
378+
await _list_objects_and_wait(s3_owner, bucket_for_replications, uploaded_prefix_objects_count - 1); //Verify that one object was deleted
354379
// sync again
355380
res1 = await scanner.run_batch();
356381
console.log('waiting for replication objects - one rule one prefix', res1);
357-
contents = await _list_objects_and_wait(s3_owner, bucket_for_replications, 5); //Check that the delete object was replicate again
382+
contents = await _list_objects_and_wait(s3_owner, bucket_for_replications, uploaded_prefix_objects_count); //Check that the delete object was replicate again
358383
const key1 = contents[0].Key;
359384
// override object in dst
360385
const dst_obj1 = await s3_owner.getObject({ Bucket: bucket_for_replications, Key: key1 }).promise();
@@ -421,7 +446,7 @@ mocha.describe('replication configuration bg worker tests', function() {
421446
});
422447

423448
mocha.it('run replication scanner and wait - no prefix - all objects should be uploaded', async function() {
424-
const contents = await _list_objects_and_wait(s3_owner, bucket_for_replications, 5);
449+
const contents = await _list_objects_and_wait(s3_owner, bucket_for_replications, uploaded_prefix_objects_count);
425450
for (const content of contents) {
426451
const key = content.Key;
427452
await s3_owner.deleteObject({ Bucket: bucket_for_replications, Key: key }).promise();
@@ -430,7 +455,7 @@ mocha.describe('replication configuration bg worker tests', function() {
430455
[{ rule_id: 'rule-1', destination_bucket: bucket_for_replications, sync_versions: false }], false);
431456
const res1 = await scanner.run_batch();
432457
console.log('waiting for replication objects - one rule no prefix', res1);
433-
await _list_objects_and_wait(s3_owner, bucket_for_replications, 10);
458+
await _list_objects_and_wait(s3_owner, bucket_for_replications, uploaded_objects_count);
434459
});
435460

436461
mocha.it('run replication scanner and wait - 2 prefixes - all objects should be uploaded', async function() {
@@ -440,14 +465,14 @@ mocha.describe('replication configuration bg worker tests', function() {
440465
{ rule_id: 'rule-2', destination_bucket: bucket2, sync_versions: false, filter: { prefix: 'pref' } }
441466
], false);
442467

443-
const res = await _list_objects_and_wait(s3_owner, bucket1, 10);
468+
const res = await _list_objects_and_wait(s3_owner, bucket1, uploaded_objects_count);
444469
console.log('waiting for replication objects original bucket ', res);
445470
let res1 = await scanner.run_batch();
446471
console.log('waiting for replication objects - 2 rules 1 prefix1 ', res1);
447-
await _list_objects_and_wait(s3_owner, bucket2, 5);
472+
await _list_objects_and_wait(s3_owner, bucket2, 5 + special_character_items.length);
448473
res1 = await scanner.run_batch();
449474
console.log('waiting for replication objects - 2 rules 1 prefix2 ', res1);
450-
await _list_objects_and_wait(s3_owner, bucket2, 10);
475+
await _list_objects_and_wait(s3_owner, bucket2, uploaded_objects_count);
451476
});
452477

453478
mocha.it('run replication scanner and wait - 2 buckets - all objects should be uploaded', async function() {
@@ -458,18 +483,20 @@ mocha.describe('replication configuration bg worker tests', function() {
458483
], false);
459484

460485
await _put_replication(bucket2,
461-
[{ rule_id: 'rule-1', destination_bucket: bucket4, sync_versions: false, filter: { prefix: 'key' } },
486+
[
487+
{ rule_id: 'rule-1', destination_bucket: bucket4, sync_versions: false, filter: { prefix: 'key' } },
462488
{ rule_id: 'rule-2', destination_bucket: bucket3, sync_versions: false, filter: { prefix: 'pref' } }
463489
], false);
464490
let res1 = await scanner.run_batch();
465491
console.log('waiting for replication objects - 2 rules 1 prefix1 ', res1);
466-
await _list_objects_and_wait(s3_owner, bucket3, 5);
467-
await _list_objects_and_wait(s3_owner, bucket4, 5);
492+
await _list_objects_and_wait(s3_owner, bucket3, 5 + special_character_items.length);
493+
await _list_objects_and_wait(s3_owner, bucket4, uploaded_prefix_objects_count);
468494

469495
res1 = await scanner.run_batch();
470496
console.log('waiting for replication objects - 2 rules 1 prefix2 ', res1);
471-
await _list_objects_and_wait(s3_owner, bucket3, 10);
472-
await _list_objects_and_wait(s3_owner, bucket4, 10);
497+
// everything is uploaded by combination of above 2
498+
await _list_objects_and_wait(s3_owner, bucket3, uploaded_objects_count);
499+
await _list_objects_and_wait(s3_owner, bucket4, uploaded_objects_count);
473500
});
474501

475502
});

0 commit comments

Comments
 (0)