Merge pull request #8844 from naveenpaul1/lifecycle-md-multipart

lifecycle | lifecycle pre-test setup

Author: naveenpaul1

src/sdk/object_io.js

@@ -290,7 +290,9 @@ class ObjectIO {
                 await this._upload_stream(params, complete_params);
             }
             dbg.log0('upload_multipart: complete upload', complete_params);
-            return params.client.object.complete_multipart(complete_params);
+            const multipart_params = await params.client.object.complete_multipart(complete_params);
+            multipart_params.multipart_id = complete_params.multipart_id;
+            return multipart_params;
         } catch (err) {
             dbg.warn('upload_multipart: failed', complete_params, err);
             // we leave the cleanup of failed multiparts to complete_object_upload or abort_object_upload

src/test/lifecycle/common.js

@@ -389,6 +389,51 @@ function duplicate_id_lifecycle_configuration(Bucket, Key) {
     };
 }
 
+function version_lifecycle_configuration(Bucket, Key, Days, ExpiredDeleteMarker, NewnonCurrentVersion, NonCurrentDays) {
+    const ID = 'rule_id';
+    return {
+        Bucket,
+        LifecycleConfiguration: {
+            Rules: [{
+                ID,
+                Filter: {
+                    Prefix: Key,
+                },
+                Expiration: {
+                    Days: Days,
+                    ExpiredObjectDeleteMarker: ExpiredDeleteMarker,
+                },
+                NoncurrentVersionExpiration: {
+                    NewerNoncurrentVersions: NewnonCurrentVersion,
+                    NoncurrentDays: NonCurrentDays,
+                },
+                Status: 'Enabled',
+            }, ],
+        },
+    };
+}
+exports.version_lifecycle_configuration = version_lifecycle_configuration;
+
+function multipart_lifecycle_configuration(Bucket, Key, Days) {
+    const ID = 'rule_id';
+    return {
+        Bucket,
+        LifecycleConfiguration: {
+            Rules: [{
+                ID,
+                Filter: {
+                    Prefix: Key,
+                },
+                AbortIncompleteMultipartUpload: {
+                    DaysAfterInitiation: Days,
+                },
+                Status: 'Enabled',
+            }, ],
+        },
+    };
+}
+exports.multipart_lifecycle_configuration = multipart_lifecycle_configuration;
+
 async function put_get_lifecycle_configuration(Bucket, putLifecycleParams, s3) {
     const putLifecycleResult = await s3.putBucketLifecycleConfiguration(putLifecycleParams);
     console.log('put lifecycle params:', putLifecycleParams, 'result', putLifecycleResult);
@@ -409,6 +454,27 @@ async function put_get_lifecycle_configuration(Bucket, putLifecycleParams, s3) {
     return getLifecycleResult;
 }
 
+exports.test_multipart = async function(Bucket, Key, s3) {
+    const putLifecycleParams = multipart_lifecycle_configuration(Bucket, Key, 10);
+    const getLifecycleResult = await put_get_lifecycle_configuration(Bucket, putLifecycleParams, s3);
+
+    const expirationDays = getLifecycleResult.Rules[0].AbortIncompleteMultipartUpload.DaysAfterInitiation.Days;
+    const expectedExpirationDays = putLifecycleParams.LifecycleConfiguration.Rules[0]
+                                        .AbortIncompleteMultipartUpload.DaysAfterInitiation.Days;
+    console.log('get lifecycle multipart expiration:', expirationDays, ' expected:', expectedExpirationDays);
+    assert(expirationDays === expectedExpirationDays, 'Multipart Expiration days do not match');
+};
+
+exports.test_version = async function(Bucket, Key, s3) {
+    const putLifecycleParams = version_lifecycle_configuration(Bucket, Key, 10, true, 5, 10);
+    const getLifecycleResult = await put_get_lifecycle_configuration(Bucket, putLifecycleParams, s3);
+
+    const expirationDays = getLifecycleResult.Rules[0].Expiration.Days;
+    const expectedExpirationDays = putLifecycleParams.LifecycleConfiguration.Rules[0].Expiration.Days;
+    console.log('get lifecycle version expiration:', expirationDays, ' expected:', expectedExpirationDays);
+    assert(expirationDays === expectedExpirationDays, 'Expiration days do not match');
+};
+
 exports.test_rules_length = async function(Bucket, Key, s3) {
     const putLifecycleParams = rules_length_lifecycle_configuration(Bucket, Key);
     const getLifecycleResult = await put_get_lifecycle_configuration(Bucket, putLifecycleParams, s3);

src/test/unit_tests/test_lifecycle.js

@@ -10,14 +10,20 @@ const mocha = require('mocha');
 const assert = require('assert');
 const mongodb = require('mongodb');
 const { v4: uuid } = require('uuid');
+const _ = require('lodash');
+const crypto = require('crypto');
+const stream = require('stream');
 
+const ObjectIO = require('../../sdk/object_io');
 const P = require('../../util/promise');
 const config = require('../../../config');
 const MDStore = require('../../server/object_services/md_store').MDStore;
 const coretest = require('./coretest');
 const lifecycle = require('../../server/bg_services/lifecycle');
 const http_utils = require('../../util/http_utils');
 const commonTests = require('../lifecycle/common');
+const seed = crypto.randomBytes(16);
+const generator = crypto.createCipheriv('aes-128-gcm', seed, Buffer.alloc(12));
 
 const { rpc_client, EMAIL } = coretest;
 const Bucket = 'first.bucket';
@@ -27,6 +33,10 @@ const TagName2 = 'tagname2';
 const TagValue = 'tagvalue';
 const TagValue2 = 'tagvalue2';
 
+const object_io = new ObjectIO();
+object_io.set_verification_mode();
+
+
 mocha.describe('lifecycle', () => {
 
     let s3;
@@ -85,6 +95,12 @@ mocha.describe('lifecycle', () => {
         mocha.it('test and prefix size', async () => {
             await commonTests.test_and_prefix_size(Bucket, Key, s3);
         });
+        mocha.it('test version', async () => {
+            await commonTests.test_version(Bucket, Key, s3);
+        });
+        mocha.it('test multipath', async () => {
+            await commonTests.test_multipart(Bucket, Key, s3);
+        });
     });
 
     mocha.describe('bucket-lifecycle-bg-worker', function() {
@@ -216,4 +232,131 @@ mocha.describe('lifecycle', () => {
 
         console.log('✅ The lifecycle test was completed successfully');
     });
+
+    mocha.describe('bucket-lifecycle-multipart-upload', function() {
+        this.timeout(60000);
+        const multipart_bucket = 'test-multipart-bucket';
+        mocha.after(async function() {
+            //TODO Delete bucket
+            //await rpc_client.bucket.delete_bucket({ name: multipart_bucket });
+        });
+        async function create_mock_multipart_upload(key, bucket, age, part_size, num_parts) {
+            await rpc_client.bucket.create_bucket({ name: bucket });
+            const content_type = 'test/test';
+            const size = num_parts * part_size;
+            const data = generator.update(Buffer.alloc(size));
+            const { obj_id } = await rpc_client.object.create_object_upload({ bucket, key, content_type });
+            const mp_list_before = await rpc_client.object.list_multiparts({ obj_id, bucket, key });
+            coretest.log('list_multiparts before', mp_list_before);
+            assert.strictEqual(mp_list_before.multiparts.length, 0);
+            const multiparts_ids = [];
+
+            const get_part_slice = i => data.slice(i * part_size, (i + 1) * part_size);
+            const upload_multipart = async (i, mp_data, split, finish) => {
+                const resp = await object_io.upload_multipart({
+                    client: rpc_client,
+                    obj_id,
+                    bucket,
+                    key,
+                    num: i + 1,
+                    size: mp_data.length,
+                    source_stream: readable_buffer(mp_data, split, finish),
+                });
+                console.log("upload_multipart", resp);
+                multiparts_ids.push(new mongodb.ObjectId(resp.multipart_id));
+            };
+            // upload the real multiparts we want to complete with
+            await Promise.all(_.times(num_parts,
+                i => upload_multipart(i, get_part_slice(i))
+            ));
+
+            // go back in time
+            const create_time = new Date();
+            create_time.setDate(create_time.getDate() - age);
+            const update = {
+                create_time,
+            };
+
+            console.log('create_mock_multipart_upload bucket', bucket, 'obj_id', obj_id, 'multiparts_ids', multiparts_ids);
+            await MDStore.instance().update_multiparts_by_ids(multiparts_ids, update);
+
+            const mp_list_after = await rpc_client.object.list_multiparts({ obj_id, bucket, key });
+            coretest.log('mp_list_after after', mp_list_after);
+            assert.strictEqual(mp_list_after.multiparts.length, num_parts);
+            const actual_create_time = mp_list_after.multiparts[0].last_modified;
+            assert.strictEqual(actual_create_time, create_time.getTime(), `object create_time/getTime actual ${actual_create_time} !== expected ${create_time.getTime()}`);
+        }
+
+        mocha.it('lifecyle - listMultiPart verify', async () => {
+            await create_mock_multipart_upload('test-lifecycle-multipart', multipart_bucket, 3, 45, 7);
+        });
+    });
+
+    mocha.describe('bucket-lifecycle-version', function() {
+        this.timeout(60000);
+        const version_bucket = 'test-version-bucket';
+        mocha.after(async function() {
+            //TODO Delete bucket
+            //await rpc_client.bucket.delete_bucket({ name: version_bucket });
+        });
+
+        async function create_mock_version(version_key, bucket, age, version_count) {
+            await rpc_client.bucket.create_bucket({ name: bucket });
+            rpc_client.bucket.update_bucket({
+                name: bucket,
+                versioning: 'ENABLED'
+            });
+
+            const obj_upload_ids = [];
+            for (let i = 0; i < version_count; ++i) {
+                const content_type = 'application/octet_stream';
+                const { obj_id } = await rpc_client.object.create_object_upload({ bucket, key: version_key, content_type });
+                await rpc_client.object.complete_object_upload({ obj_id, bucket, key: version_key });
+                if (i < version_count - 2) {
+                    obj_upload_ids.push(new mongodb.ObjectId(obj_id));
+                }
+            }
+            // go back in time
+            if (age > 0) {
+                const create_time = new Date();
+                create_time.setDate(create_time.getDate() - age);
+                const update = {
+                    create_time,
+                };
+                console.log('blow_version_objects: bucket', bucket, 'multiparts_ids', obj_upload_ids, " obj_upload_ids length: ", obj_upload_ids.length, "update :", update);
+                const update_result = await MDStore.instance().update_objects_by_ids(obj_upload_ids, update);
+                console.log('blow_version_objects: update_objects_by_ids', update_result);
+            }
+
+            const obj_params = {
+                bucket,
+            };
+            const list_obj = await rpc_client.object.list_object_versions(obj_params);
+            console.log("List updated objects : ", list_obj);
+            assert.strictEqual(list_obj.objects.length, version_count, `object total count  ${list_obj.objects.length} !== expected ${version_count}`);
+        }
+
+        mocha.it('lifecyle - version expiration', async () => {
+            await create_mock_version('test-lifecycle-version', version_bucket, 30, 10);
+        });
+    });
+
+    function readable_buffer(data, split = 1, finish = 'end') {
+        const max = Math.ceil(data.length / split);
+        let pos = 0;
+        return new stream.Readable({
+            read() {
+                if (pos < data.length) {
+                    const len = Math.min(data.length - pos, max);
+                    const buf = data.slice(pos, pos + len);
+                    pos += len;
+                    setImmediate(() => this.push(buf));
+                } else if (finish === 'fail') {
+                    this.emit('error', new Error('TEST_OBJECT_IO FAIL ON FINISH'));
+                } else {
+                    this.push(null);
+                }
+            }
+        });
+    }
 });