Merge pull request #8961 from nadavMiz/lifecycle-delete-marker-expiration1

NC | lifecycle | add expire delete marker rule

Author: nadavMiz

src/manage_nsfs/nc_lifecycle.js

@@ -369,7 +369,12 @@ class NCLifecycle {
             candidates.delete_candidates = await this.get_candidates_by_expiration_rule(lifecycle_rule, bucket_json,
                 object_sdk);
             if (lifecycle_rule.expiration.days || lifecycle_rule.expiration.expired_object_delete_marker) {
-                const dm_candidates = await this.get_candidates_by_expiration_delete_marker_rule(lifecycle_rule, bucket_json);
+                const dm_candidates = await this.get_candidates_by_expiration_delete_marker_rule(
+                    lifecycle_rule,
+                    bucket_json,
+                    object_sdk,
+                    {versions_list}
+                );
                 candidates.delete_candidates = candidates.delete_candidates.concat(dm_candidates);
             }
         }
@@ -439,21 +444,21 @@ class NCLifecycle {
      * @param {Object} object_sdk
      * @param {Object} lifecycle_rule
      * @param {Object} bucket_json
-     * @param {Object} rule_state
+     * @param {Object} expire_state
      * @returns
      */
-    async load_objects_list(object_sdk, lifecycle_rule, bucket_json, rule_state) {
+    async load_objects_list(object_sdk, lifecycle_rule, bucket_json, expire_state) {
         const objects_list = await object_sdk.list_objects({
             bucket: bucket_json.name,
             prefix: lifecycle_rule.filter?.prefix,
-            key_marker: rule_state.key_marker,
+            key_marker: expire_state.key_marker,
             limit: config.NC_LIFECYCLE_LIST_BATCH_SIZE
         });
         if (objects_list.is_truncated) {
-            rule_state.key_marker = objects_list.next_marker;
-            rule_state.is_finished = false;
+            expire_state.key_marker = objects_list.next_marker;
+            expire_state.is_finished = false;
         } else {
-            rule_state.is_finished = true;
+            expire_state.is_finished = true;
         }
         const bucket_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].state;
         bucket_state.num_processed_objects += objects_list.objects.length;
@@ -503,15 +508,51 @@ class NCLifecycle {
         return parsed_candidates;
     }
 
+    /**
+     * check if delete candidate based on expired delete marker rule
+     * @param {Object} object
+     * @param {Object} next_object
+     * @param {Function} filter_func
+     * @returns
+     */
+    filter_expired_delete_marker(object, next_object, filter_func) {
+        const lifecycle_info = this._get_lifecycle_object_info_from_list_object_entry(object);
+        if (!filter_func(lifecycle_info)) return false;
+        return object.is_latest && object.delete_marker && object.key !== next_object.key;
+    }
+
     /**
      * get_candidates_by_expiration_delete_marker_rule processes the expiration delete marker rule
      * @param {*} lifecycle_rule
      * @param {Object} bucket_json
      * @returns {Promise<Object[]>}
      */
-    async get_candidates_by_expiration_delete_marker_rule(lifecycle_rule, bucket_json) {
-        // TODO - implement
-        return [];
+    async get_candidates_by_expiration_delete_marker_rule(lifecycle_rule, bucket_json, object_sdk, {versions_list}) {
+        const rule_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state.noncurrent;
+        if (rule_state.is_finished) return [];
+        if (!versions_list) {
+            versions_list = await this.load_versions_list(object_sdk, lifecycle_rule, bucket_json, rule_state);
+        }
+        const candidates = [];
+        const expiration = lifecycle_rule.expiration?.days ? this._get_expiration_time(lifecycle_rule.expiration) : 0;
+        const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration});
+        for (let i = 0; i < versions_list.objects.length - 1; i++) {
+            if (this.filter_expired_delete_marker(versions_list.objects[i], versions_list.objects[i + 1], filter_func)) {
+                candidates.push(versions_list.objects[i]);
+            }
+        }
+        const last_item = versions_list.objects.length > 0 && versions_list.objects[versions_list.objects.length - 1];
+        const lifecycle_info = this._get_lifecycle_object_info_from_list_object_entry(last_item);
+        if (last_item.is_latest && last_item.delete_marker && filter_func(lifecycle_info)) {
+            if (rule_state.is_finished) {
+                candidates.push(last_item);
+            } else {
+                //need the next item to decide if we need to delete. start the next cycle from this key latest
+                rule_state.key_marker_versioned = last_item.key;
+                rule_state.version_id_marker = undefined;
+            }
+        }
+        return candidates;
     }
 
     /////////////////////////////////////////////
@@ -522,23 +563,23 @@ class NCLifecycle {
      * @param {Object} object_sdk
      * @param {Object} lifecycle_rule
      * @param {Object} bucket_json
-     * @param {Object} rule_state
+     * @param {Object} noncurrent_state
      * @returns
      */
-    async load_versions_list(object_sdk, lifecycle_rule, bucket_json, rule_state) {
+    async load_versions_list(object_sdk, lifecycle_rule, bucket_json, noncurrent_state) {
         const list_versions = await object_sdk.list_object_versions({
             bucket: bucket_json.name,
             prefix: lifecycle_rule.filter?.prefix,
             limit: config.NC_LIFECYCLE_LIST_BATCH_SIZE,
-            key_marker: rule_state.key_marker_versioned,
-            version_id_marker: rule_state.version_id_marker
+            key_marker: noncurrent_state.key_marker_versioned,
+            version_id_marker: noncurrent_state.version_id_marker
         });
         if (list_versions.is_truncated) {
-            rule_state.is_finished = false;
-            rule_state.key_marker_versioned = list_versions.next_marker;
-            rule_state.version_id_marker = list_versions.next_version_id_marker;
+            noncurrent_state.is_finished = false;
+            noncurrent_state.key_marker_versioned = list_versions.next_marker;
+            noncurrent_state.version_id_marker = list_versions.next_version_id_marker;
         } else {
-            rule_state.is_finished = true;
+            noncurrent_state.is_finished = true;
         }
         const bucket_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].state;
         bucket_state.num_processed_objects += list_versions.objects.length;

src/test/unit_tests/jest_tests/test_nc_lifecycle_cli.test.js

@@ -407,6 +407,7 @@ describe('noobaa cli - lifecycle versioning ENABLE', () => {
     const test_bucket_path = `${root_path}/${test_bucket}`;
     const test_key1 = 'test_key1';
     const test_key2 = 'test_key2';
+    const test_key3 = 'test_key3';
     const prefix = 'test/';
     const test_prefix_key = `${prefix}test_key1`;
     let object_sdk;
@@ -556,6 +557,120 @@ describe('noobaa cli - lifecycle versioning ENABLE', () => {
         });
         expect(has_delete_marker).toBe(true);
     });
+
+    it('lifecycle_cli - expiration rule - expire delete marker ', async () => {
+        const lifecycle_rule = [
+                {
+                "id": "expiration after 3 days with tags",
+                "status": "Enabled",
+                "filter": {
+                    "prefix": '',
+                },
+                "expiration": {
+                    "expired_object_delete_marker": true
+                }
+            }
+        ];
+        await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule });
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key1});
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key2});
+
+        await create_object(object_sdk, test_bucket, test_prefix_key, 100, false);
+        await object_sdk.delete_object({bucket: test_bucket, key: test_prefix_key});
+
+        await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
+        const object_list = await object_sdk.list_object_versions({bucket: test_bucket});
+        expect(object_list.objects.length).toBe(2);
+        object_list.objects.forEach(element => {
+            expect(element.key).toBe(test_prefix_key);
+        });
+    });
+
+    it('lifecycle_cli - expiration rule - expire delete marker with filter', async () => {
+        const lifecycle_rule = [
+                {
+                "id": "expiration after 3 days with tags",
+                "status": "Enabled",
+                "filter": {
+                    "prefix": prefix,
+                    "object_size_less_than": 1
+                },
+                "expiration": {
+                    "expired_object_delete_marker": true
+                }
+            }
+        ];
+        await object_sdk.set_bucket_versioning({name: test_bucket, versioning: "ENABLED"});
+        await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule });
+
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key1});
+        await object_sdk.delete_object({bucket: test_bucket, key: test_prefix_key});
+
+        await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
+        const object_list = await object_sdk.list_object_versions({bucket: test_bucket});
+        expect(object_list.objects.length).toBe(1);
+        expect(object_list.objects[0].key).toBe(test_key1);
+    });
+
+    it('lifecycle_cli - expiration rule - expire delete marker last item', async () => {
+        const lifecycle_rule = [
+                {
+                "id": "expiration after 3 days with tags",
+                "status": "Enabled",
+                "filter": {
+                    "prefix": '',
+                    "object_size_less_than": 1
+                },
+                "expiration": {
+                    "expired_object_delete_marker": true
+                }
+            }
+        ];
+        await object_sdk.set_bucket_versioning({name: test_bucket, versioning: "ENABLED"});
+        await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule });
+
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key1});
+
+        await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
+        const object_list = await object_sdk.list_object_versions({bucket: test_bucket});
+        expect(object_list.objects.length).toBe(0);
+    });
+
+    it('lifecycle_cli - expiration rule - last item in batch is latest delete marker', async () => {
+        const lifecycle_rule = [
+                {
+                "id": "expiration after 3 days with tags",
+                "status": "Enabled",
+                "filter": {
+                    "prefix": '',
+                    "object_size_less_than": 1
+                },
+                "expiration": {
+                    "expired_object_delete_marker": true
+                }
+            }
+        ];
+        await config_fs.create_config_json_file(JSON.stringify({
+            NC_LIFECYCLE_LIST_BATCH_SIZE: 3,
+            NC_LIFECYCLE_BUCKET_BATCH_SIZE: 3,
+        }));
+        await object_sdk.set_bucket_versioning({name: test_bucket, versioning: "ENABLED"});
+        await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule });
+
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key1});
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key1});
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key2}); //last in batch should not delete
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key2});
+        await object_sdk.delete_object({bucket: test_bucket, key: test_key3}); // last in batch should delete
+        await object_sdk.delete_object({bucket: test_bucket, key: 'test_key4'});
+
+        await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined);
+        const object_list = await object_sdk.list_object_versions({bucket: test_bucket});
+        expect(object_list.objects.length).toBe(4);
+        object_list.objects.forEach(element => {
+            expect([test_key1, test_key2]).toContain(element.key);
+        });
+    });
 });