NC | Lifecycle | Adjust expire/noncurrent state properties to GPFS flow

Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com>

Author: romayalon

src/manage_nsfs/nc_lifecycle.js

@@ -465,7 +465,7 @@ class NCLifecycle {
      * @returns {Promise<Object[]>}
      */
     async get_candidates_by_expiration_rule_posix(lifecycle_rule, bucket_json, object_sdk) {
-        const rule_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state.expire;
+        const rule_state = this._get_rule_state(bucket_json, lifecycle_rule).expire;
         if (rule_state.is_finished) return [];
         const expiration = this._get_expiration_time(lifecycle_rule.expiration);
         if (expiration < 0) return [];
@@ -520,7 +520,7 @@ class NCLifecycle {
      * @returns {Promise<Object[]>}
      */
     async get_candidates_by_expiration_delete_marker_rule(lifecycle_rule, bucket_json, object_sdk, {versions_list}) {
-        const rule_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state.noncurrent;
+        const rule_state = this._get_rule_state(bucket_json, lifecycle_rule).noncurrent;
         if (rule_state.is_finished) return [];
         if (!versions_list) {
             versions_list = await this.load_versions_list(object_sdk, lifecycle_rule, bucket_json, rule_state);
@@ -619,7 +619,7 @@ class NCLifecycle {
      * @returns {Promise<Object[]>}
      */
     async get_candidates_by_noncurrent_version_expiration_rule(lifecycle_rule, bucket_json, object_sdk, {versions_list}) {
-        const rule_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state.noncurrent;
+        const rule_state = this._get_rule_state(bucket_json, lifecycle_rule).noncurrent;
         if (rule_state.is_finished) return [];
 
         if (!versions_list) {
@@ -965,7 +965,7 @@ class NCLifecycle {
      */
     init_rule_status(bucket_name, rule_id) {
         this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id] ??= {};
-        this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].state ??= {expire: {}, noncurrent: {}};
+        this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].state ??= { expire: {}, noncurrent: {} };
         this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_process_times = {};
         this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_stats ??= {};
         return this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id];
@@ -977,9 +977,9 @@ class NCLifecycle {
      * @param {string} rule_id
      */
     update_rule_status_is_finished(bucket_name, rule_id) {
-        const rule_state = this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].state;
-        rule_state.is_finished = (rule_state.expire.is_finished === undefined || rule_state.expire.is_finished === true) &&
-            (rule_state.noncurrent.is_finished === undefined || rule_state.noncurrent.is_finished === true);
+        const rule_state = this._get_rule_state({ name: bucket_name }, { id: rule_id });
+        rule_state.is_finished = (rule_state.expire?.is_finished === undefined || rule_state.expire?.is_finished === true) &&
+            (rule_state.noncurrent?.is_finished === undefined || rule_state.noncurrent?.is_finished === true);
     }
 
     /**
@@ -1011,7 +1011,10 @@ class NCLifecycle {
      * _set_rule_state sets the current rule state on the lifecycle run status
      * @param {Object} bucket_json
      * @param {*} lifecycle_rule
-     * @param {{is_finished?: Boolean | Undefined, candidates_file_offset?: number | undefined}} rule_state
+     * @param {{
+     * is_finished?: Boolean | Undefined,
+     * expire?: {is_finished?: Boolean | Undefined, key_marker?: String | Undefined, candidates_file_offset?: number | undefined}
+     * noncurrent?: {is_finished?: Boolean | Undefined, key_marker_versioned?: String | Undefined, version_id_marker?: String | Undefined }}} rule_state
      * @returns {Void}
      */
     _set_rule_state(bucket_json, lifecycle_rule, rule_state) {
@@ -1022,7 +1025,10 @@ class NCLifecycle {
      * _get_rule_state gets the current rule state on the lifecycle run status
      * @param {Object} bucket_json
      * @param {*} lifecycle_rule
-     * @returns {{is_finished?: Boolean | Undefined, candidates_file_offset?: number | undefined}} rule_state
+     * @returns {{
+     * is_finished: Boolean | Undefined,
+     * expire?: {is_finished?: Boolean | Undefined, key_marker?: String | Undefined, candidates_file_offset?: number | undefined}
+     * noncurrent?: {is_finished?: Boolean | Undefined, key_marker_versioned?: String | Undefined, version_id_marker?: String | Undefined }}} rule_state
      */
     _get_rule_state(bucket_json, lifecycle_rule) {
         return this.lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state;
@@ -1164,6 +1170,7 @@ class NCLifecycle {
         for (const bucket_name of bucket_names) {
             const bucket_json = await this.config_fs.get_bucket_by_name(bucket_name, config_fs_options);
             const bucket_mount_point = this.find_mount_point_by_bucket_path(mount_point_to_policy_map, bucket_json.path);
+            if (!bucket_json.lifecycle_configuration_rules?.length) continue;
             for (const lifecycle_rule of bucket_json.lifecycle_configuration_rules) {
                 // currently we support expiration (current version) only
                 if (lifecycle_rule.expiration) {
@@ -1377,22 +1384,28 @@ class NCLifecycle {
      * 1.2.2. parse the key from the candidate line
      * 1.2.3. push the key to the candidates array
      * 2. if candidates file does not exist, we return without error because it's valid that no candidates found
+     * GAP - when supporting noncurrent rule, we should update the state type to noncurrent based on the candidates file path
      * @param {Object} bucket_json
      * @param {*} lifecycle_rule
      * @param {String} rule_candidates_path
      * @returns {Promise<Object[]>} parsed_candidates_array
      */
     async parse_candidates_from_gpfs_ilm_policy(bucket_json, lifecycle_rule, rule_candidates_path) {
         let reader;
+        const state_type = 'expire';
+        const rule_state = this._get_rule_state(bucket_json, lifecycle_rule)?.[state_type];
+        if (rule_state?.is_finished) return [];
+        const finished_state = { [state_type]: { is_finished: true, candidates_file_offset: undefined } };
+
         try {
-            const rule_state = this._get_rule_state(bucket_json, lifecycle_rule);
-            dbg.log2(`parse_candidates_from_gpfs_ilm_policy bucket_name=${bucket_json.name}, rule_id ${lifecycle_rule.id}, existing rule_state=${util.inspect(rule_state)}`);
 
+            dbg.log2(`parse_candidates_from_gpfs_ilm_policy bucket_name=${bucket_json.name}, rule_id ${lifecycle_rule.id}, existing rule_state=${util.inspect(rule_state)}`);
             const parsed_candidates_array = [];
             reader = new NewlineReader(this.non_gpfs_fs_context, rule_candidates_path, { lock: 'SHARED', read_file_offset: rule_state?.candidates_file_offset || 0 });
+
             const [count, is_finished] = await reader.forEachFilePathEntry(async entry => {
                 if (parsed_candidates_array.length >= config.NC_LIFECYCLE_LIST_BATCH_SIZE) return false;
-                const cur_rule_state = { is_finished: false, candidates_file_offset: reader.next_line_file_offset };
+                const cur_rule_state = { [state_type]: { is_finished: false, candidates_file_offset: reader.next_line_file_offset } };
                 this._set_rule_state(bucket_json, lifecycle_rule, cur_rule_state);
                 const key = this._parse_key_from_line(entry, bucket_json);
                 // TODO - need to add etag, size, version_id
@@ -1402,14 +1415,14 @@ class NCLifecycle {
             });
 
             if (is_finished) {
-                this._set_rule_state(bucket_json, lifecycle_rule, { is_finished: true, candidates_file_offset: undefined });
+                this._set_rule_state(bucket_json, lifecycle_rule, finished_state);
             }
             dbg.log2(`parse_candidates_from_gpfs_ilm_policy: parsed_candidates_array ${util.inspect(parsed_candidates_array)}, rule_state=${util.inspect(rule_state)}, count=${count} is_finished=${is_finished}`);
             return parsed_candidates_array;
         } catch (err) {
             if (err.code === 'ENOENT') {
                 dbg.log2(`parse_candidates_from_gpfs_ilm_policy ilm_candidates_file_exists does not exist, no candidates to delete`);
-                this._set_rule_state(bucket_json, lifecycle_rule, { is_finished: true, candidates_file_offset: undefined });
+                this._set_rule_state(bucket_json, lifecycle_rule, finished_state);
                 return;
             }
             dbg.error('parse_candidates_from_gpfs_ilm_policy: error', err);