diff --git a/config.js b/config.js index 4d8fd53fd4..30651a52ec 100644 --- a/config.js +++ b/config.js @@ -951,6 +951,33 @@ config.NSFS_GLACIER_FORCE_EXPIRE_ON_GET = false; // interval config.NSFS_GLACIER_MIGRATE_LOG_THRESHOLD = 50 * 1024; +/** + * NSFS_GLACIER_RESERVED_BUCKET_TAGS defines an object of bucket tags which will be reserved + * by the system and PUT operations for them via S3 API would be limited - as in they would be + * mutable only if specified and only under certain conditions. + * + * @type {Record & { $id: string }, + * immutable: true | false | 'if-data', + * default: any, + * event: boolean + * }>} + * + * @example + * { + 'deep-archive-copies': { + schema: { + $id: 'deep-archive-copies-schema-v0', + enum: ['1', '2'] + }, // JSON Schema + immutable: 'if-data', + default: '1', + event: true + } + * } + */ +config.NSFS_GLACIER_RESERVED_BUCKET_TAGS = {}; + // anonymous account name config.ANONYMOUS_ACCOUNT_NAME = 'anonymous'; @@ -1030,7 +1057,7 @@ config.NC_LIFECYCLE_TZ = 'LOCAL'; config.NC_LIFECYCLE_LIST_BATCH_SIZE = 1000; config.NC_LIFECYCLE_BUCKET_BATCH_SIZE = 10000; -config.NC_LIFECYCLE_GPFS_ILM_ENABLED = false; +config.NC_LIFECYCLE_GPFS_ILM_ENABLED = true; ////////// GPFS ////////// config.GPFS_DOWN_DELAY = 1000; diff --git a/docs/NooBaaNonContainerized/CI&Tests.md b/docs/NooBaaNonContainerized/CI&Tests.md index 511bc92a9b..34c8f04867 100644 --- a/docs/NooBaaNonContainerized/CI&Tests.md +++ b/docs/NooBaaNonContainerized/CI&Tests.md @@ -114,6 +114,8 @@ The following is a list of `NC jest tests` files - 17. `test_nc_upgrade_manager.test.js` - Tests of the NC upgrade manager. 18. `test_cli_upgrade.test.js` - Tests of the upgrade CLI commands. 19. `test_nc_online_upgrade_cli_integrations.test.js` - Tests CLI commands during mocked config directory upgrade. +21. `test_nc_lifecycle_posix_integration.test` - Tests NC lifecycle POSIX related configuration. +(Note: in this layer we do not test the validation related to lifecycle configuration and it is done in `test_lifecycle.js` - which currently is running only in containerized deployment, but it is mutual code) #### nc_index.js File * The `nc_index.js` is a file that runs several NC and NSFS mocha related tests. diff --git a/docs/NooBaaNonContainerized/Events.md b/docs/NooBaaNonContainerized/Events.md index 57b15862a2..4b9f1b3b16 100644 --- a/docs/NooBaaNonContainerized/Events.md +++ b/docs/NooBaaNonContainerized/Events.md @@ -32,7 +32,10 @@ The following list includes events that indicate on a normal / successful operat - Description: NooBaa account was deleted successfully using NooBaa CLI. #### 4. `noobaa_bucket_created` -- Arguments: `bucket_name` +- Arguments: + - `bucket_name` + - `account_name` + - `` (if `event` is `true` for the reserved tag) - Description: NooBaa bucket was created successfully using NooBaa CLI or S3. #### 5. `noobaa_bucket_deleted` @@ -43,6 +46,11 @@ The following list includes events that indicate on a normal / successful operat - Arguments: `whitelist_ips` - Description: Whitelist Server IPs updated successfully using NooBaa CLI. +#### 7. `noobaa_bucket_reserved_tag_modified` +- Arguments: + - `bucket_name` + - `` (if `event` is `true` for the reserved tag) +- Description: NooBaa bucket reserved tag was modified successfully using NooBaa CLI or S3. ### Error Indicating Events @@ -219,4 +227,4 @@ The following list includes events that indicate on some sort of malfunction or - Reasons: - Free space in notification log dir FS is below threshold. - Resolutions: - - Free up space is FS. \ No newline at end of file + - Free up space is FS. diff --git a/docs/NooBaaNonContainerized/NooBaaCLI.md b/docs/NooBaaNonContainerized/NooBaaCLI.md index 51258c156c..4eba67a3fe 100644 --- a/docs/NooBaaNonContainerized/NooBaaCLI.md +++ b/docs/NooBaaNonContainerized/NooBaaCLI.md @@ -376,6 +376,13 @@ noobaa-cli bucket update --name [--new_name] [--owner] - Type: Boolean - Description: Set the bucket to force md5 ETag calculation. +- `tag` + - Type: String + - Description: Set the bucket tags, type is a string of valid JSON. Behaviour is similar to `put-bucket-tagging` S3 API. + +- `merge_tag` + - Type: String + - Description: Merge the bucket tags with previous bucket tags, type is a string of valid JSON. ### Bucket Status diff --git a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md index 67cee759b3..1785e49399 100644 --- a/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md +++ b/docs/dev_guide/ceph_s3_tests/ceph_s3_tests_pending_list_status.md @@ -54,4 +54,5 @@ change in our repo) - stopped passing between the update of commit hash 6861c3d8 | test_get_bucket_encryption_s3 | Faulty Test | [613](https://github.com/ceph/s3-tests/issues/613) | | test_get_bucket_encryption_kms | Faulty Test | [613](https://github.com/ceph/s3-tests/issues/613) | | test_delete_bucket_encryption_s3 | Faulty Test | [613](https://github.com/ceph/s3-tests/issues/613) | -| test_delete_bucket_encryption_kms | Faulty Test | [613](https://github.com/ceph/s3-tests/issues/613) | \ No newline at end of file +| test_delete_bucket_encryption_kms | Faulty Test | [613](https://github.com/ceph/s3-tests/issues/613) | +| test_lifecycle_expiration_tags1 | Faulty Test | [638](https://github.com/ceph/s3-tests/issues/638) | There can be more such tests having the same issue (`Filter` is not aligned with aws structure in bucket lifecycle configuration) | \ No newline at end of file diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js index c45199f0be..e955b65dca 100644 --- a/src/cmd/manage_nsfs.js +++ b/src/cmd/manage_nsfs.js @@ -28,7 +28,7 @@ const { account_id_cache } = require('../sdk/accountspace_fs'); const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const ManageCLIResponse = require('../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; const manage_nsfs_glacier = require('../manage_nsfs/manage_nsfs_glacier'); -const noobaa_cli_lifecycle = require('../manage_nsfs/nc_lifecycle'); +const { NCLifecycle } = require('../manage_nsfs/nc_lifecycle'); const manage_nsfs_logging = require('../manage_nsfs/manage_nsfs_logging'); const noobaa_cli_diagnose = require('../manage_nsfs/diagnose'); const noobaa_cli_upgrade = require('../manage_nsfs/upgrade'); @@ -40,6 +40,8 @@ const { throw_cli_error, get_bucket_owner_account_by_name, const manage_nsfs_validations = require('../manage_nsfs/manage_nsfs_validations'); const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance(); const notifications_util = require('../util/notifications_util'); +const BucketSpaceFS = require('../sdk/bucketspace_fs'); +const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; let config_fs; @@ -123,7 +125,6 @@ async function fetch_bucket_data(action, user_input) { force_md5_etag: user_input.force_md5_etag === undefined || user_input.force_md5_etag === '' ? user_input.force_md5_etag : get_boolean_or_string_value(user_input.force_md5_etag), notifications: user_input.notifications }; - if (user_input.bucket_policy !== undefined) { if (typeof user_input.bucket_policy === 'string') { // bucket_policy deletion specified with empty string '' @@ -142,6 +143,27 @@ async function fetch_bucket_data(action, user_input) { data = await merge_new_and_existing_config_data(data); } + if ((action === ACTIONS.UPDATE && user_input.tag) || (action === ACTIONS.ADD)) { + const tags = JSON.parse(user_input.tag || '[]'); + data.tag = BucketSpaceFS._merge_reserved_tags( + data.tag || BucketSpaceFS._default_bucket_tags(), + tags, + action === ACTIONS.ADD ? true : await _is_bucket_empty(data), + ); + } + + if ((action === ACTIONS.UPDATE && user_input.merge_tag) || (action === ACTIONS.ADD)) { + const merge_tags = JSON.parse(user_input.merge_tag || '[]'); + data.tag = _.merge( + data.tag, + BucketSpaceFS._merge_reserved_tags( + data.tag || BucketSpaceFS._default_bucket_tags(), + merge_tags, + action === ACTIONS.ADD ? true : await _is_bucket_empty(data), + ) + ); + } + //if we're updating the owner, needs to override owner in file with the owner from user input. //if we're adding a bucket, need to set its owner id field if ((action === ACTIONS.UPDATE && user_input.owner) || (action === ACTIONS.ADD)) { @@ -189,7 +211,14 @@ async function add_bucket(data) { data._id = mongo_utils.mongoObjectId(); const parsed_bucket_data = await config_fs.create_bucket_config_file(data); await set_bucker_owner(parsed_bucket_data); - return { code: ManageCLIResponse.BucketCreated, detail: parsed_bucket_data, event_arg: { bucket: data.name }}; + + const [reserved_tag_event_args] = BucketSpaceFS._generate_reserved_tag_event_args({}, data.tag); + + return { + code: ManageCLIResponse.BucketCreated, + detail: parsed_bucket_data, + event_arg: { ...(reserved_tag_event_args || {}), bucket: data.name, account: parsed_bucket_data.bucket_owner }, + }; } /** @@ -245,25 +274,14 @@ async function update_bucket(data) { */ async function delete_bucket(data, force) { try { - const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id); + const bucket_empty = await _is_bucket_empty(data); + if (!bucket_empty && !force) { + throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name); + } + const bucket_temp_dir_path = native_fs_utils.get_bucket_tmpdir_full_path(data.path, data._id); - // fs_contexts for bucket temp dir (storage path) const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend); - let entries; - try { - entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path); - } catch (err) { - dbg.warn(`delete_bucket: bucket name ${data.name},` + - `got an error on readdir with path: ${data.path}`, err); - // if the bucket's path was deleted first (encounter ENOENT error) - continue deletion - if (err.code !== 'ENOENT') throw err; - } - if (entries) { - const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name)); - if (object_entries.length > 0 && !force) { - throw_cli_error(ManageCLIError.BucketDeleteForbiddenHasObjects, data.name); - } - } + await native_fs_utils.folder_delete(bucket_temp_dir_path, fs_context_fs_backend, true); await config_fs.delete_bucket_config_file(data.name); return { code: ManageCLIResponse.BucketDeleted, detail: { name: data.name }, event_arg: { bucket: data.name } }; @@ -273,6 +291,33 @@ async function delete_bucket(data, force) { } } +/** + * _is_bucket_empty returns true if the given bucket is empty + * + * @param {*} data + * @returns {Promise} + */ +async function _is_bucket_empty(data) { + const temp_dir_name = native_fs_utils.get_bucket_tmpdir_name(data._id); + // fs_contexts for bucket temp dir (storage path) + const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend); + let entries; + try { + entries = await nb_native().fs.readdir(fs_context_fs_backend, data.path); + } catch (err) { + dbg.warn(`_is_bucket_empty: bucket name ${data.name},` + + `got an error on readdir with path: ${data.path}`, err); + // if the bucket's path was deleted first (encounter ENOENT error) - continue deletion + if (err.code !== 'ENOENT') throw err; + } + if (entries) { + const object_entries = entries.filter(element => !element.name.endsWith(temp_dir_name)); + return object_entries.length === 0; + } + + return true; +} + /** * bucket_management does the following - * 1. fetches the bucket data if this is not a list operation @@ -294,7 +339,24 @@ async function bucket_management(action, user_input) { } else if (action === ACTIONS.STATUS) { response = await get_bucket_status(data); } else if (action === ACTIONS.UPDATE) { - response = await update_bucket(data); + const bucket_path = config_fs.get_bucket_path_by_name(user_input.name); + const bucket_lock_file = `${bucket_path}.lock`; + await native_fs_utils.lock_and_run(config_fs.fs_context, bucket_lock_file, async () => { + const prev_bucket_info = await fetch_bucket_data(action, _.omit(user_input, ['tag', 'merge_tag'])); + const bucket_info = await fetch_bucket_data(action, user_input); + + const tagging_object = BucketSpaceFS._objectify_tagging_arr(prev_bucket_info.tag); + const [ + reserved_tag_event_args, + reserved_tag_modified, + ] = BucketSpaceFS._generate_reserved_tag_event_args(tagging_object, bucket_info.tag); + + response = await update_bucket(bucket_info); + if (reserved_tag_modified) { + new NoobaaEvent(NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED) + .create_event(undefined, { ...reserved_tag_event_args, bucket_name: user_input.name }); + } + }); } else if (action === ACTIONS.DELETE) { const force = get_boolean_or_string_value(user_input.force); response = await delete_bucket(data, force); @@ -814,9 +876,11 @@ async function lifecycle_management(args) { const disable_service_validation = get_boolean_or_string_value(args.disable_service_validation); const disable_runtime_validation = get_boolean_or_string_value(args.disable_runtime_validation); const short_status = get_boolean_or_string_value(args.short_status); + const should_continue_last_run = get_boolean_or_string_value(args.continue); try { - const options = { disable_service_validation, disable_runtime_validation, short_status }; - const { should_run, lifecycle_run_status } = await noobaa_cli_lifecycle.run_lifecycle_under_lock(config_fs, options); + const options = { disable_service_validation, disable_runtime_validation, short_status, should_continue_last_run }; + const nc_lifecycle = new NCLifecycle(config_fs, options); + const { should_run, lifecycle_run_status } = await nc_lifecycle.run_lifecycle_under_lock(); if (should_run) { write_stdout_response(ManageCLIResponse.LifecycleSuccessful, lifecycle_run_status); } else { diff --git a/src/endpoint/s3/ops/s3_put_bucket_lifecycle.js b/src/endpoint/s3/ops/s3_put_bucket_lifecycle.js index d1c7a6c532..b5d4627a67 100644 --- a/src/endpoint/s3/ops/s3_put_bucket_lifecycle.js +++ b/src/endpoint/s3/ops/s3_put_bucket_lifecycle.js @@ -9,6 +9,63 @@ const S3Error = require('../s3_errors').S3Error; const true_regex = /true/i; +/** + * validate_lifecycle_rule validates lifecycle rule structure and logical constraints + * + * validations: + * - ID must be ≤ MAX_RULE_ID_LENGTH + * - Status must be "Enabled" or "Disabled" + * - multiple Filters must be under "And" + * - only one Expiration field is allowed + * - Expiration.Date must be midnight UTC format + * - AbortIncompleteMultipartUpload cannot be combined with Tags or ObjectSize filters + * + * @param {Object} rule - lifecycle rule to validate + * @throws {S3Error} - on validation failure + */ +function validate_lifecycle_rule(rule) { + + if (rule.ID?.length === 1 && rule.ID[0].length > s3_const.MAX_RULE_ID_LENGTH) { + dbg.error('Rule should not have ID length exceed allowed limit of ', s3_const.MAX_RULE_ID_LENGTH, ' characters', rule); + throw new S3Error({ ...S3Error.InvalidArgument, message: `ID length should not exceed allowed limit of ${s3_const.MAX_RULE_ID_LENGTH}` }); + } + + if (!rule.Status || rule.Status.length !== 1 || + (rule.Status[0] !== s3_const.LIFECYCLE_STATUS.STAT_ENABLED && rule.Status[0] !== s3_const.LIFECYCLE_STATUS.STAT_DISABLED)) { + dbg.error(`Rule should have a status value of "${s3_const.LIFECYCLE_STATUS.STAT_ENABLED}" or "${s3_const.LIFECYCLE_STATUS.STAT_DISABLED}".`, rule); + throw new S3Error(S3Error.MalformedXML); + } + + if (rule.Filter?.[0] && Object.keys(rule.Filter[0]).length > 1 && !rule.Filter[0]?.And) { + dbg.error('Rule should combine multiple filters using "And"', rule); + throw new S3Error(S3Error.MalformedXML); + } + + if (rule.Expiration?.[0] && Object.keys(rule.Expiration[0]).length > 1) { + dbg.error('Rule should specify only one expiration field: Days, Date, or ExpiredObjectDeleteMarker', rule); + throw new S3Error(S3Error.MalformedXML); + } + + if (rule.Expiration?.length === 1 && rule.Expiration[0]?.Date) { + const date = new Date(rule.Expiration[0].Date[0]); + if (isNaN(date.getTime()) || date.getTime() !== Date.UTC(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())) { + dbg.error('Date value must conform to the ISO 8601 format and at midnight UTC (00:00:00). Provided:', rule.Expiration[0].Date[0]); + throw new S3Error({ ...S3Error.InvalidArgument, message: "'Date' must be at midnight GMT" }); + } + } + + if (rule.AbortIncompleteMultipartUpload?.length === 1 && rule.Filter?.length === 1) { + if (rule.Filter[0]?.Tag) { + dbg.error('Rule should not include AbortIncompleteMultipartUpload with Tags', rule); + throw new S3Error({ ...S3Error.InvalidArgument, message: 'AbortIncompleteMultipartUpload cannot be specified with Tags' }); + } + if (rule.Filter[0]?.ObjectSizeGreaterThan || rule.Filter[0]?.ObjectSizeLessThan) { + dbg.error('Rule should not include AbortIncompleteMultipartUpload with Object Size', rule); + throw new S3Error({ ...S3Error.InvalidArgument, message: 'AbortIncompleteMultipartUpload cannot be specified with Object Size' }); + } + } +} + // parse lifecycle rule filter function parse_filter(filter) { const current_rule_filter = {}; @@ -89,13 +146,11 @@ async function put_bucket_lifecycle(req) { filter: {}, }; + // validate rule + validate_lifecycle_rule(rule); + if (rule.ID?.length === 1) { - if (rule.ID[0].length > s3_const.MAX_RULE_ID_LENGTH) { - dbg.error('Rule should not have ID length exceed allowed limit of ', s3_const.MAX_RULE_ID_LENGTH, ' characters', rule); - throw new S3Error({ ...S3Error.InvalidArgument, message: `ID length should not exceed allowed limit of ${s3_const.MAX_RULE_ID_LENGTH}` }); - } else { - current_rule.id = rule.ID[0]; - } + current_rule.id = rule.ID[0]; } else { // Generate a random ID if missing current_rule.id = crypto.randomUUID(); @@ -108,11 +163,6 @@ async function put_bucket_lifecycle(req) { } id_set.add(current_rule.id); - if (!rule.Status || rule.Status.length !== 1 || - (rule.Status[0] !== s3_const.LIFECYCLE_STATUS.STAT_ENABLED && rule.Status[0] !== s3_const.LIFECYCLE_STATUS.STAT_DISABLED)) { - dbg.error(`Rule should have a status value of "${s3_const.LIFECYCLE_STATUS.STAT_ENABLED}" or "${s3_const.LIFECYCLE_STATUS.STAT_DISABLED}".`, rule); - throw new S3Error(S3Error.MalformedXML); - } current_rule.status = rule.Status[0]; if (rule.Prefix) { diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js index 5f4c07888d..b7943b1bd7 100644 --- a/src/manage_nsfs/health.js +++ b/src/manage_nsfs/health.js @@ -3,7 +3,6 @@ const dbg = require('../util/debug_module')(__filename); const _ = require('lodash'); -const path = require('path'); const P = require('../util/promise'); const config = require('../../config'); const os_util = require('../util/os_utils'); @@ -18,6 +17,7 @@ const { get_boolean_or_string_value, throw_cli_error, write_stdout_response, const { ManageCLIResponse } = require('./manage_nsfs_cli_responses'); const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError; const notifications_util = require('../util/notifications_util'); +const lifecycle_utils = require('../util/lifecycle_utils'); const HOSTNAME = 'localhost'; @@ -462,7 +462,9 @@ class NSFSHealth { * @returns {Promise} */ async get_lifecycle_health_status() { - const latest_lifecycle_run_status = await this.get_latest_lifecycle_run_status({ silent_if_missing: true }); + const latest_lifecycle_run_status = await lifecycle_utils.get_latest_nc_lifecycle_run_status( + this.config_fs, + { silent_if_missing: true }); if (!latest_lifecycle_run_status) return {}; return { total_stats: latest_lifecycle_run_status.total_stats, @@ -471,29 +473,6 @@ class NSFSHealth { }; } - - /** - * get_latest_lifecycle_run_status returns the latest lifecycle run status - * latest run can be found by maxing the lifecycle log entry names, log entry name is the lifecycle_run_{timestamp}.json of the run - * @params {{silent_if_missing: boolean}} options - * @returns {Promise} - */ - async get_latest_lifecycle_run_status(options) { - const { silent_if_missing = false } = options; - try { - const lifecycle_log_entries = await nb_native().fs.readdir(this.config_fs.fs_context, config.NC_LIFECYCLE_LOGS_DIR); - const latest_lifecycle_run = _.maxBy(lifecycle_log_entries, entry => entry.name); - const latest_lifecycle_run_status_path = path.join(config.NC_LIFECYCLE_LOGS_DIR, latest_lifecycle_run.name); - const latest_lifecycle_run_status = await this.config_fs.get_config_data(latest_lifecycle_run_status_path, options); - return latest_lifecycle_run_status; - } catch (err) { - if (err.code === 'ENOENT' && silent_if_missing) { - return; - } - throw err; - } - } - /** * get_config_file_data_or_error_object return an object containing config_data or err_obj if error occurred * @param {string} type diff --git a/src/manage_nsfs/manage_nsfs_cli_errors.js b/src/manage_nsfs/manage_nsfs_cli_errors.js index 554ede9674..8b2e461763 100644 --- a/src/manage_nsfs/manage_nsfs_cli_errors.js +++ b/src/manage_nsfs/manage_nsfs_cli_errors.js @@ -3,6 +3,7 @@ const config = require('../../config'); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; +const { TYPES } = require('../manage_nsfs/manage_nsfs_constants'); // by default NC_DISABLE_POSIX_MODE_ACCESS_CHECK=true, therefore CLI access check of account/bucket will be based on stat (open file) // which checks only read permissions. @@ -94,7 +95,7 @@ ManageCLIError.InvalidArgumentType = Object.freeze({ ManageCLIError.InvalidType = Object.freeze({ code: 'InvalidType', - message: 'Invalid type, available types are account, bucket, logging, whitelist, upgrade, notification or connection.', + message: `Invalid type, valid types are ${Object.values(TYPES).join(', ')}.`, http_code: 400, }); diff --git a/src/manage_nsfs/manage_nsfs_cli_utils.js b/src/manage_nsfs/manage_nsfs_cli_utils.js index 0a851c459e..fd69138c52 100644 --- a/src/manage_nsfs/manage_nsfs_cli_utils.js +++ b/src/manage_nsfs/manage_nsfs_cli_utils.js @@ -124,7 +124,7 @@ async function get_options_from_file(file_path) { // we don't pass neither config_root_backend nor fs_backend const fs_context = native_fs_utils.get_process_fs_context(); try { - const input_options_with_data = await native_fs_utils.read_file(fs_context, file_path); + const input_options_with_data = await native_fs_utils.read_file(fs_context, file_path, { parse_json: true }); return input_options_with_data; } catch (err) { if (err.code === 'ENOENT') throw_cli_error(ManageCLIError.InvalidFilePath, file_path); diff --git a/src/manage_nsfs/manage_nsfs_constants.js b/src/manage_nsfs/manage_nsfs_constants.js index 0aeea99b01..0f0aeca4ac 100644 --- a/src/manage_nsfs/manage_nsfs_constants.js +++ b/src/manage_nsfs/manage_nsfs_constants.js @@ -62,7 +62,7 @@ const VALID_OPTIONS_ANONYMOUS_ACCOUNT = { const VALID_OPTIONS_BUCKET = { 'add': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'force_md5_etag', 'notifications', FROM_FILE, ...CLI_MUTUAL_OPTIONS]), - 'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', ...CLI_MUTUAL_OPTIONS]), + 'update': new Set(['name', 'owner', 'path', 'bucket_policy', 'fs_backend', 'new_name', 'force_md5_etag', 'notifications', 'tag', 'merge_tag', ...CLI_MUTUAL_OPTIONS]), 'delete': new Set(['name', 'force', ...CLI_MUTUAL_OPTIONS]), 'list': new Set(['wide', 'name', ...CLI_MUTUAL_OPTIONS]), 'status': new Set(['name', ...CLI_MUTUAL_OPTIONS]), @@ -96,7 +96,7 @@ const VALID_OPTIONS_CONNECTION = { 'status': new Set(['name', 'decrypt', ...CLI_MUTUAL_OPTIONS]), }; -const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', 'short_status', ...CLI_MUTUAL_OPTIONS]); +const VALID_OPTIONS_LIFECYCLE = new Set(['disable_service_validation', 'disable_runtime_validation', 'short_status', 'continue', ...CLI_MUTUAL_OPTIONS]); const VALID_OPTIONS_WHITELIST = new Set(['ips', ...CLI_MUTUAL_OPTIONS]); @@ -159,7 +159,8 @@ const OPTION_TYPE = { // lifecycle options disable_service_validation: 'boolean', disable_runtime_validation: 'boolean', - short: 'boolean', + short_status: 'boolean', + continue: 'boolean', //connection notification_protocol: 'string', agent_request_object: 'string', @@ -170,11 +171,15 @@ const OPTION_TYPE = { key: 'string', value: 'string', remove_key: 'boolean', + // bucket tagging + tag: 'string', + merge_tag: 'string', }; const BOOLEAN_STRING_VALUES = ['true', 'false']; const BOOLEAN_STRING_OPTIONS = new Set(['allow_bucket_creation', 'regenerate', 'wide', 'show_secrets', 'force', - 'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous', 'disable_service_validation', 'disable_runtime_validation', 'short_status']); + 'force_md5_etag', 'iam_operate_on_root_account', 'all_account_details', 'all_bucket_details', 'anonymous', + 'disable_service_validation', 'disable_runtime_validation', 'short_status', 'continue']); //options that can be unset using '' const LIST_UNSETABLE_OPTIONS = ['fs_backend', 's3_policy', 'force_md5_etag']; diff --git a/src/manage_nsfs/manage_nsfs_events_utils.js b/src/manage_nsfs/manage_nsfs_events_utils.js index dfa566015a..a74a49b3a5 100644 --- a/src/manage_nsfs/manage_nsfs_events_utils.js +++ b/src/manage_nsfs/manage_nsfs_events_utils.js @@ -302,6 +302,16 @@ NoobaaEvent.UNAUTHORIZED = Object.freeze({ severity: 'ERROR', state: 'HEALTHY', }); +NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED = Object.freeze({ + event_code: 'noobaa_bucket_reserved_tag_modified', + message: 'Bucket reserved tag modified', + description: 'Noobaa bucket reserved tag modified', + entity_type: 'NODE', + event_type: 'INFO', + scope: 'NODE', + severity: 'INFO', + state: 'HEALTHY', +}); NoobaaEvent.IO_STREAM_ITEM_TIMEOUT = Object.freeze({ event_code: 'bucket_io_stream_item_timeout', diff --git a/src/manage_nsfs/nc_lifecycle.js b/src/manage_nsfs/nc_lifecycle.js index af1ad041e6..f679aab3c3 100644 --- a/src/manage_nsfs/nc_lifecycle.js +++ b/src/manage_nsfs/nc_lifecycle.js @@ -8,38 +8,37 @@ const path = require('path'); const util = require('util'); const P = require('../util/promise'); const config = require('../../config'); +const os_utils = require('../util/os_utils'); const nb_native = require('../util/nb_native'); +const { CONFIG_TYPES } = require('../sdk/config_fs'); const NsfsObjectSDK = require('../sdk/nsfs_object_sdk'); +const { NewlineReader } = require('../util/file_reader'); +const lifecycle_utils = require('../util/lifecycle_utils'); const native_fs_utils = require('../util/native_fs_utils'); +const SensitiveString = require('../util/sensitive_string'); const { NoobaaEvent } = require('./manage_nsfs_events_utils'); const notifications_util = require('../util/notifications_util'); const ManageCLIError = require('./manage_nsfs_cli_errors').ManageCLIError; const { throw_cli_error, get_service_status, NOOBAA_SERVICE_NAME, is_desired_time, record_current_time } = require('./manage_nsfs_cli_utils'); -const SensitiveString = require('../util/sensitive_string'); // TODO: // implement -// 1. notifications -// 2. POSIX scanning and filtering per rule +// 2. POSIX scanning and filtering per rule - non current + expired delete marker // 3. GPFS ILM policy and apply for scanning and filtering optimization // TODO - we will filter during the scan except for get_candidates_by_expiration_rule on GPFS that does the filter on the file system const LIFECYCLE_CLUSTER_LOCK = 'lifecycle_cluster.lock'; const LIFECYLE_TIMESTAMP_FILE = 'lifecycle.timestamp'; const config_fs_options = { silent_if_missing: true }; +const ILM_POLICIES_TMP_DIR = path.join(config.NC_LIFECYCLE_LOGS_DIR, 'lifecycle_ilm_policies'); +const ILM_CANDIDATES_TMP_DIR = path.join(config.NC_LIFECYCLE_LOGS_DIR, 'lifecycle_ilm_candidates'); -const lifecycle_run_status = { - running_host: os.hostname(), lifecycle_run_times: {}, - total_stats: _get_default_stats(), buckets_statuses: {}, - state: {is_finished: false} -}; - -let return_short_status = false; const TIMED_OPS = Object.freeze({ RUN_LIFECYLE: 'run_lifecycle', LIST_BUCKETS: 'list_buckets', + CREATE_GPFS_CANDIDATES_FILES: 'create_gpfs_candidates_files', PROCESS_BUCKETS: 'process_buckets', PROCESS_BUCKET: 'process_bucket', PROCESS_RULE: 'process_rule', @@ -49,802 +48,1461 @@ const TIMED_OPS = Object.freeze({ }); /** - * run_lifecycle_under_lock runs the lifecycle workflow under a file system lock - * lifecycle workflow is being locked to prevent multiple instances from running the lifecycle workflow - * @param {import('../sdk/config_fs').ConfigFS} config_fs - * @param {{disable_service_validation?: boolean, disable_runtime_validation?: boolean, short_status?: boolean}} flags - * @returns {Promise<{should_run: Boolean, lifecycle_run_status: Object}>} - */ -async function run_lifecycle_under_lock(config_fs, flags) { - const { disable_service_validation = false, disable_runtime_validation = false, short_status = false } = flags; - return_short_status = short_status; - const fs_context = config_fs.fs_context; - const lifecyle_logs_dir_path = config.NC_LIFECYCLE_LOGS_DIR; - const lifecycle_config_dir_path = path.join(config_fs.config_root, config.NC_LIFECYCLE_CONFIG_DIR_NAME); - const lock_path = path.join(lifecycle_config_dir_path, LIFECYCLE_CLUSTER_LOCK); - const lifecycle_timestamp_file_path = path.join(lifecycle_config_dir_path, LIFECYLE_TIMESTAMP_FILE); - await config_fs.create_dir_if_missing(lifecyle_logs_dir_path); - await config_fs.create_dir_if_missing(lifecycle_config_dir_path); - - let should_run = true; - await native_fs_utils.lock_and_run(fs_context, lock_path, async () => { - dbg.log0('run_lifecycle_under_lock acquired lock - verifying'); - should_run = await _should_lifecycle_run(fs_context, lifecycle_timestamp_file_path, disable_runtime_validation); - if (!should_run) return; + * @typedef {{ + * is_finished?: Boolean | Undefined, + * expire?: { is_finished?: Boolean | Undefined, key_marker?: String | Undefined, candidates_file_offset?: number | undefined} + * noncurrent?: { is_finished?: Boolean | Undefined, key_marker_versioned?: String | Undefined, version_id_marker?: String | Undefined } + * }} RuleState +*/ - try { - dbg.log0('run_lifecycle_under_lock acquired lock - start lifecycle'); - new NoobaaEvent(NoobaaEvent.LIFECYCLE_STARTED).create_event(); - await run_lifecycle_or_timeout(config_fs, disable_service_validation); - } catch (err) { - dbg.error('run_lifecycle_under_lock failed with error', err, err.code, err.message); - throw err; - } finally { - await record_current_time(fs_context, lifecycle_timestamp_file_path); - await write_lifecycle_log_file(config_fs.fs_context, lifecyle_logs_dir_path); - dbg.log0('run_lifecycle_under_lock done lifecycle - released lock'); +class NCLifecycle { + constructor(config_fs, options = {}) { + this.lifecycle_config_dir_path = path.join(config_fs.config_root, config.NC_LIFECYCLE_CONFIG_DIR_NAME); + this.lifecyle_logs_dir_path = config.NC_LIFECYCLE_LOGS_DIR; + this.config_fs = config_fs; + this.fs_context = config_fs.fs_context; + // used for reading/writing policies/candidates file which are not on gpfs file system + this.non_gpfs_fs_context = { ...this.fs_context, backend: undefined }; + this.lock_path = path.join(this.lifecycle_config_dir_path, LIFECYCLE_CLUSTER_LOCK); + this.lifecycle_timestamp_file_path = path.join(this.lifecycle_config_dir_path, LIFECYLE_TIMESTAMP_FILE); + + this.lifecycle_run_status = { + running_host: os.hostname(), + lifecycle_run_times: {}, + total_stats: this._get_default_stats(), + state: { is_finished: false }, + buckets_statuses: {} + }; + this.return_short_status = options.short_status || false; + this.disable_service_validation = options.disable_service_validation || false; + this.disable_runtime_validation = options.disable_runtime_validation || false; + this.should_continue_last_run = options.should_continue_last_run || false; + } + + /** + * run_lifecycle_under_lock runs the lifecycle workflow under a file system lock + * lifecycle workflow is being locked to prevent multiple instances from running the lifecycle workflow + * @returns {Promise<{should_run: Boolean, lifecycle_run_status: Object}>} + */ + async run_lifecycle_under_lock() { + await native_fs_utils._create_path(this.lifecyle_logs_dir_path, this.non_gpfs_fs_context, config.BASE_MODE_CONFIG_DIR); + await this.config_fs.create_dir_if_missing(this.lifecycle_config_dir_path); + + let should_run = true; + await native_fs_utils.lock_and_run(this.fs_context, this.lock_path, async () => { + dbg.log0('run_lifecycle_under_lock acquired lock - verifying'); + should_run = await this._should_lifecycle_run(); + if (!should_run) return; + + try { + dbg.log0('run_lifecycle_under_lock acquired lock - start lifecycle'); + new NoobaaEvent(NoobaaEvent.LIFECYCLE_STARTED).create_event(); + await this.run_lifecycle_or_timeout(); + } catch (err) { + dbg.error('run_lifecycle_under_lock failed with error', err, err.code, err.message); + throw err; + } finally { + await record_current_time(this.fs_context, this.lifecycle_timestamp_file_path); + await this.write_lifecycle_log_file(); + dbg.log0('run_lifecycle_under_lock done lifecycle - released lock'); + } + }); + return { should_run, lifecycle_run_status: this.lifecycle_run_status }; + } + + /** + * run_lifecycle_or_timeout runs the lifecycle workflow or times out while calculating + * and saving times and stats of the run on the global lifecycle status + * @returns {Promise} + */ + async run_lifecycle_or_timeout() { + await this._call_op_and_update_status({ + op_name: TIMED_OPS.RUN_LIFECYLE, + op_func: async () => { + await P.timeout( + config.NC_LIFECYCLE_TIMEOUT_MS, + this.run_lifecycle(), + () => ManageCLIError.LifecycleWorkerReachedTimeout + ); + } + }); + } + + /** + * run_lifecycle runs the lifecycle workflow + * @returns {Promise} + */ + async run_lifecycle() { + const system_json = await this.config_fs.get_system_config_file(config_fs_options); + if (!this.disable_service_validation) await this.throw_if_noobaa_not_active(system_json); + + const bucket_names = await this._call_op_and_update_status({ + op_name: TIMED_OPS.LIST_BUCKETS, + op_func: async () => this.config_fs.list_buckets() + }); + + if (this.should_continue_last_run) { + await this.load_previous_run_state(bucket_names); } - }); - return { should_run, lifecycle_run_status }; -} -/** - * run_lifecycle_or_timeout runs the lifecycle workflow or times out while calculating - * and saving times and stats of the run on the global lifecycle status - * @param {import('../sdk/config_fs').ConfigFS} config_fs - * @param {boolean} disable_service_validation - * @returns {Promise} - */ -async function run_lifecycle_or_timeout(config_fs, disable_service_validation) { - await _call_op_and_update_status({ - op_name: TIMED_OPS.RUN_LIFECYLE, - op_func: async () => { - await P.timeout( - config.NC_LIFECYCLE_TIMEOUT_MS, - run_lifecycle(config_fs, disable_service_validation), - () => ManageCLIError.LifecycleWorkerReachedTimeout + await this._call_op_and_update_status({ + op_name: TIMED_OPS.PROCESS_BUCKETS, + op_func: async () => this.process_buckets(bucket_names, system_json) + }); + } + + /** + * process_buckets does the following - + * 1. if it's a GPFS optimization - create candidates files + * 2. iterates over buckets and handles their rules + * @param {String[]} bucket_names + * @param {Object} system_json + * @returns {Promise} + */ + async process_buckets(bucket_names, system_json) { + const buckets_concurrency = 10; // TODO - think about it + + if (this._should_use_gpfs_optimization()) { + await this._call_op_and_update_status({ + op_name: TIMED_OPS.CREATE_GPFS_CANDIDATES_FILES, + op_func: async () => this.create_gpfs_candidates_files(bucket_names) + }); + } + + while (!this.lifecycle_run_status.state.is_finished) { + await P.map_with_concurrency(buckets_concurrency, bucket_names, async bucket_name => + await this._call_op_and_update_status({ + bucket_name, + op_name: TIMED_OPS.PROCESS_BUCKET, + op_func: async () => this.process_bucket(bucket_name, system_json) + }) + ); + this.lifecycle_run_status.state.is_finished = Object.values(this.lifecycle_run_status.buckets_statuses).reduce( + (acc, bucket) => acc && (bucket.state?.is_finished), + true ); } - }); -} + } -/** - * run_lifecycle runs the lifecycle workflow - * @param {import('../sdk/config_fs').ConfigFS} config_fs - * @param {boolean} disable_service_validation - * @returns {Promise} - */ -async function run_lifecycle(config_fs, disable_service_validation) { - const system_json = await config_fs.get_system_config_file(config_fs_options); - if (!disable_service_validation) await throw_if_noobaa_not_active(config_fs, system_json); - - const bucket_names = await _call_op_and_update_status({ - op_name: TIMED_OPS.LIST_BUCKETS, - op_func: async () => config_fs.list_buckets() - }); - - await _call_op_and_update_status({ - op_name: TIMED_OPS.PROCESS_BUCKETS, - op_func: async () => process_buckets(config_fs, bucket_names, system_json) - }); -} + /** + * process_bucket processes the lifecycle rules for a bucket + * @param {string} bucket_name + * @param {Object} system_json + */ + async process_bucket(bucket_name, system_json) { + const bucket_json = await this.config_fs.get_bucket_by_name(bucket_name, config_fs_options); + const account = await this.config_fs.get_identity_by_id(bucket_json.owner_account, CONFIG_TYPES.ACCOUNT, + { silent_if_missing: true }); + if (!account) { + dbg.warn(`process_bucket - bucket owner ${bucket_json.owner_account} does not exist for bucket ${bucket_name}. skipping lifecycle for this bucket`); + return; + } + const object_sdk = new NsfsObjectSDK('', this.config_fs, account, bucket_json.versioning, this.config_fs.config_root, system_json); + await object_sdk._simple_load_requesting_account(); + const should_notify = notifications_util.should_notify_on_event(bucket_json, notifications_util.OP_TO_EVENT.lifecycle_delete.name); + if (!bucket_json.lifecycle_configuration_rules) { + this.lifecycle_run_status.buckets_statuses[bucket_json.name].state = { is_finished: true }; + return; + } + await this.process_rules(bucket_json, object_sdk, should_notify); + } -/** - * process_buckets iterates over buckets and handles their rules - * @param {import('../sdk/config_fs').ConfigFS} config_fs - * @param {String[]} bucket_names - * @param {Object} system_json - * @returns {Promise} - */ -async function process_buckets(config_fs, bucket_names, system_json) { - const buckets_concurrency = 10; // TODO - think about it - while (!lifecycle_run_status.state.is_finished) { - await P.map_with_concurrency(buckets_concurrency, bucket_names, async bucket_name => - await _call_op_and_update_status({ + /** + * process_rules processes the lifecycle rules for a bucket + * @param {Object} bucket_json + * @param {nb.ObjectSDK} object_sdk + */ + async process_rules(bucket_json, object_sdk, should_notify) { + try { + const bucket_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].state; + bucket_state.num_processed_objects = 0; + while (!bucket_state.is_finished && bucket_state.num_processed_objects < config.NC_LIFECYCLE_BUCKET_BATCH_SIZE) { + await P.all(_.map(bucket_json.lifecycle_configuration_rules, + async (lifecycle_rule, index) => + await this._call_op_and_update_status({ + bucket_name: bucket_json.name, + rule_id: lifecycle_rule.id, + op_name: TIMED_OPS.PROCESS_RULE, + op_func: async () => this.process_rule( + lifecycle_rule, + index, + bucket_json, + object_sdk, + should_notify + ) + }) + ) + ); + bucket_state.is_finished = Object.values(this.lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses) + .reduce( + (acc, rule) => acc && (_.isEmpty(rule.state) || rule.state.is_finished), + true + ); + } + } catch (err) { + dbg.error('process_rules failed with error', err, err.code, err.message); + } + } + + /** + * process_rule processes the lifecycle rule for a bucket + * TODO - implement notifications for the deleted objects (check if needed for abort mpus as well) + * @param {Object} lifecycle_rule + * @param {number} index + * @param {Object} bucket_json + * @param {nb.ObjectSDK} object_sdk + * @returns {Promise} + */ + async process_rule(lifecycle_rule, index, bucket_json, object_sdk, should_notify) { + dbg.log0('nc_lifecycle.process_rule: start bucket name:', bucket_json.name, 'rule', lifecycle_rule, 'index', index); + const bucket_name = bucket_json.name; + const rule_id = lifecycle_rule.id; + const should_process_lifecycle_rule = this.validate_rule_enabled(lifecycle_rule, bucket_json); + if (!should_process_lifecycle_rule) return; + + dbg.log0('nc_lifecycle.process_rule: processing rule:', bucket_name, '(bucket id:', bucket_json._id, ') rule', util.inspect(lifecycle_rule)); + try { + const candidates = await this._call_op_and_update_status({ bucket_name, - op_name: TIMED_OPS.PROCESS_BUCKET, - op_func: async () => process_bucket(config_fs, bucket_name, system_json) - }) - ); - lifecycle_run_status.state.is_finished = Object.values(lifecycle_run_status.buckets_statuses).reduce( - (acc, bucket) => acc && (bucket.state?.is_finished), - true + rule_id, + op_name: TIMED_OPS.GET_CANDIDATES, + op_func: async () => this.get_candidates(bucket_json, lifecycle_rule, object_sdk) + }); + + if (candidates.delete_candidates?.length > 0) { + const expiration = lifecycle_rule.expiration ? this._get_expiration_time(lifecycle_rule.expiration) : 0; + const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration}); + dbg.log0('process_rule: calling delete_multiple_objects, num of objects to be deleted', candidates.delete_candidates.length); + const delete_res = await this._call_op_and_update_status({ + bucket_name, + rule_id, + op_name: TIMED_OPS.DELETE_MULTIPLE_OBJECTS, + op_func: async () => object_sdk.delete_multiple_objects({ + bucket: bucket_json.name, + objects: candidates.delete_candidates, + filter_func + }) + }); + if (should_notify) { + await this.send_lifecycle_notifications(delete_res, candidates.delete_candidates, bucket_json, object_sdk); + } + } + + if (candidates.abort_mpu_candidates?.length > 0) { + dbg.log0('process_rule: calling delete_multiple_objects, num of mpu to be aborted', candidates.delete_candidates.length); + await this._call_op_and_update_status({ + bucket_name, + rule_id, + op_name: TIMED_OPS.ABORT_MPUS, + op_func: async () => this.abort_mpus(candidates, object_sdk) + }); + } + } catch (err) { + dbg.error('process_rule failed with error', err, err.code, err.message); + } + } + + /** + * abort_mpus iterates over the abort mpu candidates and calls abort_object_upload + * since abort_object_upload is not returning anything, we catch it in case of an error and assign err_code + * so it can be translated to error on stats + * @param {*} candidates + * @param {nb.ObjectSDK} object_sdk + * @returns {Promise} + */ + async abort_mpus(candidates, object_sdk) { + const aborts_concurrency = 10; // TODO - think about it + const abort_mpus_reply = await P.map_with_concurrency(aborts_concurrency, candidates.abort_mpu_candidates, + async candidate => { + const candidate_info = { key: candidate.key, upload_id: candidate.obj_id }; + try { + await object_sdk.abort_object_upload(candidate); + } catch (err) { + candidate_info.err_code = err.code || err.message; + dbg.log0('process_rule: abort_mpu_failed candidate_info', candidate_info); + } + return candidate_info; + } ); + return abort_mpus_reply; } -} -/** - * process_bucket processes the lifecycle rules for a bucket - * @param {import('../sdk/config_fs').ConfigFS} config_fs - * @param {string} bucket_name - * @param {Object} system_json - */ -async function process_bucket(config_fs, bucket_name, system_json) { - const bucket_json = await config_fs.get_bucket_by_name(bucket_name, config_fs_options); - const account = { email: '', nsfs_account_config: config_fs.fs_context, access_keys: [] }; - const object_sdk = new NsfsObjectSDK('', config_fs, account, bucket_json.versioning, config_fs.config_root, system_json); - await object_sdk._simple_load_requesting_account(); - const should_notify = notifications_util.should_notify_on_event(bucket_json, notifications_util.OP_TO_EVENT.lifecycle_delete.name); - if (!bucket_json.lifecycle_configuration_rules) { - lifecycle_run_status.buckets_statuses[bucket_json.name].state = {is_finished: true}; - return; - } - await process_rules(config_fs, bucket_json, object_sdk, should_notify); -} + ///////////////////////////////// + //////// GENERAL HELPERS //////// + ///////////////////////////////// -/** - * process_rules processes the lifecycle rules for a bucket - * @param {import('../sdk/config_fs').ConfigFS} config_fs - * @param {Object} bucket_json - * @param {nb.ObjectSDK} object_sdk - */ -async function process_rules(config_fs, bucket_json, object_sdk, should_notify) { - try { - lifecycle_run_status.buckets_statuses[bucket_json.name].state ??= {}; - const bucket_state = lifecycle_run_status.buckets_statuses[bucket_json.name].state; - bucket_state.num_processed_objects = 0; - while (!bucket_state.is_finished && bucket_state.num_processed_objects < config.NC_LIFECYCLE_BUCKET_BATCH_SIZE) { - await P.all(_.map(bucket_json.lifecycle_configuration_rules, - async (lifecycle_rule, index) => - await _call_op_and_update_status({ - bucket_name: bucket_json.name, - rule_id: lifecycle_rule.id, - op_name: TIMED_OPS.PROCESS_RULE, - op_func: async () => process_rule(config_fs, - lifecycle_rule, - index, - bucket_json, - object_sdk, - should_notify - ) - }) - ) - ); - bucket_state.is_finished = Object.values(lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses).reduce( - (acc, rule) => acc && (_.isEmpty(rule.state) || rule.state.is_finished), - true - ); + /** + * _should_lifecycle_run checks if lifecycle worker should run based on the followings - + * 1. lifecycle workrer can be disabled + * 2. lifecycle worker might run at time that does not match config.NC_LIFECYCLE_RUN_TIME + * 3. previous run was in the delay time frame + * @returns {Promise} + */ + async _should_lifecycle_run() { + const should_run = this.disable_runtime_validation ? true : await is_desired_time( + this.fs_context, + new Date(), + config.NC_LIFECYCLE_RUN_TIME, + config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS, + this.lifecycle_timestamp_file_path, + config.NC_LIFECYCLE_TZ); + dbg.log0('_should_lifecycle_run should_run', should_run); + return should_run; + } + + /** + * throw_if_noobaa_not_active checks if system.json exists and the noobaa service is active + * @param {Object} system_json + */ + async throw_if_noobaa_not_active(system_json) { + if (!system_json) { + dbg.error('throw_if_noobaa_not_active: system.json is missing'); + throw_cli_error(ManageCLIError.SystemJsonIsMissing); + } + + const service_status = await get_service_status(NOOBAA_SERVICE_NAME); + if (service_status !== 'active') { + dbg.error('throw_if_noobaa_not_active: noobaa service is not active'); + throw_cli_error(ManageCLIError.NooBaaServiceIsNotActive); } - } catch (err) { - dbg.error('process_rules failed with error', err, err.code, err.message); } -} -async function send_lifecycle_notifications(delete_res, bucket_json, object_sdk) { - const writes = []; - for (const deleted_obj of delete_res) { - if (delete_res.err_code) continue; - for (const notif of bucket_json.notifications) { - if (notifications_util.check_notif_relevant(notif, { - op_name: 'lifecycle_delete', - s3_event_method: deleted_obj.delete_marker_created ? 'DeleteMarkerCreated' : 'Delete', - })) { - //remember that this deletion needs a notif for this specific notification conf - writes.push({notif, deleted_obj}); + /** + * get_candidates gets the delete and abort candidates for the lifecycle rule + * @param {Object} bucket_json + * @param {*} lifecycle_rule + * @param {nb.ObjectSDK} object_sdk + * @reutrns {Promise} + */ + async get_candidates(bucket_json, lifecycle_rule, object_sdk) { + const candidates = { abort_mpu_candidates: [], delete_candidates: [] }; + const params = {versions_list: undefined}; + if (lifecycle_rule.expiration) { + candidates.delete_candidates = await this.get_candidates_by_expiration_rule(lifecycle_rule, bucket_json, + object_sdk); + if (lifecycle_rule.expiration.days || lifecycle_rule.expiration.expired_object_delete_marker) { + const dm_candidates = await this.get_candidates_by_expiration_delete_marker_rule( + lifecycle_rule, + bucket_json, + object_sdk, + params + ); + candidates.delete_candidates = candidates.delete_candidates.concat(dm_candidates); } } + if (lifecycle_rule.noncurrent_version_expiration) { + const non_current_candidates = await this.get_candidates_by_noncurrent_version_expiration_rule( + lifecycle_rule, + bucket_json, + object_sdk, + params + ); + candidates.delete_candidates = candidates.delete_candidates.concat(non_current_candidates); + } + if (lifecycle_rule.abort_incomplete_multipart_upload) { + candidates.abort_mpu_candidates = await this.get_candidates_by_abort_incomplete_multipart_upload_rule( + lifecycle_rule, bucket_json, object_sdk); + } + return candidates; } - //required format by compose_notification_lifecycle - bucket_json.bucket_owner = new SensitiveString(bucket_json.bucket_owner); - //if any notifications are needed, write them in notification log file - //(otherwise don't do any unnecessary filesystem actions) - if (writes.length > 0) { - let logger; - try { - logger = notifications_util.get_notification_logger('SHARED'); - await P.map_with_concurrency(100, writes, async write => { - const notif = notifications_util.compose_notification_lifecycle(write.deleted_obj, write.notif, bucket_json, object_sdk); - logger.append(JSON.stringify(notif)); - }); - } finally { - if (logger) logger.close(); + /** + * validate_rule_enabled checks if the rule is enabled and should be processed + * @param {*} rule + * @param {Object} bucket + * @returns {boolean} + */ + validate_rule_enabled(rule, bucket) { + if (rule.status !== 'Enabled') { + dbg.log0('validate_rule_enabled: SKIP bucket:', bucket.name, '(bucket id:', bucket._id, ') rule', util.inspect(rule), 'not Enabled'); + return false; } + return true; } -} -/** - * process_rule processes the lifecycle rule for a bucket - * TODO - implement notifications for the deleted objects (check if needed for abort mpus as well) - * @param {import('../sdk/config_fs').ConfigFS} config_fs - * @param {Object} lifecycle_rule - * @param {number} index - * @param {Object} bucket_json - * @param {nb.ObjectSDK} object_sdk - * @returns {Promise} - */ -async function process_rule(config_fs, lifecycle_rule, index, bucket_json, object_sdk, should_notify) { - dbg.log0('nc_lifecycle.process_rule: start bucket name:', bucket_json.name, 'rule', lifecycle_rule, 'index', index); - const bucket_name = bucket_json.name; - const rule_id = lifecycle_rule.id; - const should_process_lifecycle_rule = validate_rule_enabled(lifecycle_rule, bucket_json); - if (!should_process_lifecycle_rule) return; - - dbg.log0('nc_lifecycle.process_rule: processing rule:', bucket_name, '(bucket id:', bucket_json._id, ') rule', util.inspect(lifecycle_rule)); - try { - const candidates = await _call_op_and_update_status({ - bucket_name, - rule_id, - op_name: TIMED_OPS.GET_CANDIDATES, - op_func: async () => get_candidates(bucket_json, lifecycle_rule, object_sdk, config_fs.fs_context) + //////////////////////////////////// + //////// EXPIRATION HELPERS //////// + //////////////////////////////////// + + /** + * get_candidates_by_expiration_rule processes the expiration rule + * @param {*} lifecycle_rule + * @param {Object} bucket_json + * @returns {Promise} + */ + async get_candidates_by_expiration_rule(lifecycle_rule, bucket_json, object_sdk) { + if (this._should_use_gpfs_optimization()) { + return this.get_candidates_by_expiration_rule_gpfs(lifecycle_rule, bucket_json); + } else { + return this.get_candidates_by_expiration_rule_posix(lifecycle_rule, bucket_json, object_sdk); + } + } + + /** + * load objects list batch and update state for the next cycle + * @param {Object} object_sdk + * @param {Object} lifecycle_rule + * @param {Object} bucket_json + * @param {Object} expire_state + * @returns + */ + async load_objects_list(object_sdk, lifecycle_rule, bucket_json, expire_state) { + const objects_list = await object_sdk.list_objects({ + bucket: bucket_json.name, + prefix: lifecycle_rule.filter?.prefix, + key_marker: expire_state.key_marker, + limit: config.NC_LIFECYCLE_LIST_BATCH_SIZE }); + if (objects_list.is_truncated) { + expire_state.key_marker = objects_list.next_marker; + expire_state.is_finished = false; + } else { + expire_state.key_marker = undefined; + expire_state.is_finished = true; + } + const bucket_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].state; + bucket_state.num_processed_objects += objects_list.objects.length; + return objects_list; + } - if (candidates.delete_candidates?.length > 0) { - const delete_res = await _call_op_and_update_status({ - bucket_name, - rule_id, - op_name: TIMED_OPS.DELETE_MULTIPLE_OBJECTS, - op_func: async () => object_sdk.delete_multiple_objects({ - bucket: bucket_json.name, - objects: candidates.delete_candidates - }) - }); - if (should_notify) { - await send_lifecycle_notifications(delete_res, bucket_json, object_sdk); + /** + * + * @param {*} lifecycle_rule + * @param {Object} bucket_json + * @param {nb.ObjectSDK} object_sdk + * @returns {Promise} + */ + async get_candidates_by_expiration_rule_posix(lifecycle_rule, bucket_json, object_sdk) { + const rule_state = this._get_rule_state(bucket_json, lifecycle_rule).expire; + if (rule_state.is_finished) return []; + const expiration = this._get_expiration_time(lifecycle_rule.expiration); + if (expiration < 0) return []; + const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration}); + + const filtered_objects = []; + // TODO list_objects does not accept a filter and works in batch sizes of 1000. should handle batching + // also should maybe create a helper function or add argument for a filter in list object + const objects_list = await this.load_objects_list(object_sdk, lifecycle_rule, bucket_json, rule_state); + objects_list.objects.forEach(obj => { + const should_delete = lifecycle_utils.file_matches_filter({ obj_info: obj, filter_func }); + if (should_delete) { + //need to delete latest. so remove version_id if exists + const candidate = _.omit(obj, ['version_id']); + filtered_objects.push(candidate); + } + }); + return filtered_objects; + } + + /** + * get_candidates_by_expiration_rule_gpfs does the following - + * 1. gets the ilm candidates file path + * 2. parses and returns the candidates from the files + * @param {*} lifecycle_rule + * @param {Object} bucket_json + * @returns {Promise} + */ + async get_candidates_by_expiration_rule_gpfs(lifecycle_rule, bucket_json) { + const ilm_candidates_file_path = this.get_gpfs_ilm_candidates_file_path(bucket_json, lifecycle_rule); + const parsed_candidates = await this.parse_candidates_from_gpfs_ilm_policy(bucket_json, lifecycle_rule, ilm_candidates_file_path); + return parsed_candidates; + } + + /** + * check if delete candidate based on expired delete marker rule + * @param {Object} object + * @param {Object} next_object + * @param {Function} filter_func + * @returns + */ + filter_expired_delete_marker(object, next_object, filter_func) { + const lifecycle_info = lifecycle_utils.get_lifecycle_object_info_for_filter(object); + if (!filter_func(lifecycle_info)) return false; + return object.is_latest && object.delete_marker && object.key !== next_object.key; + } + + /** + * get_candidates_by_expiration_delete_marker_rule processes the expiration delete marker rule + * @param {*} lifecycle_rule + * @param {Object} bucket_json + * @returns {Promise} + */ + async get_candidates_by_expiration_delete_marker_rule(lifecycle_rule, bucket_json, object_sdk, params) { + const rule_state = this._get_rule_state(bucket_json, lifecycle_rule).noncurrent; + if (!params.versions_list) { + if (rule_state.is_finished) return []; + params.versions_list = await this.load_versions_list(object_sdk, lifecycle_rule, bucket_json, rule_state); + } + const versions_list = params.versions_list; + const candidates = []; + const expiration = lifecycle_rule.expiration?.days ? this._get_expiration_time(lifecycle_rule.expiration) : 0; + const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration}); + for (let i = 0; i < versions_list.objects.length - 1; i++) { + if (this.filter_expired_delete_marker(versions_list.objects[i], versions_list.objects[i + 1], filter_func)) { + candidates.push(versions_list.objects[i]); + } + } + const last_item = versions_list.objects.length > 0 && versions_list.objects[versions_list.objects.length - 1]; + const lifecycle_info = lifecycle_utils.get_lifecycle_object_info_for_filter(last_item); + if (last_item.is_latest && last_item.delete_marker && filter_func(lifecycle_info)) { + if (rule_state.is_finished) { + candidates.push(last_item); + } else { + //need the next item to decide if we need to delete. start the next cycle from this key latest + rule_state.key_marker_versioned = last_item.key; + rule_state.version_id_marker = undefined; } } + return candidates; + } - if (candidates.abort_mpu_candidates?.length > 0) { - await _call_op_and_update_status({ - bucket_name, - rule_id, - op_name: TIMED_OPS.ABORT_MPUS, - op_func: async () => abort_mpus(candidates, object_sdk) - }); + ///////////////////////////////////////////// + //////// NON CURRENT VERSION HELPERS //////// + ///////////////////////////////////////////// + /** + * load versions list batch and update state for the next cycle + * @param {Object} object_sdk + * @param {Object} lifecycle_rule + * @param {Object} bucket_json + * @param {Object} noncurrent_state + * @returns + */ + async load_versions_list(object_sdk, lifecycle_rule, bucket_json, noncurrent_state) { + const list_versions = await object_sdk.list_object_versions({ + bucket: bucket_json.name, + prefix: lifecycle_rule.filter?.prefix, + limit: config.NC_LIFECYCLE_LIST_BATCH_SIZE, + key_marker: noncurrent_state.key_marker_versioned, + version_id_marker: noncurrent_state.version_id_marker + }); + if (list_versions.is_truncated) { + noncurrent_state.is_finished = false; + noncurrent_state.key_marker_versioned = list_versions.next_marker; + noncurrent_state.version_id_marker = list_versions.next_version_id_marker; + } else { + noncurrent_state.key_marker_versioned = undefined; + noncurrent_state.version_id_marker = undefined; + noncurrent_state.is_finished = true; } - } catch (err) { - dbg.error('process_rule failed with error', err, err.code, err.message); + const bucket_state = this.lifecycle_run_status.buckets_statuses[bucket_json.name].state; + bucket_state.num_processed_objects += list_versions.objects.length; + return list_versions; } -} -/** - * abort_mpus iterates over the abort mpu candidates and calls abort_object_upload - * since abort_object_upload is not returning anything, we catch it in case of an error and assign err_code - * so it can be translated to error on stats - * @param {*} candidates - * @param {nb.ObjectSDK} object_sdk - * @returns {Promise} - */ -async function abort_mpus(candidates, object_sdk) { - const aborts_concurrency = 10; // TODO - think about it - const abort_mpus_reply = await P.map_with_concurrency(aborts_concurrency, candidates.abort_mpu_candidates, - async candidate => { - const candidate_info = { key: candidate.key, upload_id: candidate.obj_id }; + /** + * check if object is delete candidate based on newer noncurrent versions rule + * @param {nb.ObjectInfo} object_info + * @param {Object} newer_noncurrent_state + * @param {Number} num_newer_versions + * @returns + */ + filter_newer_versions(object_info, newer_noncurrent_state, num_newer_versions) { + if (object_info.is_latest) { + newer_noncurrent_state.version_count = 0; //latest + newer_noncurrent_state.current_version = object_info.key; + return false; + } + newer_noncurrent_state.version_count += 1; + if (newer_noncurrent_state.version_count > num_newer_versions) { + return true; + } + return false; + } + + /** + * check if object is delete candidate based on number of noncurrent days rule + * @param {nb.ObjectInfo} object_info + * @param {Number} num_non_current_days + * @returns + */ + filter_noncurrent_days(object_info, num_non_current_days) { + if (object_info.is_latest) return false; + const noncurrent_time = object_info.nc_noncurrent_time; + return lifecycle_utils.get_file_age_days(noncurrent_time) >= num_non_current_days; + } + + /** + * get_candidates_by_noncurrent_version_expiration_rule processes the noncurrent version expiration rule + * TODO: + * POSIX - need to support both noncurrent_days and newer_noncurrent_versions + * GPFS - implement noncurrent_days using GPFS ILM policy as an optimization + * @param {Object} lifecycle_rule + * @param {Object} bucket_json + * @returns {Promise} + */ + async get_candidates_by_noncurrent_version_expiration_rule(lifecycle_rule, bucket_json, object_sdk, params) { + const rule_state = this._get_rule_state(bucket_json, lifecycle_rule).noncurrent; + + if (!params.versions_list) { + if (rule_state.is_finished) return []; + params.versions_list = await this.load_versions_list(object_sdk, lifecycle_rule, bucket_json, rule_state); + } + const versions_list = params.versions_list; + + const filter_func = this._build_lifecycle_filter({filter: lifecycle_rule.filter, expiration: 0}); + const num_newer_versions = lifecycle_rule.noncurrent_version_expiration.newer_noncurrent_versions; + const num_non_current_days = lifecycle_rule.noncurrent_version_expiration.noncurrent_days; + const delete_candidates = []; + + for (const entry of versions_list.objects) { + const lifecycle_info = lifecycle_utils.get_lifecycle_object_info_for_filter(entry); + if ((num_newer_versions === undefined || this.filter_newer_versions(entry, rule_state, num_newer_versions)) && + (num_non_current_days === undefined || this.filter_noncurrent_days(entry, num_non_current_days))) { + if (filter_func(lifecycle_info)) { + delete_candidates.push({key: entry.key, version_id: entry.version_id}); + } + } + } + return delete_candidates; + } + //////////////////////////////////// + ///////// ABORT MPU HELPERS //////// + //////////////////////////////////// + + /** + * get_candidates_by_abort_incomplete_multipart_upload_rule processes the abort incomplete multipart upload rule + * @param {*} lifecycle_rule + * @param {Object} bucket_json + * @param {nb.ObjectSDK} object_sdk + * @returns {Promise} + */ + async get_candidates_by_abort_incomplete_multipart_upload_rule(lifecycle_rule, bucket_json, object_sdk) { + const nsfs = await object_sdk._get_bucket_namespace(bucket_json.name); + const mpu_path = nsfs._mpu_root_path(); + const filter = lifecycle_rule.filter; + const expiration = lifecycle_rule.abort_incomplete_multipart_upload.days_after_initiation; + const res = []; + + const filter_func = this._build_lifecycle_filter({filter, expiration}); + let dir_handle; + //TODO this is almost identical to list_uploads except for error handling and support for pagination. should modify list-upload and use it in here instead + try { + dir_handle = await nb_native().fs.opendir(this.fs_context, mpu_path); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + return; + } + for (;;) { try { - await object_sdk.abort_object_upload(candidate); + const dir_entry = await dir_handle.read(this.fs_context); + if (!dir_entry) break; + const create_path = path.join(mpu_path, dir_entry.name, 'create_object_upload'); + const { data: create_params_buffer } = await nb_native().fs.readFile(this.fs_context, create_path); + const create_params_parsed = JSON.parse(create_params_buffer.toString()); + const stat = await nb_native().fs.stat(this.fs_context, path.join(mpu_path, dir_entry.name)); + const object_lifecycle_info = this._get_lifecycle_object_info_for_mpu(create_params_parsed, stat); + if (filter_func(object_lifecycle_info)) { + res.push({ obj_id: dir_entry.name, key: create_params_parsed.key, bucket: bucket_json.name}); + } } catch (err) { - candidate_info.err_code = err.code || err.message; - dbg.log0('process_rule: abort_mpu_failed candidate_info', candidate_info); + if (err.code !== 'ENOENT' || err.code !== 'ENOTDIR') throw err; } - return candidate_info; } - ); - return abort_mpus_reply; -} + await dir_handle.close(this.fs_context); + dir_handle = null; + return res; + } -///////////////////////////////// -//////// GENERAL HELPERS //////// -///////////////////////////////// + /** + * @param {Object} create_params_parsed + * @param {nb.NativeFSStats} stat + */ + _get_lifecycle_object_info_for_mpu(create_params_parsed, stat) { + return { + key: create_params_parsed.key, + age: lifecycle_utils.get_file_age_days(stat.mtime.getTime()), + tags: create_params_parsed.tagging, + }; + } -/** - * _should_lifecycle_run checks if lifecycle worker should run based on the followings - - * 1. lifecycle workrer can be disabled - * 2. lifecycle worker might run at time that does not match config.NC_LIFECYCLE_RUN_TIME - * 3. previous run was in the delay time frame - * @param {nb.NativeFSContext} fs_context - * @param {String} lifecycle_timestamp_file_path - * @param {Boolean} disable_runtime_validation - * @returns {Promise} - */ -async function _should_lifecycle_run(fs_context, lifecycle_timestamp_file_path, disable_runtime_validation) { - const should_run = disable_runtime_validation ? true : await is_desired_time( - fs_context, - new Date(), - config.NC_LIFECYCLE_RUN_TIME, - config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS, - lifecycle_timestamp_file_path, - config.NC_LIFECYCLE_TZ); - dbg.log0('_should_lifecycle_run should_run', should_run); - return should_run; -} + //////////////////////////////////// + ///////// FILTER HELPERS //////// + //////////////////////////////////// -/** - * throw_if_noobaa_not_active checks if system.json exists and the noobaa service is active - * @param {import('../sdk/config_fs').ConfigFS} config_fs - * @param {Object} system_json - */ -async function throw_if_noobaa_not_active(config_fs, system_json) { - if (!system_json) { - dbg.error('throw_if_noobaa_not_active: system.json is missing'); - throw_cli_error(ManageCLIError.SystemJsonIsMissing); + /** + * @typedef {{ + * filter: Object + * expiration: Number + * }} filter_params + * + * @param {filter_params} params + * @returns + */ + _build_lifecycle_filter(params) { + /** + * @param {Object} object_info + */ + return function(object_info) { + if (params.filter?.prefix && !object_info.key.startsWith(params.filter.prefix)) return false; + if (params.expiration && object_info.age < params.expiration) return false; + if (params.filter?.tags && !_file_contain_tags(object_info, params.filter.tags)) return false; + if (params.filter?.object_size_greater_than && object_info.size < params.filter.object_size_greater_than) return false; + if (params.filter?.object_size_less_than && object_info.size > params.filter.object_size_less_than) return false; + return true; + }; } - const service_status = await get_service_status(NOOBAA_SERVICE_NAME); - if (service_status !== 'active') { - dbg.error('throw_if_noobaa_not_active: noobaa service is not active'); - throw_cli_error(ManageCLIError.NooBaaServiceIsNotActive); + /** + * get the expiration time in days of an object + * if rule is set with date, then rule is applied for all objects after that date + * return -1 to indicate that the date hasn't arrived, so rule should not be applied + * return 0 in case date has arrived so expiration is true for all elements + * return days in case days was defined and not date + * @param {Object} expiration_rule + * @returns {Number} + */ + _get_expiration_time(expiration_rule) { + if (expiration_rule.date) { + const expiration_date = new Date(expiration_rule.date).getTime(); + if (Date.now() < expiration_date) return -1; + return 0; + } else if (expiration_rule.days) { + return expiration_rule.days; + } else { + //expiration delete marker rule + return -1; + } } -} -/** - * get_candidates gets the delete and abort candidates for the lifecycle rule - * @param {Object} bucket_json - * @param {*} lifecycle_rule - */ -async function get_candidates(bucket_json, lifecycle_rule, object_sdk, fs_context) { - const candidates = { abort_mpu_candidates: [], delete_candidates: [] }; - const rule_state = lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id]?.state || {}; - if (lifecycle_rule.expiration) { - candidates.delete_candidates = await get_candidates_by_expiration_rule(lifecycle_rule, bucket_json, object_sdk, rule_state); - if (lifecycle_rule.expiration.days || lifecycle_rule.expiration.expired_object_delete_marker) { - const dm_candidates = await get_candidates_by_expiration_delete_marker_rule(lifecycle_rule, bucket_json); - candidates.delete_candidates = candidates.delete_candidates.concat(dm_candidates); - } - } - if (lifecycle_rule.noncurrent_version_expiration) { - const non_current_candidates = await get_candidates_by_noncurrent_version_expiration_rule(lifecycle_rule, bucket_json); - candidates.delete_candidates = candidates.delete_candidates.concat(non_current_candidates); - } - if (lifecycle_rule.abort_incomplete_multipart_upload) { - candidates.abort_mpu_candidates = await get_candidates_by_abort_incomplete_multipart_upload_rule( - lifecycle_rule, bucket_json, object_sdk, fs_context); - } - lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state = rule_state; - return candidates; -} + //////////////////////////////// + //////// STATUS HELPERS //////// + //////////////////////////////// + /** + * _call_op_and_update_status calls the op and report time and error to the lifecycle status. + * + * @template T + * @param {{ + * op_name: string; + * op_func: () => Promise; + * bucket_name?: string, + * rule_id?: string + * }} params + * @returns {Promise} + */ + async _call_op_and_update_status({ bucket_name = undefined, rule_id = undefined, op_name, op_func }) { + const start_time = Date.now(); + const update_options = { op_name, bucket_name, rule_id }; + let end_time; + let took_ms; + let error; + let reply; + try { + if (!this.return_short_status) this.update_status({ ...update_options, op_times: { start_time } }); + reply = await op_func(); + return reply; + } catch (e) { + error = e; + throw e; + } finally { + end_time = Date.now(); + took_ms = end_time - start_time; + const op_times = this.return_short_status ? { took_ms } : { end_time, took_ms }; + this.update_status({ ...update_options, op_times, reply, error }); + } + } -/** - * validate_rule_enabled checks if the rule is enabled and should be processed - * @param {*} rule - * @param {Object} bucket - * @returns {boolean} - */ -function validate_rule_enabled(rule, bucket) { - if (rule.status !== 'Enabled') { - dbg.log0('validate_rule_enabled: SKIP bucket:', bucket.name, '(bucket id:', bucket._id, ') rule', util.inspect(rule), 'not Enabled'); - return false; + /** + * update_status updates rule/bucket/global based on the given parameters + * 1. initalize statuses/times/stats per level + * 2. update times + * 3. update errors + * 4. update stats if the op is at rule level + * @param {{ + * op_name: string, + * bucket_name?: string, + * rule_id?: string, + * op_times: { start_time?: number, end_time?: number, took_ms?: number }, + * reply?: Object[], + * error?: Error} + * } params + * @returns {Void} + */ + update_status({ bucket_name, rule_id, op_name, op_times, reply = [], error = undefined }) { + if (op_times.start_time) { + if (op_name === TIMED_OPS.PROCESS_RULE) { + this.init_rule_status(bucket_name, rule_id); + } else if (op_name === TIMED_OPS.PROCESS_BUCKET) { + this.init_bucket_status(bucket_name); + } + } + if (op_times.end_time) { + if (op_name === TIMED_OPS.PROCESS_RULE) { + this.update_rule_status_is_finished(bucket_name, rule_id); + } + } + this._update_times_on_status({ op_name, op_times, bucket_name, rule_id }); + this._update_error_on_status({ error, bucket_name, rule_id }); + if (bucket_name && rule_id) { + this.update_stats_on_status({ bucket_name, rule_id, op_name, op_times, reply }); + } } - return true; -} -//////////////////////////////////// -//////// EXPIRATION HELPERS //////// -//////////////////////////////////// + /** + * _calc_stats accumulates stats for global/bucket stats + * @param {Object} stats_acc + * @param {Object} [cur_op_stats] + * @returns {Object} + */ + _acc_stats(stats_acc, cur_op_stats = {}) { + const stats_res = stats_acc; + + for (const [stat_key, stat_value] of Object.entries(cur_op_stats)) { + if (typeof stat_value === 'number') { + if (stats_res[stat_key]) stats_res[stat_key] += stat_value; + else stats_res[stat_key] = stat_value; + } + if (Array.isArray(stat_value)) { + if (stats_res[stat_key]) stats_res[stat_key].concat(stat_value); + else stats_res[stat_key] = stat_value; + } + } + return stats_res; + } -/** - * @param {Object} entry list object entry - */ -function _get_lifecycle_object_info_from_list_object_entry(entry) { - return { - key: entry.key, - age: _get_file_age_days(entry.create_time), - size: entry.size, - tags: entry.tagging, - }; -} + /** + * update_stats_on_status updates stats on rule context status and adds the rule status to the summarized bucket/global context stats + * @param {{ + * op_name: string, + * bucket_name: string, + * rule_id: string, + * op_times: { + * start_time?: number, + * end_time?: number, + * took_ms?: number + * }, + * reply?: Object[], + * }} params + * @returns {Void} + */ + update_stats_on_status({ bucket_name, rule_id, op_name, op_times, reply = [] }) { + if (op_times.end_time === undefined || ![TIMED_OPS.DELETE_MULTIPLE_OBJECTS, TIMED_OPS.ABORT_MPUS].includes(op_name)) return; + + const rule_stats_acc = this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_stats || + this._get_default_stats(); + const bucket_stats_acc = this.lifecycle_run_status.buckets_statuses[bucket_name].bucket_stats || this._get_default_stats(); + const lifecycle_stats_acc = this.lifecycle_run_status.total_stats || this._get_default_stats(); + + let cur_op_stats; + if (op_name === TIMED_OPS.DELETE_MULTIPLE_OBJECTS) { + const objects_delete_errors = reply.filter(obj => obj.err_code); + const num_objects_delete_failed = objects_delete_errors.length; + const num_objects_deleted = reply.length - num_objects_delete_failed; + cur_op_stats = { num_objects_deleted, num_objects_delete_failed, objects_delete_errors }; + } + if (op_name === TIMED_OPS.ABORT_MPUS) { + const mpu_abort_errors = reply.filter(obj => obj.err_code); + const num_mpu_abort_failed = mpu_abort_errors.length; + const num_mpu_aborted = reply.length - num_mpu_abort_failed; + cur_op_stats = { num_mpu_aborted, num_mpu_abort_failed, mpu_abort_errors }; + } + this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_stats = this._acc_stats( + rule_stats_acc, cur_op_stats); + this.lifecycle_run_status.buckets_statuses[bucket_name].bucket_stats = this._acc_stats( + bucket_stats_acc, cur_op_stats); + this.lifecycle_run_status.total_stats = this._acc_stats(lifecycle_stats_acc, cur_op_stats); + } -/** - * get_candidates_by_expiration_rule processes the expiration rule - * @param {*} lifecycle_rule - * @param {Object} bucket_json - * @returns {Promise} - */ -async function get_candidates_by_expiration_rule(lifecycle_rule, bucket_json, object_sdk, rule_state) { - const is_gpfs = nb_native().fs.gpfs; - if (is_gpfs && config.NC_LIFECYCLE_GPFS_ILM_ENABLED) { - return get_candidates_by_expiration_rule_gpfs(lifecycle_rule, bucket_json); - } else { - return get_candidates_by_expiration_rule_posix(lifecycle_rule, bucket_json, object_sdk, rule_state); + /** + * _update_times_on_status updates start/end & took times in lifecycle status + * @param {{op_name: String, op_times: {start_time?: number, end_time?: number, took_ms?: number }, + * bucket_name?: String, rule_id?: String}} params + * @returns + */ + _update_times_on_status({ op_name, op_times, bucket_name = undefined, rule_id = undefined }) { + for (const [key, value] of Object.entries(op_times)) { + const status_key = op_name + '_' + key; + if (bucket_name && rule_id) { + this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_process_times[status_key] = value; + } else if (bucket_name) { + this.lifecycle_run_status.buckets_statuses[bucket_name].bucket_process_times[status_key] = value; + } else { + this.lifecycle_run_status.lifecycle_run_times[status_key] = value; + } + } } -} -/** - * - * @param {*} lifecycle_rule - * @param {Object} bucket_json - * @returns {Promise} - */ -async function get_candidates_by_expiration_rule_gpfs(lifecycle_rule, bucket_json) { - // TODO - implement - return []; -} + /** + * _update_error_on_status updates an error occured in lifecycle status + * @param {{error: Error, bucket_name?: string, rule_id?: string}} params + * @returns + */ + _update_error_on_status({ error, bucket_name = undefined, rule_id = undefined }) { + if (!error) return; + if (bucket_name && rule_id) { + (this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].errors ??= []).push(error.message); + } else if (bucket_name) { + (this.lifecycle_run_status.buckets_statuses[bucket_name].errors ??= []).push(error.message); + } else { + (this.lifecycle_run_status.errors ??= []).push(error.message); + } + } -/** - * - * @param {*} lifecycle_rule - * @param {Object} bucket_json - * @returns {Promise} - */ -async function get_candidates_by_expiration_rule_posix(lifecycle_rule, bucket_json, object_sdk, rule_state) { - const expiration = _get_expiration_time(lifecycle_rule.expiration); - if (expiration < 0) return []; - const filter_func = _build_lifecycle_filter({filter: lifecycle_rule.filter, expiration}); - - const filtered_objects = []; - // TODO list_objects does not accept a filter and works in batch sizes of 1000. should handle batching - // also should maybe create a helper function or add argument for a filter in list object - const objects_list = await object_sdk.list_objects({ - bucket: bucket_json.name, - prefix: lifecycle_rule.filter?.prefix, - key_marker: rule_state.key_marker, - limit: config.NC_LIFECYCLE_LIST_BATCH_SIZE - }); - objects_list.objects.forEach(obj => { - const object_info = _get_lifecycle_object_info_from_list_object_entry(obj); - if (filter_func(object_info)) { - filtered_objects.push({key: object_info.key}); - } - }); - - const bucket_state = lifecycle_run_status.buckets_statuses[bucket_json.name].state; - bucket_state.num_processed_objects += objects_list.objects.length; - if (objects_list.is_truncated) { - rule_state.key_marker = objects_list.next_marker; - } else { - rule_state.is_finished = true; - } - return filtered_objects; + _get_default_stats() { + return { + num_objects_deleted: 0, num_objects_delete_failed: 0, objects_delete_errors: [], + num_mpu_aborted: 0, num_mpu_abort_failed: 0, mpu_abort_errors: [] + }; + } -} + /** + * write_lifecycle_log_file writes the lifecycle log file to the lifecycle logs directory + * @returns {Promise} + */ + async write_lifecycle_log_file() { + const log_file_name = `lifecycle_run_${this.lifecycle_run_status.lifecycle_run_times.run_lifecycle_start_time}.json`; + await nb_native().fs.writeFile( + this.fs_context, + path.join(this.lifecyle_logs_dir_path, log_file_name), + Buffer.from(JSON.stringify(this.lifecycle_run_status)), + { mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_FILE) } + ); + } -/** - * get_candidates_by_expiration_delete_marker_rule processes the expiration delete marker rule - * @param {*} lifecycle_rule - * @param {Object} bucket_json - * @returns {Promise} - */ -async function get_candidates_by_expiration_delete_marker_rule(lifecycle_rule, bucket_json) { - // TODO - implement - return []; -} + /** + * init the bucket status object statuses if they don't exist + * @param {string} bucket_name + * @returns {Object} created or existing bucket status + */ + init_bucket_status(bucket_name) { + this.lifecycle_run_status.buckets_statuses[bucket_name] ??= {}; + this.lifecycle_run_status.buckets_statuses[bucket_name].bucket_process_times ??= {}; + this.lifecycle_run_status.buckets_statuses[bucket_name].bucket_stats ??= {}; + this.lifecycle_run_status.buckets_statuses[bucket_name].state ??= {}; + this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses ??= {}; + return this.lifecycle_run_status.buckets_statuses[bucket_name]; + } -///////////////////////////////////////////// -//////// NON CURRENT VERSION HELPERS //////// -///////////////////////////////////////////// + /** + * init the rule status object statuses if they don't exist + * @param {string} bucket_name + * @param {string} rule_id + * @returns {Object} created or existing rule status + */ + init_rule_status(bucket_name, rule_id) { + this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id] ??= {}; + this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].state ??= { expire: {}, noncurrent: {} }; + this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_process_times = {}; + this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_stats ??= {}; + return this.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id]; + } -/** - * get_candidates_by_noncurrent_version_expiration_rule processes the noncurrent version expiration rule - * TODO: - * POSIX - need to support both noncurrent_days and newer_noncurrent_versions - * GPFS - implement noncurrent_days using GPFS ILM policy as an optimization - * @param {*} lifecycle_rule - * @param {Object} bucket_json - * @returns {Promise} - */ -async function get_candidates_by_noncurrent_version_expiration_rule(lifecycle_rule, bucket_json) { - // TODO - implement - return []; -} + /** + * update_rule_status_is_finished updates the rule state if all actions finished + * notice that expire and noncurrent properties are initiated in init_rule_status() + * therefore they should not be undefined + * @param {string} bucket_name + * @param {string} rule_id + * @returns {Void} + */ + update_rule_status_is_finished(bucket_name, rule_id) { + const rule_state = this._get_rule_state({ name: bucket_name }, { id: rule_id }); + rule_state.is_finished = (rule_state.expire.is_finished === undefined || rule_state.expire.is_finished === true) && + (rule_state.noncurrent.is_finished === undefined || rule_state.noncurrent.is_finished === true); + } -//////////////////////////////////// -///////// ABORT MPU HELPERS //////// -//////////////////////////////////// + /** + * + * @param {Object[]} buckets + * @returns + */ + async load_previous_run_state(buckets) { + const previous_run = await lifecycle_utils.get_latest_nc_lifecycle_run_status(this.config_fs, { silent_if_missing: true }); + if (previous_run) { + this.lifecycle_run_status.state = previous_run.state; + for (const [bucket_name, prev_bucket_status] of Object.entries(previous_run.buckets_statuses)) { + if (!buckets.includes(bucket_name)) continue; + const bucket_json = await this.config_fs.get_bucket_by_name(bucket_name, config_fs_options); + if (!bucket_json.lifecycle_configuration_rules) continue; + const bucket_status = this.init_bucket_status(bucket_name); + bucket_status.state = prev_bucket_status.state; + const bucket_rules = bucket_json.lifecycle_configuration_rules.map(rule => rule.id); + for (const [rule_id, prev_rule_status] of Object.entries(prev_bucket_status.rules_statuses)) { + if (!bucket_rules.includes(rule_id)) return; + const rule_status = this.init_rule_status(bucket_name, rule_id); + rule_status.state = prev_rule_status.state; + } + } + } + } -/** - * get_candidates_by_abort_incomplete_multipart_upload_rule processes the abort incomplete multipart upload rule - * @param {*} lifecycle_rule - * @param {Object} bucket_json - * @returns {Promise} - */ -async function get_candidates_by_abort_incomplete_multipart_upload_rule(lifecycle_rule, bucket_json, object_sdk, fs_context) { - const nsfs = await object_sdk._get_bucket_namespace(bucket_json.name); - const mpu_path = nsfs._mpu_root_path(); - const filter = lifecycle_rule.filter; - const expiration = lifecycle_rule.abort_incomplete_multipart_upload.days_after_initiation; - const res = []; - - const filter_func = _build_lifecycle_filter({filter, expiration}); - let dir_handle; - //TODO this is almost identical to list_uploads except for error handling and support for pagination. should modify list-upload and use it in here instead - try { - dir_handle = await nb_native().fs.opendir(fs_context, mpu_path); - } catch (err) { - if (err.code !== 'ENOENT') throw err; - return; - } - for (;;) { + /** + * _set_rule_state sets the current rule state on the lifecycle run status + * @param {Object} bucket_json + * @param {*} lifecycle_rule + * @param {RuleState} rule_state + * @returns {Void} + */ + _set_rule_state(bucket_json, lifecycle_rule, rule_state) { + const existing_state = this._get_rule_state(bucket_json, lifecycle_rule); + const new_state = { ...existing_state, ...rule_state }; + this.lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state = new_state; + } + + /** + * _get_rule_state gets the current rule state on the lifecycle run status + * @param {Object} bucket_json + * @param {*} lifecycle_rule + * @returns {RuleState} + */ + _get_rule_state(bucket_json, lifecycle_rule) { + return this.lifecycle_run_status.buckets_statuses[bucket_json.name].rules_statuses[lifecycle_rule.id].state; + } + + ///////////////////////////////// + ////// NOTIFICATION HELPERS ///// + ///////////////////////////////// + + /** + * + * @param {Object} delete_res + * @param {Object} delete_obj_info + * @returns + */ + create_notification_delete_object(delete_res, delete_obj_info) { + return { + ...delete_obj_info, + created_delete_marker: delete_res.created_delete_marker, + version_id: delete_res.created_delete_marker ? delete_res.created_version_id : delete_obj_info.version_id, + }; + } + + /** + * + * @param {Object[]} delete_res array of delete results + * @param {Object[]} delete_candidates array of delete candidates object info + * @param {Object} bucket_json + * @param {nb.ObjectSDK} object_sdk + * @returns {Promise} + * NOTE implementation assumes delete_candidates and delete_res uses the same index. this assumption is also made in + * s3_post_bucket_delete.js. + */ + async send_lifecycle_notifications(delete_res, delete_candidates, bucket_json, object_sdk) { + const writes = []; + for (let i = 0; i < delete_res.length; ++i) { + if (delete_res[i].err_code) continue; + for (const notif of bucket_json.notifications) { + if (notifications_util.check_notif_relevant(notif, { + op_name: 'lifecycle_delete', + s3_event_method: delete_res[i].created_delete_marker ? 'DeleteMarkerCreated' : 'Delete', + })) { + const deleted_obj = this.create_notification_delete_object(delete_res[i], delete_candidates[i]); + //remember that this deletion needs a notif for this specific notification conf + writes.push({notif, deleted_obj}); + } + } + } + + //required format by compose_notification_lifecycle + bucket_json.bucket_owner = new SensitiveString(object_sdk.requesting_account.name); + + //if any notifications are needed, write them in notification log file + //(otherwise don't do any unnecessary filesystem actions) + if (writes.length > 0) { + let logger; + try { + logger = notifications_util.get_notification_logger('SHARED'); + await P.map_with_concurrency(100, writes, async write => { + const notif = notifications_util.compose_notification_lifecycle(write.deleted_obj, write.notif, + bucket_json, object_sdk); + await logger.append(JSON.stringify(notif)); + }); + } finally { + if (logger) await logger.close(); + } + } + } + + //////////////////////////////////////////// + // GPFS ILM POLICIES OPTIMIZATION HELPERS // + //////////////////////////////////////////// + + /** + * _should_use_gpfs_optimization returns true is gpfs optimization should be used + * @returns {Boolean} + */ + _should_use_gpfs_optimization() { + const is_gpfs = nb_native().fs.gpfs; + return is_gpfs && config.NC_LIFECYCLE_GPFS_ILM_ENABLED; + } + + /** + * get_mount_points returns a map of the following format - + * { mount_point_path1: {}, mount_point_path2: {} } + * @returns {Promise} + */ + async get_mount_points_map() { try { - const dir_entry = await dir_handle.read(fs_context); - if (!dir_entry) break; - const create_path = path.join(mpu_path, dir_entry.name, 'create_object_upload'); - const { data: create_params_buffer } = await nb_native().fs.readFile(fs_context, create_path); - const create_params_parsed = JSON.parse(create_params_buffer.toString()); - const stat = await nb_native().fs.stat(fs_context, path.join(mpu_path, dir_entry.name)); - const object_lifecycle_info = _get_lifecycle_object_info_for_mpu(create_params_parsed, stat); - if (filter_func(object_lifecycle_info)) { - res.push({ obj_id: dir_entry.name, key: create_params_parsed.key, bucket: bucket_json.name}); + const fs_list = await os_utils.exec(`mmlsfs all -T -Y`, { return_stdout: true }); + dbg.log2('get_mount_points fs_list res ', fs_list); + const lines = fs_list.trim().split('\n'); + const res = {}; + + for (let idx = 1; idx < lines.length; idx++) { + const line = lines[idx]; + const parts = line.split(':'); + const mount_name = decodeURIComponent(parts[8]); + res[mount_name] = ''; } + return res; } catch (err) { - if (err.code !== 'ENOENT' || err.code !== 'ENOTDIR') throw err; + throw new Error(`get_mount_points failed with error ${err}`); } } - await dir_handle.close(fs_context); - dir_handle = null; - return res; -} - -/** - * @param {Object} create_params_parsed - * @param {nb.NativeFSStats} stat - */ -function _get_lifecycle_object_info_for_mpu(create_params_parsed, stat) { - return { - key: create_params_parsed.key, - age: _get_file_age_days(stat.mtime.getTime()), - tags: create_params_parsed.tagging, - }; -} -//////////////////////////////////// -///////// FILTER HELPERS //////// -//////////////////////////////////// + /** + * find_mount_point_by_bucket_path finds the mount point of a given bucket path + * @param {Object} mount_point_to_policy_map + * @param {String} bucket_path + * @returns {String} + */ + find_mount_point_by_bucket_path(mount_point_to_policy_map, bucket_path) { + const sorted_mounts = Object.keys(mount_point_to_policy_map).sort((a, b) => b.length - a.length); + dbg.log2(`find_mount_point_by_bucket_path bucket_path=${bucket_path} mount_point_path=${util.inspect(mount_point_to_policy_map)}`); + for (const mount_point_path of sorted_mounts) { + if (bucket_path === mount_point_path || bucket_path.startsWith(mount_point_path + '/')) { + return mount_point_path; + } + } + throw new Error(`can not find mount path of bucket in the mount lists ${bucket_path}, ${util.inspect(mount_point_to_policy_map)}`); + } -/** - * @typedef {{ - * filter: Object - * expiration: Number - * }} filter_params - * - * @param {filter_params} params - * @returns - */ -function _build_lifecycle_filter(params) { /** - * @param {Object} object_info + * create_gpfs_candidates_files creates a candidates file per mount point that is used by at least one bucket + * 1. creates a map of mount point to buckets + * 2. for each bucket - + * 2.1. finds the mount point it belongs to + * 2.2. convert the bucket's lifecycle policy to a GPFS ILM policy + * 2.3. concat the bucket's GPFS ILM policy to the mount point policy file string + * 3. for each mount point - + * 3.1. writes the ILM policy to a tmp file + * 3. creates the candidates file by applying the ILM policy + * @param {String[]} bucket_names + * @returns {Promise} */ - return function(object_info) { - if (params.filter?.prefix && !object_info.key.startsWith(params.filter.prefix)) return false; - if (params.expiration && object_info.age < params.expiration) return false; - if (params.filter?.tags && !_file_contain_tags(object_info, params.filter.tags)) return false; - if (params.filter?.object_size_greater_than && object_info.size < params.filter.object_size_greater_than) return false; - if (params.filter?.object_size_less_than && object_info.size > params.filter.object_size_less_than) return false; - return true; - }; -} + async create_gpfs_candidates_files(bucket_names) { + const mount_point_to_policy_map = await this.get_mount_points_map(); + for (const bucket_name of bucket_names) { + const bucket_json = await this.config_fs.get_bucket_by_name(bucket_name, config_fs_options); + const bucket_mount_point = this.find_mount_point_by_bucket_path(mount_point_to_policy_map, bucket_json.path); + if (!bucket_json.lifecycle_configuration_rules?.length) continue; + for (const lifecycle_rule of bucket_json.lifecycle_configuration_rules) { + // currently we support expiration (current version) only + if (lifecycle_rule.expiration) { + const should_expire = this._get_expiration_time(lifecycle_rule.expiration) >= 0; + if (!should_expire) continue; + const ilm_rule = this.convert_lifecycle_policy_to_gpfs_ilm_policy(lifecycle_rule, bucket_json); + mount_point_to_policy_map[bucket_mount_point] += ilm_rule + '\n'; + } + } + } -/** - * get file time since last modified in days - * @param {Number} mtime - * @returns {Number} days since object was last modified - */ -function _get_file_age_days(mtime) { - return Math.floor((Date.now() - mtime) / 24 / 60 / 60 / 1000); -} + await native_fs_utils._create_path(ILM_POLICIES_TMP_DIR, this.non_gpfs_fs_context, config.BASE_MODE_CONFIG_DIR); + await native_fs_utils._create_path(ILM_CANDIDATES_TMP_DIR, this.non_gpfs_fs_context, config.BASE_MODE_CONFIG_DIR); + for (const [mount_point, policy] of Object.entries(mount_point_to_policy_map)) { + if (policy === '') continue; + const ilm_policy_path = await this.write_tmp_ilm_policy(mount_point, policy); + await this.create_candidates_file_by_gpfs_ilm_policy(mount_point, ilm_policy_path); + } + } -/** - * get the expiration time in days of an object - * if rule is set with date, then rule is applied for all objects after that date - * return -1 to indicate that the date hasn't arrived, so rule should not be applied - * return 0 in case date has arrived so expiration is true for all elements - * return days in case days was defined and not date - * @param {Object} expiration_rule - * @returns {Number} - */ -function _get_expiration_time(expiration_rule) { - if (expiration_rule.date) { - const expiration_date = new Date(expiration_rule.date).getTime(); - if (Date.now() < expiration_date) return -1; - return 0; + /** + * convert_lifecycle_policy_to_gpfs_ilm_policy converts the lifecycle rule to GPFS ILM policy + * currently we support expiration (current version) only + * TODO - implement gpfs optimization for non_current_days - + * non current can't be on the same policy, when implementing non current we should split the policies + * @param {*} lifecycle_rule + * @param {Object} bucket_json + * @returns {String} + */ + convert_lifecycle_policy_to_gpfs_ilm_policy(lifecycle_rule, bucket_json) { + const bucket_path = bucket_json.path; + const bucket_rule_id = this.get_lifecycle_ilm_candidate_file_suffix(bucket_json.name, lifecycle_rule); + const in_bucket_path = path.join(bucket_path, '/%'); + const in_bucket_internal_dir = path.join(bucket_path, '/.noobaa_nsfs%/%'); + const in_versions_dir = path.join(bucket_path, '/.versions/%'); + const in_nested_versions_dir = path.join(bucket_path, '/%/.versions/%'); + const ilm_policy_helpers = { bucket_rule_id, in_bucket_path, in_bucket_internal_dir, in_versions_dir, in_nested_versions_dir }; + + const policy_base = this._get_gpfs_ilm_policy_base(ilm_policy_helpers); + const expiry_string = this.convert_expiry_rule_to_gpfs_ilm_policy(lifecycle_rule, ilm_policy_helpers); + const non_current_days_string = this.convert_noncurrent_version_by_days_to_gpfs_ilm_policy(lifecycle_rule, ilm_policy_helpers); + const filter_policy = this.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, bucket_json); + return policy_base + non_current_days_string + expiry_string + filter_policy; } - return expiration_rule.days; -} + /** + * _get_gpfs_ilm_policy_base returns policy base definitions and bucket path phrase + * @param {{bucket_rule_id: String, in_bucket_path: String, in_bucket_internal_dir: String}} ilm_policy_helpers + * @returns {String} + */ + _get_gpfs_ilm_policy_base(ilm_policy_helpers) { + const { bucket_rule_id, in_bucket_path, in_bucket_internal_dir } = ilm_policy_helpers; + const mod_age_definition = `define( mod_age, (DAYS(CURRENT_TIMESTAMP) - DAYS(MODIFICATION_TIME)) )\n`; + const change_age_definition = `define( change_age, (DAYS(CURRENT_TIMESTAMP) - DAYS(CHANGE_TIME)) )\n`; + const rule_id_definition = `RULE '${bucket_rule_id}' LIST '${bucket_rule_id}'\n`; + const policy_path_base = `WHERE PATH_NAME LIKE '${in_bucket_path}'\n` + + `AND PATH_NAME NOT LIKE '${in_bucket_internal_dir}'\n`; + + return mod_age_definition + change_age_definition + rule_id_definition + policy_path_base; + } -/** - * checks if tag query_tag is in the list tag_set - * @param {Object} query_tag - * @param {Array} tag_set - */ -function _list_contain_tag(query_tag, tag_set) { - for (const t of tag_set) { - if (t.key === query_tag.key && t.value === query_tag.value) return true; + /** + * convert_expiry_rule_to_gpfs_ilm_policy converts the expiry rule to GPFS ILM policy + * expiration rule works on latest version path (not inside .versions or in nested .versions) + * @param {*} lifecycle_rule + * @param {{in_versions_dir: String, in_nested_versions_dir: String}} ilm_policy_paths + * @returns {String} + */ + convert_expiry_rule_to_gpfs_ilm_policy(lifecycle_rule, { in_versions_dir, in_nested_versions_dir }) { + const { expiration = undefined } = lifecycle_rule; + if (!expiration) return ''; + const current_path_policy = `AND PATH_NAME NOT LIKE '${in_versions_dir}'\n` + + `AND PATH_NAME NOT LIKE '${in_nested_versions_dir}'\n`; + + const expiry_policy = expiration.days ? `AND mod_age > ${expiration.days}\n` : ''; + return current_path_policy + expiry_policy; } - return false; -} -/** - * checks if object has all the tags in filter_tags - * @param {Object} object_info - * @param {Array} filter_tags - * @returns - */ -function _file_contain_tags(object_info, filter_tags) { - if (object_info.tags === undefined) return false; - for (const tag of filter_tags) { - if (!_list_contain_tag(tag, object_info.tags)) { - return false; + /** + * convert_noncurrent_version_to_gpfs_ilm_policy converts the noncurrent version by days to GPFS ILM policy + * @param {*} lifecycle_rule + * @param {{in_versions_dir: String, in_nested_versions_dir: String}} ilm_policy_paths + * @returns {String} + */ + convert_noncurrent_version_by_days_to_gpfs_ilm_policy(lifecycle_rule, { in_versions_dir, in_nested_versions_dir }) { + return ''; + // TODO - add implementation + } + + /** + * convert_filter_to_gpfs_ilm_policy converts the filter to GPFS ILM policy + * @param {*} lifecycle_rule + * @param {Object} bucket_json + * @returns {String} + */ + convert_filter_to_gpfs_ilm_policy(lifecycle_rule, bucket_json) { + const { prefix = undefined, filter = {} } = lifecycle_rule; + const bucket_path = bucket_json.path; + let filter_policy = ''; + if (prefix || Object.keys(filter).length > 0) { + const { object_size_greater_than = undefined, object_size_less_than = undefined, tags = undefined } = filter; + const rule_prefix = prefix || filter.prefix; + filter_policy += rule_prefix ? `AND PATH_NAME LIKE '${path.join(bucket_path, rule_prefix)}%'\n` : ''; + filter_policy += object_size_greater_than === undefined ? '' : `AND FILE_SIZE > ${object_size_greater_than}\n`; + filter_policy += object_size_less_than === undefined ? '' : `AND FILE_SIZE < ${object_size_less_than}\n`; + filter_policy += tags ? tags.map(tag => `AND XATTR('user.noobaa.tag.${tag.key}') LIKE ${tag.value}\n`).join('') : ''; } + return filter_policy; } - return true; -} -///////////////////////////////// -//////// STATUS HELPERS //////// -///////////////////////////////// + /** + * get_lifecycle_ilm_candidates_file_name gets the ILM policy file name + * @param {String} bucket_name + * @param {*} lifecycle_rule + * @returns {String} + */ + get_lifecycle_ilm_candidates_file_name(bucket_name, lifecycle_rule) { + const lifecycle_ilm_candidates_file_suffix = this.get_lifecycle_ilm_candidate_file_suffix(bucket_name, lifecycle_rule); + return `list.${lifecycle_ilm_candidates_file_suffix}`; + } -/** - * _call_op_and_update_status calls the op and report time and error to the lifecycle status. - * - * @template T - * @param {{ -* op_name: string; -* op_func: () => Promise; -* bucket_name?: string, -* rule_id?: string -* }} params -* @returns {Promise} -*/ -async function _call_op_and_update_status({ bucket_name = undefined, rule_id = undefined, op_name, op_func }) { - const start_time = Date.now(); - const update_options = { op_name, bucket_name, rule_id }; - let end_time; - let took_ms; - let error; - let reply; - try { - if (!return_short_status) update_status({ ...update_options, op_times: { start_time } }); - reply = await op_func(); - return reply; - } catch (e) { - error = e; - throw e; - } finally { - end_time = Date.now(); - took_ms = end_time - start_time; - const op_times = return_short_status ? { took_ms } : { end_time, took_ms }; - update_status({ ...update_options, op_times, reply, error }); + /** + * get_lifecycle_ilm_candidate_file_suffix returns the suffix of a candidates file based on bucket name, rule id and lifecycle run start + * TODO - when noncurrent_version is supported, suffix should contain expiration/non_current_version_expiration rule type + * @param {String} bucket_name + * @param {*} lifecycle_rule + * @returns {String} + */ + get_lifecycle_ilm_candidate_file_suffix(bucket_name, lifecycle_rule) { + const rule_id = lifecycle_rule.id; + return `${bucket_name}_${rule_id}_${this.lifecycle_run_status.lifecycle_run_times.run_lifecycle_start_time}`; } -} -/** - * update_status updates rule/bucket/global based on the given parameters - * 1. initalize statuses/times/stats per level - * 2. update times - * 3. update errors - * 4. update stats if the op is at rule level - * @param {{ - * op_name: string, - * bucket_name?: string, - * rule_id?: string, - * op_times: { start_time?: number, end_time?: number, took_ms?: number }, - * reply?: Object[], - * error?: Error} - * } params - * @returns {Void} -*/ -function update_status({ bucket_name, rule_id, op_name, op_times, reply = [], error = undefined }) { - // TODO - check errors - if (op_times.start_time) { - if (op_name === TIMED_OPS.PROCESS_RULE) { - lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id] = { rule_process_times: {}, rule_stats: {} }; - } else if (op_name === TIMED_OPS.PROCESS_BUCKET) { - lifecycle_run_status.buckets_statuses[bucket_name] = { bucket_process_times: {}, bucket_stats: {}, rules_statuses: {} }; + /** + * write_tmp_ilm_policy writes the ILM policy string to a tmp file + * TODO - delete the policy on restart and on is_finished of the rule + * TODO - should we unlink the policy file if the file already exists? - might be dangerous + * @param {String} mount_point_path + * @param {String} ilm_policy_string + * @returns {Promise} + */ + async write_tmp_ilm_policy(mount_point_path, ilm_policy_string) { + try { + const ilm_policy_tmp_path = this.get_gpfs_ilm_policy_file_path(mount_point_path); + const ilm_policy_stat = await native_fs_utils.stat_ignore_enoent(this.non_gpfs_fs_context, ilm_policy_tmp_path); + if (ilm_policy_stat) { + dbg.log2('write_tmp_ilm_policy: policy already exists, ', ilm_policy_tmp_path); + } else { + // TODO - maybe we should write to tmp file and then link so we won't override the file + await nb_native().fs.writeFile( + this.non_gpfs_fs_context, + ilm_policy_tmp_path, + Buffer.from(ilm_policy_string), { + mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_FILE), + }, + ); + } + return ilm_policy_tmp_path; + } catch (err) { + throw new Error(`write_tmp_ilm_policy failed with error ${err}`); } } - _update_times_on_status({ op_name, op_times, bucket_name, rule_id }); - _update_error_on_status({ error, bucket_name, rule_id }); - if (bucket_name && rule_id) { - update_stats_on_status({ bucket_name, rule_id, op_name, op_times, reply }); + + /** + * lifecycle_ilm_policy_path returns ilm policy file path based on bucket name and rule_id + * @param {String} mount_point_path + * @returns {String} + */ + get_gpfs_ilm_policy_file_path(mount_point_path) { + const encoded_mount_point_path = encodeURIComponent(mount_point_path); + const lifecycle_ilm_policy_path = path.join(ILM_POLICIES_TMP_DIR, `noobaa_ilm_policy_${encoded_mount_point_path}_${this.lifecycle_run_status.lifecycle_run_times.run_lifecycle_start_time}`); + return lifecycle_ilm_policy_path; } -} -/** - * _calc_stats accumulates stats for global/bucket stats - * @param {Object} stats_acc - * @param {Object} [cur_op_stats] - * @returns {Object} - */ -function _acc_stats(stats_acc, cur_op_stats = {}) { - const stats_res = stats_acc; + /** + * get_gpfs_ilm_candidates_file_path returns ilm policy file path based on bucket name and rule_id + * @param {*} bucket_json + * @param {*} lifecycle_rule + * @returns {String} + */ + get_gpfs_ilm_candidates_file_path(bucket_json, lifecycle_rule) { + const ilm_candidates_file_name = this.get_lifecycle_ilm_candidates_file_name(bucket_json.name, lifecycle_rule); + const ilm_candidates_file_path = path.join(ILM_CANDIDATES_TMP_DIR, ilm_candidates_file_name); + return ilm_candidates_file_path; + } - for (const [stat_key, stat_value] of Object.entries(cur_op_stats)) { - if (typeof stat_value === 'number') { - stats_res[stat_key] += stat_value; - } - if (Array.isArray(stat_value)) { - stats_res[stat_key].concat(stat_value); + /** + * create_candidates_file_by_gpfs_ilm_policy gets the candidates by applying the ILM policy using mmapplypolicy + * the return value is a path to the output file that contains the candidates + * TODO - check if the output file is created - this is probablt not the correct path + * @param {String} mount_point_path + * @param {String} ilm_policy_tmp_path + * @returns {Promise} + */ + async create_candidates_file_by_gpfs_ilm_policy(mount_point_path, ilm_policy_tmp_path) { + try { + // TODO - understand which is better defer or prepare + const mmapply_policy_res = await os_utils.exec(`mmapplypolicy ${mount_point_path} -P ${ilm_policy_tmp_path} -f ${ILM_CANDIDATES_TMP_DIR} -I defer`, { return_stdout: true }); + dbg.log2('create_candidates_file_by_gpfs_ilm_policy mmapplypolicy res ', mmapply_policy_res); + } catch (err) { + throw new Error(`create_candidates_file_by_gpfs_ilm_policy failed with error ${err}`); } } - return stats_res; -} -/** - * update_stats_on_status updates stats on rule context status and adds the rule status to the summarized bucket/global context stats - * @param {{ - * op_name: string, - * bucket_name: string, - * rule_id: string, - * op_times: { - * start_time?: number, - * end_time?: number, - * took_ms?: number - * }, - * reply?: Object[], - * }} params - * @returns {Void} - */ -function update_stats_on_status({ bucket_name, rule_id, op_name, op_times, reply = [] }) { - if (op_times.end_time === undefined || ![TIMED_OPS.DELETE_MULTIPLE_OBJECTS, TIMED_OPS.ABORT_MPUS].includes(op_name)) return; - - const rule_stats_acc = lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_stats || _get_default_stats(); - const bucket_stats_acc = lifecycle_run_status.buckets_statuses[bucket_name].bucket_stats || _get_default_stats(); - const lifecycle_stats_acc = lifecycle_run_status.total_stats || _get_default_stats(); - - let cur_op_stats; - if (op_name === TIMED_OPS.DELETE_MULTIPLE_OBJECTS) { - const objects_delete_errors = reply.filter(obj => obj.err_code); - const num_objects_delete_failed = objects_delete_errors.length; - const num_objects_deleted = reply.length - num_objects_delete_failed; - cur_op_stats = { num_objects_deleted, num_objects_delete_failed, objects_delete_errors }; - } - if (op_name === TIMED_OPS.ABORT_MPUS) { - const mpu_abort_errors = reply.filter(obj => obj.err_code); - const num_mpu_abort_failed = mpu_abort_errors.length; - const num_mpu_aborted = reply.length - num_mpu_abort_failed; - cur_op_stats = { num_mpu_aborted, num_mpu_abort_failed, mpu_abort_errors }; - } - lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_stats = { ...rule_stats_acc, ...cur_op_stats }; - lifecycle_run_status.buckets_statuses[bucket_name].bucket_stats = _acc_stats({ stats_acc: bucket_stats_acc, cur_op_stats }); - lifecycle_run_status.total_stats = _acc_stats({ stats_acc: lifecycle_stats_acc, cur_op_stats }); -} + /** + * parse_candidates_from_gpfs_ilm_policy does the following - + * 1. reads the candidates file line by line (Note - we set read_file_offset so we will read the file from the line we stopped last iteration)- + * 1.1. if number of parsed candidates is above the batch size - break the loop and stop reading the candidates file + * 1.2. else - + * 1.2.1. update the new rule state + * 1.2.2. parse the key from the candidate line + * 1.2.3. push the key to the candidates array + * 2. if candidates file does not exist, we return without error because it's valid that no candidates found + * GAP - when supporting noncurrent rule, we should update the state type to noncurrent based on the candidates file path + * @param {Object} bucket_json + * @param {*} lifecycle_rule + * @param {String} rule_candidates_path + * @returns {Promise} parsed_candidates_array + */ + async parse_candidates_from_gpfs_ilm_policy(bucket_json, lifecycle_rule, rule_candidates_path) { + let reader; + const state_type = 'expire'; + const rule_state = this._get_rule_state(bucket_json, lifecycle_rule)?.[state_type]; + dbg.log2(`parse_candidates_from_gpfs_ilm_policy rule_state=${rule_state} state_type=${state_type}, currently on gpfs ilm flow - we support only expiration rule`); + if (rule_state?.is_finished) return []; + const finished_state = { [state_type]: { is_finished: true, candidates_file_offset: undefined } }; -/** - * _update_times_on_status updates start/end & took times in lifecycle status - * @param {{op_name: String, op_times: {start_time?: number, end_time?: number, took_ms?: number }, - * bucket_name?: String, rule_id?: String}} params - * @returns - */ -function _update_times_on_status({ op_name, op_times, bucket_name = undefined, rule_id = undefined }) { - for (const [key, value] of Object.entries(op_times)) { - const status_key = op_name + '_' + key; - if (bucket_name && rule_id) { - lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].rule_process_times[status_key] = value; - } else if (bucket_name) { - lifecycle_run_status.buckets_statuses[bucket_name].bucket_process_times[status_key] = value; - } else { - lifecycle_run_status.lifecycle_run_times[status_key] = value; + try { + dbg.log2(`parse_candidates_from_gpfs_ilm_policy bucket_name=${bucket_json.name}, rule_id ${lifecycle_rule.id}, existing rule_state=${util.inspect(rule_state)}`); + const parsed_candidates_array = []; + reader = new NewlineReader(this.non_gpfs_fs_context, rule_candidates_path, { lock: 'SHARED', read_file_offset: rule_state?.candidates_file_offset || 0 }); + + const [count, is_finished] = await reader.forEachFilePathEntry(async entry => { + if (parsed_candidates_array.length >= config.NC_LIFECYCLE_LIST_BATCH_SIZE) return false; + const cur_rule_state = { [state_type]: { is_finished: false, candidates_file_offset: reader.next_line_file_offset } }; + this._set_rule_state(bucket_json, lifecycle_rule, cur_rule_state); + const key = this._parse_key_from_line(entry, bucket_json); + // TODO - need to add etag, size, version_id + parsed_candidates_array.push({ key }); + dbg.log2(`parse_candidates_from_gpfs_ilm_policy: file_key=${key}, entry_path=${entry.path}, reader.next_line_file_offset=${reader.next_line_file_offset}, rule_state=${rule_state}`); + return true; + }); + + if (is_finished) { + this._set_rule_state(bucket_json, lifecycle_rule, finished_state); + } + dbg.log2(`parse_candidates_from_gpfs_ilm_policy: parsed_candidates_array ${util.inspect(parsed_candidates_array)}, rule_state=${util.inspect(rule_state)}, count=${count} is_finished=${is_finished}`); + return parsed_candidates_array; + } catch (err) { + if (err.code === 'ENOENT') { + dbg.log2(`parse_candidates_from_gpfs_ilm_policy ilm_candidates_file_exists does not exist, no candidates to delete`); + this._set_rule_state(bucket_json, lifecycle_rule, finished_state); + return []; + } + dbg.error('parse_candidates_from_gpfs_ilm_policy: error', err); + throw err; + } finally { + if (reader) await reader.close(); } } + + /** + * _parse_key_from_line parses the object key from a candidate line + * candidate line (when using mmapplypolicy defer) is of the following format - + * example - + * 17460 1316236366 0 -- /mnt/gpfs0/account1_new_buckets_path/bucket1_storage/key1.txt + * if file is .folder (directory object) we need to return its parent directory + * @param {*} entry + */ + _parse_key_from_line(entry, bucket_json) { + const line_array = entry.path.split(' '); + const file_path = line_array[line_array.length - 1]; + let file_key = file_path.replace(path.join(bucket_json.path, '/'), ''); + const basename = path.basename(file_key); + if (basename.startsWith(config.NSFS_FOLDER_OBJECT_NAME)) { + file_key = path.join(path.dirname(file_key), '/'); + } + return file_key; + } } +////////////////// +// TAGS HELPERS // +////////////////// + /** - * _update_error_on_status updates an error occured in lifecycle status - * @param {{error: Error, bucket_name?: string, rule_id?: string}} params - * @returns + * checks if tag query_tag is in the list tag_set + * @param {Object} query_tag + * @param {Array} tag_set */ -function _update_error_on_status({ error, bucket_name = undefined, rule_id = undefined }) { - if (!error) return; - if (bucket_name && rule_id) { - (lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[rule_id].errors ??= []).push(error.message); - } else if (bucket_name) { - (lifecycle_run_status.buckets_statuses[bucket_name].errors ??= []).push(error.message); - } else { - (lifecycle_run_status.errors ??= []).push(error.message); +function _list_contain_tag(query_tag, tag_set) { + for (const t of tag_set) { + if (t.key === query_tag.key && t.value === query_tag.value) return true; } -} - -function _get_default_stats() { - return { - num_objects_deleted: 0, num_objects_delete_failed: 0, objects_delete_errors: [], - num_mpu_aborted: 0, num_mpu_abort_failed: 0, mpu_abort_errors: [] - }; + return false; } /** - * write_lifecycle_log_file + * checks if object has all the tags in filter_tags + * @param {Object} object_info + * @param {Array} filter_tags + * @returns */ -async function write_lifecycle_log_file(fs_context, lifecyle_logs_dir_path) { - const log_file_name = `lifecycle_run_${lifecycle_run_status.lifecycle_run_times.run_lifecycle_start_time}.json`; - await nb_native().fs.writeFile( - fs_context, - path.join(lifecyle_logs_dir_path, log_file_name), - Buffer.from(JSON.stringify(lifecycle_run_status)), - { mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_FILE) } - ); +function _file_contain_tags(object_info, filter_tags) { + if (object_info.tags === undefined) return false; + for (const tag of filter_tags) { + if (!_list_contain_tag(tag, object_info.tags)) { + return false; + } + } + return true; } // EXPORTS -exports.run_lifecycle_under_lock = run_lifecycle_under_lock; +exports.NCLifecycle = NCLifecycle; +exports.ILM_POLICIES_TMP_DIR = ILM_POLICIES_TMP_DIR; +exports.ILM_CANDIDATES_TMP_DIR = ILM_CANDIDATES_TMP_DIR; + diff --git a/src/manage_nsfs/nc_master_key_manager.js b/src/manage_nsfs/nc_master_key_manager.js index dff0760de6..9c0ea3adca 100644 --- a/src/manage_nsfs/nc_master_key_manager.js +++ b/src/manage_nsfs/nc_master_key_manager.js @@ -140,7 +140,7 @@ class NCMasterKeysManager { try { const stat = await nb_native().fs.stat(fs_context, master_keys_path); if (stat.ctime.getTime() === this.last_init_time) return; - const master_keys = await native_fs_utils.read_file(fs_context, master_keys_path); + const master_keys = await native_fs_utils.read_file(fs_context, master_keys_path, { parse_json: true }); this._set_keys(master_keys); this.last_init_time = stat.ctime.getTime(); diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js index 76df15167b..81ffac863a 100644 --- a/src/sdk/bucketspace_fs.js +++ b/src/sdk/bucketspace_fs.js @@ -4,6 +4,7 @@ const _ = require('lodash'); const util = require('util'); const path = require('path'); +const { default: Ajv } = require('ajv'); const P = require('../util/promise'); const config = require('../../config'); const RpcError = require('../rpc/rpc_error'); @@ -31,10 +32,12 @@ const nsfs_schema_utils = require('../manage_nsfs/nsfs_schema_utils'); const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils'); const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance(); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; +const native_fs_utils = require('../util/native_fs_utils'); const dbg = require('../util/debug_module')(__filename); const bucket_semaphore = new KeysSemaphore(1); +const ajv = new Ajv(); class BucketSpaceFS extends BucketSpaceSimpleFS { constructor({ config_root }, stats) { @@ -323,7 +326,15 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { // create bucket's underlying storage directory try { await nb_native().fs.mkdir(fs_context, bucket_storage_path, get_umasked_mode(config.BASE_MODE_DIR)); - new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, { bucket_name: name }); + const reserved_tag_event_args = Object.keys(config.NSFS_GLACIER_RESERVED_BUCKET_TAGS).reduce((curr, tag) => { + const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag]; + if (tag_info.event) return Object.assign(curr, { [tag]: tag_info.default }); + return curr; + }, {}); + + new NoobaaEvent(NoobaaEvent.BUCKET_CREATED).create_event(name, { + ...reserved_tag_event_args, bucket_name: name, account: sdk.requesting_account.name + }); } catch (err) { dbg.error('BucketSpaceFS: create_bucket could not create underlying directory - nsfs, deleting bucket', err); new NoobaaEvent(NoobaaEvent.BUCKET_DIR_CREATION_FAILED) @@ -339,7 +350,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { return { _id: mongo_utils.mongoObjectId(), name, - tag: js_utils.default_value(tag, undefined), + tag: js_utils.default_value(tag, BucketSpaceFS._default_bucket_tags()), owner_account: account.owner ? account.owner : account._id, // The account is the owner of the buckets that were created by it or by its users. creator: account._id, versioning: config.NSFS_VERSIONING_ENABLED && lock_enabled ? 'ENABLED' : 'DISABLED', @@ -478,13 +489,39 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { // BUCKET TAGGING // //////////////////// - async put_bucket_tagging(params) { + /** + * @param {*} params + * @param {nb.ObjectSDK} object_sdk + */ + async put_bucket_tagging(params, object_sdk) { try { const { name, tagging } = params; dbg.log0('BucketSpaceFS.put_bucket_tagging: Bucket name, tagging', name, tagging); - const bucket = await this.config_fs.get_bucket_by_name(name); - bucket.tag = tagging; - await this.config_fs.update_bucket_config_file(bucket); + const bucket_path = this.config_fs.get_bucket_path_by_name(name); + + const bucket_lock_file = `${bucket_path}.lock`; + await native_fs_utils.lock_and_run(this.fs_context, bucket_lock_file, async () => { + const bucket = await this.config_fs.get_bucket_by_name(name); + const { ns } = await object_sdk.read_bucket_full_info(name); + const is_bucket_empty = await BucketSpaceFS._is_bucket_empty(name, null, ns, object_sdk); + + const tagging_object = BucketSpaceFS._objectify_tagging_arr(bucket.tag); + const merged_tags = BucketSpaceFS._merge_reserved_tags( + bucket.tag || BucketSpaceFS._default_bucket_tags(), tagging, is_bucket_empty + ); + + const [ + reserved_tag_event_args, + reserved_tag_modified, + ] = BucketSpaceFS._generate_reserved_tag_event_args(tagging_object, merged_tags); + + bucket.tag = merged_tags; + await this.config_fs.update_bucket_config_file(bucket); + if (reserved_tag_modified) { + new NoobaaEvent(NoobaaEvent.BUCKET_RESERVED_TAG_MODIFIED) + .create_event(undefined, { ...reserved_tag_event_args, bucket_name: name }); + } + }); } catch (error) { throw translate_error_codes(error, entity_enum.BUCKET); } @@ -495,7 +532,16 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { const { name } = params; dbg.log0('BucketSpaceFS.delete_bucket_tagging: Bucket name', name); const bucket = await this.config_fs.get_bucket_by_name(name); - delete bucket.tag; + + const preserved_tags = []; + for (const tag of bucket.tag) { + const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key]; + if (tag_info?.immutable) { + preserved_tags.push(tag); + } + } + + bucket.tag = preserved_tags; await this.config_fs.update_bucket_config_file(bucket); } catch (error) { throw translate_error_codes(error, entity_enum.BUCKET); @@ -513,6 +559,56 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { } } + /** + * _default_bucket_tags returns the default bucket tags (primarily reserved tags) + * in the AWS tagging format + * + * @returns {Array<{key: string, value: string}>} + */ + static _default_bucket_tags() { + return Object.keys( + config.NSFS_GLACIER_RESERVED_BUCKET_TAGS + ).map(key => ({ key, value: config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[key].default })); + } + + /** + * _merge_reserved_tags takes original tags, new_tags and a boolean indicating whether + * the bucket is empty or not and returns an array of merged tags based on the rules + * of reserved tags. + * + * @param {Array<{key: string, value: string}>} new_tags + * @param {Array<{key: string, value: string}>} original_tags + * @param {boolean} is_bucket_empty + * @returns {Array<{key: string, value: string}>} + */ + static _merge_reserved_tags(original_tags, new_tags, is_bucket_empty) { + const merged_tags = original_tags.reduce((curr, tag) => { + if (!config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key]) return curr; + return Object.assign(curr, { [tag.key]: tag.value }); + }, {}); + + for (const tag of new_tags) { + const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key]; + if (!tag_info) { + merged_tags[tag.key] = tag.value; + continue; + } + if (!tag_info.immutable || (tag_info.immutable === 'if-data' && is_bucket_empty)) { + let validator = ajv.getSchema(tag_info.schema.$id); + if (!validator) { + ajv.addSchema(tag_info.schema); + validator = ajv.getSchema(tag_info.schema.$id); + } + + if (validator(tag.value)) { + merged_tags[tag.key] = tag.value; + } + } + } + + return Object.keys(merged_tags).map(key => ({ key, value: merged_tags[key] })); + } + //////////////////// // BUCKET LOGGING // //////////////////// @@ -875,6 +971,71 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { return storage_classes; } + + /** + * _is_bucket_empty returns true if the given bucket is empty + * + * @param {*} ns + * @param {*} params + * @param {string} name + * @param {nb.ObjectSDK} object_sdk + * + * @returns {Promise} + */ + static async _is_bucket_empty(name, params, ns, object_sdk) { + params = params || {}; + + let list; + try { + if (ns._is_versioning_disabled()) { + list = await ns.list_objects({ ...params, bucket: name, limit: 1 }, object_sdk); + } else { + list = await ns.list_object_versions({ ...params, bucket: name, limit: 1 }, object_sdk); + } + } catch (err) { + dbg.warn('_is_bucket_empty: bucket name', name, 'got an error while trying to list_objects', err); + // in case the ULS was deleted - we will continue + if (err.rpc_code !== 'NO_SUCH_BUCKET') throw err; + } + + return !(list && list.objects && list.objects.length > 0); + } + + /** + * _generate_reserved_tag_event_args returns the list of reserved + * bucket tags which have been modified and also returns a variable + * indicating if there has been any modifications at all. + * @param {Record} prev_tags_objectified + * @param {Array<{key: string, value: string}>} new_tags + */ + static _generate_reserved_tag_event_args(prev_tags_objectified, new_tags) { + let reserved_tag_modified = false; + const reserved_tag_event_args = new_tags?.reduce((curr, tag) => { + const tag_info = config.NSFS_GLACIER_RESERVED_BUCKET_TAGS[tag.key]; + + // If not a reserved tag - skip + if (!tag_info) return curr; + + // If no event is requested - skip + if (!tag_info.event) return curr; + + // If value didn't change - skip + if (_.isEqual(prev_tags_objectified[tag.key], tag.value)) return curr; + + reserved_tag_modified = true; + return Object.assign(curr, { [tag.key]: tag.value }); + }, {}); + + return [reserved_tag_event_args, reserved_tag_modified]; + } + + /** + * @param {Array<{key: string, value: string}>} tagging + * @returns {Record} + */ + static _objectify_tagging_arr(tagging) { + return (tagging || []).reduce((curr, tag) => Object.assign(curr, { [tag.key]: tag.value }), {}); + } } module.exports = BucketSpaceFS; diff --git a/src/sdk/bucketspace_simple_fs.js b/src/sdk/bucketspace_simple_fs.js index 39ac38ad41..eda1c037eb 100644 --- a/src/sdk/bucketspace_simple_fs.js +++ b/src/sdk/bucketspace_simple_fs.js @@ -168,7 +168,12 @@ class BucketSpaceSimpleFS { // BUCKET TAGGING // //////////////////// - async put_bucket_tagging(params) { + /** + * + * @param {*} params + * @param {nb.ObjectSDK} object_sdk + */ + async put_bucket_tagging(params, object_sdk) { // TODO } diff --git a/src/sdk/namespace_fs.js b/src/sdk/namespace_fs.js index b58652d68b..29d9455dd0 100644 --- a/src/sdk/namespace_fs.js +++ b/src/sdk/namespace_fs.js @@ -23,6 +23,7 @@ const LRUCache = require('../util/lru_cache'); const nb_native = require('../util/nb_native'); const RpcError = require('../rpc/rpc_error'); const { S3Error } = require('../endpoint/s3/s3_errors'); +const lifecycle_utils = require('../util/lifecycle_utils'); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; const { PersistentLogger } = require('../util/persistent_logger'); const { Glacier } = require('./glacier'); @@ -61,6 +62,7 @@ const XATTR_PART_ETAG = XATTR_NOOBAA_INTERNAL_PREFIX + 'part_etag'; const XATTR_VERSION_ID = XATTR_NOOBAA_INTERNAL_PREFIX + 'version_id'; const XATTR_DELETE_MARKER = XATTR_NOOBAA_INTERNAL_PREFIX + 'delete_marker'; const XATTR_DIR_CONTENT = XATTR_NOOBAA_INTERNAL_PREFIX + 'dir_content'; +const XATTR_NON_CURRENT_TIMESTASMP = XATTR_NOOBAA_INTERNAL_PREFIX + 'non_current_timestamp'; const XATTR_TAG = XATTR_NOOBAA_INTERNAL_PREFIX + 'tag.'; const HIDDEN_VERSIONS_PATH = '.versions'; const NULL_VERSION_ID = 'null'; @@ -292,6 +294,11 @@ function to_fs_xattr(xattr) { return _.mapKeys(xattr, (val, key) => XATTR_USER_PREFIX + key); } +/** + * get_tags_from_xattr converts relevant xattr to tags format + * @param {Object} xattr + * @returns {Object} + */ function get_tags_from_xattr(xattr) { const tag_set = []; for (const [xattr_key, xattr_value] of Object.entries(xattr)) { @@ -730,6 +737,8 @@ class NamespaceFS { const use_lstat = !(await this._is_path_in_bucket_boundaries(fs_context, entry_path)); const stat = await native_fs_utils.stat_if_exists(fs_context, entry_path, use_lstat, config.NSFS_LIST_IGNORE_ENTRY_ON_EACCES); + // TODO - GAP of .folder files - we return stat of the directory for the + // xattr, but the creation time should be of the .folder files (and maybe more ) if (stat) { r.stat = stat; // add the result only if we have the stat information @@ -1385,10 +1394,10 @@ class NamespaceFS { fs_xattr = this._assign_md5_to_fs_xattr(digest, fs_xattr); } if (part_upload) { - fs_xattr = await this._assign_part_props_to_fs_xattr(fs_context, params.size, digest, offset, fs_xattr); + fs_xattr = this._assign_part_props_to_fs_xattr(params.size, digest, offset, fs_xattr); } if (!part_upload && (this._is_versioning_enabled() || this._is_versioning_suspended())) { - fs_xattr = await this._assign_versions_to_fs_xattr(stat, fs_xattr, undefined); + fs_xattr = this._assign_versions_to_fs_xattr(stat, fs_xattr, undefined); } if (!part_upload && params.storage_class) { fs_xattr = Object.assign(fs_xattr || {}, { @@ -1506,8 +1515,10 @@ class NamespaceFS { await native_fs_utils._make_path_dirs(latest_ver_path, fs_context); if (is_gpfs) { const latest_ver_info_exist = await native_fs_utils.is_path_exists(fs_context, latest_ver_path); - gpfs_options = await this._open_files_gpfs(fs_context, new_ver_tmp_path, latest_ver_path, upload_file, - latest_ver_info_exist, open_mode, undefined, undefined); + gpfs_options = await this._open_files(fs_context, { + src_path: new_ver_tmp_path, dst_path: latest_ver_path, upload_or_dir_file: upload_file, + dst_ver_exist: latest_ver_info_exist, open_mode + }); //get latest version if exists const latest_fd = gpfs_options?.move_to_dst?.dst_file; @@ -1540,6 +1551,7 @@ class NamespaceFS { await native_fs_utils._make_path_dirs(versioned_path, fs_context); await native_fs_utils.safe_move(fs_context, latest_ver_path, versioned_path, latest_ver_info, gpfs_options?.move_to_versions, bucket_tmp_dir_path); + await this._set_non_current_timestamp_on_past_version(fs_context, versioned_path); } try { // move new version to latest_ver_path (key path) @@ -1559,7 +1571,7 @@ class NamespaceFS { if (!should_retry || retries <= 0) throw err; await P.delay(get_random_delay(config.NSFS_RANDOM_DELAY_BASE, 0, 50)); } finally { - if (gpfs_options) await this._close_files_gpfs(fs_context, gpfs_options.move_to_dst, open_mode); + if (gpfs_options) await this._close_files(fs_context, gpfs_options.move_to_dst, open_mode); } } } @@ -1987,7 +1999,7 @@ class NamespaceFS { const file_path = this._get_file_path({ key }); await this._check_path_in_bucket_boundaries(fs_context, file_path); dbg.log1('NamespaceFS: delete_multiple_objects', file_path); - await this._delete_single_object(fs_context, file_path, { key }); + await this._delete_single_object(fs_context, file_path, { key, filter_func: params.filter_func }); res.push({ key }); } catch (err) { res.push({ err_code: err.code, err_message: err.message }); @@ -2002,7 +2014,7 @@ class NamespaceFS { } dbg.log3('NamespaceFS: versions_by_key_map', versions_by_key_map); for (const key of Object.keys(versions_by_key_map)) { - const key_res = await this._delete_objects_versioned(fs_context, key, versions_by_key_map[key]); + const key_res = await this._delete_objects_versioned(fs_context, key, versions_by_key_map[key], params.filter_func); res = res.concat(key_res); } } @@ -2012,9 +2024,32 @@ class NamespaceFS { } } - + /** + * _delete_single_object does the following before deleting the object + * 1. if is_lifecycle_deletion - + * 1.1. open dir_file and src_file fd + * 1.2. _verify_lifecycle_filter_and_unlink - which means it stats the to be deleted file, validate filter if exists, unlink safely + * 2. else - unlink_ignore_enoent + * 3. deleted parent directories if they are empty + * 4. clears directory object xattr if relevant + * 5. closes file and dir_file + */ async _delete_single_object(fs_context, file_path, params) { - await native_fs_utils.unlink_ignore_enoent(fs_context, file_path); + const is_lifecycle_deletion = this.is_lifecycle_deletion_flow(params); + if (is_lifecycle_deletion) { + let files; + try { + files = await this._open_files(fs_context, { src_path: file_path, delete_version: true }); + await this._verify_lifecycle_filter_and_unlink(fs_context, params, file_path, files.delete_version); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + } finally { + if (files) await this._close_files(fs_context, files.delete_version, undefined, true); + } + } else { + await native_fs_utils.unlink_ignore_enoent(fs_context, file_path); + } + await this._delete_path_dirs(file_path, fs_context); // when deleting the data of a directory object, we need to remove the directory dir object xattr // if the dir still exists - occurs when deleting dir while the dir still has entries in it @@ -2312,17 +2347,30 @@ class NamespaceFS { return fs_xattr; } - async _assign_versions_to_fs_xattr(new_ver_stat, fs_xattr, delete_marker) { + /** + * _assign_versions_to_fs_xattr assigns version related xattrs to the file + * 1. assign version_id xattr + * 2. if delete_marker - + * 2.1. assigns delete_marker xattr + * 2.2. assigns non_current_timestamp xattr - on the current structure - delete marker is under .versions/ + * @param {nb.NativeFSStats} new_ver_stat + * @param {nb.NativeFSXattr} fs_xattr + * @param {Boolean} [delete_marker] + * @returns {nb.NativeFSXattr} + */ + _assign_versions_to_fs_xattr(new_ver_stat, fs_xattr, delete_marker = undefined) { fs_xattr = Object.assign(fs_xattr || {}, { [XATTR_VERSION_ID]: this._get_version_id_by_mode(new_ver_stat) }); - if (delete_marker) fs_xattr[XATTR_DELETE_MARKER] = delete_marker; - + if (delete_marker) { + fs_xattr[XATTR_DELETE_MARKER] = String(delete_marker); + fs_xattr = this._assign_non_current_timestamp_xattr(fs_xattr); + } return fs_xattr; } - async _assign_part_props_to_fs_xattr(fs_context, size, digest, offset, fs_xattr) { + _assign_part_props_to_fs_xattr(size, digest, offset, fs_xattr) { fs_xattr = Object.assign(fs_xattr || {}, { [XATTR_PART_SIZE]: size, [XATTR_PART_OFFSET]: offset, @@ -2332,6 +2380,39 @@ class NamespaceFS { return fs_xattr; } + /** + * _assign_non_current_timestamp_xattr assigns non current timestamp xattr to file xattr + * @param {nb.NativeFSXattr} fs_xattr + * @returns {nb.NativeFSXattr} + */ + _assign_non_current_timestamp_xattr(fs_xattr = {}) { + fs_xattr = Object.assign(fs_xattr, { + [XATTR_NON_CURRENT_TIMESTASMP]: String(Date.now()) + }); + return fs_xattr; + } + + /** + * _set_non_current_timestamp_on_past_version sets non current timestamp on past version - used as a hint for lifecycle process + * @param {nb.NativeFSContext} fs_context + * @param {String} versioned_path + * @returns {Promise} + */ + async _set_non_current_timestamp_on_past_version(fs_context, versioned_path) { + const xattr = this._assign_non_current_timestamp_xattr(); + await this.set_fs_xattr_op(fs_context, versioned_path, xattr); + } + + /** + * _unset_non_current_timestamp_on_past_version unsets non current timestamp on past version - used as a hint for lifecycle process + * @param {nb.NativeFSContext} fs_context + * @param {String} versioned_path + * @returns {Promise} + */ + async _unset_non_current_timestamp_on_past_version(fs_context, versioned_path) { + await this._clear_user_xattr(fs_context, versioned_path, XATTR_NON_CURRENT_TIMESTASMP); + } + /** * * @param {*} fs_context - fs context object @@ -2464,6 +2545,8 @@ class NamespaceFS { const storage_class = Glacier.storage_class_from_xattr(stat.xattr); const size = Number(stat.xattr?.[XATTR_DIR_CONTENT] || stat.size); const tag_count = stat.xattr ? this._number_of_tags_fs_xttr(stat.xattr) : 0; + const nc_noncurrent_time = (stat.xattr?.[XATTR_NON_CURRENT_TIMESTASMP] && Number(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP])) || + stat.ctime.getTime(); return { obj_id: etag, @@ -2482,6 +2565,7 @@ class NamespaceFS { xattr: to_xattr(stat.xattr), tag_count, tagging: get_tags_from_xattr(stat.xattr), + nc_noncurrent_time, // temp: lock_settings: undefined, @@ -2813,20 +2897,26 @@ class NamespaceFS { return path.normalize(path.join(this.bucket_path, path.dirname(key), HIDDEN_VERSIONS_PATH, key_version)); } - // this function returns the following version information - - // version_id_str - mtime-{mtimeNsBigint}-ino-{ino} | explicit null - // mtimeNsBigint - modified timestmap in bigint - last time the content of the file was modified - // ino - refers to the data stored in a particular location - // delete_marker - specifies if the version is a delete marker - // path - specifies the path to version - // if version xattr contains version info - return info by xattr - // else - it's a null version - return stat - // if fd is passed, will use fd instead of path to stat - async _get_version_info(fs_context, version_path, fd) { + + /** + * this function returns the following version information - + * version_id_str - mtime-{mtimeNsBigint}-ino-{ino} | explicit null + * mtimeNsBigint - modified timestmap in bigint - last time the content of the file was modified + * ino - refers to the data stored in a particular location + * delete_marker - specifies if the version is a delete marker + * path - specifies the path to version + * if version xattr contains version info - return info by xattr + * else - it's a null version - return stat + * if file is passed, will use file instead of path to stat + * @param {nb.NativeFSContext} fs_context + * @param {String} version_path + * @param {nb.NativeFile} [file] + * @returns {Promise} + */ + async _get_version_info(fs_context, version_path, file = undefined) { try { - const stat = fd ? await fd.stat(fs_context, { skip_user_xattr: true }) : - await nb_native().fs.stat(fs_context, version_path, { skip_user_xattr: true }); - dbg.log1('NamespaceFS._get_version_info stat ', stat, version_path, fd); + const stat = file ? await file.stat(fs_context) : await nb_native().fs.stat(fs_context, version_path); + dbg.log1('NamespaceFS._get_version_info stat ', stat, version_path, file); const version_id_str = this._get_version_id_by_xattr(stat); const ver_info_by_xattr = this._extract_version_info_from_xattr(version_id_str); @@ -2914,8 +3004,15 @@ class NamespaceFS { } } - _is_mismatch_version_id(stat, version_id) { - return version_id && !this._is_versioning_disabled() && this._get_version_id_by_xattr(stat) !== version_id; + /** + * _is_mismatch_version_id checks if the expected_version_id equals to the version_id_str received by version_info or by version_id xattr coming from stat + * @param {nb.NativeFSStats} stat + * @param {String} expected_version_id + * @returns {Boolean} + */ + _is_mismatch_version_id(stat, expected_version_id) { + const actual_version_id = this._get_version_id_by_xattr(stat); + return expected_version_id && !this._is_versioning_disabled() && expected_version_id !== actual_version_id; } /** @@ -2925,8 +3022,7 @@ class NamespaceFS { * we call check_version_moved() in case of concurrent puts, the version might move to .versions/ * if the version moved we will retry * @param {nb.NativeFSContext} fs_context - * @param {string} key - * @param {string} version_id + * @param {{key: string, version_id: string, filter_func: function}} params * @returns {Promise<{ * version_id_str: any; * delete_marker: string; @@ -2936,35 +3032,48 @@ class NamespaceFS { * latest?: boolean; * }>} */ - async _delete_single_object_versioned(fs_context, key, version_id) { + async _delete_single_object_versioned(fs_context, params) { let retries = config.NSFS_RENAME_RETRIES; + const { key, version_id } = params; const latest_version_path = this._get_file_path({ key }); + const is_lifecycle_deletion = this.is_lifecycle_deletion_flow(params); + const is_gpfs = native_fs_utils._is_gpfs(fs_context); + for (;;) { let file_path; - let gpfs_options; + let files; try { - file_path = await this._find_version_path(fs_context, { key, version_id }); + file_path = await this._find_version_path(fs_context, params); await this._check_path_in_bucket_boundaries(fs_context, file_path); const version_info = await this._get_version_info(fs_context, file_path); if (!version_info) return; const deleted_latest = file_path === latest_version_path; if (deleted_latest) { - gpfs_options = await this._open_files_gpfs(fs_context, file_path, undefined, undefined, undefined, undefined, true); - if (gpfs_options) { - const src_stat = await gpfs_options.delete_version.src_file.stat(fs_context); + if (is_gpfs) { + files = await this._open_files(fs_context, { src_path: file_path, delete_version: true }); + const src_stat = await files.delete_version.src_file.stat(fs_context); if (this._is_mismatch_version_id(src_stat, version_id)) { dbg.warn('NamespaceFS._delete_single_object_versioned mismatch version_id', file_path, version_id, this._get_version_id_by_xattr(src_stat)); throw error_utils.new_error_code('MISMATCH_VERSION', 'file version does not match the version we asked for'); } } const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path(); - await native_fs_utils.safe_unlink(fs_context, file_path, version_info, - gpfs_options?.delete_version, bucket_tmp_dir_path); + if (is_lifecycle_deletion) { + files = await this._open_files(fs_context, { src_path: file_path, delete_version: true }); + await this._verify_lifecycle_filter_and_unlink(fs_context, params, file_path, files.delete_version); + } else { + await native_fs_utils.safe_unlink(fs_context, file_path, version_info, files?.delete_version, bucket_tmp_dir_path); + } await this._check_version_moved(fs_context, key, version_id); return { ...version_info, latest: true }; } else { - await native_fs_utils.unlink_ignore_enoent(fs_context, file_path); + if (is_lifecycle_deletion) { + files = await this._open_files(fs_context, { src_path: file_path, delete_version: true }); + await this._verify_lifecycle_filter_and_unlink(fs_context, params, file_path, files.delete_version); + } else { + await native_fs_utils.unlink_ignore_enoent(fs_context, file_path); + } await this._check_version_moved(fs_context, key, version_id); } return version_info; @@ -2981,7 +3090,7 @@ class NamespaceFS { if (retries <= 0 || !native_fs_utils.should_retry_link_unlink(err)) throw err; await P.delay(get_random_delay(config.NSFS_RANDOM_DELAY_BASE, 0, 50)); } finally { - if (gpfs_options) await this._close_files_gpfs(fs_context, gpfs_options.delete_version, undefined, true); + if (files) await this._close_files(fs_context, files.delete_version, undefined, true); } } } @@ -2990,7 +3099,7 @@ class NamespaceFS { // 1.1 if version_id is undefined, delete latest // 1.2 if version exists - unlink version // 2. try promote second latest to latest if one of the deleted versions is the latest version (with version id specified) or a delete marker - async _delete_objects_versioned(fs_context, key, versions) { + async _delete_objects_versioned(fs_context, key, versions, filter_func) { dbg.log1('NamespaceFS._delete_objects_versioned', key, versions); const res = []; let deleted_delete_marker; @@ -3001,7 +3110,7 @@ class NamespaceFS { for (const version_id of versions) { try { if (version_id) { - const del_ver_info = await this._delete_single_object_versioned(fs_context, key, version_id); + const del_ver_info = await this._delete_single_object_versioned(fs_context, { key, version_id, filter_func }); if (!del_ver_info) { res.push({}); continue; @@ -3013,7 +3122,8 @@ class NamespaceFS { } res.push({ deleted_delete_marker: del_ver_info.delete_marker }); } else { - const version_res = await this._delete_latest_version(fs_context, latest_version_path, { key, version_id }); + const version_res = await this._delete_latest_version(fs_context, latest_version_path, + { key, version_id, filter_func }); res.push(version_res); delete_marker_created = true; } @@ -3046,7 +3156,7 @@ class NamespaceFS { */ async _delete_version_id(fs_context, file_path, params) { // TODO optimization - GPFS link overrides, no need to unlink before promoting, but if there is nothing to promote we should unlink - const del_obj_version_info = await this._delete_single_object_versioned(fs_context, params.key, params.version_id); + const del_obj_version_info = await this._delete_single_object_versioned(fs_context, params); if (!del_obj_version_info) return {}; // we try promote only if the latest version was deleted or we deleted a delete marker @@ -3088,6 +3198,8 @@ class NamespaceFS { const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path(); await native_fs_utils.safe_move_posix(fs_context, max_past_ver_info.path, latest_ver_path, max_past_ver_info, bucket_tmp_dir_path); + // TODO - catch error if no such xattr + await this._unset_non_current_timestamp_on_past_version(fs_context, latest_ver_path); break; } catch (err) { dbg.warn(`NamespaceFS: _promote_version_to_latest failed error: retries=${retries}`, err); @@ -3120,22 +3232,25 @@ class NamespaceFS { async _delete_latest_version(fs_context, latest_ver_path, params) { dbg.log0('Namespace_fs._delete_latest_version:', latest_ver_path, params); - let gpfs_options; + let files; const is_gpfs = native_fs_utils._is_gpfs(fs_context); let retries = config.NSFS_RENAME_RETRIES; let latest_ver_info; for (;;) { try { - // TODO get latest version from file in POSIX like in GPFS path latest_ver_info = await this._get_version_info(fs_context, latest_ver_path); dbg.log1('Namespace_fs._delete_latest_version:', latest_ver_info); if (latest_ver_info) { - if (is_gpfs) { - gpfs_options = await this._open_files_gpfs(fs_context, latest_ver_path, undefined, undefined, undefined, - undefined, true); - const latest_fd = gpfs_options?.delete_version?.src_file; - latest_ver_info = latest_fd && await this._get_version_info(fs_context, undefined, latest_fd); + const is_lifecycle_deletion = this.is_lifecycle_deletion_flow(params); + if (is_gpfs || is_lifecycle_deletion) { + files = await this._open_files(fs_context, { src_path: latest_ver_path, delete_version: true }); + const latest_file = files?.delete_version?.src_file; + latest_ver_info = latest_file && await this._get_version_info(fs_context, undefined, latest_file); if (!latest_ver_info) break; + if (is_lifecycle_deletion) { + const stat = await latest_file.stat(fs_context); + this._check_lifecycle_filter_before_deletion(params, stat); + } } const versioned_path = this._get_version_path(params.key, latest_ver_info.version_id_str); @@ -3144,8 +3259,9 @@ class NamespaceFS { const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path(); if (this._is_versioning_enabled() || suspended_and_latest_is_not_null) { await native_fs_utils._make_path_dirs(versioned_path, fs_context); - await native_fs_utils.safe_move_posix(fs_context, latest_ver_path, versioned_path, latest_ver_info, + await native_fs_utils.safe_move_posix(fs_context, latest_ver_path, versioned_path, latest_ver_info, bucket_tmp_dir_path); + await this._set_non_current_timestamp_on_past_version(fs_context, versioned_path); if (suspended_and_latest_is_not_null) { // remove a version (or delete marker) with null version ID from .versions/ (if exists) await this._delete_null_version_from_versions_directory(params.key, fs_context); @@ -3154,7 +3270,7 @@ class NamespaceFS { // versioning suspended and version_id is null dbg.log1('NamespaceFS._delete_latest_version: suspended mode version ID of the latest version is null - file will be unlinked'); await native_fs_utils.safe_unlink(fs_context, latest_ver_path, latest_ver_info, - gpfs_options?.delete_version, bucket_tmp_dir_path); + files?.delete_version, bucket_tmp_dir_path); } } break; @@ -3164,7 +3280,7 @@ class NamespaceFS { if (retries <= 0 || !native_fs_utils.should_retry_link_unlink(err)) throw err; await P.delay(get_random_delay(config.NSFS_RANDOM_DELAY_BASE, 0, 50)); } finally { - if (gpfs_options) await this._close_files_gpfs(fs_context, gpfs_options.delete_version, undefined, true); + if (files) await this._close_files(fs_context, files.delete_version, undefined, true); } } // create delete marker and move it to .versions/key_{delete_marker_version_id} @@ -3182,15 +3298,17 @@ class NamespaceFS { const null_versioned_path = this._get_version_path(key, NULL_VERSION_ID); await this._check_path_in_bucket_boundaries(fs_context, null_versioned_path); let gpfs_options; + const is_gpfs = native_fs_utils._is_gpfs(fs_context); + let retries = config.NSFS_RENAME_RETRIES; for (;;) { try { const null_versioned_path_info = await this._get_version_info(fs_context, null_versioned_path); dbg.log1('Namespace_fs._delete_null_version_from_versions_directory:', null_versioned_path, null_versioned_path_info); if (!null_versioned_path_info) return; - - gpfs_options = await this._open_files_gpfs(fs_context, null_versioned_path, undefined, undefined, undefined, - undefined, true); + gpfs_options = is_gpfs ? + await this._open_files(fs_context, { src_path: null_versioned_path, delete_version: true }) : + undefined; const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path(); await native_fs_utils.safe_unlink(fs_context, null_versioned_path, null_versioned_path_info, gpfs_options?.delete_version, bucket_tmp_dir_path); @@ -3201,7 +3319,7 @@ class NamespaceFS { if (retries <= 0 || !native_fs_utils.should_retry_link_unlink(err)) throw err; await P.delay(get_random_delay(config.NSFS_RANDOM_DELAY_BASE, 0, 50)); } finally { - if (gpfs_options) await this._close_files_gpfs(fs_context, gpfs_options.delete_version, undefined, true); + if (gpfs_options) await this._close_files(fs_context, gpfs_options.delete_version, undefined, true); } } } @@ -3225,7 +3343,7 @@ class NamespaceFS { } const file_path = this._get_version_path(params.key, delete_marker_version_id); - const fs_xattr = await this._assign_versions_to_fs_xattr(stat, undefined, true); + const fs_xattr = this._assign_versions_to_fs_xattr(stat, undefined, true); if (fs_xattr) await upload_params.target_file.replacexattr(fs_context, fs_xattr); // create .version in case we don't have it yet await native_fs_utils._make_path_dirs(file_path, fs_context); @@ -3282,11 +3400,10 @@ class NamespaceFS { // opens the unopened files involved in the version move during upload/deletion // returns an object contains the relevant options for the move/unlink flow - // eslint-disable-next-line max-params - async _open_files_gpfs(fs_context, src_path, dst_path, upload_or_dir_file, dst_ver_exist, open_mode, delete_version, versioned_info) { - dbg.log1('Namespace_fs._open_files_gpfs:', src_path, src_path && path.dirname(src_path), dst_path, upload_or_dir_file, Boolean(dst_ver_exist), open_mode, delete_version, versioned_info); - const is_gpfs = native_fs_utils._is_gpfs(fs_context); - if (!is_gpfs) return; + async _open_files(fs_context, options) { + const { src_path = undefined, dst_path = undefined, upload_or_dir_file = undefined, + dst_ver_exist = false, open_mode = undefined, delete_version = false, versioned_info = undefined } = options; + dbg.log1('Namespace_fs._open_files:', src_path, src_path && path.dirname(src_path), dst_path, upload_or_dir_file, Boolean(dst_ver_exist), open_mode, delete_version, versioned_info); let src_file; let dst_file; @@ -3311,7 +3428,7 @@ class NamespaceFS { dir_file = await native_fs_utils.open_file(fs_context, this.bucket_path, path.dirname(src_path), 'r'); } if (dst_ver_exist) { - dbg.log1('NamespaceFS._open_files_gpfs dst version exist - opening dst version file...'); + dbg.log1('NamespaceFS._open_files dst version exist - opening dst version file...'); dst_file = await native_fs_utils.open_file(fs_context, this.bucket_path, dst_path, 'r'); } return { @@ -3319,34 +3436,34 @@ class NamespaceFS { move_to_dst: { src_file, dst_file, dir_file} }; } catch (err) { - dbg.warn('NamespaceFS._open_files_gpfs couldn\'t open files', err); - await this._close_files_gpfs(fs_context, { src_file, dst_file, dir_file, versioned_file }, open_mode, delete_version); + dbg.warn('NamespaceFS._open_files couldn\'t open files', err); + await this._close_files(fs_context, { src_file, dst_file, dir_file, versioned_file }, open_mode, delete_version); throw err; } } // closes files opened during gpfs upload / deletion, avoiding closing files that opened sooner in the process - async _close_files_gpfs(fs_context, files_to_close, open_mode, delete_version) { + async _close_files(fs_context, files_to_close, open_mode, delete_version) { const { src_file, dst_file = undefined, dir_file, versioned_file = undefined } = files_to_close; try { if (src_file && (delete_version || open_mode === 'wt')) await src_file.close(fs_context); } catch (err) { - dbg.warn('NamespaceFS: _close_files_gpfs src_file error', err); + dbg.warn('NamespaceFS: _close_files src_file error', err); } try { if (dst_file) await dst_file.close(fs_context); } catch (err) { - dbg.warn('NamespaceFS: _close_files_gpfs dst_file error', err); + dbg.warn('NamespaceFS: _close_files dst_file error', err); } try { if (dir_file && (delete_version || open_mode !== 'wt')) await dir_file.close(fs_context); } catch (err) { - dbg.warn('NamespaceFS: _close_files_gpfs dir_file error', err); + dbg.warn('NamespaceFS: _close_files dir_file error', err); } try { if (versioned_file) await versioned_file.close(fs_context); } catch (err) { - dbg.warn('NamespaceFS: _close_files_gpfs versioned_file error', err); + dbg.warn('NamespaceFS: _close_files versioned_file error', err); } } @@ -3366,6 +3483,10 @@ class NamespaceFS { if (latest_ver_info && latest_ver_info.version_id_str === version_id) throw error_utils.new_error_code('VERSION_MOVED', `version file moved from .versions/ ${versioned_path} to latest ${latest_version_path}, retrying`); } + ///////////////////////// + // GLACIER HELPERS // + ///////////////////////// + async _throw_if_storage_class_not_supported(storage_class) { if (!await this._is_storage_class_supported(storage_class)) { throw new S3Error(S3Error.InvalidStorageClass); @@ -3493,6 +3614,71 @@ class NamespaceFS { return NamespaceFS._restore_wal; } + + //////////////////////////// + // LIFECYLE HELPERS // + //////////////////////////// + + /** + * _verify_lifecycle_filter_and_unlink does the following - + * 1. stat the to be deleted file + * 2. checks that the file should be deleted based on lifecycle filter (if the flow is not lifecycle the check will be skipped) + * 3. calls safe_unlink that inside checks that the path to be deleted has the same inode/fd of the file that should be deleted + * GAP - in .folder if exists we should take mtime from the file and not from the directory, this is a bug in get_object_info we should fix + * @param {nb.NativeFSContext} fs_context + * @param {Object} params + * @param {String} file_path + * @param {{dir_file: nb.NativeFile, src_file: nb.NativeFile }} files + */ + async _verify_lifecycle_filter_and_unlink(fs_context, params, file_path, { dir_file, src_file }) { + try { + const is_dir_content = this._is_directory_content(file_path, params.key); + const dir_stat = is_dir_content && await dir_file.stat(fs_context); + const is_empty_directory_content = dir_stat && dir_stat.xattr && dir_stat.xattr[XATTR_DIR_CONTENT] === '0'; + const src_stat = !is_empty_directory_content && await src_file.stat(fs_context); + const stat = is_empty_directory_content ? dir_stat : is_dir_content && { ...src_stat, xattr: dir_stat.xattr } || src_stat; + + this._check_lifecycle_filter_before_deletion(params, stat); + const bucket_tmp_dir_path = this.get_bucket_tmpdir_full_path(); + await native_fs_utils.safe_unlink(fs_context, file_path, stat, { dir_file, src_file }, bucket_tmp_dir_path); + } catch (err) { + dbg.log0('_verify_lifecycle_filter_and_unlink err', err.code, err, file_path); + if (err.code !== 'ENOENT' && err.code !== 'EISDIR') throw err; + } + } + + /** + * _check_lifecycle_filter_before_deletion checks if filter_func provided that we want to delete the object + * @param {Object} params + * @param {nb.NativeFSStats} stat + * @returns {Void} + */ + _check_lifecycle_filter_before_deletion(params, stat) { + if (!params.filter_func) return; + const obj_info = { + key: params.key, + create_time: stat.mtime.getTime(), + size: stat.size, + tagging: get_tags_from_xattr(stat.xattr) + }; + const should_delete = lifecycle_utils.file_matches_filter({ obj_info, filter_func: params.filter_func }); + if (!should_delete) { + const err = new RpcError('FILTER_MATCH_FAILED', `file_matches_filter lifecycle - filter on file returned false ${obj_info.key}`); + dbg.error(err.message); + err.code = 'FILTER_MATCH_FAILED'; + throw err; + } + } + + /** + * is_lifecycle_deletion_flow returns true if params contain filter_func which occurs when calling the function + * from lifecycle deletion flow + * @param {Object} params + * @returns {Boolean} + */ + is_lifecycle_deletion_flow(params) { + return params.filter_func; + } } /** @type {PersistentLogger} */ @@ -3504,3 +3690,4 @@ NamespaceFS._restore_wal = null; module.exports = NamespaceFS; module.exports.multi_buffer_pool = multi_buffer_pool; + diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index 3ef7d0635d..6f7132517b 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -442,6 +442,7 @@ interface ObjectInfo { restore_status?: RestoreStatus; checksum?: Checksum; object_parts?: GetObjectAttributesParts; + nc_noncurrent_time ?: number; } @@ -838,7 +839,7 @@ interface BucketSpace { set_bucket_versioning(params: object, object_sdk: ObjectSDK): Promise; - put_bucket_tagging(params: object): Promise; + put_bucket_tagging(params: object, object_sdk: ObjectSDK): Promise; delete_bucket_tagging(params: object): Promise; get_bucket_tagging(params: object): Promise; diff --git a/src/sdk/object_sdk.js b/src/sdk/object_sdk.js index ae10d267cf..9870c2ae07 100644 --- a/src/sdk/object_sdk.js +++ b/src/sdk/object_sdk.js @@ -986,7 +986,7 @@ class ObjectSDK { async put_bucket_tagging(params) { const bs = this._get_bucketspace(); - return bs.put_bucket_tagging(params); + return bs.put_bucket_tagging(params, this); } async delete_bucket_tagging(params) { diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js index 35689b560d..9ae5bc9fc9 100644 --- a/src/server/object_services/md_store.js +++ b/src/server/object_services/md_store.js @@ -483,7 +483,7 @@ class MDStore { $lt: new Date(moment.unix(max_create_time).toISOString()), $exists: true } : undefined, - tagging: tagging ? { + tagging: (tagging?.length > 0) ? { $all: tagging, } : undefined, size: (max_size || min_size) ? diff --git a/src/test/lifecycle/common.js b/src/test/lifecycle/common.js index 18c0763af9..333b4c0fac 100644 --- a/src/test/lifecycle/common.js +++ b/src/test/lifecycle/common.js @@ -161,8 +161,10 @@ function size_gt_lt_lifecycle_configuration(Bucket, gt, lt) { Date: midnight, }, Filter: { - ObjectSizeLessThan: lt, - ObjectSizeGreaterThan: gt + And: { + ObjectSizeLessThan: lt, + ObjectSizeGreaterThan: gt, + }, }, Status: 'Enabled', }, ], @@ -366,7 +368,7 @@ function duplicate_id_lifecycle_configuration(Bucket, Key) { Bucket, LifecycleConfiguration: { Rules: [{ - ID1, + ID: ID1, Expiration: { Days: 17, }, @@ -376,7 +378,7 @@ function duplicate_id_lifecycle_configuration(Bucket, Key) { Status: 'Enabled', }, { - ID2, + ID: ID2, Expiration: { Days: 18, }, @@ -555,7 +557,7 @@ exports.test_rule_id_length = async function(Bucket, Key, s3) { await s3.putBucketLifecycleConfiguration(putLifecycleParams); assert.fail(`Expected error for ID length exceeding maximum allowed characters ${s3_const.MAX_RULE_ID_LENGTH}, but request was successful`); } catch (error) { - assert(error.code === 'InvalidArgument', `Expected InvalidArgument: id length exceeding ${s3_const.MAX_RULE_ID_LENGTH} characters`); + assert(error.Code === 'InvalidArgument', `Expected InvalidArgument: id length exceeding ${s3_const.MAX_RULE_ID_LENGTH} characters`); } }; @@ -566,7 +568,7 @@ exports.test_rule_duplicate_id = async function(Bucket, Key, s3) { await s3.putBucketLifecycleConfiguration(putLifecycleParams); assert.fail('Expected error for duplicate rule ID, but request was successful'); } catch (error) { - assert(error.code === 'InvalidArgument', 'Expected InvalidArgument: duplicate ID found in the rules'); + assert(error.Code === 'InvalidArgument', 'Expected InvalidArgument: duplicate ID found in the rules'); } }; @@ -580,6 +582,80 @@ exports.test_rule_status_value = async function(Bucket, Key, s3) { await s3.putBucketLifecycleConfiguration(putLifecycleParams); assert.fail('Expected MalformedXML error due to wrong status value, but received a different response'); } catch (error) { - assert(error.code === 'MalformedXML', `Expected MalformedXML error: due to invalid status value`); + assert(error.Code === 'MalformedXML', `Expected MalformedXML error: due to invalid status value`); + } +}; + +exports.test_invalid_filter_format = async function(Bucket, Key, s3) { + const putLifecycleParams = tags_lifecycle_configuration(Bucket, Key); + + // append prefix for invalid filter: "And" condition is missing, but multiple filters are present + putLifecycleParams.LifecycleConfiguration.Rules[0].Filter.Prefix = 'test-prefix'; + + try { + await s3.putBucketLifecycleConfiguration(putLifecycleParams); + assert.fail('Expected MalformedXML error due to missing "And" condition for multiple filters'); + } catch (error) { + assert(error.Code === 'MalformedXML', 'Expected MalformedXML error: due to missing "And" condition'); + } +}; + +exports.test_invalid_expiration_date_format = async function(Bucket, Key, s3) { + const putLifecycleParams = date_lifecycle_configuration(Bucket, Key); + + // set expiration with a Date that is not at midnight UTC (incorrect time specified) + putLifecycleParams.LifecycleConfiguration.Rules[0].Expiration.Date = new Date('2025-01-01T15:30:00Z'); + + try { + await s3.putBucketLifecycleConfiguration(putLifecycleParams); + assert.fail('Expected error due to incorrect date format (not at midnight UTC), but request was successful'); + } catch (error) { + assert(error.Code === 'InvalidArgument', 'Expected InvalidArgument error: date must be at midnight UTC'); + } +}; + +exports.test_expiration_multiple_fields = async function(Bucket, Key, s3) { + const putLifecycleParams = days_lifecycle_configuration(Bucket, Key); + + // append ExpiredObjectDeleteMarker for invalid expiration with multiple fields + putLifecycleParams.LifecycleConfiguration.Rules[0].Expiration.ExpiredObjectDeleteMarker = false; + + try { + await s3.putBucketLifecycleConfiguration(putLifecycleParams); + assert.fail('Expected MalformedXML error due to multiple expiration fields'); + } catch (error) { + assert(error.Code === 'MalformedXML', 'Expected MalformedXML error: due to multiple expiration fields'); + } +}; + +exports.test_abortincompletemultipartupload_with_tags = async function(Bucket, Key, s3) { + const putLifecycleParams = tags_lifecycle_configuration(Bucket); + + // invalid combination of AbortIncompleteMultipartUpload with tags + putLifecycleParams.LifecycleConfiguration.Rules[0].AbortIncompleteMultipartUpload = { + DaysAfterInitiation: 5 + }; + + try { + await s3.putBucketLifecycleConfiguration(putLifecycleParams); + assert.fail('Expected InvalidArgument error due to AbortIncompleteMultipartUpload specified with tags'); + } catch (error) { + assert(error.Code === 'InvalidArgument', 'Expected InvalidArgument: AbortIncompleteMultipartUpload cannot be specified with tags'); + } +}; + +exports.test_abortincompletemultipartupload_with_sizes = async function(Bucket, Key, s3) { + const putLifecycleParams = filter_size_lifecycle_configuration(Bucket); + + // invalid combination of AbortIncompleteMultipartUpload with object size filters + putLifecycleParams.LifecycleConfiguration.Rules[0].AbortIncompleteMultipartUpload = { + DaysAfterInitiation: 5 + }; + + try { + await s3.putBucketLifecycleConfiguration(putLifecycleParams); + assert.fail('Expected InvalidArgument error due to AbortIncompleteMultipartUpload specified with object size'); + } catch (error) { + assert(error.Code === 'InvalidArgument', 'Expected InvalidArgument: AbortIncompleteMultipartUpload cannot be specified with object size'); } }; diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_pending_list.txt index 5bd8af3f2d..4ad88518be 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_pending_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/nsfs_s3_tests_pending_list.txt @@ -157,4 +157,5 @@ s3tests/functional/test_headers.py::test_bucket_create_bad_date_none_aws2 s3tests_boto3/functional/test_s3.py::test_versioned_concurrent_object_create_and_remove s3tests_boto3/functional/test_s3.py::test_object_presigned_put_object_with_acl_tenant s3tests_boto3/functional/test_s3.py::test_get_undefined_public_block -s3tests_boto3/functional/test_s3.py::test_get_public_block_deny_bucket_policy \ No newline at end of file +s3tests_boto3/functional/test_s3.py::test_get_public_block_deny_bucket_policy +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_tags1 \ No newline at end of file diff --git a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt index 193ce41448..4b90e632f4 100644 --- a/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt +++ b/src/test/system_tests/ceph_s3_tests/s3-tests-lists/s3_tests_pending_list.txt @@ -144,4 +144,5 @@ s3tests_boto3/functional/test_s3.py::test_get_public_block_deny_bucket_policy s3tests_boto3/functional/test_s3.py::test_get_bucket_encryption_s3 s3tests_boto3/functional/test_s3.py::test_get_bucket_encryption_kms s3tests_boto3/functional/test_s3.py::test_delete_bucket_encryption_s3 -s3tests_boto3/functional/test_s3.py::test_delete_bucket_encryption_kms \ No newline at end of file +s3tests_boto3/functional/test_s3.py::test_delete_bucket_encryption_kms +s3tests_boto3/functional/test_s3.py::test_lifecycle_expiration_tags1 \ No newline at end of file diff --git a/src/test/system_tests/test_lifecycle.js b/src/test/system_tests/test_lifecycle.js index b2693e8363..e30acde165 100644 --- a/src/test/system_tests/test_lifecycle.js +++ b/src/test/system_tests/test_lifecycle.js @@ -54,9 +54,6 @@ async function main() { await commonTests.test_rule_id(Bucket, Key, s3); await commonTests.test_filter_size(Bucket, s3); await commonTests.test_and_prefix_size(Bucket, Key, s3); - await commonTests.test_rule_id_length(Bucket, Key, s3); - await commonTests.test_rule_duplicate_id(Bucket, Key, s3); - await commonTests.test_rule_status_value(Bucket, Key, s3); const getObjectParams = { Bucket, diff --git a/src/test/system_tests/test_utils.js b/src/test/system_tests/test_utils.js index 7a6b097b5c..0ccbe6c730 100644 --- a/src/test/system_tests/test_utils.js +++ b/src/test/system_tests/test_utils.js @@ -691,12 +691,14 @@ async function clean_config_dir(config_fs, custom_config_dir_path) { * @param {nb.NativeFSContext} fs_context * @param {String} file_path * @param {Object} file_data + * @param {{stringify_json?: Boolean}} [options={}] */ -async function create_file(fs_context, file_path, file_data) { +async function create_file(fs_context, file_path, file_data, options = {}) { + const buf = Buffer.from(options?.stringify_json ? JSON.stringify(file_data) : file_data); await nb_native().fs.writeFile( fs_context, file_path, - Buffer.from(JSON.stringify(file_data)), + buf, { mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_FILE) } @@ -729,6 +731,20 @@ async function update_system_json(config_fs, mock_config_dir_version) { await config_fs.update_system_config_file(JSON.stringify(system_data)); } +/** + * run_or_skip_test checks - + * 1. if cond condition evaluated to true - run test + * 2. else - skip test + * @param {*} cond + * @returns {*} + */ +const run_or_skip_test = cond => { + if (cond) { + return it; + } else return it.skip; +}; + +exports.run_or_skip_test = run_or_skip_test; exports.blocks_exist_on_cloud = blocks_exist_on_cloud; exports.create_hosts_pool = create_hosts_pool; exports.delete_hosts_pool = delete_hosts_pool; diff --git a/src/test/unit_tests/jest_tests/test_config_dir_restructure_upgrade_script.test.js b/src/test/unit_tests/jest_tests/test_config_dir_restructure_upgrade_script.test.js index dc68c6a1c3..150f2e54c1 100644 --- a/src/test/unit_tests/jest_tests/test_config_dir_restructure_upgrade_script.test.js +++ b/src/test/unit_tests/jest_tests/test_config_dir_restructure_upgrade_script.test.js @@ -101,7 +101,7 @@ describe('move_old_accounts_dir', () => { account_names_obj[account_data.name] = account_data; if (account_id % 2 === 0) { const backup_file_path = path.join(hidden_old_accounts_path, default_config_fs.json(account_data.name)); - await create_file(default_config_fs.fs_context, backup_file_path, account_data); + await create_file(default_config_fs.fs_context, backup_file_path, account_data, { stringify_json: true }); } }); await move_old_accounts_dir(default_config_fs, Object.keys(account_names_obj), mock_old_version, dbg); @@ -183,7 +183,7 @@ describe('create_account_access_keys_index_if_missing', () => { const mock_id_dir = path.join(default_config_fs.identities_dir_path, 'mock_id_dir'); const mock_link_id_path = path.join(mock_id_dir, 'identity.json'); await create_fresh_path(mock_id_dir); - await create_file(default_config_fs.fs_context, mock_link_id_path, { mock_key: 'mock_value' }); + await create_file(default_config_fs.fs_context, mock_link_id_path, { mock_key: 'mock_value' }, { stringify_json: true }); await symlink_account_access_keys(default_config_fs, account_data.access_keys, mock_link_id_path); const identity_path = default_config_fs.get_identity_path_by_id(account_data._id); const account_upgrade_params = { ...account_data, identity_path }; @@ -275,7 +275,7 @@ describe('create_account_name_index_if_missing', () => { const mock_id_dir = path.join(default_config_fs.identities_dir_path, 'mock_id_dir'); const mock_link_id_path = path.join(mock_id_dir, 'identity.json'); await create_fresh_path(mock_id_dir); - await create_file(default_config_fs.fs_context, mock_link_id_path, { mock_key: 'mock_value' }); + await create_file(default_config_fs.fs_context, mock_link_id_path, { mock_key: 'mock_value' }, { stringify_json: true }); await symlink_account_name(default_config_fs, account_data.name, mock_link_id_path); const identity_path = default_config_fs.get_identity_path_by_id(account_data._id); const account_upgrade_params = { ...account_data, account_name: account_data.name, identity_path }; diff --git a/src/test/unit_tests/jest_tests/test_nc_lifecycle.test.js b/src/test/unit_tests/jest_tests/test_nc_lifecycle.test.js new file mode 100644 index 0000000000..cc2c5ce4d2 --- /dev/null +++ b/src/test/unit_tests/jest_tests/test_nc_lifecycle.test.js @@ -0,0 +1,613 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; +/* eslint-disable max-lines-per-function */ + +// disabling init_rand_seed as it takes longer than the actual test execution +process.env.DISABLE_INIT_RANDOM_SEED = 'true'; + +const path = require('path'); +const crypto = require('crypto'); +const config = require('../../../../config'); +const fs_utils = require('../../../util/fs_utils'); +const { ConfigFS } = require('../../../sdk/config_fs'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const buffer_utils = require('../../../util/buffer_utils'); +const { NCLifecycle } = require('../../../manage_nsfs/nc_lifecycle'); +const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); +const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT } = require('../../system_tests/test_utils'); + + +const config_root = path.join(TMP_PATH, 'config_root_nc_lifecycle'); +const root_path = path.join(TMP_PATH, 'root_path_nc_lifecycle/'); +const bucket_name = 'lifecycle_bucket'; +const bucket_path = path.join(root_path, bucket_name); +const config_fs = new ConfigFS(config_root); +const dummy_object_sdk = make_dummy_object_sdk(); +const nc_lifecycle = new NCLifecycle(config_fs); +const key = 'obj1.txt'; +const data = crypto.randomBytes(100); + +function make_dummy_object_sdk() { + return { + requesting_account: { + force_md5_etag: false, + nsfs_account_config: { + uid: process.getuid(), + gid: process.getgid(), + } + }, + abort_controller: new AbortController(), + throw_if_aborted() { + if (this.abort_controller.signal.aborted) throw new Error('request aborted signal'); + }, + + read_bucket_sdk_config_info(name) { + return undefined; + }, + }; +} + +const nsfs = new NamespaceFS({ + bucket_path: bucket_path, + bucket_id: '1', + namespace_resource_id: undefined, + access_mode: undefined, + versioning: undefined, + force_md5_etag: false, + stats: endpoint_stats_collector.instance(), +}); + +describe('delete_multiple_objects + filter', () => { + const original_lifecycle_run_time = config.NC_LIFECYCLE_RUN_TIME; + const original_lifecycle_run_delay = config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS; + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + await fs_utils.create_fresh_path(bucket_path, 0o777); + }); + + beforeEach(async () => { + await fs_utils.create_fresh_path(root_path, 0o777); + await fs_utils.create_fresh_path(bucket_path, 0o777); + }); + + afterEach(async () => { + config.NC_LIFECYCLE_RUN_TIME = original_lifecycle_run_time; + config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS = original_lifecycle_run_delay; + await fs_utils.folder_delete(config.NC_LIFECYCLE_LOGS_DIR); + await fs_utils.folder_delete(config_root); + }); + + afterAll(async () => { + await fs_utils.folder_delete(root_path); + await fs_utils.folder_delete(config_root); + }, TEST_TIMEOUT); + + describe('filter should fail - versioning DISABLED', () => { + + it('delete_multiple_objects - filter should fail on wrong prefix - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'd' }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects - filter should fail on wrong object_size_less_than - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_less_than: 99 }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects - filter should fail on wrong object_size_greater_than - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 101 }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects - filter should fail on wrong tags - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: [{ key: 'a', value: 'b'}] }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects - filter should fail on expiration - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 5 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + }); + + describe('filter should pass - versioning DISABLED', () => { + + it('delete_multiple_objects - filter should pass on wrong prefix - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'ob' }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + + it('delete_multiple_objects - filter should pass on wrong object_size_less_than - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_less_than: 101 }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + + it('delete_multiple_objects - filter should pass on wrong object_size_greater_than - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 1 }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + + it('delete_multiple_objects - filter should pass on wrong tags - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const tagging = [{ key: 'a', value: 'b' }]; + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer, tagging }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: tagging }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + + it('delete_multiple_objects - filter should pass on expiration - versioning DISABLED bucket', async () => { + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + }); + + describe('filter should fail - versioning ENABLED', () => { + + it('delete_multiple_objects - filter should fail on wrong prefix - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'd' }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects - filter should fail on wrong object_size_less_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_less_than: 99 }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects - filter should fail on wrong object_size_greater_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 101 }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects - filter should fail on wrong tags - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: [{ key: 'a', value: 'b'}] }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects - filter should fail on expiration - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 5 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + }); + + describe('filter should pass - versioning ENABLED', () => { + + it('delete_multiple_objects - filter should pass on wrong prefix - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'ob' }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { should_create_a_delete_marker: true }); + }); + + it('delete_multiple_objects - filter should pass on wrong object_size_less_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_less_than: 101 }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { should_create_a_delete_marker: true }); + }); + + it('delete_multiple_objects - filter should pass on wrong object_size_greater_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 1 }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { should_create_a_delete_marker: true }); + }); + + it('delete_multiple_objects - filter should pass on wrong tags - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const tagging = [{ key: 'a', value: 'b' }]; + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer, tagging }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: tagging }, expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { should_create_a_delete_marker: true }); + }); + + it('delete_multiple_objects - filter should pass on expiration - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 0 }); + const delete_res = await nsfs.delete_multiple_objects({ objects: [{ key }], filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { should_create_a_delete_marker: true }); + }); + }); + + describe('delete multiple objects + version_id + version is latest - filter should fail - versioning ENABLED', () => { + + it('delete_multiple_objects + version_id + version is latest - filter should fail on wrong prefix - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'd' }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects + version_id + version is latest - filter should fail on wrong object_size_less_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_less_than: 99 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects + version_id + version is latest - filter should fail on wrong object_size_greater_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 101 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects + version_id + version is latest - filter should fail on wrong tags - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: [{ key: 'a', value: 'b'}] }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects + version_id + version is latest - filter should fail on expiration - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 5 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + }); + + + describe('filter should pass + version_id + version is latest - versioning ENABLED', () => { + + it('delete_multiple_objects + version_id + version is latest - filter should pass on prefix - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'ob' }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + + it('delete_multiple_objects + version_id + version is latest - filter should pass on object_size_less_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_less_than: 101 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + + it('delete_multiple_objects + version_id + version is latest - filter should pass on object_size_greater_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 1 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + + it('delete_multiple_objects + version_id + version is latest - filter should pass on tags - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const tagging = [{ key: 'a', value: 'b' }]; + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer, tagging }, + dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: tagging }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + + it('delete_multiple_objects + version_id + version is latest - filter should pass on expiration - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer = buffer_utils.buffer_to_read_stream(data); + const upload_res = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res); + }); + }); + + describe('delete multiple objects + version_id + version is not latest - filter should fail - versioning ENABLED', () => { + + it('delete_multiple_objects + version_id + version is not latest - filter should fail on wrong prefix - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'd' }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects + version_id + version is not latest - filter should fail on wrong object_size_less_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_less_than: 99 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects + version_id + version is not latest - filter should fail on wrong object_size_greater_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 101 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects + version_id + version is not latest - filter should fail on wrong tags - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: [{ key: 'a', value: 'b'}] }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + + it('delete_multiple_objects + version_id + version is not latest - filter should fail on expiration - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 5 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res); + }); + }); + + describe('filter should pass + version_id + version is not latest - versioning ENABLED', () => { + + it('delete_multiple_objects + version_id + version is not latest - filter should pass on prefix - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const upload_res2 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'ob' }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { new_latest_version: upload_res2.version_id}); + }); + + it('delete_multiple_objects + version_id + version is not latest - filter should pass on object_size_less_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const upload_res2 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_less_than: 101 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { new_latest_version: upload_res2.version_id}); + }); + + it('delete_multiple_objects + version_id + version is not latest - filter should pass on object_size_greater_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const upload_res2 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 1 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { new_latest_version: upload_res2.version_id}); + }); + + it('delete_multiple_objects + version_id + version is not latest - filter should pass on tags - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const tagging = [{ key: 'a', value: 'b' }]; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const upload_res2 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: tagging }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { new_latest_version: upload_res2.version_id}); + }); + + it('delete_multiple_objects + version_id + version is not latest - filter should pass on expiration - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + const data_buffer2 = buffer_utils.buffer_to_read_stream(data); + const upload_res1 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const upload_res2 = await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer2 }, dummy_object_sdk); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 0 }); + const objects_to_delete = [{ key, version_id: upload_res1.version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deleted(delete_res, { new_latest_version: upload_res2.version_id}); + }); + }); + + describe('delete multiple objects + version_id + version is latest delete_marker - filter should fail - versioning ENABLED', () => { + + it('delete_multiple_objects + version_id + version is latest delete_marker - filter should fail on wrong prefix - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const delete_res1 = await nsfs.delete_object({ bucket: bucket_name, key: key }, dummy_object_sdk); + expect(delete_res1.created_delete_marker).toBe(true); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { prefix: 'd' }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: delete_res1.created_version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res, { latest_delete_marker: true }); + }); + + it('delete_multiple_objects + version_id + version is latest delete_marker - filter should fail on wrong object_size_greater_than - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const delete_res1 = await nsfs.delete_object({ bucket: bucket_name, key: key }, dummy_object_sdk); + expect(delete_res1.created_delete_marker).toBe(true); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { object_size_greater_than: 101 }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: delete_res1.created_version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res, { latest_delete_marker: true }); + }); + + it('delete_multiple_objects + version_id + version is latest delete_marker - filter should fail on wrong tags - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const delete_res1 = await nsfs.delete_object({ bucket: bucket_name, key: key }, dummy_object_sdk); + expect(delete_res1.created_delete_marker).toBe(true); + const filter_func = nc_lifecycle._build_lifecycle_filter({ filter: { tags: [{ key: 'a', value: 'b' }] }, expiration: 0 }); + const objects_to_delete = [{ key, version_id: delete_res1.created_version_id }]; + const delete_res = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res, { latest_delete_marker: true }); + }); + + it('delete_multiple_objects + version_id + version is latest delete_marker - filter should fail on expiration - versioning ENABLED bucket', async () => { + nsfs.versioning = 'ENABLED'; + const data_buffer1 = buffer_utils.buffer_to_read_stream(data); + await nsfs.upload_object({ bucket: bucket_name, key: key, source_stream: data_buffer1 }, dummy_object_sdk); + const delete_res1 = await nsfs.delete_object({ bucket: bucket_name, key: key }, dummy_object_sdk); + expect(delete_res1.created_delete_marker).toBe(true); + const filter_func = nc_lifecycle._build_lifecycle_filter({ expiration: 5 }); + const objects_to_delete = [{ key, version_id: delete_res1.created_version_id }]; + const delete_res2 = await nsfs.delete_multiple_objects({ objects: objects_to_delete, filter_func }, dummy_object_sdk); + await assert_object_deletion_failed(delete_res2, { latest_delete_marker: true }); + }); + }); +}); + +// , { deleted_delete_marker: true, new_latest_version: upload_res1.version_id } +/** + * assert_object_deletion_failed asserts that delete_res contains an error and that the file still exists + * @param {Object} delete_res + * @param {{latest_delete_marker?: boolean}} [options] + * @returns {Promise} + */ +async function assert_object_deletion_failed(delete_res, { latest_delete_marker = false } = {}) { + expect(delete_res.length).toBe(1); + expect(delete_res[0].err_code).toBeDefined(); + expect(delete_res[0].err_message).toBe('file_matches_filter lifecycle - filter on file returned false ' + key); + // file was not deleted + if (latest_delete_marker) { + await expect(nsfs.read_object_md({ bucket: bucket_name, key: key }, dummy_object_sdk)).rejects.toThrow('No such file or directory'); + } else { + const object_metadata = await nsfs.read_object_md({ bucket: bucket_name, key: key }, dummy_object_sdk); + expect(object_metadata.key).toBe(key); + } +} + +/** + * assert_object_deleted asserts that delete_res contains the deleted key and that the file was deleted + * @param {Object} delete_res + * @param {{should_create_a_delete_marker?: Boolean, + * should_delete_a_delete_marker?: Boolean, + * new_latest_version?: String + * }} [options] + * @returns {Promise} + */ +async function assert_object_deleted(delete_res, options = {}) { + const { should_create_a_delete_marker = false, should_delete_a_delete_marker = false, new_latest_version = undefined } = options; + expect(delete_res.length).toBe(1); + if (nsfs.versioning === 'ENABLED') { + if (should_create_a_delete_marker) { + expect(delete_res[0].created_delete_marker).toBe(true); + expect(delete_res[0].created_version_id).toBeDefined(); + } else if (should_delete_a_delete_marker) { + expect(delete_res[0].deleted_delete_marker).toBeDefined(); + } else { + expect(delete_res[0].created_delete_marker).toBeUndefined(); + expect(delete_res[0].deleted_delete_marker).toBeUndefined(); + expect(delete_res[0].created_version_id).toBeUndefined(); + } + } else { + expect(delete_res[0].key).toBe(key); + } + // file was deleted + if (new_latest_version) { + const actual_latest_version = await nsfs.read_object_md({ bucket: bucket_name, key: key }, dummy_object_sdk); + expect(new_latest_version).toBe(actual_latest_version.version_id); + } else { + await expect(nsfs.read_object_md({ bucket: bucket_name, key: key }, dummy_object_sdk)).rejects.toThrow('No such file or directory'); + } +} diff --git a/src/test/unit_tests/jest_tests/test_nc_lifecycle_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_lifecycle_cli.test.js deleted file mode 100644 index bf0d777fe9..0000000000 --- a/src/test/unit_tests/jest_tests/test_nc_lifecycle_cli.test.js +++ /dev/null @@ -1,659 +0,0 @@ -/* Copyright (C) 2016 NooBaa */ -'use strict'; - -// disabling init_rand_seed as it takes longer than the actual test execution -process.env.DISABLE_INIT_RANDOM_SEED = 'true'; - -const path = require('path'); -const config = require('../../../../config'); -const fs_utils = require('../../../util/fs_utils'); -const { ConfigFS } = require('../../../sdk/config_fs'); -const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT, exec_manage_cli } = require('../../system_tests/test_utils'); -const BucketSpaceFS = require('../../../sdk/bucketspace_fs'); -const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); -const NamespaceFS = require('../../../sdk/namespace_fs'); -const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); -const os_utils = require('../../../util/os_utils'); -const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses'); -const { ManageCLIError } = require('../../../manage_nsfs/manage_nsfs_cli_errors'); -const buffer_utils = require('../../../util/buffer_utils'); -const crypto = require('crypto'); -const NsfsObjectSDK = require('../../../sdk/nsfs_object_sdk'); -const nb_native = require('../../../util/nb_native'); - -const new_umask = process.env.NOOBAA_ENDPOINT_UMASK || 0o000; -const old_umask = process.umask(new_umask); -console.log('test_nc_lifecycle_cli: replacing old umask: ', old_umask.toString(8), 'with new umask: ', new_umask.toString(8)); - -const config_root = path.join(TMP_PATH, 'config_root_nc_lifecycle'); -const root_path = path.join(TMP_PATH, 'root_path_nc_lifecycle/'); -const config_fs = new ConfigFS(config_root); - -function make_dummy_object_sdk(account_json) { - return { - requesting_account: account_json - }; -} - -describe('noobaa cli - lifecycle - lock check', () => { - const original_lifecycle_run_time = config.NC_LIFECYCLE_RUN_TIME; - const original_lifecycle_run_delay = config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS; - - beforeAll(async () => { - await fs_utils.create_fresh_path(config_root, 0o777); - set_nc_config_dir_in_config(config_root); - await fs_utils.create_fresh_path(root_path, 0o777); - }); - - afterEach(async () => { - config.NC_LIFECYCLE_RUN_TIME = original_lifecycle_run_time; - config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS = original_lifecycle_run_delay; - await fs_utils.folder_delete(config.NC_LIFECYCLE_LOGS_DIR); - await fs_utils.folder_delete(config_root); - }); - - afterAll(async () => { - await fs_utils.folder_delete(root_path); - await fs_utils.folder_delete(config_root); - }, TEST_TIMEOUT); - - it('lifecycle_cli - change run time to now - 2 locks - the second should fail ', async () => { - await config_fs.create_config_json_file(JSON.stringify({ NC_LIFECYCLE_RUN_TIME: date_to_run_time_format()})); - const res1 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); - const res2 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); - await config_fs.delete_config_json_file(); - const parsed_res1 = JSON.parse(res1).response; - expect(parsed_res1.code).toBe(ManageCLIResponse.LifecycleSuccessful.code); - expect(parsed_res1.message).toBe(ManageCLIResponse.LifecycleSuccessful.message); - const parsed_res2 = JSON.parse(res2).response; - expect(parsed_res2.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); - expect(parsed_res2.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); - }); - - it('lifecycle_cli - no run time change - 2 locks - both should fail ', async () => { - const res1 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); - const res2 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); - const parsed_res1 = JSON.parse(res1).response; - expect(parsed_res1.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); - expect(parsed_res1.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); - const parsed_res2 = JSON.parse(res2).response; - expect(parsed_res2.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); - expect(parsed_res2.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); - }); - - it('lifecycle_cli - change run time to 4 minutes ago - should fail ', async () => { - const lifecyle_run_date = new Date(); - lifecyle_run_date.setMinutes(lifecyle_run_date.getMinutes() - 4); - await config_fs.create_config_json_file(JSON.stringify({ - NC_LIFECYCLE_RUN_TIME: date_to_run_time_format(lifecyle_run_date) - })); - const res = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); - await config_fs.delete_config_json_file(); - const parsed_res = JSON.parse(res).response; - expect(parsed_res.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); - expect(parsed_res.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); - }); - - it('lifecycle_cli - change run time to 1 minutes ago - should succeed ', async () => { - const lifecyle_run_date = new Date(); - lifecyle_run_date.setMinutes(lifecyle_run_date.getMinutes() - 1); - await config_fs.create_config_json_file(JSON.stringify({ - NC_LIFECYCLE_RUN_TIME: date_to_run_time_format(lifecyle_run_date) - })); - const res = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); - await config_fs.delete_config_json_file(); - const parsed_res = JSON.parse(res).response; - expect(parsed_res.code).toBe(ManageCLIResponse.LifecycleSuccessful.code); - expect(parsed_res.message).toBe(ManageCLIResponse.LifecycleSuccessful.message); - }); - - it('lifecycle_cli - change run time to 1 minute in the future - should fail ', async () => { - const lifecyle_run_date = new Date(); - lifecyle_run_date.setMinutes(lifecyle_run_date.getMinutes() + 1); - await config_fs.create_config_json_file(JSON.stringify({ - NC_LIFECYCLE_RUN_TIME: date_to_run_time_format(lifecyle_run_date) - })); - const res = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); - await config_fs.delete_config_json_file(); - const parsed_res = JSON.parse(res).response; - expect(parsed_res.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); - expect(parsed_res.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); - }); -}); - -describe('noobaa cli - lifecycle', () => { - const bucketspace_fs = new BucketSpaceFS({ config_root }, undefined); - const test_bucket = 'test-bucket'; - const test_bucket_path = `${root_path}/${test_bucket}`; - const test_bucket2 = 'test-bucket2'; - const test_bucket2_path = `${root_path}/${test_bucket2}`; - const test_key1 = 'test_key1'; - const test_key2 = 'test_key2'; - const prefix = 'test/'; - const test_prefix_key = `${prefix}/test_key1`; - const account_options1 = {uid: 2002, gid: 2002, new_buckets_path: root_path, name: 'user2', config_root, allow_bucket_creation: 'true'}; - let dummy_sdk; - let nsfs; - - beforeAll(async () => { - await fs_utils.create_fresh_path(config_root, 0o777); - set_nc_config_dir_in_config(config_root); - await fs_utils.create_fresh_path(root_path, 0o777); - const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); - const json_account = JSON.parse(res).response.reply; - console.log(json_account); - dummy_sdk = make_dummy_object_sdk(json_account); - await bucketspace_fs.create_bucket({ name: test_bucket }, dummy_sdk); - await bucketspace_fs.create_bucket({ name: test_bucket2 }, dummy_sdk); - const bucket_json = await config_fs.get_bucket_by_name(test_bucket, undefined); - - nsfs = new NamespaceFS({ - bucket_path: test_bucket_path, - bucket_id: bucket_json._id, - namespace_resource_id: undefined, - access_mode: undefined, - versioning: undefined, - force_md5_etag: false, - stats: endpoint_stats_collector.instance(), - }); - - }); - - afterEach(async () => { - await bucketspace_fs.delete_bucket_lifecycle({ name: test_bucket }); - await bucketspace_fs.delete_bucket_lifecycle({ name: test_bucket2 }); - await fs_utils.create_fresh_path(test_bucket_path); - }); - - afterAll(async () => { - await fs_utils.folder_delete(test_bucket_path); - await fs_utils.folder_delete(test_bucket2_path); - await fs_utils.folder_delete(root_path); - await fs_utils.folder_delete(config_root); - }, TEST_TIMEOUT); - - it('lifecycle_cli - abort mpu by number of days ', async () => { - const lifecycle_rule = [ - { - "id": "abort mpu after 3 days", - "status": "Enabled", - "filter": {"prefix": ""}, - "abort_incomplete_multipart_upload": { - "days_after_initiation": 3 - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - const res = await nsfs.create_object_upload({ key: test_key1, bucket: test_bucket }, dummy_sdk); - await nsfs.create_object_upload({ key: test_key1, bucket: test_bucket }, dummy_sdk); - await update_mpu_mtime(res.obj_id); - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const mpu_list = await nsfs.list_uploads({ bucket: test_bucket }, dummy_sdk); - expect(mpu_list.objects.length).toBe(1); //removed the mpu that was created 5 days ago - }); - - it('lifecycle_cli - abort mpu by prefix and number of days ', async () => { - const lifecycle_rule = [ - { - "id": "abort mpu after 3 days for prefix", - "status": "Enabled", - "filter": {"prefix": prefix}, - "abort_incomplete_multipart_upload": { - "days_after_initiation": 3 - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - let res = await nsfs.create_object_upload({ key: test_key1, bucket: test_bucket }, dummy_sdk); - await update_mpu_mtime(res.obj_id); - res = await nsfs.create_object_upload({ key: test_prefix_key, bucket: test_bucket }, dummy_sdk); - await update_mpu_mtime(res.obj_id); - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const mpu_list = await nsfs.list_uploads({ bucket: test_bucket }, dummy_sdk); - expect(mpu_list.objects.length).toBe(1); //only removed test_prefix_key - }); - - it('lifecycle_cli - abort mpu by tags ', async () => { - const tag_set = [{key: "key1", value: "val1"}, {key: "key2", value: "val2"}]; - const different_tag_set = [{key: "key5", value: "val5"}]; - const lifecycle_rule = [ - { - "id": "abort mpu after 3 days for tags", - "status": "Enabled", - "filter": { - "prefix": '', - "tags": tag_set - }, - "abort_incomplete_multipart_upload": { - "days_after_initiation": 3 - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - let res = await nsfs.create_object_upload( - {key: test_key1, bucket: test_bucket, tagging: [...tag_set, ...different_tag_set]}, - dummy_sdk); - await update_mpu_mtime(res.obj_id); - res = await nsfs.create_object_upload({ key: test_key1, bucket: test_bucket, tagging: different_tag_set}, dummy_sdk); - await update_mpu_mtime(res.obj_id); - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const mpu_list = await nsfs.list_uploads({ bucket: test_bucket }, dummy_sdk); - expect(mpu_list.objects.length).toBe(1); - }); - - it('lifecycle_cli - expiration rule - by number of days ', async () => { - const lifecycle_rule = [ - { - "id": "expiration after 3 days", - "status": "Enabled", - "filter": { - "prefix": '', - }, - "expiration": { - "days": 3 - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - create_object(test_bucket, test_key1, 100, true); - create_object(test_bucket, test_key2, 100, false); - - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const object_list = await nsfs.list_objects({bucket: test_bucket}, dummy_sdk); - expect(object_list.objects.length).toBe(1); - expect(object_list.objects[0].key).toBe(test_key2); - }); - - it('lifecycle_cli - expiration rule - by date - after the date ', async () => { - const date = new Date(); - date.setDate(date.getDate() - 1); // yesterday - const lifecycle_rule = [ - { - "id": "expiration by date", - "status": "Enabled", - "filter": { - "prefix": '', - }, - "expiration": { - "date": date.getTime() - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - create_object(test_bucket, test_key1, 100, false); - create_object(test_bucket, test_key2, 100, false); - - await update_file_mtime(path.join(test_bucket_path, test_key1)); - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const object_list = await nsfs.list_objects({bucket: test_bucket}, dummy_sdk); - expect(object_list.objects.length).toBe(0); //should delete all objects - }); - - it('lifecycle_cli - expiration rule - by date - before the date ', async () => { - const date = new Date(); - date.setDate(date.getDate() + 1); // tommorow - const lifecycle_rule = [ - { - "id": "expiration by date", - "status": "Enabled", - "filter": { - "prefix": '', - }, - "expiration": { - "date": date.getTime() - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - create_object(test_bucket, test_key1, 100, false); - create_object(test_bucket, test_key2, 100, false); - - await update_file_mtime(path.join(test_bucket_path, test_key1)); - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const object_list = await nsfs.list_objects({bucket: test_bucket}, dummy_sdk); - expect(object_list.objects.length).toBe(2); //should not delete any entry - }); - - it('lifecycle_cli - expiration rule - with prefix ', async () => { - const date = new Date(); - date.setDate(date.getDate() - 1); // yesterday - const lifecycle_rule = [ - { - "id": "expiration by prefix", - "status": "Enabled", - "filter": { - "prefix": prefix, - }, - "expiration": { - "date": date.getTime() - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - create_object(test_bucket, test_key1, 100, true); - create_object(test_bucket, test_prefix_key, 100, false); - - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const object_list = await nsfs.list_objects({bucket: test_bucket}, dummy_sdk); - expect(object_list.objects.length).toBe(1); - expect(object_list.objects[0].key).toBe(test_key1); - }); - - it('lifecycle_cli - expiration rule - filter by tags ', async () => { - const tag_set = [{key: "key1", value: "val1"}, {key: "key2", value: "val2"}]; - const different_tag_set = [{key: "key5", value: "val5"}]; - const lifecycle_rule = [ - { - "id": "expiration after 3 days with tags", - "status": "Enabled", - "filter": { - "prefix": '', - "tags": tag_set - }, - "expiration": { - "days": 3 - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - const test_key1_tags = [...tag_set, ...different_tag_set]; - create_object(test_bucket, test_key1, 100, true, test_key1_tags); - create_object(test_bucket, test_key2, 100, true, different_tag_set); - - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const object_list = await nsfs.list_objects({bucket: test_bucket}, dummy_sdk); - expect(object_list.objects.length).toBe(1); - expect(object_list.objects[0].key).toBe(test_key2); - }); - - it('lifecycle_cli - expiration rule - filter by size ', async () => { - const lifecycle_rule = [ - { - "id": "expiration after 3 days with tags", - "status": "Enabled", - "filter": { - "prefix": '', - "object_size_greater_than": 30, - "object_size_less_than": 90 - }, - "expiration": { - "days": 3 - } - } - ]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - - create_object(test_bucket, test_key1, 100, true); - create_object(test_bucket, test_key2, 80, true); - create_object(test_bucket, test_prefix_key, 20, true); - - await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const object_list = await nsfs.list_objects({bucket: test_bucket}, dummy_sdk); - expect(object_list.objects.length).toBe(2); - object_list.objects.forEach(element => { - expect(element.key).not.toBe(test_key2); - }); - }); - - async function create_object(bucket, key, size, is_old, tagging) { - const data = crypto.randomBytes(size); - await nsfs.upload_object({ - bucket, - key, - source_stream: buffer_utils.buffer_to_read_stream(data), - size, - tagging - }, dummy_sdk); - if (is_old) await update_file_mtime(path.join(root_path, bucket, key)); - } - - async function update_mpu_mtime(obj_id) { - const mpu_path = nsfs._mpu_path({obj_id}); - return await update_file_mtime(mpu_path); - } - -}); - - -describe('noobaa cli lifecycle - timeout check', () => { - const original_lifecycle_timeout = config.NC_LIFECYCLE_TIMEOUT_MS; - - beforeAll(async () => { - await fs_utils.create_fresh_path(config_root, 0o777); - set_nc_config_dir_in_config(config_root); - await fs_utils.create_fresh_path(root_path, 0o777); - }); - - afterEach(async () => { - config.NC_LIFECYCLE_TIMEOUT_MS = original_lifecycle_timeout; - await fs_utils.folder_delete(config.NC_LIFECYCLE_LOGS_DIR); - }); - - afterAll(async () => { - await fs_utils.folder_delete(root_path); - await fs_utils.folder_delete(config_root); - }, TEST_TIMEOUT); - - it('lifecycle_cli - change timeout to 1 ms - should fail', async () => { - await config_fs.create_config_json_file(JSON.stringify({ NC_LIFECYCLE_TIMEOUT_MS: 1 })); - const res1 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); - await config_fs.delete_config_json_file(); - const parsed_res1 = JSON.parse(res1.stdout).error; - expect(parsed_res1.code).toBe(ManageCLIError.LifecycleWorkerReachedTimeout.code); - expect(parsed_res1.message).toBe(ManageCLIError.LifecycleWorkerReachedTimeout.message); - }); -}); - -describe('noobaa cli - lifecycle batching', () => { - const bucketspace_fs = new BucketSpaceFS({ config_root }, undefined); - const test_bucket = 'test-bucket'; - const test_bucket_path = `${root_path}/${test_bucket}`; - const test_key1 = 'test_key1'; - const test_key2 = 'test_key2'; - const account_options1 = {uid: 2002, gid: 2002, new_buckets_path: root_path, name: 'user2', config_root, allow_bucket_creation: 'true'}; - let object_sdk; - const tmp_lifecycle_logs_dir_path = path.join(TMP_PATH, 'test_lifecycle_logs'); - - beforeAll(async () => { - await fs_utils.create_fresh_path(config_root, 0o777); - set_nc_config_dir_in_config(config_root); - await fs_utils.create_fresh_path(root_path, 0o777); - const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); - const json_account = JSON.parse(res).response.reply; - console.log(json_account); - object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); - object_sdk.requesting_account = json_account; - await object_sdk.create_bucket({ name: test_bucket }); - config.NC_LIFECYCLE_LOGS_DIR = tmp_lifecycle_logs_dir_path; - config.NC_LIFECYCLE_LIST_BATCH_SIZE = 2; - config.NC_LIFECYCLE_BUCKET_BATCH_SIZE = 5; - }); - - beforeEach(async () => { - await config_fs.create_config_json_file(JSON.stringify({ NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path })); - }); - - afterEach(async () => { - await bucketspace_fs.delete_bucket_lifecycle({ name: test_bucket }); - await fs_utils.create_fresh_path(test_bucket_path); - fs_utils.folder_delete(tmp_lifecycle_logs_dir_path); - await config_fs.delete_config_json_file(); - }); - - it("lifecycle batching - no lifecycle rules", async () => { - const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); - expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); - }); - - it("lifecycle batching - with lifecycle rule, one batch", async () => { - const lifecycle_rule = [ - { - "id": "expiration after 3 days with tags", - "status": "Enabled", - "filter": { - "prefix": '', - }, - "expiration": { - "days": 3 - } - }]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - - create_object(object_sdk, test_bucket, test_key1, 100, true); - create_object(object_sdk, test_bucket, test_key2, 100, true); - const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); - expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); - Object.values(parsed_res_latest_lifecycle.response.reply.buckets_statuses).forEach(bucket_stat => { - expect(bucket_stat.state.is_finished).toBe(true); - Object.values(bucket_stat.rules_statuses).forEach(rule_status => { - expect(rule_status.state.is_finished).toBe(true); - }); - }); - }); - - it("lifecycle batching - with lifecycle rule, no expire statement", async () => { - const lifecycle_rule = [ - { - "id": "expiration after 3 days with tags", - "status": "Enabled", - "filter": { - "prefix": '', - }, - "abort_incomplete_multipart_upload": { - "days_after_initiation": 3 - } - }]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - - await object_sdk.create_object_upload({ key: test_key1, bucket: test_bucket }); - await object_sdk.create_object_upload({ key: test_key2, bucket: test_bucket }); - const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); - expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); - Object.values(parsed_res_latest_lifecycle.response.reply.buckets_statuses).forEach(bucket_stat => { - expect(bucket_stat.state.is_finished).toBe(true); - }); - }); - - it("lifecycle batching - with lifecycle rule, multiple list batches, one bucket batch", async () => { - const lifecycle_rule = [ - { - "id": "expiration after 3 days with tags", - "status": "Enabled", - "filter": { - "prefix": '', - }, - "expiration": { - "days": 3 - } - }]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - - create_object(object_sdk, test_bucket, test_key1, 100, true); - create_object(object_sdk, test_bucket, test_key2, 100, true); - create_object(object_sdk, test_bucket, "key3", 100, true); - create_object(object_sdk, test_bucket, "key4", 100, true); - create_object(object_sdk, test_bucket, "key5", 100, true); - const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); - expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); - const object_list = await object_sdk.list_objects({bucket: test_bucket}); - expect(object_list.objects.length).toBe(0); - }); - - it("lifecycle batching - with lifecycle rule, multiple list batches, multiple bucket batches", async () => { - const lifecycle_rule = [ - { - "id": "expiration after 3 days with tags", - "status": "Enabled", - "filter": { - "prefix": '', - }, - "expiration": { - "days": 3 - } - }]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - - create_object(object_sdk, test_bucket, test_key1, 100, true); - create_object(object_sdk, test_bucket, test_key2, 100, true); - create_object(object_sdk, test_bucket, "key3", 100, true); - create_object(object_sdk, test_bucket, "key4", 100, true); - create_object(object_sdk, test_bucket, "key5", 100, true); - create_object(object_sdk, test_bucket, "key6", 100, true); - create_object(object_sdk, test_bucket, "key7", 100, true); - const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', {disable_service_validation: 'true', disable_runtime_validation: 'true', config_root}, undefined, undefined); - const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); - expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); - const object_list = await object_sdk.list_objects({bucket: test_bucket}); - expect(object_list.objects.length).toBe(0); - }); - - it("lifecycle batching - with lifecycle rule, multiple list batches, one bucket batch", async () => { - const lifecycle_rule = [ - { - "id": "expiration after 3 days with tags", - "status": "Enabled", - "filter": { - "prefix": '', - }, - "expiration": { - "days": 3 - } - }]; - await bucketspace_fs.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); - - create_object(object_sdk, test_bucket, test_key1, 100, true); - create_object(object_sdk, test_bucket, test_key2, 100, true); - - await config_fs.update_config_json_file(JSON.stringify({ - NC_LIFECYCLE_TIMEOUT_MS: 1, - NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path - })); - await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); - - const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); - expect(lifecycle_log_entries.length).toBe(1); - const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name); - const lifecycle_log_json = await config_fs.get_config_data(log_file_path, {silent_if_missing: true}); - expect(lifecycle_log_json.state.is_finished).toBe(false); - const object_list = await object_sdk.list_objects({bucket: test_bucket}); - expect(object_list.objects.length).not.toBe(0); - }); - - //TODO should move outside scope and change also in lifecycle tests. already done in non current lifecycle rule PR - async function create_object(sdk, bucket, key, size, is_old, tagging) { - const data = crypto.randomBytes(size); - const res = await sdk.upload_object({ - bucket, - key, - source_stream: buffer_utils.buffer_to_read_stream(data), - size, - tagging - }); - if (is_old) await update_file_mtime(path.join(root_path, bucket, key)); - return res; - } -}); - - -/** - * update_file_mtime updates the mtime of the target path - * @param {String} target_path - * @returns {Promise} - */ -async function update_file_mtime(target_path) { - const update_file_mtime_cmp = os_utils.IS_MAC ? `touch -t $(date -v -5d +"%Y%m%d%H%M.%S") ${target_path}` : `touch -d "5 days ago" ${target_path}`; - await os_utils.exec(update_file_mtime_cmp, { return_stdout: true }); -} - -/** - * date_to_run_time_format coverts a date to run time format HH:MM - * @param {Date} date - * @returns {String} - */ -function date_to_run_time_format(date = new Date()) { - return date.getHours() + ':' + date.getMinutes(); -} diff --git a/src/test/unit_tests/jest_tests/test_nc_lifecycle_gpfs_ilm_integration.test.js b/src/test/unit_tests/jest_tests/test_nc_lifecycle_gpfs_ilm_integration.test.js new file mode 100644 index 0000000000..4ca71060f7 --- /dev/null +++ b/src/test/unit_tests/jest_tests/test_nc_lifecycle_gpfs_ilm_integration.test.js @@ -0,0 +1,547 @@ +/* Copyright (C) 2016 NooBaa */ +'use strict'; + +// disabling init_rand_seed as it takes longer than the actual test execution +process.env.DISABLE_INIT_RANDOM_SEED = 'true'; + +const fs = require('fs'); +const path = require('path'); +const { ConfigFS } = require('../../../sdk/config_fs'); +const { file_delete, create_fresh_path } = require('../../../util/fs_utils'); +const { read_file } = require('../../../util/native_fs_utils'); +const { run_or_skip_test, TMP_PATH, create_file, IS_GPFS } = require('../../system_tests/test_utils'); +const { NCLifecycle, ILM_POLICIES_TMP_DIR, ILM_CANDIDATES_TMP_DIR } = require('../../../manage_nsfs/nc_lifecycle'); + +const config_root = path.join(TMP_PATH, 'config_root_nc_lifecycle'); +const config_fs = new ConfigFS(config_root); +const nc_lifecycle = new NCLifecycle(config_fs); +const fs_context = config_fs.fs_context; +const mock_start_time = Date.now(); +nc_lifecycle.lifecycle_run_status = { + lifecycle_run_times: { + run_lifecycle_start_time: mock_start_time, + } +}; + +const mock_mount_point = 'mock/mount/path'; +const now = new Date(); +const two_days_ago = now.setDate(now.getDate() - 2); +const two_days_from_now = now.setDate(now.getDate() + 2); + +const bucket_name = 'mock_bucket_name'; +const bucket_path = path.join(TMP_PATH, 'mock_bucket_path'); +const prefix = 'mock_prefix'; +const mock_content = `mock_content`; +const mock_bucket_json = { _id: 'mock_bucket_id', name: bucket_name, path: bucket_path }; + +const days = 3; +const default_lifecycle_rule = { + id: 'abort mpu and expire all objects after 3 days', + status: 'Enabled', + abort_incomplete_multipart_upload: { + days_after_initiation: days + }, +}; + + +// TODO - move to test utils + + +describe('convert_expiry_rule_to_gpfs_ilm_policy unit tests', () => { + const lifecycle_rule_base = { + ...default_lifecycle_rule, + filter: { 'prefix': '' } + }; + const convertion_helpers = { + in_versions_dir: path.join(bucket_path, '/.versions/%'), + in_nested_versions_dir: path.join(bucket_path, '/%/.versions/%') + }; + it('convert_expiry_rule_to_gpfs_ilm_policy - expiry days', () => { + const lifecycle_rule = { ...lifecycle_rule_base, expiration: { days: days } }; + + const ilm_policy = nc_lifecycle.convert_expiry_rule_to_gpfs_ilm_policy(lifecycle_rule, convertion_helpers); + expect(ilm_policy).toBe(get_expected_ilm_expiry(days, bucket_path)); + }); + + it('convert_expiry_rule_to_gpfs_ilm_policy - expiry date', () => { + const lifecycle_rule = { ...lifecycle_rule_base, expiration: { date: Date.now() } }; + const ilm_policy = nc_lifecycle.convert_expiry_rule_to_gpfs_ilm_policy(lifecycle_rule, convertion_helpers); + expect(ilm_policy).toBe(get_expected_ilm_expiry(undefined, bucket_path)); + }); + + it('convert_expiry_rule_to_gpfs_ilm_policy - no expiry', () => { + const lifecycle_rule = lifecycle_rule_base; + const ilm_policy = nc_lifecycle.convert_expiry_rule_to_gpfs_ilm_policy(lifecycle_rule, convertion_helpers); + expect(ilm_policy).toBe(''); + }); +}); + +describe('convert_filter_to_gpfs_ilm_policy unit tests', () => { + const tags = [{ key: 'key1', value: 'val1' }, { key: 'key3', value: 'val4' }]; + const object_size_greater_than = 5; + const object_size_less_than = 10; + const lifecycle_rule_base = { + ...default_lifecycle_rule, + expiration: { days: days } + }; + + it('convert_filter_to_gpfs_ilm_policy - filter empty', () => { + const lifecycle_rule = lifecycle_rule_base; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + expect(ilm_policy).toBe(''); + }); + + it('convert_filter_to_gpfs_ilm_policy - inline prefix', () => { + const lifecycle_rule = { ...lifecycle_rule_base, prefix }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + expect(ilm_policy).toBe(get_expected_ilm_prefix(bucket_path, prefix)); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with prefix', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { prefix } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + expect(ilm_policy).toBe(get_expected_ilm_prefix(bucket_path, prefix)); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with size gt', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { object_size_greater_than } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + expect(ilm_policy).toBe(get_expected_ilm_size_greater_than(object_size_greater_than)); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with size lt', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { object_size_less_than } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + expect(ilm_policy).toBe(get_expected_ilm_size_less_than(object_size_less_than)); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with tags', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { tags } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + expect(ilm_policy).toBe(get_expected_ilm_tags(tags)); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with prefix + size gt', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { prefix, object_size_greater_than } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_prefix(bucket_path, prefix) + + get_expected_ilm_size_greater_than(object_size_greater_than); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with prefix + size lt', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { prefix, object_size_less_than } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_prefix(bucket_path, prefix) + get_expected_ilm_size_less_than(object_size_less_than); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with prefix + tags', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { prefix, tags } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_prefix(bucket_path, prefix) + get_expected_ilm_tags(tags); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with size gt + size lt', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { object_size_less_than, object_size_greater_than } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_size_less_than(object_size_less_than); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with size gt + tags', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { object_size_greater_than, tags } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_tags(tags); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with size lt + tags', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { object_size_less_than, tags } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_size_less_than(object_size_less_than) + + get_expected_ilm_tags(tags); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with prefix + size gt + size lt', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { prefix, object_size_greater_than, object_size_less_than } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_prefix(bucket_path, prefix) + + get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_size_less_than(object_size_less_than); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with prefix + size lt + tags', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { prefix, object_size_less_than, tags } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_prefix(bucket_path, prefix) + + get_expected_ilm_size_less_than(object_size_less_than) + + get_expected_ilm_tags(tags); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with size gt + size lt + tags', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { object_size_greater_than, object_size_less_than, tags } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_size_less_than(object_size_less_than) + + get_expected_ilm_tags(tags); + expect(ilm_policy).toBe(expected_ilm_filter); + }); + + it('convert_filter_to_gpfs_ilm_policy - filter with prefix + size gt + size lt + tags', () => { + const lifecycle_rule = { ...lifecycle_rule_base, filter: { prefix, object_size_greater_than, object_size_less_than, tags } }; + const ilm_policy = nc_lifecycle.convert_filter_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_filter = get_expected_ilm_prefix(bucket_path, prefix) + + get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_size_less_than(object_size_less_than) + + get_expected_ilm_tags(tags); + expect(ilm_policy).toBe(expected_ilm_filter); + }); +}); + + +describe('write_tmp_ilm_policy unit tests', () => { + const tags = [{ key: 'key1', value: 'val1' }, { key: 'key3', value: 'val4' }]; + const object_size_greater_than = 5; + const object_size_less_than = 10; + + const expected_ilm_policy_path = nc_lifecycle.get_gpfs_ilm_policy_file_path(mock_mount_point); + const ilm_policy_mock_string = get_expected_ilm_prefix(bucket_path, prefix) + + get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_size_less_than(object_size_less_than) + + get_expected_ilm_tags(tags); + + beforeAll(async () => { + await create_fresh_path(ILM_POLICIES_TMP_DIR); + }); + + afterEach(async () => { + await file_delete(expected_ilm_policy_path); + }); + + it('write_tmp_ilm_policy - ilm file does not exist', async () => { + const ilm_policy_tmp_path = await nc_lifecycle.write_tmp_ilm_policy(mock_mount_point, ilm_policy_mock_string); + expect(ilm_policy_tmp_path).toBe(expected_ilm_policy_path); + const data = await read_file(fs_context, ilm_policy_tmp_path); + expect(data).toBe(ilm_policy_mock_string); + }); + + it('write_tmp_ilm_policy - already exists', async () => { + await create_file(fs_context, expected_ilm_policy_path, mock_content); + const ilm_policy_tmp_path = await nc_lifecycle.write_tmp_ilm_policy(mock_mount_point, ilm_policy_mock_string); + expect(ilm_policy_tmp_path).toBe(expected_ilm_policy_path); + const data = await read_file(fs_context, ilm_policy_tmp_path); + expect(data).toBe(mock_content); + }); +}); + +describe('get_candidates_by_gpfs_ilm_policy unit tests', () => { + const tags = [{ key: 'key1', value: 'val1' }, { key: 'key3', value: 'val4' }]; + const object_size_greater_than = 0; + const object_size_less_than = 10; + const lifecycle_rule_base = { + ...default_lifecycle_rule, + expiration: { days: days } + }; + const ilm_policy_path = nc_lifecycle.get_gpfs_ilm_policy_file_path(mock_mount_point); + const expected_candidates_suffix = `${bucket_name}_${lifecycle_rule_base.id}_${nc_lifecycle.lifecycle_run_status.lifecycle_run_times.run_lifecycle_start_time}`; + const expected_candidates_path = path.join(ILM_CANDIDATES_TMP_DIR, 'list.' + expected_candidates_suffix); + + beforeEach(async () => { + const ilm_policy_mock_string = get_mock_base_ilm_policy(bucket_path, lifecycle_rule_base.id, nc_lifecycle.lifecycle_run_status) + + get_expected_ilm_prefix(bucket_path, prefix) + + get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_size_less_than(object_size_less_than) + + get_expected_ilm_tags(tags); + await create_file(fs_context, ilm_policy_path, ilm_policy_mock_string); + }); + + afterEach(async () => { + await file_delete(ilm_policy_path); + await file_delete(expected_candidates_path); + }); + + run_or_skip_test(IS_GPFS)('get_candidates_by_gpfs_ilm_policy - ilm file does not exist - no objects returned', async () => { + await nc_lifecycle.create_candidates_file_by_gpfs_ilm_policy(mock_mount_point, ilm_policy_path); + const data = await read_file(fs_context, expected_candidates_path); + console.log('GPFS empty bucket path test data', data); + expect(data).toBe(''); + }); + + run_or_skip_test(IS_GPFS)('create_candidates_file_by_gpfs_ilm_policy - ilm file does not exist - some objects returned', async () => { + const expired_obj_path = 'expired_mock_prefix.txt'; + await create_file(fs_context, path.join(bucket_path, expired_obj_path), mock_content); + await nc_lifecycle.create_candidates_file_by_gpfs_ilm_policy(mock_mount_point, ilm_policy_path); + const data = await read_file(fs_context, expected_candidates_path); + console.log('GPFS non empty bucket path test data', data); + expect(data).toContain(expired_obj_path); + }); +}); + + +describe('get_candidates_by_expiration_rule_gpfs unit tests', () => { + const lifecycle_rule_base = default_lifecycle_rule; + + beforeEach(() => { + nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule_base.id].state = {}; + }); + + run_or_skip_test(IS_GPFS)('get_candidates_by_expiration_rule_gpfs - expire by date - should not expire', async () => { + const lifecycle_rule = lifecycle_rule_base; + lifecycle_rule.expiration = { date: two_days_from_now }; + const candidates = await nc_lifecycle.get_candidates_by_expiration_rule_gpfs(lifecycle_rule, mock_bucket_json); + expect(candidates).toEqual([]); + const actual_rule_state = nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state; + const expected_rule_state = { is_finished: true, key_marker: undefined, key_marker_count: undefined }; + assert_rule_state(actual_rule_state, expected_rule_state); + }); + + run_or_skip_test(IS_GPFS)('get_candidates_by_expiration_rule_gpfs - expire by days, nothing should expire ', async () => { + const lifecycle_rule = lifecycle_rule_base; + lifecycle_rule.expiration = { days: two_days_ago }; + const candidates = await nc_lifecycle.get_candidates_by_expiration_rule_gpfs(lifecycle_rule, mock_bucket_json); + expect(candidates).toEqual([]); + const actual_rule_state = nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state; + const expected_rule_state = { is_finished: true, key_marker: undefined, key_marker_count: undefined }; + assert_rule_state(actual_rule_state, expected_rule_state); + }); + + run_or_skip_test(IS_GPFS)('get_candidates_by_expiration_rule_gpfs - expire by days, expire by days, some objects should expire', async () => { + const lifecycle_rule = lifecycle_rule_base; + lifecycle_rule.expiration = { days: two_days_ago }; + const expired_file_path = path.join(mock_bucket_json.path, 'expired_file.txt'); + const unexpired_file_path = path.join(mock_bucket_json.path, 'unexpired_file.txt'); + + await create_file(fs_context, expired_file_path, 'expired mock content'); + await create_file(fs_context, unexpired_file_path, 'unexpired mock content'); + + const candidates = await nc_lifecycle.get_candidates_by_expiration_rule_gpfs(lifecycle_rule, mock_bucket_json); + expect(candidates).toEqual([{ key: expired_file_path }]); + const actual_rule_state = nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state; + const expected_rule_state = { is_finished: true, key_marker: undefined, key_marker_count: undefined }; + assert_rule_state(actual_rule_state, expected_rule_state); + }); + // TODO - add more with more than 1000 objects and check the key marker etc +}); + +describe('convert_lifecycle_policy_to_gpfs_ilm_policy unit tests', () => { + const tags = [{ key: 'key1', value: 'val1' }, { key: 'key3', value: 'val4' }]; + const object_size_greater_than = 5; + const object_size_less_than = 10; + const lifecycle_rule_base = default_lifecycle_rule; + + it('convert_lifecycle_policy_to_gpfs_ilm_policy - expire date + filter empty', () => { + const lifecycle_rule = { ...lifecycle_rule_base, expiration: { date: new Date() } }; + const actual_ilm_policy = nc_lifecycle.convert_lifecycle_policy_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_policy = get_mock_base_ilm_policy(bucket_path, lifecycle_rule_base.id, nc_lifecycle.lifecycle_run_status) + + get_expected_ilm_expiry(undefined, bucket_path); + expect(actual_ilm_policy).toBe(expected_ilm_policy); + }); + + it('convert_lifecycle_policy_to_gpfs_ilm_policy - expire days + filter empty', () => { + const lifecycle_rule = { ...lifecycle_rule_base, expiration: { days } }; + const actual_ilm_policy = nc_lifecycle.convert_lifecycle_policy_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_policy = get_mock_base_ilm_policy(bucket_path, lifecycle_rule_base.id, nc_lifecycle.lifecycle_run_status) + + get_expected_ilm_expiry(days, bucket_path); + expect(actual_ilm_policy).toBe(expected_ilm_policy); + }); + + it('convert_lifecycle_policy_to_gpfs_ilm_policy - expire date + filter with prefix + size gt + size lt + tags', () => { + const lifecycle_rule = { + ...lifecycle_rule_base, + expiration: { date: new Date() }, + filter: { prefix, object_size_greater_than, object_size_less_than, tags } + }; + const actual_ilm_policy = nc_lifecycle.convert_lifecycle_policy_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_policy = get_mock_base_ilm_policy(bucket_path, lifecycle_rule_base.id, nc_lifecycle.lifecycle_run_status) + + get_expected_ilm_expiry(undefined, bucket_path) + + get_expected_ilm_prefix(bucket_path, prefix) + + get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_size_less_than(object_size_less_than) + + get_expected_ilm_tags(tags); + expect(actual_ilm_policy).toBe(expected_ilm_policy); + }); + + it('convert_lifecycle_policy_to_gpfs_ilm_policy - expire days + filter with prefix + size gt + size lt + tags', () => { + const lifecycle_rule = { + ...lifecycle_rule_base, + expiration: { days }, + filter: { prefix, object_size_greater_than, object_size_less_than, tags } + }; + const actual_ilm_policy = nc_lifecycle.convert_lifecycle_policy_to_gpfs_ilm_policy(lifecycle_rule, mock_bucket_json); + const expected_ilm_policy = get_mock_base_ilm_policy(bucket_path, lifecycle_rule_base.id, nc_lifecycle.lifecycle_run_status) + + get_expected_ilm_expiry(days, bucket_path) + + get_expected_ilm_prefix(bucket_path, prefix) + + get_expected_ilm_size_greater_than(object_size_greater_than) + + get_expected_ilm_size_less_than(object_size_less_than) + + get_expected_ilm_tags(tags); + expect(actual_ilm_policy).toBe(expected_ilm_policy); + }); +}); + +describe('parse_candidates_from_gpfs_ilm_policy unit tests', () => { + const lifecycle_rule_base = default_lifecycle_rule; + const candidates_path = path.join(TMP_PATH, 'mock_candidates_file'); + + beforeEach(() => { + nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule_base.id].state = {}; + }); + + afterEach(async () => { + await file_delete(candidates_path); + }); + + run_or_skip_test(IS_GPFS)('parse_candidates_from_gpfs_ilm_policy - candidates file is empty', async () => { + const lifecycle_rule = lifecycle_rule_base; + const candidates = await nc_lifecycle.parse_candidates_from_gpfs_ilm_policy(mock_bucket_json, lifecycle_rule, candidates_path); + expect(candidates).toEqual([]); + }); + + run_or_skip_test(IS_GPFS)('parse_candidates_from_gpfs_ilm_policy - candidates file contains less than 1000 files', async () => { + const lifecycle_rule = lifecycle_rule_base; + const expected_expired_objects_list = await create_mock_candidates_file(candidates_path, 'file_parse', 600); + const candidates = await nc_lifecycle.parse_candidates_from_gpfs_ilm_policy(mock_bucket_json, lifecycle_rule, candidates_path); + expect(candidates).toEqual(expected_expired_objects_list); + const actual_rule_state = nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state; + const expected_rule_state = { is_finished: true, key_marker: undefined, key_marker_count: undefined }; + assert_rule_state(actual_rule_state, expected_rule_state); + }); + + run_or_skip_test(IS_GPFS)('parse_candidates_from_gpfs_ilm_policy - candidates file contains more than 1000 files', async () => { + const lifecycle_rule = lifecycle_rule_base; + const expected_expired_objects_list = await create_mock_candidates_file(candidates_path, 'file_parse', 1500); + const candidates = await nc_lifecycle.parse_candidates_from_gpfs_ilm_policy(mock_bucket_json, lifecycle_rule, candidates_path); + expect(candidates).toEqual(expected_expired_objects_list); + const actual_rule_state = nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state; + const expected_rule_state = { is_finished: false, key_marker: expected_expired_objects_list[1001].path, key_marker_count: 1001 }; + assert_rule_state(actual_rule_state, expected_rule_state); + }); + + run_or_skip_test(IS_GPFS)('parse_candidates_from_gpfs_ilm_policy - candidates file contains more than 1000 files', async () => { + const lifecycle_rule = lifecycle_rule_base; + const expected_expired_objects_list = await create_mock_candidates_file(candidates_path, 'file_parse', 2500); + nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state = { + is_finished: false, key_marker: expected_expired_objects_list[1001].key, key_marker_count: 1001 + }; + const candidates = await nc_lifecycle.parse_candidates_from_gpfs_ilm_policy(mock_bucket_json, lifecycle_rule, candidates_path); + expect(candidates).toEqual(expected_expired_objects_list); + const actual_rule_state = nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state; + const expected_rule_state = { is_finished: false, key_marker: expected_expired_objects_list[2001].key, key_marker_count: 2001 }; + assert_rule_state(actual_rule_state, expected_rule_state); + }); + + run_or_skip_test(IS_GPFS)('parse_candidates_from_gpfs_ilm_policy - candidates file contains more than 1000 files', async () => { + const lifecycle_rule = lifecycle_rule_base; + const expected_expired_objects_list = await create_mock_candidates_file(candidates_path, 'file_parse', 2500); + nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state = { + is_finished: false, key_marker: expected_expired_objects_list[2001].key, key_marker_count: 2001 + }; + const candidates = await nc_lifecycle.parse_candidates_from_gpfs_ilm_policy(mock_bucket_json, lifecycle_rule, candidates_path); + expect(candidates).toEqual(expected_expired_objects_list); + const actual_rule_state = nc_lifecycle.lifecycle_run_status.buckets_statuses[bucket_name].rules_statuses[lifecycle_rule.id].state; + const expected_rule_state = { is_finished: true, key_marker: undefined, key_marker_count: undefined }; + assert_rule_state(actual_rule_state, expected_rule_state); + }); +}); + +/** + * + * @param {String} candidates_path + * @param {String} file_prefix + * @param {Number} num_of_expired_objects + * @return {Promise} + */ +async function create_mock_candidates_file(candidates_path, file_prefix, num_of_expired_objects) { + const res = []; + for (let i = 0; i < num_of_expired_objects; i++) { + const file_path = file_prefix + i; + await fs.promises.appendFile(candidates_path, file_path + '\n'); + res.push({ key: file_path }); + } + return res; +} +/** + * get_mock_base_ilm_policy returns the base ilm policy for testing + * @param {String} bucket_storage_path + * @param {String} rule_id + * @returns + */ +function get_mock_base_ilm_policy(bucket_storage_path, rule_id, lifecycle_run_status) { + const definitions_base = `define( mod_age, (DAYS(CURRENT_TIMESTAMP) - DAYS(MODIFICATION_TIME)) )\n` + + `define( change_age, (DAYS(CURRENT_TIMESTAMP) - DAYS(CHANGE_TIME)) )\n`; + + const policy_rule_id = `${bucket_name}_${rule_id}_${lifecycle_run_status.lifecycle_run_times.run_lifecycle_start_time}`; + const policy_base = `RULE '${policy_rule_id}' LIST '${policy_rule_id}'\n` + + `WHERE PATH_NAME LIKE '${bucket_storage_path}/%'\n` + + `AND PATH_NAME NOT LIKE '${bucket_storage_path}/.noobaa_nsfs%/%'\n`; + + return definitions_base + policy_base; +} + +/** + * get_expected_ilm_expiry returns the expected ilm policy of expiry days + * @param {number} expiry_days + * @param {String} bucket_storage_path + * @returns {String} + */ +function get_expected_ilm_expiry(expiry_days, bucket_storage_path) { + const path_policy = `AND PATH_NAME NOT LIKE '${bucket_storage_path}/.versions/%'\n` + + `AND PATH_NAME NOT LIKE '${bucket_storage_path}/%/.versions/%'\n`; + const expiration_policy = expiry_days ? `AND mod_age > ${expiry_days}\n` : ''; + return path_policy + expiration_policy; +} + +/** + * get_expected_ilm_prefix returns the expected ilm policy of filter prefix + * @param {String} bucket_storage_path + * @param {String} obj_prefix + * @returns {String} + */ +function get_expected_ilm_prefix(bucket_storage_path, obj_prefix) { + return `AND PATH_NAME LIKE '${bucket_storage_path}/${obj_prefix}%'\n`; +} + +/** + * get_expected_ilm_size_less_than returns the expected ilm size greater policy of phrase + * @param {number} size + * @returns {String} + */ +function get_expected_ilm_size_greater_than(size) { + return `AND FILE_SIZE > ${size}\n`; +} + +/** + * get_expected_ilm_size_less_than returns the expected ilm size less policy of phrase + * @param {number} size + * @returns {String} + */ +function get_expected_ilm_size_less_than(size) { + return `AND FILE_SIZE < ${size}\n`; +} + +/** + * get_expected_ilm_tags returns the expected ilm tags policy phrase + * @param {{key: string, value: string}[]} tags + * @returns {String} + */ +function get_expected_ilm_tags(tags) { + return tags.map(tag => `AND XATTR('user.noobaa.tag.${tag.key}') LIKE ${tag.value}\n`).join(''); +} + +/** + * assert_rule_state asserts that the actual rule state matches the expected rule state + * @param {{is_finished: Boolean, key_marker: String, key_marker_count: Number}} actual_rule_state + * @param {{is_finished: Boolean, key_marker: String, key_marker_count: Number}} expected_rule_state + * @returns {Void} + */ +function assert_rule_state(actual_rule_state, expected_rule_state) { + expect(actual_rule_state.is_finished).toBe(expected_rule_state.is_finished); + expect(actual_rule_state.key_marker).toBe(expected_rule_state.key_marker); + expect(actual_rule_state.key_marker_count).toBe(expected_rule_state.key_marker_count); +} diff --git a/src/test/unit_tests/jest_tests/test_nc_lifecycle_posix_integration.test.js b/src/test/unit_tests/jest_tests/test_nc_lifecycle_posix_integration.test.js new file mode 100644 index 0000000000..4e9e3fc9b7 --- /dev/null +++ b/src/test/unit_tests/jest_tests/test_nc_lifecycle_posix_integration.test.js @@ -0,0 +1,2237 @@ +/* Copyright (C) 2016 NooBaa */ +/* eslint-disable max-lines-per-function */ +/* eslint-disable max-lines */ +'use strict'; + +// disabling init_rand_seed as it takes longer than the actual test execution +process.env.DISABLE_INIT_RANDOM_SEED = 'true'; + +const path = require('path'); +const fs = require('fs'); +const config = require('../../../../config'); +const fs_utils = require('../../../util/fs_utils'); +const { ConfigFS } = require('../../../sdk/config_fs'); +const { TMP_PATH, set_nc_config_dir_in_config, TEST_TIMEOUT, exec_manage_cli, create_system_json } = require('../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); +const NamespaceFS = require('../../../sdk/namespace_fs'); +const endpoint_stats_collector = require('../../../sdk/endpoint_stats_collector'); +const os_utils = require('../../../util/os_utils'); +const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses'); +const { ManageCLIError } = require('../../../manage_nsfs/manage_nsfs_cli_errors'); +const buffer_utils = require('../../../util/buffer_utils'); +const crypto = require('crypto'); +const NsfsObjectSDK = require('../../../sdk/nsfs_object_sdk'); +const nb_native = require('../../../util/nb_native'); +const native_fs_utils = require('../../../util/native_fs_utils'); + +const LIFECYCLE_RULE_STATUS_ENUM = Object.freeze({ + ENABLED: 'Enabled', + DISABLED: 'Disabled' +}); + +const new_umask = process.env.NOOBAA_ENDPOINT_UMASK || 0o000; +const old_umask = process.umask(new_umask); +console.log('test_nc_lifecycle_cli: replacing old umask: ', old_umask.toString(8), 'with new umask: ', new_umask.toString(8)); + +const config_root = path.join(TMP_PATH, 'config_root_nc_lifecycle_posix_integration'); +const root_path = path.join(TMP_PATH, 'root_path_nc_lifecycle_posix_integration/'); +const config_fs = new ConfigFS(config_root); +const account_options1 = { uid: 2002, gid: 2002, new_buckets_path: root_path, name: 'user2', config_root, allow_bucket_creation: 'true' }; +const test_bucket = 'test-bucket'; +const test_bucket1 = 'test-bucket1'; + +const yesterday = new Date(); +yesterday.setDate(yesterday.getDate() - 1); // yesterday +const lifecycle_rule_delete_all = [{ + "id": "expiration after day to all objects", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "expiration": { + "date": yesterday.getTime() + }, +}]; + +describe('noobaa nc - lifecycle - lock check', () => { + const original_lifecycle_run_time = config.NC_LIFECYCLE_RUN_TIME; + const original_lifecycle_run_delay = config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS; + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + }); + + afterEach(async () => { + config.NC_LIFECYCLE_RUN_TIME = original_lifecycle_run_time; + config.NC_LIFECYCLE_RUN_DELAY_LIMIT_MINS = original_lifecycle_run_delay; + await fs_utils.folder_delete(config.NC_LIFECYCLE_LOGS_DIR); + await fs_utils.folder_delete(config_root); + }); + + afterAll(async () => { + await fs_utils.folder_delete(root_path); + await fs_utils.folder_delete(config_root); + }, TEST_TIMEOUT); + + it('nc lifecycle - change run time to now - 2 locks - the second should fail ', async () => { + await config_fs.create_config_json_file(JSON.stringify({ NC_LIFECYCLE_RUN_TIME: date_to_run_time_format() })); + const res1 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); + const res2 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); + await config_fs.delete_config_json_file(); + const parsed_res1 = JSON.parse(res1).response; + expect(parsed_res1.code).toBe(ManageCLIResponse.LifecycleSuccessful.code); + expect(parsed_res1.message).toBe(ManageCLIResponse.LifecycleSuccessful.message); + const parsed_res2 = JSON.parse(res2).response; + expect(parsed_res2.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); + expect(parsed_res2.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); + }); + + it('nc lifecycle - no run time change - 2 locks - both should fail ', async () => { + const res1 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); + const res2 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); + const parsed_res1 = JSON.parse(res1).response; + expect(parsed_res1.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); + expect(parsed_res1.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); + const parsed_res2 = JSON.parse(res2).response; + expect(parsed_res2.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); + expect(parsed_res2.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); + }); + + it('nc lifecycle - change run time to 4 minutes ago - should fail ', async () => { + const lifecyle_run_date = new Date(); + lifecyle_run_date.setMinutes(lifecyle_run_date.getMinutes() - 4); + await config_fs.create_config_json_file(JSON.stringify({ + NC_LIFECYCLE_RUN_TIME: date_to_run_time_format(lifecyle_run_date) + })); + const res = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); + await config_fs.delete_config_json_file(); + const parsed_res = JSON.parse(res).response; + expect(parsed_res.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); + expect(parsed_res.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); + }); + + it('nc lifecycle - change run time to 1 minutes ago - should succeed ', async () => { + const lifecyle_run_date = new Date(); + lifecyle_run_date.setMinutes(lifecyle_run_date.getMinutes() - 1); + await config_fs.create_config_json_file(JSON.stringify({ + NC_LIFECYCLE_RUN_TIME: date_to_run_time_format(lifecyle_run_date) + })); + const res = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); + await config_fs.delete_config_json_file(); + const parsed_res = JSON.parse(res).response; + expect(parsed_res.code).toBe(ManageCLIResponse.LifecycleSuccessful.code); + expect(parsed_res.message).toBe(ManageCLIResponse.LifecycleSuccessful.message); + }); + + it('nc lifecycle - change run time to 1 minute in the future - should fail ', async () => { + const lifecyle_run_date = new Date(); + lifecyle_run_date.setMinutes(lifecyle_run_date.getMinutes() + 1); + await config_fs.create_config_json_file(JSON.stringify({ + NC_LIFECYCLE_RUN_TIME: date_to_run_time_format(lifecyle_run_date) + })); + const res = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', config_root }, undefined, undefined); + await config_fs.delete_config_json_file(); + const parsed_res = JSON.parse(res).response; + expect(parsed_res.code).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.code); + expect(parsed_res.message).toBe(ManageCLIResponse.LifecycleWorkerNotRunning.message); + }); +}); + +describe('noobaa nc - lifecycle versioning DISABLED', () => { + const test_bucket_path = `${root_path}/${test_bucket}`; + const test_key1_regular = 'test_key1'; + const test_key2_regular = 'test_key2'; + const test_key1_nested = 'nested/test_key1'; + const test_key2_nested = 'nested/test_key2'; + const test_key1_object_dir = 'test_key1_dir/'; + const test_key2_object_dir = 'test_key2_dir/'; + const prefix = 'test/'; + const test_prefix_key_regular = `${prefix}${test_key1_regular}`; + const test_prefix_key_nested = `${prefix}${test_key1_nested}`; + const test_prefix_key_object_dir = `${prefix}${test_key1_object_dir}`; + let object_sdk; + let nsfs; + const test_cases = [ + { description: 'regular key', test_key1: test_key1_regular, test_key2: test_key2_regular, test_prefix_key: test_prefix_key_regular }, + { description: 'nested key', test_key1: test_key1_nested, test_key2: test_key2_nested, test_prefix_key: test_prefix_key_nested }, + { description: 'object dir (key with suffix of "/")', test_key1: test_key1_object_dir, test_key2: test_key2_object_dir, test_prefix_key: test_prefix_key_object_dir }, + ]; + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); + const json_account = JSON.parse(res).response.reply; + console.log(json_account); + object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); + object_sdk.requesting_account = json_account; + await object_sdk.create_bucket({ name: test_bucket }); + const bucket_json = await config_fs.get_bucket_by_name(test_bucket, undefined); + + nsfs = new NamespaceFS({ + bucket_path: test_bucket_path, + bucket_id: bucket_json._id, + namespace_resource_id: undefined, + access_mode: undefined, + versioning: undefined, + force_md5_etag: false, + stats: endpoint_stats_collector.instance(), + }); + + }); + + afterEach(async () => { + await object_sdk.delete_bucket_lifecycle({ name: test_bucket }); + await fs_utils.create_fresh_path(test_bucket_path); + }); + + afterAll(async () => { + await fs_utils.folder_delete(test_bucket_path); + await fs_utils.folder_delete(root_path); + await fs_utils.folder_delete(config_root); + }, TEST_TIMEOUT); + + describe('noobaa nc - lifecycle - abort mpu', () => { + it('nc lifecycle - abort mpu by number of days - regular', async () => { + const lifecycle_rule = [{ + "id": "abort mpu after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { "prefix": "" }, + "abort_incomplete_multipart_upload": { + "days_after_initiation": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + const res = await object_sdk.create_object_upload({ key: test_key1_regular, bucket: test_bucket }); + await object_sdk.create_object_upload({ key: test_key1_regular, bucket: test_bucket }); + await update_mpu_mtime(res.obj_id); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const mpu_list = await object_sdk.list_uploads({ bucket: test_bucket }); + expect(mpu_list.objects.length).toBe(1); //removed the mpu that was created 5 days ago + }); + + it('nc lifecycle - abort mpu by number of days - status is Disabled (do not perform the deletion) - regular', async () => { + const lifecycle_rule = [{ + "id": "abort mpu after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.DISABLED, + "filter": { "prefix": "" }, + "abort_incomplete_multipart_upload": { + "days_after_initiation": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + const res = await object_sdk.create_object_upload({ key: test_key1_regular, bucket: test_bucket }); + await object_sdk.create_object_upload({ key: test_key1_regular, bucket: test_bucket }); + await update_mpu_mtime(res.obj_id); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const mpu_list = await object_sdk.list_uploads({ bucket: test_bucket }); + expect(mpu_list.objects.length).toBe(2); // nothing was removed + }); + + it('nc lifecycle - abort mpu by prefix and number of days - regular', async () => { + const lifecycle_rule = [{ + "id": "abort mpu after 3 days for prefix", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { "prefix": prefix }, + "abort_incomplete_multipart_upload": { + "days_after_initiation": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + let res = await object_sdk.create_object_upload({ key: test_key1_regular, bucket: test_bucket }); + await update_mpu_mtime(res.obj_id); + res = await object_sdk.create_object_upload({ key: test_prefix_key_regular, bucket: test_bucket }); + await update_mpu_mtime(res.obj_id); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const mpu_list = await object_sdk.list_uploads({ bucket: test_bucket }); + // only removed test_prefix_key (test/test_key1) as the rule contains the prefix (test/) + expect(mpu_list.objects.length).toBe(1); + expect(mpu_list.objects[0].key).not.toBe(test_prefix_key_regular); + }); + }); + + describe('noobaa nc - lifecycle - expiration rule', () => { + it.each(test_cases)('nc lifecycle - expiration rule - by number of days - $description', async ({ description, test_key1, test_key2, test_prefix_key }) => { + const lifecycle_rule = [{ + "id": "expiration after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "expiration": { + "days": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, false); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(1); + expect(object_list.objects[0].key).toBe(test_key2); + }); + + it('nc lifecycle - expiration rule - by number of days - status is Disabled (do not perform the deletion) - regular key', async () => { + const lifecycle_rule = [{ + "id": "expiration after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.DISABLED, + "filter": { + "prefix": '', + }, + "expiration": { + "days": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, true); + await create_object(object_sdk, test_bucket, test_key2_regular, 100, false); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); //should not delete any entry + }); + + it('nc lifecycle - expiration rule - by date - after the date - regular key', async () => { + const date = new Date(); + date.setDate(date.getDate() - 1); // yesterday + const lifecycle_rule = [{ + "id": "expiration by date", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "expiration": { + "date": date.getTime() + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key2_regular, 100, false); + + await update_file_mtime(path.join(test_bucket_path, test_key1_regular)); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); //should delete all objects + }); + + it('nc lifecycle - expiration rule - by date - before the date - regular key', async () => { + const date = new Date(); + date.setDate(date.getDate() + 1); // tomorrow + const lifecycle_rule = [{ + "id": "expiration by date", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "expiration": { + "date": date.getTime() + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, true); + await create_object(object_sdk, test_bucket, test_key2_regular, 100, false); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); //should not delete any entry + }); + + it('nc lifecycle - expiration rule - with prefix - regular', async () => { + const date = new Date(); + date.setDate(date.getDate() - 1); // yesterday + const lifecycle_rule = [{ + "id": "expiration by prefix", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": prefix, + }, + "expiration": { + "date": date.getTime() + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, true); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(1); + expect(object_list.objects[0].key).toBe(test_key1_regular); + }); + + it.each(test_cases)('nc lifecycle - expiration rule - filter by tags- $description', async ({ description, test_key1, test_key2, test_prefix_key }) => { + const tag_set = [{ key: "key1", value: "val1" }, { key: "key2", value: "val2" }]; + const different_tag_set = [{ key: "key5", value: "val5" }]; + const lifecycle_rule = [{ + "id": "expiration after 3 days with tags", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + "tags": tag_set + }, + "expiration": { + "days": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + const test_key1_tags = [...tag_set, ...different_tag_set]; + await create_object(object_sdk, test_bucket, test_key1, 100, true, test_key1_tags); + await create_object(object_sdk, test_bucket, test_key2, 100, true, different_tag_set); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(1); + expect(object_list.objects[0].key).toBe(test_key2); + }); + + it('nc lifecycle - expiration rule - filter by size - regular key', async () => { + const lifecycle_rule = [{ + "id": "filter by size and expiration after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + "object_size_greater_than": 30, + "object_size_less_than": 90 + }, + "expiration": { + "days": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await create_object(object_sdk, test_bucket, test_key1_regular, 100, true); + await create_object(object_sdk, test_bucket, test_key2_regular, 80, true); // expected to be deleted according to filtering by size (30 < 80 < 90) + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 20, true); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); + object_list.objects.forEach(element => { + expect(element.key).not.toBe(test_key2_regular); + }); + }); + }); + + async function update_mpu_mtime(obj_id) { + const mpu_path = nsfs._mpu_path({ obj_id }); + return await update_file_mtime(mpu_path); + } +}); + +describe('noobaa nc - lifecycle versioning ENABLED', () => { + const test_bucket_path = `${root_path}/${test_bucket}`; + const test_key1_regular = 'test_key1'; + const test_key2_regular = 'test_key2'; + const test_key3_regular = 'test_key3'; + const test_key1_nested = 'nested/test_key1'; + const test_key2_nested = 'nested/test_key2'; + const test_key3_nested = 'nested/test_key3'; + const prefix = 'test/'; + const test_prefix_key_regular = `${prefix}${test_key1_regular}`; + const test_prefix_key_nested = `${prefix}${test_key1_nested}`; + const test_cases = [ + { description: 'regular key', test_key1: test_key1_regular, test_key2: test_key2_regular, test_key3: test_key3_regular, test_prefix_key: test_prefix_key_regular }, + { description: 'nested key', test_key1: test_key1_nested, test_key2: test_key2_nested, test_key3: test_key3_nested, test_prefix_key: test_prefix_key_nested }, + ]; + let object_sdk; + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); + const json_account = JSON.parse(res).response.reply; + console.log(json_account); + object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); + object_sdk.requesting_account = json_account; + await object_sdk.create_bucket({ name: test_bucket }); + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "ENABLED" }); + }); + + afterEach(async () => { + await object_sdk.delete_bucket_lifecycle({ name: test_bucket }); + await fs_utils.create_fresh_path(test_bucket_path); + }); + + afterAll(async () => { + await fs_utils.folder_delete(test_bucket_path); + await fs_utils.folder_delete(root_path); + await fs_utils.folder_delete(config_root); + }, TEST_TIMEOUT); + + it('nc lifecycle - versioning ENABLED - expiration rule - regular key', async () => { + const date = new Date(); + date.setDate(date.getDate() - 1); // yesterday + const lifecycle_rule = [{ + "id": "expiration after 3 days", + "status": "Enabled", + "filter": { + "prefix": prefix, + }, + "expiration": { + "date": date.getTime() + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key2_regular, 100, false); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(4); //added delete marker + let has_delete_marker = false; + object_list.objects.forEach(element => { + if (element.delete_marker) has_delete_marker = true; + }); + expect(has_delete_marker).toBe(true); + }); + + + describe('noobaa nc - lifecycle versioning ENABLED - noncurrent expiration rule', () => { + it.each(test_cases)('nc lifecycle - versioning ENABLED - noncurrent expiration rule - expire older versions - $description', async ({ description, test_key1, test_key2, test_key3, test_prefix_key }) => { + const lifecycle_rule = [{ + "id": "keep 2 noncurrent versions", + "status": "Enabled", + "filter": { + "prefix": "", + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 2, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + await create_object(object_sdk, test_bucket, test_key1, 100, false); + await create_object(object_sdk, test_bucket, test_key1, 100, false); + await create_object(object_sdk, test_bucket, test_key1, 100, false); + await update_version_xattr(test_bucket, test_key1, res.version_id); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(3); + object_list.objects.forEach(element => { + expect(element.version_id).not.toBe(res.version_id); + }); + }); + + it('nc lifecycle - versioning ENABLED - noncurrent expiration rule - expire older versions with filter - regular key', async () => { + const lifecycle_rule = [{ + "id": "keep 1 noncurrent version with filter", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": prefix, + "object_size_greater_than": 80, + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 1, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await update_version_xattr(test_bucket, test_key1_regular, res.version_id); + + const res_prefix = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 60, false); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await update_version_xattr(test_bucket, test_prefix_key_regular, res_prefix.version_id); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(6); + object_list.objects.forEach(element => { + expect(element.version_id).not.toBe(res_prefix.version_id); + }); + }); + + it('nc lifecycle - versioning ENABLED - noncurrent expiration rule - expire older versions only delete markers - regular key', async () => { + const lifecycle_rule = [{ + "id": "keep 1 noncurrent with size filter", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + "object_size_less_than": 1, + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 1, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + let res1 = await object_sdk.delete_object({ bucket: test_bucket, key: test_prefix_key_regular }); + await update_version_xattr(test_bucket, test_prefix_key_regular, res1.created_version_id); + let res2 = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + res1 = await object_sdk.delete_object({ bucket: test_bucket, key: test_prefix_key_regular }); + await update_version_xattr(test_bucket, test_prefix_key_regular, res1.created_version_id); + await update_version_xattr(test_bucket, test_prefix_key_regular, res2.version_id); + res2 = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + res1 = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await update_version_xattr(test_bucket, test_prefix_key_regular, res2.version_id); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(3); + object_list.objects.forEach(element => { + expect(element.delete_marker).not.toBe(true); + }); + }); + + it('nc lifecycle - versioning ENABLED - noncurrent expiration rule - expire older versions by number of days with filter - regular key', async () => { + const lifecycle_rule = [{ + "id": "expire noncurrent versions after 3 days with size ", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": prefix, + "object_size_greater_than": 80, + }, + "noncurrent_version_expiration": { + "noncurrent_days": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + let res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await update_version_xattr(test_bucket, test_key1_regular, res.version_id); + + const expected_res = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + res = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 60, false); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await update_version_xattr(test_bucket, test_prefix_key_regular, expected_res.version_id); + await update_version_xattr(test_bucket, test_prefix_key_regular, res.version_id); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(5); + object_list.objects.forEach(element => { + expect(element.version_id).not.toBe(expected_res.version_id); + }); + }); + + it('nc lifecycle - versioning ENABLED - noncurrent expiration rule - expire older versions by number of days whith expire delete marker rule', async () => { + const lifecycle_rule = [{ + "id": "expire noncurrent versions after 3 days with size ", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "expiration": { + "expired_object_delete_marker": true + }, + "noncurrent_version_expiration": { + "noncurrent_days": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await update_version_xattr(test_bucket, test_key1_regular, res.version_id); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(1); + object_list.objects.forEach(element => { + expect(element.version_id).not.toBe(res.version_id); + }); + }); + + it('nc lifecycle - versioning ENABLED - noncurrent expiration rule - both noncurrent days and older versions', async () => { + const lifecycle_rule = [{ + "id": "expire noncurrent versions after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "noncurrent_version_expiration": { + "noncurrent_days": 3, + "newer_noncurrent_versions": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const expected_res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); // the second newest noncurrent version (will be deleted) + const res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); // the newest noncurrent version + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); // latest version + await update_version_xattr(test_bucket, test_key1_regular, expected_res.version_id); + await update_version_xattr(test_bucket, test_key1_regular, res.version_id); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); + object_list.objects.forEach(element => { + expect(element.version_id).not.toBe(expected_res.version_id); + }); + }); + + it('nc lifecycle - versioning ENABLED - noncurrent expiration rule - older versions valid but noncurrent_days not valid', async () => { + const lifecycle_rule = [{ + "id": "expire noncurrent versions after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "noncurrent_version_expiration": { + "noncurrent_days": 3, + "newer_noncurrent_versions": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + // more than one noncurrent version but not older than 3 days - don't delete + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(3); + }); + }); + + describe('noobaa nc - lifecycle versioning ENABLED - expiration rule - delete marker', () => { + it.each(test_cases)('nc lifecycle - versioning ENABLED - expiration rule - expire delete marker - $description', async ({ description, test_key1, test_key2, test_key3, test_prefix_key }) => { + const lifecycle_rule = [{ + "id": "expired_object_delete_marker no filters", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "expiration": { + "expired_object_delete_marker": true + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1 }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key2 }); + + await create_object(object_sdk, test_bucket, test_prefix_key, 100, false); + await object_sdk.delete_object({ bucket: test_bucket, key: test_prefix_key }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); // test_prefix_key as older version and a delete marker, hence it doesn't considered expired + object_list.objects.forEach(element => { + expect(element.key).toBe(test_prefix_key); + }); + }); + + it('nc lifecycle - versioning ENABLED - expiration rule - expire delete marker with filter - regular key', async () => { + const lifecycle_rule = [{ + "id": "expired_object_delete_marker with filter by prefix and size", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": prefix, + "object_size_less_than": 1 + }, + "expiration": { + "expired_object_delete_marker": true + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1_regular }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_prefix_key_regular }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(1); + expect(object_list.objects[0].key).toBe(test_key1_regular); + }); + + it('nc lifecycle - versioning ENABLED - expiration rule - expire delete marker last item', async () => { + const lifecycle_rule = [{ + "id": "expiration of delete marker with filter by size and prefix", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + "object_size_less_than": 1 + }, + "expiration": { + "expired_object_delete_marker": true + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1_regular }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + + it('nc lifecycle - versioning ENABLED - expiration rule - last item in batch is latest delete marker', async () => { + const lifecycle_rule = [{ + "id": "filter by size and no filter by prefix with expiration of delete marker", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + "object_size_less_than": 1 + }, + "expiration": { + "expired_object_delete_marker": true + } + }]; + await config_fs.create_config_json_file(JSON.stringify({ + NC_LIFECYCLE_LIST_BATCH_SIZE: 3, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 3, + })); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1_regular }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1_regular }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key2_regular }); //last in batch should not delete + await object_sdk.delete_object({ bucket: test_bucket, key: test_key2_regular }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key3_regular }); // last in batch should delete + await object_sdk.delete_object({ bucket: test_bucket, key: 'test_key4' }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(4); + object_list.objects.forEach(element => { + expect([test_key1_regular, test_key2_regular]).toContain(element.key); + }); + }); + }); + +}); + +describe('noobaa nc - lifecycle versioning SUSPENDED', () => { + const test_bucket_path = `${root_path}/${test_bucket}`; + const test_key1_regular = 'test_key1'; + const test_key2_regular = 'test_key2'; + const test_key3_regular = 'test_key3'; + const test_key1_nested = 'nested/test_key1'; + const test_key2_nested = 'nested/test_key2'; + const test_key3_nested = 'nested/test_key3'; + const prefix = 'test/'; + const test_prefix_key_regular = `${prefix}${test_key1_regular}`; + const test_prefix_key_nested = `${prefix}${test_key1_nested}`; + const test_cases = [ + { description: 'regular key', test_key1: test_key1_regular, test_key2: test_key2_regular, test_key3: test_key3_regular, test_prefix_key: test_prefix_key_regular }, + { description: 'nested key', test_key1: test_key1_nested, test_key2: test_key2_nested, test_key3: test_key3_nested, test_prefix_key: test_prefix_key_nested }, + ]; + let object_sdk; + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); + const json_account = JSON.parse(res).response.reply; + console.log(json_account); + object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); + object_sdk.requesting_account = json_account; + await object_sdk.create_bucket({ name: test_bucket }); + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "SUSPENDED" }); + }); + + afterEach(async () => { + await object_sdk.delete_bucket_lifecycle({ name: test_bucket }); + await fs_utils.create_fresh_path(test_bucket_path); + }); + + afterAll(async () => { + await fs_utils.folder_delete(test_bucket_path); + await fs_utils.folder_delete(root_path); + await fs_utils.folder_delete(config_root); + }, TEST_TIMEOUT); + + it('nc lifecycle - versioning SUSPENDED - expiration rule - regular key', async () => { + const date = new Date(); + date.setDate(date.getDate() - 1); // yesterday + const lifecycle_rule = [{ + "id": "expiration after 3 days", + "status": "Enabled", + "filter": { + "prefix": prefix, + }, + "expiration": { + "date": date.getTime() + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); // matches prefix filter - will effectively deletes the object + await create_object(object_sdk, test_bucket, test_key2_regular, 100, false); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); // one that didn't match and a delete marker of the one that matched + const matched_filter_object = object_list.objects.find(element => element.key === test_prefix_key_regular); + expect(matched_filter_object.version_id).toBe('null'); + expect(matched_filter_object.delete_marker).toBe(true); + const non_matched_filter_object = object_list.objects.find(element => element.key === test_key2_regular); + expect(non_matched_filter_object.version_id).toBe('null'); + expect(non_matched_filter_object.delete_marker).toBe(false); + }); + + describe('noobaa nc - lifecycle versioning SUSPENDED - noncurrent expiration rule', () => { + it.each(test_cases)('nc lifecycle - versioning SUSPENDED - noncurrent expiration rule - expire older versions - $description', async ({ description, test_key1, test_key2, test_prefix_key }) => { + const lifecycle_rule = [{ + "id": "keep 2 noncurrent versions", + "status": "Enabled", + "filter": { + "prefix": "", + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 2, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + for (let i = 0; i < 5; i++) { + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + } + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + // in case the versioning is SUSPENDED the number of newer_noncurrent_versions is not relevant and we would have only the latest version (with version_id null) + expect(object_list.objects.length).toBe(1); + expect(object_list.objects[0].version_id).toBe('null'); + expect(object_list.objects[0].is_latest).toBe(true); + }); + + it('nc lifecycle - versioning SUSPENDED - noncurrent expiration rule - expire older versions with filter - regular key', async () => { + const lifecycle_rule = [{ + "id": "keep 1 noncurrent version with filter", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": prefix, + "object_size_greater_than": 80, + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 1, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "ENABLED" }); + const version_arr = []; + let res = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); // filter by prefix + size + version_arr.push(res.version_id); + for (let i = 0; i < 2; i++) { + const prev_res = res; + res = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); // filter by prefix + size + await update_version_xattr(test_bucket, test_prefix_key_regular, prev_res.version_id); + version_arr.push(res.version_id); + } + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "SUSPENDED" }); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); // doesn't match prefix filter + await create_object(object_sdk, test_bucket, `${prefix}${test_key2_regular}`, 60, false); // doesn't match size filter + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); // filter by prefix + size + + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(4); // 2 of the mismatched keys (1 version) + 2 versions the filtered key + + const filtered_versions = object_list.objects.filter(element => element.key === test_prefix_key_regular); + expect(filtered_versions.length).toBe(2); + expect(['null', version_arr[2]]).toContain(filtered_versions[0].version_id); + expect(['null', version_arr[2]]).toContain(filtered_versions[1].version_id); + }); + + it('nc lifecycle - versioning SUSPENDED - noncurrent expiration rule - expire older versions by number of days with filter - regular key', async () => { + const lifecycle_rule = [{ + "id": "expire noncurrent versions after 3 days with size ", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": prefix, + "object_size_greater_than": 80, + }, + "noncurrent_version_expiration": { + "noncurrent_days": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "ENABLED" }); + let res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await update_version_xattr(test_bucket, test_key1_regular, res.version_id); + + const expected_res = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + res = await create_object(object_sdk, test_bucket, test_prefix_key_regular, 60, false); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await create_object(object_sdk, test_bucket, test_prefix_key_regular, 100, false); + await update_version_xattr(test_bucket, test_prefix_key_regular, expected_res.version_id); + await update_version_xattr(test_bucket, test_prefix_key_regular, res.version_id); + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "SUSPENDED" }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(5); + object_list.objects.forEach(element => { + expect(element.version_id).not.toBe(expected_res.version_id); + }); + }); + + it('nc lifecycle - versioning SUSPENDED - noncurrent expiration rule - both noncurrent days and older versions', async () => { + const lifecycle_rule = [{ + "id": "expire noncurrent versions after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "noncurrent_version_expiration": { + "noncurrent_days": 3, + "newer_noncurrent_versions": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "ENABLED" }); + const expected_res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); // the second newest noncurrent version (will be deleted) + const res = await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); // the newest noncurrent version + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); // latest version + await update_version_xattr(test_bucket, test_key1_regular, expected_res.version_id); + await update_version_xattr(test_bucket, test_key1_regular, res.version_id); + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "SUSPENDED" }); + + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); + object_list.objects.forEach(element => { + expect(element.version_id).not.toBe(expected_res.version_id); + }); + }); + + it('nc lifecycle - versioning SUSPENDED - noncurrent expiration rule - older versions valid but noncurrent_days not valid', async () => { + const lifecycle_rule = [{ + "id": "expire noncurrent versions after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "noncurrent_version_expiration": { + "noncurrent_days": 3, + "newer_noncurrent_versions": 1 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "ENABLED" }); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + // more than one noncurrent version but not older than 3 days - don't delete + await create_object(object_sdk, test_bucket, test_key1_regular, 100, false); + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: "SUSPENDED" }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(3); + }); + + }); + + describe('noobaa nc - lifecycle versioning SUSPENDED - expiration rule - delete marker', () => { + it.each(test_cases)('nc lifecycle - versioning SUSPENDED - expiration rule - expire delete marker - $description', async ({ description, test_key1, test_key2, test_prefix_key }) => { + const lifecycle_rule = [{ + "id": "expired_object_delete_marker no filters", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "expiration": { + "expired_object_delete_marker": true + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1 }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key2 }); + + await create_object(object_sdk, test_bucket, test_prefix_key, 100, false); + await object_sdk.delete_object({ bucket: test_bucket, key: test_prefix_key }); // in SUSPENDED mode there would be only the delete marker of test_prefix_key with version id id of null + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); // all the objects doesn't have an older version and we have only delete markers + }); + + it('nc lifecycle - versioning SUSPENDED - expiration rule - expire delete marker with filter - regular key', async () => { + const lifecycle_rule = [{ + "id": "expired_object_delete_marker with filter by prefix and size", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": prefix, + "object_size_less_than": 1 + }, + "expiration": { + "expired_object_delete_marker": true + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1_regular }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_prefix_key_regular }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(1); + expect(object_list.objects[0].key).toBe(test_key1_regular); + }); + + it('nc lifecycle - versioning SUSPENDED - expiration rule - expire delete marker last item', async () => { + const lifecycle_rule = [{ + "id": "expiration of delete marker with filter by size and prefix", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + "object_size_less_than": 1 + }, + "expiration": { + "expired_object_delete_marker": true + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1_regular }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + + it('nc lifecycle - versioning SUSPENDED - expiration rule - last item in batch is latest delete marker', async () => { + const lifecycle_rule = [{ + "id": "filter by size and no filter by prefix with expiration of delete marker", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + "object_size_less_than": 1 + }, + "expiration": { + "expired_object_delete_marker": true + } + }]; + await config_fs.create_config_json_file(JSON.stringify({ + NC_LIFECYCLE_LIST_BATCH_SIZE: 3, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 3, + })); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1_regular }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key1_regular }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key2_regular }); // last in batch should not delete + await object_sdk.delete_object({ bucket: test_bucket, key: test_key2_regular }); + await object_sdk.delete_object({ bucket: test_bucket, key: test_key3_regular }); // last in batch should delete + await object_sdk.delete_object({ bucket: test_bucket, key: 'test_key4' }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + }); +}); + +describe('noobaa nc lifecycle - timeout check', () => { + const original_lifecycle_timeout = config.NC_LIFECYCLE_TIMEOUT_MS; + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); + const json_account = JSON.parse(res).response.reply; + console.log(json_account); + const object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); + object_sdk.requesting_account = json_account; + // don't delete this bucket creation - it's being used for making sure that the lifecycle run will take more than 1 ms + await object_sdk.create_bucket({ name: test_bucket }); + await object_sdk.create_bucket({ name: test_bucket1 }); + + }); + + afterEach(async () => { + config.NC_LIFECYCLE_TIMEOUT_MS = original_lifecycle_timeout; + await fs_utils.folder_delete(config.NC_LIFECYCLE_LOGS_DIR); + }); + + afterAll(async () => { + await fs_utils.folder_delete(root_path); + await fs_utils.folder_delete(config_root); + }, TEST_TIMEOUT); + + it('nc lifecycle - change timeout to 1 ms - should fail', async () => { + await config_fs.create_config_json_file(JSON.stringify({ NC_LIFECYCLE_TIMEOUT_MS: 1 })); + const res1 = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + await config_fs.delete_config_json_file(); + const parsed_res1 = JSON.parse(res1.stdout); + const actual_error = parsed_res1.error; + expect(actual_error.code).toBe(ManageCLIError.LifecycleWorkerReachedTimeout.code); + expect(actual_error.message).toBe(ManageCLIError.LifecycleWorkerReachedTimeout.message); + }); +}); + +describe('noobaa nc - lifecycle batching', () => { + describe('noobaa nc - lifecycle batching - bucket batch size is bigger than list batch size', () => { + const test_bucket_path = `${root_path}/${test_bucket}`; + const test_key1 = 'test_key1'; + const test_key2 = 'test_key2'; + let object_sdk; + const tmp_lifecycle_logs_dir_path = path.join(root_path, 'test_lifecycle_logs'); + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); + const json_account = JSON.parse(res).response.reply; + console.log(json_account); + object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); + object_sdk.requesting_account = json_account; + await object_sdk.create_bucket({ name: test_bucket }); + }); + + beforeEach(async () => { + await config_fs.create_config_json_file(JSON.stringify({ + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_LIST_BATCH_SIZE: 2, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5, // bucket batch size is bigger than list batch size + })); + }); + + afterEach(async () => { + await object_sdk.delete_bucket_lifecycle({ name: test_bucket }); + await fs_utils.create_fresh_path(test_bucket_path); + fs_utils.folder_delete(tmp_lifecycle_logs_dir_path); + await config_fs.delete_config_json_file(); + }); + + it("lifecycle batching - no lifecycle rules", async () => { + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + }); + + it("lifecycle batching - with lifecycle rule, one batch of bucket and list", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + Object.values(parsed_res_latest_lifecycle.response.reply.buckets_statuses).forEach(bucket_status => { + expect(bucket_status.state.is_finished).toBe(true); + Object.values(bucket_status.rules_statuses).forEach(rule_status => { + expect(rule_status.state.is_finished).toBe(true); + }); + }); + }); + + it("lifecycle batching - with lifecycle rule, one batch of bucket and list, no expire statement", async () => { + const lifecycle_rule = [{ + "id": "abort mpu after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "abort_incomplete_multipart_upload": { + "days_after_initiation": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.create_object_upload({ key: test_key1, bucket: test_bucket }); + await object_sdk.create_object_upload({ key: test_key2, bucket: test_bucket }); + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + Object.values(parsed_res_latest_lifecycle.response.reply.buckets_statuses).forEach(bucket_status => { + expect(bucket_status.state.is_finished).toBe(true); + }); + }, TEST_TIMEOUT); + + it("lifecycle batching - with lifecycle rule, multiple list batches, one bucket batch", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + await create_object(object_sdk, test_bucket, "key3", 100, true); + await create_object(object_sdk, test_bucket, "key4", 100, true); + await create_object(object_sdk, test_bucket, "key5", 100, true); + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + + it("lifecycle batching - with lifecycle rule, multiple list batches, multiple bucket batches", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + await create_object(object_sdk, test_bucket, "key3", 100, true); + await create_object(object_sdk, test_bucket, "key4", 100, true); + await create_object(object_sdk, test_bucket, "key5", 100, true); + await create_object(object_sdk, test_bucket, "key6", 100, true); + await create_object(object_sdk, test_bucket, "key7", 100, true); + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + + it("lifecycle batching - with lifecycle rule, multiple list batches, one bucket batch - worker did not finish", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + + await config_fs.update_config_json_file(JSON.stringify({ + // set short timeout so the lifecycle run will not finish + NC_LIFECYCLE_TIMEOUT_MS: 1, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5, + NC_LIFECYCLE_LIST_BATCH_SIZE: 2 + })); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + + const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); + expect(lifecycle_log_entries.length).toBe(1); + const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name); + const lifecycle_log_json = await config_fs.get_config_data(log_file_path, { silent_if_missing: true }); + expect(lifecycle_log_json.state.is_finished).toBe(false); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).not.toBe(0); + }); + + it("lifecycle batching - continue finished lifecycle should do nothing", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + + //continue finished run + await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + + const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); + expect(lifecycle_log_entries.length).toBe(2); + const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name); + const lifecycle_log_json = await config_fs.get_config_data(log_file_path, { silent_if_missing: true }); + expect(lifecycle_log_json.state.is_finished).toBe(true); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); + }); + + it("continue lifecycle batching (running twice: worker not finish and continued worker) - should finish the run - delete all", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + await config_fs.update_config_json_file(JSON.stringify({ + // set short timeout so the lifecycle run will not finish + NC_LIFECYCLE_TIMEOUT_MS: 5, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5, + NC_LIFECYCLE_LIST_BATCH_SIZE: 2 + + })); + const keys = [test_key1, test_key2, "key3", "key4", "key5", "key6", "key7"]; + for (const key of keys) { + await create_object(object_sdk, test_bucket, key, 100, false); + } + try { + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + } catch (e) { + //ignore timeout error + } + await config_fs.update_config_json_file(JSON.stringify({ + // set default timeout so the lifecycle run will continue to run + NC_LIFECYCLE_TIMEOUT_MS: config.NC_LIFECYCLE_TIMEOUT_MS, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 5, + NC_LIFECYCLE_LIST_BATCH_SIZE: 2 + + })); + await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + + it("continue lifecycle batching should finish the run - validate new run. don't delete already deleted items", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + await config_fs.update_config_json_file(JSON.stringify({ + NC_LIFECYCLE_TIMEOUT_MS: 70, + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 4, + NC_LIFECYCLE_LIST_BATCH_SIZE: 2 + })); + const keys = []; + for (let i = 0; i < 100; i++) { + const new_key = `key${i}`; + await create_object(object_sdk, test_bucket, new_key, 100, false); + keys.push(new_key); + } + try { + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + } catch (e) { + //ignore error + } + + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + const intermediate_key_list = object_list.objects.map(object => object.key); + const new_keys = []; + //recreate deleted key. next run should skip those keys + for (const key of keys) { + if (!intermediate_key_list.includes(key)) { + await create_object(object_sdk, test_bucket, key, 100, false); + new_keys.push(key); + } + } + await config_fs.update_config_json_file(JSON.stringify({ + NC_LIFECYCLE_TIMEOUT_MS: 9999, + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 4, + NC_LIFECYCLE_LIST_BATCH_SIZE: 2, + })); + await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list2 = await object_sdk.list_objects({ bucket: test_bucket }); + const res_keys = object_list2.objects.map(object => object.key); + for (const key of new_keys) { + expect(res_keys).toContain(key); + } + }, TEST_TIMEOUT); + + it("lifecycle batching - with lifecycle rule, multiple list batches, multiple bucket batches - newer noncurrent versions", async () => { + const lifecycle_rule = [{ + "id": "keep 2 noncurrent versions", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": "", + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 2, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: 'ENABLED' }); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const version_arr = []; + let res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + version_arr.push(res.version_id); + for (let i = 0; i < 9; i++) { + const prev_res = res; + res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + await update_version_xattr(test_bucket, test_key1, prev_res.version_id); + version_arr.push(res.version_id); + } + const last_3_versions = new Set(version_arr.slice(-3)); // latest version + 2 noncurrent versions + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + const object_list_versions = new Set(object_list.objects.map(object => object.version_id)); + expect(object_list.objects.length).toBe(3); + expect(object_list_versions).toEqual(last_3_versions); + }); + + it("lifecycle rule, multiple list batches, multiple bucket batches - both expire and noncurrent actions", async () => { + const lifecycle_rule = [{ + "id": "keep 2 noncurrent versions and expire after 1 day", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": "", + }, + "expiration": { + "date": yesterday.getTime() + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 2, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: 'ENABLED' }); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const keys = [test_key1, test_key2, "key3", "key4", "key5", "key6", "key7"]; + for (const key of keys) { + if (key === test_key1) continue; //test_key1 is initialized in his own loop + await create_object(object_sdk, test_bucket, key, 100, false); + } + + let prev_res; + let res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + for (let i = 0; i < 9; i++) { + prev_res = res; + res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + await update_version_xattr(test_bucket, test_key1, prev_res.version_id); + } + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + const expected_length = keys.length * 2 + 1; //all keys + delete marker for each key + 1 noncurrent versions for test_key1 + expect(object_list.objects.length).toBe(expected_length); + }); + }); + + describe('noobaa nc - lifecycle batching - bucket batch size is smaller than list batch size', () => { + const test_bucket_path = `${root_path}/${test_bucket}`; + const test_key1 = 'test_key1'; + const test_key2 = 'test_key2'; + let object_sdk; + const tmp_lifecycle_logs_dir_path = path.join(root_path, 'test_lifecycle_logs'); + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); + const json_account = JSON.parse(res).response.reply; + console.log(json_account); + object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); + object_sdk.requesting_account = json_account; + await object_sdk.create_bucket({ name: test_bucket }); + }); + + beforeEach(async () => { + await config_fs.create_config_json_file(JSON.stringify({ + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_LIST_BATCH_SIZE: 5, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 2, // bucket batch size is smaller than list batch size + })); + }); + + afterEach(async () => { + await object_sdk.delete_bucket_lifecycle({ name: test_bucket }); + await fs_utils.create_fresh_path(test_bucket_path); + fs_utils.folder_delete(tmp_lifecycle_logs_dir_path); + await config_fs.delete_config_json_file(); + }); + + it("lifecycle batching - no lifecycle rules", async () => { + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + }); + + it("lifecycle batching - with lifecycle rule, one batch of bucket and list", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + Object.values(parsed_res_latest_lifecycle.response.reply.buckets_statuses).forEach(bucket_status => { + expect(bucket_status.state.is_finished).toBe(true); + Object.values(bucket_status.rules_statuses).forEach(rule_status => { + expect(rule_status.state.is_finished).toBe(true); + }); + }); + }); + + it("lifecycle batching - with lifecycle rule, one batch of bucket and list, no expire statement", async () => { + const lifecycle_rule = [{ + "id": "abort mpu after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "abort_incomplete_multipart_upload": { + "days_after_initiation": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + await object_sdk.create_object_upload({ key: test_key1, bucket: test_bucket }); + await object_sdk.create_object_upload({ key: test_key2, bucket: test_bucket }); + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + Object.values(parsed_res_latest_lifecycle.response.reply.buckets_statuses).forEach(bucket_status => { + expect(bucket_status.state.is_finished).toBe(true); + }); + }); + + it("lifecycle batching - with lifecycle rule, one list batch, multiple bucket batches", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + await create_object(object_sdk, test_bucket, "key3", 100, true); + await create_object(object_sdk, test_bucket, "key4", 100, true); + await create_object(object_sdk, test_bucket, "key5", 100, true); + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + + it("lifecycle batching - with lifecycle rule, multiple list batches, multiple bucket batches", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + await create_object(object_sdk, test_bucket, "key3", 100, true); + await create_object(object_sdk, test_bucket, "key4", 100, true); + await create_object(object_sdk, test_bucket, "key5", 100, true); + await create_object(object_sdk, test_bucket, "key6", 100, true); + await create_object(object_sdk, test_bucket, "key7", 100, true); + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + + it("lifecycle batching - with lifecycle rule, one list batch, multiple bucket batches - worker did not finish", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + + await config_fs.update_config_json_file(JSON.stringify({ + // set short timeout so the lifecycle run will not finish + NC_LIFECYCLE_TIMEOUT_MS: 1, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 2, + NC_LIFECYCLE_LIST_BATCH_SIZE: 5 + })); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + + const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); + expect(lifecycle_log_entries.length).toBe(1); + const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name); + const lifecycle_log_json = await config_fs.get_config_data(log_file_path, { silent_if_missing: true }); + expect(lifecycle_log_json.state.is_finished).toBe(false); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).not.toBe(0); + }); + + it("lifecycle batching - continue finished lifecycle should do nothing", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + + //continue finished run + await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + + const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); + expect(lifecycle_log_entries.length).toBe(2); + const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name); + const lifecycle_log_json = await config_fs.get_config_data(log_file_path, { silent_if_missing: true }); + expect(lifecycle_log_json.state.is_finished).toBe(true); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); + }); + + it("continue lifecycle batching (running twice: worker not finish and continued worker) - should finish the run - delete all", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + await config_fs.update_config_json_file(JSON.stringify({ + // set short timeout so the lifecycle run will not finish + NC_LIFECYCLE_TIMEOUT_MS: 5, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 2, + NC_LIFECYCLE_LIST_BATCH_SIZE: 5 + + })); + const keys = [test_key1, test_key2, "key3", "key4", "key5", "key6", "key7"]; + for (const key of keys) { + await create_object(object_sdk, test_bucket, key, 100, false); + } + try { + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + } catch (e) { + //ignore timeout error + } + await config_fs.update_config_json_file(JSON.stringify({ + // set default timeout so the lifecycle run will continue to run + NC_LIFECYCLE_TIMEOUT_MS: config.NC_LIFECYCLE_TIMEOUT_MS, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 2, + NC_LIFECYCLE_LIST_BATCH_SIZE: 5 + + })); + await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }); + + it("continue lifecycle batching should finish the run - validate new run. don't delete already deleted items", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + await config_fs.update_config_json_file(JSON.stringify({ + NC_LIFECYCLE_TIMEOUT_MS: 70, + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 2, + NC_LIFECYCLE_LIST_BATCH_SIZE: 4 + })); + const keys = []; + for (let i = 0; i < 100; i++) { + const new_key = `key${i}`; + await create_object(object_sdk, test_bucket, new_key, 100, false); + keys.push(new_key); + } + try { + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + } catch (e) { + //ignore error + } + + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + const intermediate_key_list = object_list.objects.map(object => object.key); + const new_keys = []; + //recreate deleted key. next run should skip those keys + for (const key of keys) { + if (!intermediate_key_list.includes(key)) { + await create_object(object_sdk, test_bucket, key, 100, false); + new_keys.push(key); + } + } + await config_fs.update_config_json_file(JSON.stringify({ + NC_LIFECYCLE_TIMEOUT_MS: 9999, + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + NC_LIFECYCLE_BUCKET_BATCH_SIZE: 2, + NC_LIFECYCLE_LIST_BATCH_SIZE: 4, + })); + await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list2 = await object_sdk.list_objects({ bucket: test_bucket }); + const res_keys = object_list2.objects.map(object => object.key); + for (const key of new_keys) { + expect(res_keys).toContain(key); + } + }, TEST_TIMEOUT); + + it("lifecycle batching - with lifecycle rule, multiple list batches, multiple bucket batches - newer noncurrent versions", async () => { + const lifecycle_rule = [{ + "id": "keep 2 noncurrent versions", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": "", + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 2, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: 'ENABLED' }); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const version_arr = []; + let res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + version_arr.push(res.version_id); + for (let i = 0; i < 9; i++) { + const prev_res = res; + res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + await update_version_xattr(test_bucket, test_key1, prev_res.version_id); + version_arr.push(res.version_id); + } + const last_3_versions = new Set(version_arr.slice(-3)); // latest version + 2 noncurrent versions + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + const object_list_versions = new Set(object_list.objects.map(object => object.version_id)); + expect(object_list.objects.length).toBe(3); + expect(object_list_versions).toEqual(last_3_versions); + }); + + it("lifecycle rule, multiple list batches, multiple bucket batches - both expire and noncurrent actions", async () => { + const lifecycle_rule = [{ + "id": "keep 2 noncurrent versions and expire after 1 day", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": "", + }, + "expiration": { + "date": yesterday.getTime() + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 2, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: 'ENABLED' }); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const keys = [test_key1, test_key2, "key3", "key4", "key5", "key6", "key7"]; + for (const key of keys) { + if (key === test_key1) continue; //test_key1 is initialized in his own loop + await create_object(object_sdk, test_bucket, key, 100, false); + } + let prev_res; + let res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + for (let i = 0; i < 9; i++) { + prev_res = res; + res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + await update_version_xattr(test_bucket, test_key1, prev_res.version_id); + } + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + const expected_length = keys.length * 2 + 1; //all keys + delete marker for each key + 1 noncurrent versions for test_key1 + expect(object_list.objects.length).toBe(expected_length); + }); + }); + + describe('noobaa nc - lifecycle batching - thousands of objects', () => { + const TEST_TIMEOUT_FOR_LONG_BATCHING = 120 * 1000; + const test_bucket_path = `${root_path}/${test_bucket}`; + const test_key1 = 'test_key1'; + const test_key2 = 'test_key2'; + let object_sdk; + const tmp_lifecycle_logs_dir_path = path.join(root_path, 'test_lifecycle_logs'); + + beforeAll(async () => { + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); + const json_account = JSON.parse(res).response.reply; + console.log(json_account); + object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); + object_sdk.requesting_account = json_account; + await object_sdk.create_bucket({ name: test_bucket }); + }); + + beforeEach(async () => { + await config_fs.create_config_json_file(JSON.stringify({ + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path + })); + }); + + afterEach(async () => { + await object_sdk.delete_bucket_lifecycle({ name: test_bucket }); + await fs_utils.create_fresh_path(test_bucket_path); + fs_utils.folder_delete(tmp_lifecycle_logs_dir_path); + await config_fs.delete_config_json_file(); + }); + + it("lifecycle batching - with lifecycle rule, one batch of bucket and list", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + for (let i = 0; i < 1000; i++) { + await create_object(object_sdk, test_bucket, `test_key${i}`, 5, true); + } + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + Object.values(parsed_res_latest_lifecycle.response.reply.buckets_statuses).forEach(bucket_status => { + expect(bucket_status.state.is_finished).toBe(true); + Object.values(bucket_status.rules_statuses).forEach(rule_status => { + expect(rule_status.state.is_finished).toBe(true); + }); + }); + }, TEST_TIMEOUT_FOR_LONG_BATCHING); + + it("lifecycle batching - with lifecycle rule, one batch of bucket and list, no expire statement", async () => { + const lifecycle_rule = [{ + "id": "abort mpu after 3 days", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": '', + }, + "abort_incomplete_multipart_upload": { + "days_after_initiation": 3 + } + }]; + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + for (let i = 0; i < 1000; i++) { + await object_sdk.create_object_upload({ key: `test_key${i}`, bucket: test_bucket }); + } + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + Object.values(parsed_res_latest_lifecycle.response.reply.buckets_statuses).forEach(bucket_status => { + expect(bucket_status.state.is_finished).toBe(true); + }); + }); + + it("lifecycle batching - with lifecycle rule, one list batches, one bucket batch - worker did not finish", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + for (let i = 0; i < 1000; i++) { + await create_object(object_sdk, test_bucket, `test_key${i}`, 5, true); + } + + await config_fs.update_config_json_file(JSON.stringify({ + // set short timeout so the lifecycle run will not finish + NC_LIFECYCLE_TIMEOUT_MS: 1, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + })); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + + const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); + expect(lifecycle_log_entries.length).toBe(1); + const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name); + const lifecycle_log_json = await config_fs.get_config_data(log_file_path, { silent_if_missing: true }); + expect(lifecycle_log_json.state.is_finished).toBe(false); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).not.toBe(0); + }, TEST_TIMEOUT_FOR_LONG_BATCHING); + + it("lifecycle batching - continue finished lifecycle should do nothing", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + + //continue finished run + await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true, undefined); + + const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); + expect(lifecycle_log_entries.length).toBe(2); + const log_file_path = path.join(tmp_lifecycle_logs_dir_path, lifecycle_log_entries[0].name); + const lifecycle_log_json = await config_fs.get_config_data(log_file_path, { silent_if_missing: true }); + expect(lifecycle_log_json.state.is_finished).toBe(true); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(2); + }); + + it("continue lifecycle batching (running twice: worker not finish and continued worker) - should finish the run - delete all", async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + await config_fs.update_config_json_file(JSON.stringify({ + // set short timeout so the lifecycle run will not finish + NC_LIFECYCLE_TIMEOUT_MS: 5, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + + })); + for (let i = 0; i < 1000; i++) { + await create_object(object_sdk, test_bucket, `test_key${i}`, 5, false); + } + try { + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + } catch (e) { + //ignore timeout error + } + await config_fs.update_config_json_file(JSON.stringify({ + // set default timeout so the lifecycle run will continue to run + NC_LIFECYCLE_TIMEOUT_MS: config.NC_LIFECYCLE_TIMEOUT_MS, + // the configs as it was before (in the beforeEach) + NC_LIFECYCLE_LOGS_DIR: tmp_lifecycle_logs_dir_path, + + })); + await exec_manage_cli(TYPES.LIFECYCLE, '', { continue: 'true', disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + }, TEST_TIMEOUT_FOR_LONG_BATCHING); + + it("lifecycle batching - with lifecycle rule, multiple list batches, multiple bucket batches - newer noncurrent versions", async () => { + const lifecycle_rule = [{ + "id": "keep 2 noncurrent versions", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": "", + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 2, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: 'ENABLED' }); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const version_arr = []; + let res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + version_arr.push(res.version_id); + for (let i = 0; i < 1099; i++) { + const prev_res = res; + res = await create_object(object_sdk, test_bucket, test_key1, 100, false); + await update_version_xattr(test_bucket, test_key1, prev_res.version_id); + version_arr.push(res.version_id); + } + const last_3_versions = new Set(version_arr.slice(-3)); // latest version + 2 noncurrent versions + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + const object_list_versions = new Set(object_list.objects.map(object => object.version_id)); + expect(object_list.objects.length).toBe(3); + expect(object_list_versions).toEqual(last_3_versions); + }, TEST_TIMEOUT_FOR_LONG_BATCHING); + + it("lifecycle rule, multiple list batches, multiple bucket batches - both expire and noncurrent actions", async () => { + const lifecycle_rule = [{ + "id": "keep 2 noncurrent versions and expire after 1 day", + "status": LIFECYCLE_RULE_STATUS_ENUM.ENABLED, + "filter": { + "prefix": "", + }, + "expiration": { + "date": yesterday.getTime() + }, + "noncurrent_version_expiration": { + "newer_noncurrent_versions": 2, + "noncurrent_days": 1 + } + }]; + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: 'ENABLED' }); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule }); + + const keys = [test_key1, test_key2, "key3", "key4", "key5", "key6", "key7"]; + for (const key of keys) { + if (key === test_key1) continue; //test_key1 is initialized in his own loop + await create_object(object_sdk, test_bucket, key, 10, false); + } + + let prev_res; + let res = await create_object(object_sdk, test_bucket, test_key1, 10, false); + for (let i = 0; i < 1099; i++) { + prev_res = res; + res = await create_object(object_sdk, test_bucket, test_key1, 10, false); + await update_version_xattr(test_bucket, test_key1, prev_res.version_id); + } + const latest_lifecycle = await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, undefined, undefined); + const parsed_res_latest_lifecycle = JSON.parse(latest_lifecycle); + expect(parsed_res_latest_lifecycle.response.reply.state.is_finished).toBe(true); + + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + const expected_length = keys.length * 2 + 1; //all keys + delete marker for each key + 1 noncurrent versions for test_key1 + expect(object_list.objects.length).toBe(expected_length); + }, TEST_TIMEOUT_FOR_LONG_BATCHING); + }); +}); + +describe('noobaa nc - lifecycle notifications', () => { + const test_bucket_path = `${root_path}/${test_bucket}`; + const test_key1 = 'test_key1'; + const test_key2 = 'test_key2'; + let object_sdk; + const tmp_lifecycle_logs_dir_path = path.join(root_path, 'test_lifecycle_notifications_logs'); + const tmp_conn_dir_path = path.join(root_path, 'test_notification_logs'); + const http_connect_filename = 'http_connect.json'; + const http_connect_path = path.join(tmp_conn_dir_path, http_connect_filename); + //content of connect file, will be written to a file in before() + const http_connect = { + agent_request_object: { "host": "localhost", "port": 9998, "timeout": 1500 }, + request_options_object: { "auth": "amit:passw", "timeout": 1500 }, + notification_protocol: 'http', + name: 'http_notif' + }; + + const notifications_json = { + bucket_name: test_bucket, + notifications: [{ + id: ["test id"], + topic: [http_connect_filename], + event: [ + "s3:LifecycleExpiration:Delete", + "s3:LifecycleExpiration:DeleteMarkerCreated" + ] + }], + }; + + beforeAll(async () => { + //create paths + await fs_utils.create_fresh_path(config_root, 0o777); + set_nc_config_dir_in_config(config_root); + await fs_utils.create_fresh_path(root_path, 0o777); + await fs_utils.create_fresh_path(tmp_lifecycle_logs_dir_path, 0o777); + await fs_utils.create_fresh_path(tmp_conn_dir_path, 0o777); + + //create account and bucket + const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, account_options1); + const json_account = JSON.parse(res).response.reply; + await create_system_json(config_fs); + object_sdk = new NsfsObjectSDK('', config_fs, json_account, "DISABLED", config_fs.config_root, undefined); + object_sdk.requesting_account = json_account; + await object_sdk.create_bucket({ name: test_bucket }); + + //configure bucket notifications + config.NOTIFICATION_LOG_DIR = tmp_lifecycle_logs_dir_path; + config.NOTIFICATION_CONNECT_DIR = tmp_conn_dir_path; + fs.writeFileSync(http_connect_path, JSON.stringify(http_connect)); + await object_sdk.put_bucket_notification(notifications_json); + await config_fs.create_config_json_file(JSON.stringify({ NOTIFICATION_LOG_DIR: tmp_lifecycle_logs_dir_path })); + }); + + afterEach(async () => { + await object_sdk.delete_bucket_lifecycle({ name: test_bucket }); + await fs_utils.create_fresh_path(test_bucket_path); + await fs_utils.create_fresh_path(tmp_lifecycle_logs_dir_path); + }); + + afterAll(async () => { + await fs_utils.folder_delete(root_path); + await fs_utils.folder_delete(config_root); + }, TEST_TIMEOUT); + + it('nc lifecycle - lifecycle rule with Delete notification', async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + await create_object(object_sdk, test_bucket, test_key1, 100, true); + await create_object(object_sdk, test_bucket, test_key2, 100, true); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true); + + const object_list = await object_sdk.list_objects({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(0); + + const notification_log_entries = await get_notifications_obj(); + expect(notification_log_entries.length).toBe(2); + + notification_log_entries.forEach(notification => { + const record = notification.notif.Records[0]; + expect(record.eventName).toBe('LifecycleExpiration:Delete'); + expect(record.s3.bucket.name).toBe(test_bucket); + expect(record.s3.object.size).toBe(100); + expect([test_key1, test_key2]).toContain(record.s3.object.key); + }); + }); + + it('nc lifecycle - lifecycle rule with Delete marker notification', async () => { + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: 'ENABLED' }); + + await create_object(object_sdk, test_bucket, test_key1, 100, false); + await create_object(object_sdk, test_bucket, test_key2, 100, false); + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true); + + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(4); + + const notification_log_entries = await get_notifications_obj(); + expect(notification_log_entries.length).toBe(2); + + notification_log_entries.forEach(notification => { + const record = notification.notif.Records[0]; + expect(record.eventName).toBe('LifecycleExpiration:DeleteMarkerCreated'); + expect(record.s3.bucket.name).toBe(test_bucket); + expect(record.s3.object.size).toBe(100); + expect([test_key1, test_key2]).toContain(record.s3.object.key); + }); + }); + + it('nc lifecycle - should not notify', async () => { + notifications_json.notifications[0].event = ["s3:LifecycleExpiration:Delete"]; + await object_sdk.put_bucket_notification(notifications_json); + await object_sdk.set_bucket_lifecycle_configuration_rules({ name: test_bucket, rules: lifecycle_rule_delete_all }); + await object_sdk.set_bucket_versioning({ name: test_bucket, versioning: 'ENABLED' }); + + await create_object(object_sdk, test_bucket, test_key1, 100, false); + await create_object(object_sdk, test_bucket, test_key2, 100, false); + + await exec_manage_cli(TYPES.LIFECYCLE, '', { disable_service_validation: 'true', disable_runtime_validation: 'true', config_root }, true); + await config_fs.delete_config_json_file(); + + const object_list = await object_sdk.list_object_versions({ bucket: test_bucket }); + expect(object_list.objects.length).toBe(4); + + //there should not be any notifications + const notification_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); + expect(notification_log_entries.length).toBe(0); + }); + + async function get_notifications_obj() { + const notification_log_entries = await nb_native().fs.readdir(config_fs.fs_context, tmp_lifecycle_logs_dir_path); + expect(notification_log_entries.length).toBe(1); + const log_file_path = path.join(tmp_lifecycle_logs_dir_path, notification_log_entries[0].name); + const { data: notifications_buffer } = await nb_native().fs.readFile(config_fs.fs_context, log_file_path); + const notification_list = notifications_buffer.toString().split('\n'); + notification_list.pop(); //remove last empty element + const notifications_parsed = notification_list.map(str => JSON.parse(str)); + return notifications_parsed; + } +}); + +/** + * create_object creates an object with random data in the bucket + * Note: is_old - if true, would update the mtime of the file. + * @param {object} object_sdk + * @param {string} bucket + * @param {string} key + * @param {number} size + * @param {boolean} [is_old] + * @param {{ key: string; value: string; }[]} [tagging] + */ +async function create_object(object_sdk, bucket, key, size, is_old, tagging) { + const data = crypto.randomBytes(size); + const res = await object_sdk.upload_object({ + bucket, + key, + source_stream: buffer_utils.buffer_to_read_stream(data), + size, + tagging + }); + if (is_old) { + await update_file_mtime(path.join(root_path, bucket, key)); + if (key.endsWith('/') === true) { + await update_file_mtime(path.join(root_path, bucket, key, '.folder')); + } + } + return res; +} + +/** + * update_file_mtime updates the mtime of the target path + * Warnings: + * - This operation would change the mtime of the file to 5 days ago - which means that it changes the etag / obj_id of the object + * - Please do not use on versioned objects (version_id will not be changed, but the mtime will be changed) - might cause issues. + * @param {String} target_path + * @returns {Promise} + */ +async function update_file_mtime(target_path) { + const update_file_mtime_cmp = os_utils.IS_MAC ? `touch -t $(date -v -5d +"%Y%m%d%H%M.%S") ${target_path}` : `touch -d "5 days ago" ${target_path}`; + await os_utils.exec(update_file_mtime_cmp, { return_stdout: true }); +} + +/** + * updates the number of noncurrent days xattr of target path to be 5 days older. use only on noncurrent objects. + * is use this function on latest object the xattr will be changed when the object turns noncurrent + * how to use this function: + * 1. create a new object but don't change its mtime (changing mtime will cause versioning functions to fail) + * 2. create a new object with the same key to make the object noncurrent + * 3. call this function to change the xattr of the noncurrent object + * @param {String} bucket + * @param {String} key + * @param {String} version_id + * @returns {Promise} + */ +async function update_version_xattr(bucket, key, version_id) { + const older_time = new Date(); + older_time.setDate(yesterday.getDate() - 5); // 5 days ago + + const target_path = path.join(root_path, bucket, path.dirname(key), '.versions', `${path.basename(key)}_${version_id}`); + const file = await nb_native().fs.open(config_fs.fs_context, target_path, config.NSFS_OPEN_READ_MODE, + native_fs_utils.get_umasked_mode(config.BASE_MODE_FILE)); + const stat = await file.stat(config_fs.fs_context); + const xattr = Object.assign(stat.xattr, { + 'user.noobaa.non_current_timestamp': older_time.getTime(), + }); + await file.replacexattr(config_fs.fs_context, xattr, undefined); + await file.close(config_fs.fs_context); +} + +/** + * date_to_run_time_format coverts a date to run time format HH:MM + * @param {Date} date + * @returns {String} + */ +function date_to_run_time_format(date = new Date()) { + return date.getHours() + ':' + date.getMinutes(); +} diff --git a/src/test/unit_tests/jest_tests/test_newline_reader.test.js b/src/test/unit_tests/jest_tests/test_newline_reader.test.js index 2e47864188..39dc275d2d 100644 --- a/src/test/unit_tests/jest_tests/test_newline_reader.test.js +++ b/src/test/unit_tests/jest_tests/test_newline_reader.test.js @@ -51,6 +51,26 @@ describe('newline_reader', () => { expect(result).toStrictEqual(UTF8DATA_ARR); }); + it('read_file_offset - can process utf8 characters when termination with newline character', async () => { + const UTF8DATA_BUF = Buffer.from(UTF8DATA_ARR.join('\n') + '\n', 'utf8'); + + const reader = new NewlineReader({}, '', { skip_leftover_line: true, read_file_offset: 0 }); + // @ts-ignore + reader.fh = mocked_file_handler(UTF8DATA_BUF); + + const result = []; + let expected_cur_next_line_file_offset = 0; + const [processed] = await reader.forEach(async entry => { + result.push(entry); + expected_cur_next_line_file_offset += Buffer.byteLength(entry, 'utf8') + 1; + expect(reader.next_line_file_offset).toBe(expected_cur_next_line_file_offset); + return true; + }); + + expect(processed).toBe(UTF8DATA_ARR.length); + expect(result).toStrictEqual(UTF8DATA_ARR); + }); + it('can process utf8 characters when termination not with new line character', async () => { const UTF8DATA_BUF = Buffer.from(UTF8DATA_ARR.join('\n'), 'utf8'); @@ -68,7 +88,48 @@ describe('newline_reader', () => { expect(result).toStrictEqual(UTF8DATA_ARR); }); - it('can process utf8 characters when termination not with new line character [bufsize = 4]', async () => { + it('read_file_offset - can process utf8 characters when termination not with new line character', async () => { + const UTF8DATA_BUF = Buffer.from(UTF8DATA_ARR.join('\n'), 'utf8'); + + const reader = new NewlineReader({}, '', { read_file_offset: 0 }); + // @ts-ignore + reader.fh = mocked_file_handler(UTF8DATA_BUF); + + const result = []; + let expected_cur_next_line_file_offset = 0; + const [processed] = await reader.forEach(async entry => { + result.push(entry); + expected_cur_next_line_file_offset += Buffer.byteLength(entry, 'utf8') + (reader.eof ? 0 : 1); + expect(reader.next_line_file_offset).toBe(expected_cur_next_line_file_offset); + return true; + }); + + expect(processed).toBe(UTF8DATA_ARR.length); + expect(result).toStrictEqual(UTF8DATA_ARR); + }); + + it('read_file_offset starts from the second line - can process utf8 characters when termination not with new line character', async () => { + const UTF8DATA_BUF = Buffer.from(UTF8DATA_ARR.join('\n'), 'utf8'); + const expected_to_be_processed_data_array = UTF8DATA_ARR.slice(1); + const initial_next_line_file_offset = Buffer.byteLength(UTF8DATA_ARR[0], 'utf8') + 1; + const reader = new NewlineReader({}, '', { read_file_offset: initial_next_line_file_offset}); + // @ts-ignore + reader.fh = mocked_file_handler(UTF8DATA_BUF); + + const result = []; + let expected_cur_next_line_file_offset = initial_next_line_file_offset; + const [processed] = await reader.forEach(async entry => { + result.push(entry); + expected_cur_next_line_file_offset += Buffer.byteLength(entry, 'utf8') + (reader.eof ? 0 : 1); + expect(reader.next_line_file_offset).toBe(expected_cur_next_line_file_offset); + return true; + }); + + expect(processed).toBe(expected_to_be_processed_data_array.length); + expect(result).toStrictEqual(expected_to_be_processed_data_array); + }); + + it('can process utf8 characters when termination not with new line character [bufsize = 256]', async () => { const expected = "abc"; const UTF8DATA_ARR_TEMP = [ ...UTF8DATA_ARR, expected ]; const UTF8DATA_BUF = Buffer.from(UTF8DATA_ARR_TEMP.join('\n'), 'utf8'); @@ -86,5 +147,26 @@ describe('newline_reader', () => { expect(processed).toBe(1); expect(result).toStrictEqual([expected]); }); + + it('read_file_offset - can process utf8 characters when termination not with new line character [bufsize = 256]', async () => { + const expected = "abc"; + const UTF8DATA_ARR_TEMP = [ ...UTF8DATA_ARR, expected ]; + const UTF8DATA_BUF = Buffer.from(UTF8DATA_ARR_TEMP.join('\n'), 'utf8'); + + const reader = new NewlineReader({}, '', { bufsize: 256, skip_overflow_lines: true, read_file_offset: 0 }); + // @ts-ignore + reader.fh = mocked_file_handler(UTF8DATA_BUF); + + const result = []; + const [processed] = await reader.forEach(async entry => { + result.push(entry); + return true; + }); + + expect(processed).toBe(1); + expect(result).toStrictEqual([expected]); + const expected_cur_next_line_file_offset = UTF8DATA_BUF.length; + expect(reader.next_line_file_offset).toBe(expected_cur_next_line_file_offset); + }); }); }); diff --git a/src/test/unit_tests/test_bucketspace_fs.js b/src/test/unit_tests/test_bucketspace_fs.js index 203bd2d4cb..2b5b740440 100644 --- a/src/test/unit_tests/test_bucketspace_fs.js +++ b/src/test/unit_tests/test_bucketspace_fs.js @@ -552,7 +552,7 @@ mocha.describe('bucketspace_fs', function() { assert.equal(objects.buckets.length, 1); assert.equal(objects.buckets[0].name.unwrap(), expected_bucket_name); const bucket_config_path = get_config_file_path(CONFIG_SUBDIRS.BUCKETS, expected_bucket_name); - const bucket_data = await read_file(process_fs_context, bucket_config_path); + const bucket_data = await read_file(process_fs_context, bucket_config_path, { parse_json: true }); assert.equal(objects.buckets[0].creation_date, bucket_data.creation_date); }); }); @@ -710,7 +710,7 @@ mocha.describe('bucketspace_fs', function() { const param = { name: test_bucket, versioning: 'ENABLED' }; await bucketspace_fs.set_bucket_versioning(param, dummy_object_sdk); const bucket_config_path = get_config_file_path(CONFIG_SUBDIRS.BUCKETS, param.name); - const bucket = await read_file(process_fs_context, bucket_config_path); + const bucket = await read_file(process_fs_context, bucket_config_path, { parse_json: true }); assert.equal(bucket.versioning, 'ENABLED'); }); @@ -920,7 +920,7 @@ mocha.describe('bucketspace_fs', function() { mocha.describe('bucket tagging operations', function() { mocha.it('put_bucket_tagging', async function() { const param = { name: test_bucket, tagging: [{ key: 'k1', value: 'v1' }] }; - await bucketspace_fs.put_bucket_tagging(param); + await bucketspace_fs.put_bucket_tagging(param, dummy_object_sdk); const tag = await bucketspace_fs.get_bucket_tagging(param); assert.deepEqual(tag, { tagging: param.tagging }); }); diff --git a/src/test/unit_tests/test_bucketspace_versioning.js b/src/test/unit_tests/test_bucketspace_versioning.js index db0491ba1f..94865e5dc8 100644 --- a/src/test/unit_tests/test_bucketspace_versioning.js +++ b/src/test/unit_tests/test_bucketspace_versioning.js @@ -24,6 +24,7 @@ coretest.setup({}); const XATTR_INTERNAL_NOOBAA_PREFIX = 'user.noobaa.'; const XATTR_VERSION_ID = XATTR_INTERNAL_NOOBAA_PREFIX + 'version_id'; const XATTR_DELETE_MARKER = XATTR_INTERNAL_NOOBAA_PREFIX + 'delete_marker'; +const XATTR_NON_CURRENT_TIMESTASMP = XATTR_INTERNAL_NOOBAA_PREFIX + 'non_current_timestamp'; const NULL_VERSION_ID = 'null'; const HIDDEN_VERSIONS_PATH = '.versions'; const NSFS_FOLDER_OBJECT_NAME = '.folder'; @@ -1181,6 +1182,9 @@ mocha.describe('bucketspace namespace_fs - versioning', function() { const version_path = path.join(suspended_full_path, '.versions', key_to_delete3 + '_' + latest_dm_version); const version_info = await stat_and_get_all(version_path, ''); assert.equal(version_info.xattr[XATTR_VERSION_ID], NULL_VERSION_ID); + // check second latest is still non current xattr + const second_latest_version_path = path.join(suspended_full_path, '.versions', key_to_delete3 + '_' + prev_dm.VersionId); + await check_non_current_xattr_exists(second_latest_version_path); }); }); @@ -1217,6 +1221,8 @@ mocha.describe('bucketspace namespace_fs - versioning', function() { mocha.it('delete object version id - latest - second latest is null version', async function() { const upload_res_arr = await upload_object_versions(account_with_access, delete_object_test_bucket_reg, key1, ['null', 'regular']); const cur_version_id1 = await stat_and_get_version_id(full_delete_path, key1); + const second_latest_version_path = path.join(full_delete_path, '.versions', key1 + '_null'); + await check_non_current_xattr_exists(second_latest_version_path); const delete_res = await account_with_access.deleteObject({ Bucket: delete_object_test_bucket_reg, @@ -1227,6 +1233,9 @@ mocha.describe('bucketspace namespace_fs - versioning', function() { const cur_version_id2 = await stat_and_get_version_id(full_delete_path, key1); assert.notEqual(cur_version_id1, cur_version_id2); assert.equal('null', cur_version_id2); + // check second latest current xattr removed + const latest_version_path = path.join(full_delete_path, key1); + await check_non_current_xattr_does_not_exist(latest_version_path); await fs_utils.file_must_not_exist(path.join(full_delete_path, key1 + '_' + upload_res_arr[1].VersionId)); const max_version1 = await find_max_version_past(full_delete_path, key1, ''); assert.equal(max_version1, undefined); @@ -3175,9 +3184,18 @@ function _extract_version_info_from_xattr(version_id_str) { return { mtimeNsBigint: size_utils.string_to_bigint(arr[0], 36), ino: parseInt(arr[1], 36) }; } +/** + * version_file_exists returns path of version in .versions + * @param {String} full_path + * @param {String} key + * @param {String} dir + * @param {String} version_id + * @returns {Promise} + */ async function version_file_exists(full_path, key, dir, version_id) { const version_path = path.join(full_path, dir, '.versions', key + '_' + version_id); await fs_utils.file_must_exist(version_path); + await check_non_current_xattr_exists(version_path, ''); return true; } @@ -3194,10 +3212,10 @@ async function get_obj_and_compare_data(s3, bucket_name, key, expected_body) { return true; } -async function is_delete_marker(full_path, dir, key, version) { +async function is_delete_marker(full_path, dir, key, version, check_non_current_version = true) { const version_path = path.join(full_path, dir, '.versions', key + '_' + version); const stat = await nb_native().fs.stat(DEFAULT_FS_CONFIG, version_path); - return stat && stat.xattr[XATTR_DELETE_MARKER]; + return stat && stat.xattr[XATTR_DELETE_MARKER] && (check_non_current_version ? stat.xattr[XATTR_NON_CURRENT_TIMESTASMP] : true); } async function stat_and_get_version_id(full_path, key) { @@ -3260,6 +3278,28 @@ function check_null_version_id(version_id) { return version_id === NULL_VERSION_ID; } +/** + * check_non_current_xattr_exists checks that the XATTR_NON_CURRENT_TIMESTASMP xattr exists + * @param {String} full_path + * @param {String} [key] + * @returns {Promise} + */ +async function check_non_current_xattr_exists(full_path, key = '') { + const stat = await stat_and_get_all(full_path, key); + assert.ok(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP]); +} + +/** + * check_non_current_xattr_does_not_exist checks that the XATTR_NON_CURRENT_TIMESTASMP xattr does not exist + * @param {String} full_path + * @param {String} [key] + * @returns {Promise} + */ +async function check_non_current_xattr_does_not_exist(full_path, key = '') { + const stat = await stat_and_get_all(full_path, key); + assert.equal(stat.xattr[XATTR_NON_CURRENT_TIMESTASMP], undefined); +} + async function put_allow_all_bucket_policy(s3_client, bucket) { const policy = { Version: '2012-10-17', diff --git a/src/test/unit_tests/test_lifecycle.js b/src/test/unit_tests/test_lifecycle.js index 5789307001..fdfc66be84 100644 --- a/src/test/unit_tests/test_lifecycle.js +++ b/src/test/unit_tests/test_lifecycle.js @@ -84,6 +84,30 @@ mocha.describe('lifecycle', () => { mocha.it('test and prefix size', async () => { await commonTests.test_and_prefix_size(Bucket, Key, s3); }); + mocha.it('test rule ID length', async () => { + await commonTests.test_rule_id_length(Bucket, Key, s3); + }); + mocha.it('test rule duplicate ID', async () => { + await commonTests.test_rule_duplicate_id(Bucket, Key, s3); + }); + mocha.it('test rule status value', async () => { + await commonTests.test_rule_status_value(Bucket, Key, s3); + }); + mocha.it('test invalid filter format', async () => { + await commonTests.test_invalid_filter_format(Bucket, Key, s3); + }); + mocha.it('test invalid expiration date format', async () => { + await commonTests.test_invalid_expiration_date_format(Bucket, Key, s3); + }); + mocha.it('test expiration with multiple fields', async () => { + await commonTests.test_expiration_multiple_fields(Bucket, Key, s3); + }); + mocha.it('test AbortIncompleteMultipartUpload with tags', async () => { + await commonTests.test_abortincompletemultipartupload_with_tags(Bucket, Key, s3); + }); + mocha.it('test AbortIncompleteMultipartUpload with object sizes', async () => { + await commonTests.test_abortincompletemultipartupload_with_sizes(Bucket, Key, s3); + }); }); mocha.describe('bucket-lifecycle-bg-worker', function() { diff --git a/src/util/file_reader.js b/src/util/file_reader.js index 517573503c..a8be5e9184 100644 --- a/src/util/file_reader.js +++ b/src/util/file_reader.js @@ -30,6 +30,7 @@ class NewlineReader { * bufsize?: number; * skip_leftover_line?: boolean; * skip_overflow_lines?: boolean; + * read_file_offset?: number; * }} [cfg] **/ constructor(fs_context, filepath, cfg) { @@ -41,18 +42,19 @@ class NewlineReader { this.fs_context = fs_context; this.fh = null; this.eof = false; - this.readoffset = 0; + this.read_file_offset = cfg?.read_file_offset || 0; this.buf = Buffer.alloc(cfg?.bufsize || 64 * 1024); this.start = 0; this.end = 0; this.overflow_state = false; + this.next_line_file_offset = cfg?.read_file_offset || 0; } info() { return { path: this.path, - read_offset: this.readoffset, + read_offset: this.read_file_offset, overflow_state: this.overflow_state, start: this.start, end: this.end, @@ -67,6 +69,7 @@ class NewlineReader { async nextline() { if (!this.fh) await this.init(); + // TODO - in case more data will be appended to the file - after each read the reader must set reader.eof = false if someone will keep on reading from a file while it is being written. while (!this.eof) { // extract next line if terminated in current buffer if (this.start < this.end) { @@ -78,9 +81,9 @@ class NewlineReader { this.start += term_idx + 1; continue; } - const line = this.buf.toString('utf8', this.start, this.start + term_idx); this.start += term_idx + 1; + this.next_line_file_offset = this.read_file_offset - (this.end - this.start); return line; } } @@ -106,7 +109,7 @@ class NewlineReader { // read from file const avail = this.buf.length - this.end; - const read = await this.fh.read(this.fs_context, this.buf, this.end, avail, this.readoffset); + const read = await this.fh.read(this.fs_context, this.buf, this.end, avail, this.read_file_offset); if (!read) { this.eof = true; @@ -118,13 +121,15 @@ class NewlineReader { console.warn('line too long finally terminated at eof:', this.info()); } else { const line = this.buf.toString('utf8', this.start, this.end); + this.start = this.end; + this.next_line_file_offset = this.read_file_offset; return line; } } return null; } - this.readoffset += read; + this.read_file_offset += read; this.end += read; } @@ -169,7 +174,7 @@ class NewlineReader { // was moved, this will still keep on reading from the previous FD. reset() { this.eof = false; - this.readoffset = 0; + this.read_file_offset = 0; this.start = 0; this.end = 0; this.overflow_state = false; diff --git a/src/util/lifecycle_utils.js b/src/util/lifecycle_utils.js new file mode 100644 index 0000000000..dc7d22434c --- /dev/null +++ b/src/util/lifecycle_utils.js @@ -0,0 +1,75 @@ +/* Copyright (C) 2025 NooBaa */ +'use strict'; + +const _ = require('lodash'); +const path = require('path'); +const config = require('../../config'); +const nb_native = require('./nb_native'); + +/** + * get_latest_nc_lifecycle_run_status returns the latest lifecycle run status + * latest run can be found by maxing the lifecycle log entry names, log entry name is the lifecycle_run_{timestamp}.json of the run + * @param {Object} config_fs + * @param {{silent_if_missing: boolean}} options + * @returns {Promise} + */ +async function get_latest_nc_lifecycle_run_status(config_fs, options) { + const { silent_if_missing = false } = options; + try { + const lifecycle_log_entries = await nb_native().fs.readdir(config_fs.fs_context, config.NC_LIFECYCLE_LOGS_DIR); + const latest_lifecycle_run = _.maxBy(lifecycle_log_entries, entry => entry.name); + const latest_lifecycle_run_status_path = path.join(config.NC_LIFECYCLE_LOGS_DIR, latest_lifecycle_run.name); + const latest_lifecycle_run_status = await config_fs.get_config_data(latest_lifecycle_run_status_path, options); + return latest_lifecycle_run_status; + } catch (err) { + if (err.code === 'ENOENT' && silent_if_missing) { + return; + } + throw err; + } +} + +/** + * get_lifecycle_object_info_for_filter returns an object that contains properties needed for filter check + * based on list_objects/stat result + * @param {{key: String, create_time: Number, size: Number, tagging: Object}} entry list object entry + * @returns {{key: String, age: Number, size: Number, tags: Object}} + */ +function get_lifecycle_object_info_for_filter(entry) { + return { + key: entry.key, + age: get_file_age_days(entry.create_time), + size: entry.size, + tags: entry.tagging, + }; +} + + +/** + * get_file_age_days gets file time since last modified in days + * @param {Number} mtime + * @returns {Number} days since object was last modified + */ +function get_file_age_days(mtime) { + return Math.floor((Date.now() - mtime) / 24 / 60 / 60 / 1000); +} + +/** + * file_matches_filter used for checking the filter before deletion + * @param {{obj_info: { key: String, create_time: Number, size: Number, tagging: Object}, filter_func?: Function}} params + * @returns {Boolean} + */ +function file_matches_filter({obj_info, filter_func = undefined}) { + if (filter_func) { + const object_info_for_filter = get_lifecycle_object_info_for_filter(obj_info); + if (!filter_func(object_info_for_filter)) { + return false; + } + } + return true; +} + +exports.get_latest_nc_lifecycle_run_status = get_latest_nc_lifecycle_run_status; +exports.file_matches_filter = file_matches_filter; +exports.get_lifecycle_object_info_for_filter = get_lifecycle_object_info_for_filter; +exports.get_file_age_days = get_file_age_days; diff --git a/src/util/native_fs_utils.js b/src/util/native_fs_utils.js index 89e152d5c5..0af1a77c6a 100644 --- a/src/util/native_fs_utils.js +++ b/src/util/native_fs_utils.js @@ -289,6 +289,19 @@ function should_retry_link_unlink(err) { return should_retry_general || should_retry_gpfs || should_retry_posix; } +/** + * stat_ignore_enoent unlinks a file and if recieved an ENOENT error it'll not fail + * @param {nb.NativeFSContext} fs_context + * @param {string} file_path + * @returns {Promise} + */ +async function stat_ignore_enoent(fs_context, file_path) { + try { + return await nb_native().fs.stat(fs_context, file_path); + } catch (err) { + if (err.code !== 'ENOENT') throw err; + } +} /** * stat_if_exists execute stat on entry_path and ignores on certain error codes. @@ -650,12 +663,14 @@ async function folder_delete(dir, fs_context, is_temp, silent_if_missing) { * read_file reads file and returns the parsed file data as object * @param {nb.NativeFSContext} fs_context * @param {string} _path + * @param {{parse_json?: Boolean}} [options] * @return {Promise} */ -async function read_file(fs_context, _path) { +async function read_file(fs_context, _path, options = {}) { const { data } = await nb_native().fs.readFile(fs_context, _path); - const data_parsed = JSON.parse(data.toString()); - return data_parsed; + let data_parsed; + if (options?.parse_json) data_parsed = JSON.parse(data.toString()); + return data_parsed || data.toString(); } @@ -727,6 +742,7 @@ exports.finally_close_files = finally_close_files; exports.get_user_by_distinguished_name = get_user_by_distinguished_name; exports.get_config_files_tmpdir = get_config_files_tmpdir; exports.stat_if_exists = stat_if_exists; +exports.stat_ignore_enoent = stat_ignore_enoent; exports._is_gpfs = _is_gpfs; exports.safe_move = safe_move;