Merge pull request #8482 from dannyzaken/danny-dedup

Avoid dedup chunks that are relatively new

Author: dannyzaken

config.js

@@ -392,6 +392,9 @@ config.CHUNK_CODER_EC_PARITY_TYPE = 'cm256';
 config.CHUNK_CODER_EC_TOLERANCE_THRESHOLD = 2;
 config.CHUNK_CODER_EC_IS_DEFAULT = false;
 
+// DEDUP
+config.MIN_CHUNK_AGE_FOR_DEDUP = 60 * 60 * 1000; // 1 hour
+
 //////////////////////////
 // DEDUP INDEXER CONFIG //
 //////////////////////////
@@ -870,7 +873,7 @@ config.ENDPOINT_SSL_PORT = Number(process.env.ENDPOINT_SSL_PORT) || 6443;
 config.ENDPOINT_SSL_STS_PORT = Number(process.env.ENDPOINT_SSL_STS_PORT) || -1;
 config.ENDPOINT_SSL_IAM_PORT = Number(process.env.ENDPOINT_SSL_IAM_PORT) || -1;
 config.ALLOW_HTTP = false;
-// config files should allow access to the owner of the files 
+// config files should allow access to the owner of the files
 config.BASE_MODE_CONFIG_FILE = 0o600;
 config.BASE_MODE_CONFIG_DIR = 0o700;
 
@@ -1054,10 +1057,10 @@ function _get_config_root() {
 }
 
 /**
- * validate_nc_master_keys_config validates the following - 
+ * validate_nc_master_keys_config validates the following -
  * 1. if type is file -
  *    1.1. no GET/PUT executables provided
- * 2. if type is executable - 
+ * 2. if type is executable -
  *    2.1. no file location provided
  *    2.2. GET & PUT executables exist and executables
  */

src/server/object_services/mapper.js

@@ -186,7 +186,7 @@ function map_chunk(chunk, tier, tiering, tiering_status, location_info) {
         }
 
         /**
-         * @param {nb.TierMirror} mirror 
+         * @param {nb.TierMirror} mirror
          */
         function map_frag_in_mirror(mirror) {
             const used_blocks = [];
@@ -291,6 +291,13 @@ function is_chunk_good_for_dedup(chunk) {
     if (!chunk.is_accessible) return false;
     if (chunk.is_building_blocks) return false;
     if (chunk.is_building_frags) return false;
+
+    // reject chunks that are not at least config.MIN_CHUNK_AGE_FOR_DEDUP old
+    // this is to avoid deduping chunks that might be from a previous attempt to write the same object
+    // see https://bugzilla.redhat.com/show_bug.cgi?id=2256223
+    const chunk_age = Date.now() - chunk._id.getTimestamp().getTime();
+    if (chunk_age < config.MIN_CHUNK_AGE_FOR_DEDUP) return false;
+
     return true;
 }
 
@@ -360,15 +367,15 @@ function _block_sort_newer_first(block1, block2) {
 
 
 /**
- * @param {nb.Pool} pool 
+ * @param {nb.Pool} pool
  * @returns {boolean}
  */
 function _pool_has_redundancy(pool) {
     return Boolean(pool.cloud_pool_info || pool.mongo_pool_info);
 }
 
 // /**
-//  * 
+//  *
 //  * @param {nb.Chunk} chunk
 //  * @param {nb.LocationInfo} [location_info]
 //  */

src/test/unit_tests/test_encryption.js

@@ -29,6 +29,8 @@ let wrapped_coretest_secret_key;
 const BKT = `bucket.example`;
 const key_rotator = new KeyRotator({ name: 'kr'});
 
+config.MIN_CHUNK_AGE_FOR_DEDUP = 0;
+
 mocha.describe('Encryption tests', function() {
     const { rpc_client, EMAIL, SYSTEM } = coretest;
     let response_account;
@@ -991,10 +993,10 @@ mocha.describe('Rotation tests', function() {
             compare_secrets(secrets, system_store_account.master_key_id._id);
         });
 });
-// TODO: 
+// TODO:
         // 1. add more tests for checking namespace resources
         // 2. add tests for enable/disable account that has pool/namespace resource
-////////////// HELPERS 
+////////////// HELPERS
 
 async function multipart_upload(bucket, key, s3_conf) {
     let res = await s3_conf.createMultipartUpload({Bucket: bucket, Key: key, ContentType: 'text/plain'});