Import wiredtiger: 3c4bad55a5e5ea77a5547381f777757349375e2a from branch mongodb-master

ref: a9c02719fb..3c4bad55a5
for: 7.2.0-rc0

WT-11346 Create cppsuite test for background compaction

Author: ruby-oujo

src/third_party/wiredtiger/dist/test_data.py

@@ -126,7 +126,7 @@ def __ge__(self, other):
         The rate at which checkpoint is executed.''')
 ]
 
-background_compact_thread_config = [
+background_compact_thread_config = throttle_config + [
     Config('thread_count', 0, r'''
         Specifies the number of threads that will be used to perform background compaction
            operation.''',
@@ -228,6 +228,9 @@ def __ge__(self, other):
         type='category', subconfig=operation_tracker),
 
 # Non component top level configuration.
+    Config('background_compact_debug_mode', 'false', r'''
+        If true, background compact aggressively removes compact statistics for a file and decreases
+        the max amount of time a file can be skipped for.''', type='boolean'),
     Config('cache_max_wait_ms', 0, r'''
         The strict equivalent of cache_max_wait_ms defined in wiredtiger.''', min=0),
     Config('cache_size_mb', 0, r'''
@@ -258,6 +261,7 @@ def __ge__(self, other):
 # Test and their respective configuration sorted alphabetically.
 #
 methods = {
+    'background_compact' : Method(test_config),
     'bounded_cursor_perf' : Method(test_config),
     'bounded_cursor_prefix_indices' : Method(test_config),
     'bounded_cursor_prefix_search_near' : Method(test_config),

src/third_party/wiredtiger/import.data

@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "a9c02719fbef960e017ab03e789f6a947aaee5c1"
+    "commit": "3c4bad55a5e5ea77a5547381f777757349375e2a"
 }

src/third_party/wiredtiger/src/config/test_config.c

@@ -2090,7 +2090,7 @@ __debug_mode_background_compact_config(WT_SESSION_IMPL *session, const char *cfg
         conn->background_compact.full_iteration_wait_time = WT_BACKGROUND_COMPACT_WAIT_TIME_DEBUG;
     } else {
         conn->background_compact.max_file_idle_time = WT_BACKGROUND_COMPACT_MAX_IDLE_TIME;
-        conn->background_compact.max_file_skip_time = 60 * WT_BACKGROUND_COMPACT_MAX_SKIP_TIME;
+        conn->background_compact.max_file_skip_time = WT_BACKGROUND_COMPACT_MAX_SKIP_TIME;
         conn->background_compact.full_iteration_wait_time = WT_BACKGROUND_COMPACT_WAIT_TIME;
     }
 

src/third_party/wiredtiger/src/conn/conn_api.c

@@ -0,0 +1,51 @@
+# Configuration for background_compact.
+background_compact_debug_mode=true,
+cache_size_mb=10000,
+duration_seconds=30,
+# We don't need to validate anything with the default config.
+validate=false,
+workload_manager=
+(
+    populate_config=
+    (
+        collection_count=3,
+        key_count_per_collection=1000000,
+        thread_count=3,
+        key_size=10,
+        value_size=1000
+    ),
+    insert_config=
+    (
+        thread_count=3,
+        op_rate=10ms,
+        ops_per_transaction=(min=5,max=20),
+        key_size=10,
+        value_size=1000
+    ),
+    remove_config=
+    (
+        op_rate=2s,
+        ops_per_transaction=(max=1,min=0),
+        thread_count=1
+    ),
+    # Set the length of the maintenance window period.
+    custom_config=
+    (
+        op_rate=10s,
+        thread_count=1
+    ),
+    background_compact_config=
+    (
+        thread_count=1,
+        free_space_target_mb=1,
+        op_rate=60s
+    ),
+    checkpoint_config=
+    (
+        op_rate=5s,
+    )
+),
+operation_tracker=
+(
+    enabled=false,
+)

src/third_party/wiredtiger/test/cppsuite/configs/background_compact_default.txt

@@ -0,0 +1,47 @@
+# Configuration for background_compact.
+duration_seconds=7200,
+cache_size_mb=10000,
+workload_manager=
+(
+    populate_config=
+    (
+        collection_count=100,
+        key_count_per_collection=1000000,
+        thread_count=10,
+        key_size=10,
+        value_size=1000
+    ),
+    insert_config=
+    (
+        thread_count=50,
+        op_rate=10ms,
+        ops_per_transaction=(min=5,max=20),
+        key_size=10,
+        value_size=1000
+    ),
+    remove_config=
+    (
+        op_rate=1s,
+        ops_per_transaction=( max=1,min=0),
+        thread_count=1
+    ),
+    custom_config=
+    (
+        op_rate=600s,
+        thread_count=1
+    ),
+    background_compact_config=
+    (
+        thread_count=1,
+        free_space_target_mb=1,
+        op_rate=1200s
+    ),
+    checkpoint_config=
+    (
+        op_rate=60s,
+    )
+),
+operation_tracker=
+(
+    enabled=false,
+)

src/third_party/wiredtiger/test/cppsuite/configs/background_compact_long.txt

@@ -32,19 +32,20 @@
 namespace test_harness {
 
 /* Component names. */
-const std::string OPERATION_TRACKER = "operation_tracker";
 const std::string METRICS_MONITOR = "metrics_monitor";
+const std::string OPERATION_TRACKER = "operation_tracker";
 const std::string TIMESTAMP_MANAGER = "timestamp_manager";
 const std::string WORKLOAD_MANAGER = "workload_manager";
 
 /* Configuration API consts. */
+const std::string BACKGROUND_COMPACT_DEBUG_MODE = "background_compact_debug_mode";
 const std::string BACKGROUND_COMPACT_OP_CONFIG = "background_compact_config";
 const std::string CACHE_HS_INSERT = "cache_hs_insert";
 const std::string CACHE_MAX_WAIT_MS = "cache_max_wait_ms";
 const std::string CACHE_SIZE_MB = "cache_size_mb";
 const std::string CC_PAGES_REMOVED = "cc_pages_removed";
-const std::string COLLECTION_COUNT = "collection_count";
 const std::string CHECKPOINT_OP_CONFIG = "checkpoint_config";
+const std::string COLLECTION_COUNT = "collection_count";
 const std::string COMPRESSION_ENABLED = "compression_enabled";
 const std::string CUSTOM_OP_CONFIG = "custom_config";
 const std::string DURATION_SECONDS = "duration_seconds";
@@ -88,8 +89,8 @@ const std::string STATISTICS_LOG = "statistics_log=(json,wait=1)";
 
 /* Test harness consts. */
 const std::string DEFAULT_FRAMEWORK_SCHEMA = "key_format=S,value_format=S,";
+const std::string STATISTICS_URI = "statistics:";
 const std::string TABLE_OPERATION_TRACKING = "table:operation_tracking";
 const std::string TABLE_SCHEMA_TRACKING = "table:schema_tracking";
-const std::string STATISTICS_URI = "statistics:";
 
 } // namespace test_harness

src/third_party/wiredtiger/test/cppsuite/src/common/constants.cpp

@@ -35,19 +35,20 @@
 namespace test_harness {
 
 /* Component names. */
-extern const std::string OPERATION_TRACKER;
 extern const std::string METRICS_MONITOR;
+extern const std::string OPERATION_TRACKER;
 extern const std::string TIMESTAMP_MANAGER;
 extern const std::string WORKLOAD_MANAGER;
 
 /* Configuration API consts. */
+extern const std::string BACKGROUND_COMPACT_DEBUG_MODE;
 extern const std::string BACKGROUND_COMPACT_OP_CONFIG;
 extern const std::string CACHE_HS_INSERT;
 extern const std::string CACHE_MAX_WAIT_MS;
 extern const std::string CACHE_SIZE_MB;
 extern const std::string CC_PAGES_REMOVED;
-extern const std::string COLLECTION_COUNT;
 extern const std::string CHECKPOINT_OP_CONFIG;
+extern const std::string COLLECTION_COUNT;
 extern const std::string COMPRESSION_ENABLED;
 extern const std::string CUSTOM_OP_CONFIG;
 extern const std::string DURATION_SECONDS;
@@ -110,9 +111,9 @@ extern const std::string STATISTICS_LOG;
 
 /* Test harness consts. */
 extern const std::string DEFAULT_FRAMEWORK_SCHEMA;
+extern const std::string STATISTICS_URI;
 extern const std::string TABLE_OPERATION_TRACKING;
 extern const std::string TABLE_SCHEMA_TRACKING;
-extern const std::string STATISTICS_URI;
 
 } // namespace test_harness
 

src/third_party/wiredtiger/test/cppsuite/src/common/constants.h

@@ -123,6 +123,10 @@ test::run()
     enable_logging = _config->get_bool(ENABLE_LOGGING);
     db_create_config += ",log=(enabled=" + std::string(enable_logging ? "true" : "false") + ")";
 
+    /* Enable or disable background compact debug mode. */
+    if (_config->get_bool(BACKGROUND_COMPACT_DEBUG_MODE))
+        db_create_config += ",debug_mode=(background_compact)";
+
     /* Maximum waiting time for the cache to get unstuck. */
     cache_max_wait_ms = _config->get_int(CACHE_MAX_WAIT_MS);
     db_create_config += ",cache_max_wait_ms=" + std::to_string(cache_max_wait_ms);