Import wiredtiger: 5b5f12635bdc5ce6f1c68bc58fbae13ec6a042fd from branch mongodb-master

ref: c1dde2389d..5b5f12635b
for: 7.2.0-rc0

WT-11045 Create the mirror table before its base in Workgen

Author: ruby-oujo

src/third_party/wiredtiger/bench/workgen/workgen.cpp

@@ -132,6 +132,8 @@ thread_runner_main(void *arg)
         runner->_errno = runner->run();
     } catch (WorkgenException &wge) {
         runner->_exception = wge;
+        std::cerr << "Exception in runner->run(): " << wge._str << std::endl;
+        ASSERT(false);
     }
     return (nullptr);
 }
@@ -148,6 +150,7 @@ thread_workload(void *arg)
         runner->increment_timestamp(connection);
     } catch (WorkgenException &wge) {
         std::cerr << "Exception while incrementing timestamp: " << wge._str << std::endl;
+        ASSERT(false);
     }
 
     return (nullptr);
@@ -164,6 +167,7 @@ thread_idle_table_cycle_workload(void *arg)
         runner->start_table_idle_cycle(connection);
     } catch (WorkgenException &wge) {
         std::cerr << "Exception while create/drop tables: " << wge._str << std::endl;
+        ASSERT(false);
     }
 
     return (nullptr);
@@ -180,6 +184,7 @@ thread_tables_create_workload(void *arg)
         runner->start_tables_create(connection);
     } catch (WorkgenException &wge) {
         std::cerr << "Exception while creating tables: " << wge._str << std::endl;
+        ASSERT(false);
     }
 
     return (nullptr);
@@ -196,6 +201,7 @@ thread_tables_drop_workload(void *arg)
         runner->start_tables_drop(connection);
     } catch (WorkgenException &wge) {
         std::cerr << "Exception while dropping tables: " << wge._str << std::endl;
+        ASSERT(false);
     }
 
     return (nullptr);
@@ -353,42 +359,104 @@ gen_random_table_name(char *name, workgen_random_state volatile *rand_state)
  * fails.
  */
 int
-WorkloadRunner::create_table(WT_SESSION *session, const std::string &config, const std::string &uri,
-  const std::string &mirror_uri, const bool is_base)
+WorkloadRunner::create_table(
+  WT_SESSION *session, const std::string &config, const std::string &uri, bool mirror_enabled)
 {
-    // Check if a table with this name already exists. Return if it does. Use a shared lock to
-    // read the dynamic table structure.
+    const std::string mirror_uri(mirror_enabled ? uri + _workload->options.mirror_suffix : "");
+
+    // Return if a table with the same name already exists. If mirror is enabled, make sure it does
+    // not exist either.
     ContextInternal *icontext = _workload->_context->_internal;
     {
+        // Use a shared lock to read the dynamic table structure.
         const std::shared_lock lock(*icontext->_dyn_mutex);
         if (icontext->_tint.count(uri) > 0 || icontext->_dyn_tint.count(uri) > 0)
             return EEXIST;
+        if (mirror_enabled &&
+          (icontext->_tint.count(mirror_uri) > 0 || icontext->_dyn_tint.count(mirror_uri) > 0))
+            return EEXIST;
     }
 
-    // Create the table.
-    WT_DECL_RET;
-    if ((ret = session->create(session, uri.c_str(), config.c_str())) != 0) {
-        if (ret != EBUSY)
-            THROW("Failed to create table '" << uri << "'.");
+    /*
+     * When mirror is enabled, create the mirror first and then the base. If we create the base
+     * first, threads may start working on the base while the mirror is not fully created.
+     *
+     * The config has to be updated with the following information:
+     *  - The name of the table's mirror if mirroring is enabled.
+     *  - If this table is a base table or a mirror.
+     *
+     * See below an example when creating the table 'a' with mirror enabled:
+     *  - Configuration expected for the mirror:
+     * app_metadata="workgen_dynamic_table=true,workgen_table_mirror=table:a,workgen_base_table=false"
+     *  - Configuration expected for the base:
+     * app_metadata="workgen_dynamic_table=true,workgen_table_mirror=table:a_mirror,workgen_base_table=true"
+     *
+     * Without mirror enabled:
+     * app_metadata="workgen_dynamic_table=true,workgen_base_table=true"
+     *
+     */
+    std::string mirror_config;
+    if (mirror_enabled) {
+        mirror_config = config + "," + MIRROR_TABLE_APP_METADATA + uri + "," +
+          BASE_TABLE_APP_METADATA + "false\"";
+        int ret = session->create(session, mirror_uri.c_str(), mirror_config.c_str());
+        if (ret != 0) {
+            VERBOSE(*_workload, "Failed to create mirror table '" << mirror_uri << "'");
+            return ret;
+        }
+        // This will be used when creating the base table.
+        mirror_config = MIRROR_TABLE_APP_METADATA + mirror_uri + ",";
+    }
+
+    // If mirror is enabled, we don't want to fail when creating the base. Getting spurious EBUSY
+    // errors is ok though, retry in that case.
+    const std::string base_config(
+      config + "," + mirror_config + BASE_TABLE_APP_METADATA + "true\"");
+    int ret, retries = 0;
+
+    do {
+        ret = session->create(session, uri.c_str(), base_config.c_str());
+    } while (ret != 0 && ret == EBUSY && mirror_enabled && ++retries < TABLE_MAX_RETRIES);
+
+    if (ret != 0) {
+        const std::string err_msg("Failed to create table '" + uri + "'");
+        VERBOSE(*_workload, err_msg);
+        // Fail if we have failed at creating the base of a mirror.
+        if (mirror_enabled)
+            THROW_ERRNO(ret, err_msg);
         return ret;
     }
 
-    // The data structures for the dynamic table set are protected by a mutex.
+    // All the required tables have been created, update the data structures for the dynamic tables
+    // which are protected by a mutex.
     {
         const std::lock_guard<std::shared_mutex> lock(*icontext->_dyn_mutex);
-
-        // Add the table into the list of dynamic set.
-        tint_t tint = icontext->_dyn_tint_last;
-        icontext->_dyn_tint[uri] = tint;
-        icontext->_dyn_table_names[tint] = uri;
-        icontext->_dyn_table_runtime[tint] = TableRuntime(is_base, mirror_uri);
-        ++icontext->_dyn_tint_last;
-        VERBOSE(*_workload, "Created table and added to the dynamic set: " << uri);
+        update_dyn_struct_locked(uri, true, mirror_uri);
+        if (mirror_enabled)
+            update_dyn_struct_locked(mirror_uri, false, uri);
     }
 
     return 0;
 }
 
+/*
+ * Update the structures dedicated to tables that can be created or removed during the workload. The
+ * caller should hold the mutex that protects those structures.
+ */
+void
+WorkloadRunner::update_dyn_struct_locked(
+  const std::string &uri, bool is_base, const std::string &mirror_uri)
+{
+    ContextInternal *icontext = _workload->_context->_internal;
+
+    // This should be safe as we are supposed to be under a lock.
+    tint_t tint = icontext->_dyn_tint_last++;
+    icontext->_dyn_tint[uri] = tint;
+    icontext->_dyn_table_names[tint] = uri;
+    icontext->_dyn_table_runtime[tint] = TableRuntime(is_base, mirror_uri);
+    VERBOSE(*_workload, "Created table and added to the dynamic set: " << uri);
+}
+
 /*
  * This function creates one or more tables at regular intervals, where the interval length and
  * number of tables are specified in the workload options. It also monitors the database size and
@@ -415,25 +483,6 @@ WorkloadRunner::start_tables_create(WT_CONNECTION *conn)
         creating = db_size < _workload->options.create_target;
     }
 
-    std::string uri;
-    std::string mirror_uri = std::string();
-    int creates, retries, status;
-    char rand_chars[DYNAMIC_TABLE_LEN];
-
-    /*
-     * Add app_metadata to the config to indicate the table was created dynamically (can be selected
-     * for random deletion), the name of the table's mirror if mirroring is enabled, and if this
-     * table is a base table or a mirror. We want these settings to persist over restarts.
-     */
-    std::string base_config =
-      "key_format=S,value_format=S,app_metadata=\"" + DYN_TABLE_APP_METADATA;
-    if (_workload->options.mirror_tables) {
-        base_config += "," + MIRROR_TABLE_APP_METADATA;
-    } else {
-        base_config += "," + BASE_TABLE_APP_METADATA + "true\"";
-    }
-    std::string config = base_config;
-
     while (!stopping) {
         /*
          * When managing the database size: If we are creating tables, continue until we reach the
@@ -465,40 +514,27 @@ WorkloadRunner::start_tables_create(WT_CONNECTION *conn)
             continue;
         }
 
-        retries = 0;
-        creates = 0;
-        while (creates < _workload->options.create_count) {
+        // Add app_metadata to the config to indicate the table was created dynamically which means
+        // it can be selected for random deletion. We want this information to persist over restart.
+        const std::string config(
+          "key_format=S,value_format=S,app_metadata=\"" + DYN_TABLE_APP_METADATA);
 
+        int creates = 0, retries = 0;
+        while (
+          !stopping && creates < _workload->options.create_count && retries < TABLE_MAX_RETRIES) {
             // Generate a table name from the user specified prefix and a random alphanumeric
             // sequence.
+            char rand_chars[DYNAMIC_TABLE_LEN];
             gen_random_table_name(rand_chars, _rand_state);
-            uri = "table:";
-            uri += _workload->options.create_prefix;
-            uri += rand_chars;
-
-            if (_workload->options.mirror_tables) {
-                // The mirror table name is the table name with the user specified suffix.
-                mirror_uri = uri + _workload->options.mirror_suffix;
-                config = base_config + mirror_uri + "," + BASE_TABLE_APP_METADATA + "true\"";
-            }
+            const std::string uri("table:" + _workload->options.create_prefix + rand_chars);
 
-            // Create the table. Simply continue on failure.
-            if (create_table(session, config, uri, mirror_uri, true) == 0) {
-                VERBOSE(*_workload, "Created base table '" << uri << "'");
-                if (_workload->options.mirror_tables) {
-                    // Create the mirror. Retry on failure and throw an exception after
-                    // making too many retry attempts.
-                    config = base_config + uri + "," + BASE_TABLE_APP_METADATA + "false\"";
-                    do
-                        status = create_table(session, config, mirror_uri, uri, false);
-                    while (status == EBUSY && ++retries < TABLE_MAX_RETRIES);
-                    if (status != 0)
-                        THROW_ERRNO(
-                          status, "Failed to create mirror table '" << mirror_uri << "'.");
-                    VERBOSE(*_workload, "Created mirror table '" << mirror_uri << "'");
-                }
+            // Create the table and its mirror if enabled.
+            int ret = create_table(session, config, uri, _workload->options.mirror_tables);
+            ASSERT(ret == 0 || ret == EBUSY || ret == EEXIST);
+            if (ret == 0)
                 ++creates;
-            }
+            else
+                ++retries;
         }
         sleep(_workload->options.create_interval);
     }
@@ -709,6 +745,8 @@ monitor_main(void *arg)
         monitor->_errno = monitor->run();
     } catch (WorkgenException &wge) {
         monitor->_exception = wge;
+        std::cerr << "Exception in monitor->run(): " << wge._str << std::endl;
+        ASSERT(false);
     }
     return (nullptr);
 }
@@ -1478,7 +1516,7 @@ ThreadRunner::op_get_table(Operation *op) const
 
     std::string uri = op->_tables[_number];
     tint_t tint = 0;
-    if (uri != std::string()) {
+    if (!uri.empty()) {
         const std::shared_lock lock(*_icontext->_dyn_mutex);
         tint = _icontext->_dyn_tint[uri];
     }
@@ -1612,13 +1650,12 @@ ThreadRunner::op_run_setup(Operation *op)
             std::advance(itr, random_value() % num_tables);
 
             if (_icontext->_dyn_table_runtime[itr->second]._is_base &&
-              !_icontext->_dyn_table_runtime[itr->second]._pending_delete) {
+              !_icontext->_dyn_table_runtime[itr->second]._pending_delete)
                 break;
-            }
         }
-        if (num_tables == 0 || retries >= TABLE_MAX_RETRIES) { // Try again next time.
+        // Try again next time.
+        if (num_tables == 0 || retries >= TABLE_MAX_RETRIES)
             return 0;
-        }
 
         std::string op_uri = itr->first; // Get the table name.
         tint_t op_tint = itr->second;    // Get the tint.

src/third_party/wiredtiger/bench/workgen/workgen_int.h

@@ -341,8 +341,8 @@ struct WorkloadRunner {
 private:
     int close_all();
     int create_all(WT_CONNECTION *conn, Context *context);
-    int create_table(WT_SESSION *session, const std::string &config, const std::string &uri,
-      const std::string &mirror_uri, const bool is_base);
+    int create_table(
+      WT_SESSION *session, const std::string &config, const std::string &uri, bool mirror_enabled);
     void final_report(timespec &);
     void schedule_table_for_drop(const std::map<std::string, tint_t>::iterator &itr,
       std::vector<std::string> &pending_delete);
@@ -352,6 +352,7 @@ struct WorkloadRunner {
     void report(time_t, time_t, Stats *stats);
     int run_all(WT_CONNECTION *conn);
     int select_table_for_drop(std::vector<std::string> &pending_delete);
+    void update_dyn_struct_locked(const std::string &uri, bool is_base, const std::string &mirror_uri);
 
     WorkloadRunner(const WorkloadRunner &);                 // disallowed
     WorkloadRunner& operator=(const WorkloadRunner &other); // disallowed

src/third_party/wiredtiger/import.data

@@ -2,5 +2,5 @@
     "vendor": "wiredtiger",
     "github": "wiredtiger/wiredtiger.git",
     "branch": "mongodb-master",
-    "commit": "c1dde2389d34cf4bc0ccfbc5e6e1136f3dbc8ae9"
+    "commit": "5b5f12635bdc5ce6f1c68bc58fbae13ec6a042fd"
 }