Merge pull request #8915 from hppritcha/topic/swat_issue_8860

hppritcha · web-flow · commit 5f49b2feb2a6 · 2021-05-17T17:47:11.000-06:00
Topic/swat issue 8860
diff --git a/opal/mca/common/ofi/common_ofi.c b/opal/mca/common/ofi/common_ofi.c
@@ -2,7 +2,7 @@
  * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2021 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
@@ -32,6 +32,7 @@ OPAL_DECLSPEC opal_common_ofi_module_t opal_common_ofi = {.prov_include = NULL,
                                                           .verbose = 0};
 
 static const char default_prov_exclude_list[] = "shm,sockets,tcp,udp,rstream";
+static opal_mutex_t opal_common_ofi_mutex = OPAL_MUTEX_STATIC_INIT;
 
 OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item)
 {
@@ -54,23 +55,29 @@ OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item)
 
 OPAL_DECLSPEC int opal_common_ofi_register_mca_variables(const mca_base_component_t *component)
 {
-    static int registered = 0;
     static int include_index;
     static int exclude_index;
     static int verbose_index;
+    int param;
 
     if (fi_version() < FI_VERSION(1, 0)) {
         return OPAL_ERROR;
     }
 
-    if (!registered) {
+    OPAL_THREAD_LOCK(&opal_common_ofi_mutex);
+
+    param = mca_base_var_find("opal", "opal_common", "ofi", "provider_incude");
+    if (0 > param) {
         /*
          * this monkey business is needed because of the way the MCA VARs stuff tries to handle
          * pointers to strings when when destructing the MCA var database.  If you don't do
          * something like this,the MCA var framework will try to dereference a pointer which itself
          * is no longer a valid address owing to having been previously dlclosed.
          */
-        opal_common_ofi.prov_include = (char **) malloc(sizeof(char *));
+        if (NULL == opal_common_ofi.prov_include) {
+            opal_common_ofi.prov_include = (char **) malloc(sizeof(char *));
+            assert(NULL != opal_common_ofi.prov_include);
+        }
         *opal_common_ofi.prov_include = NULL;
         include_index = mca_base_var_register(
             "opal", "opal_common", "ofi", "provider_include",
@@ -79,7 +86,14 @@ OPAL_DECLSPEC int opal_common_ofi_register_mca_variables(const mca_base_componen
             "exclusive with mtl_ofi_provider_exclude.",
             MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_1, MCA_BASE_VAR_SCOPE_READONLY,
             opal_common_ofi.prov_include);
-        opal_common_ofi.prov_exclude = (char **) malloc(sizeof(char *));
+    }
+
+    param = mca_base_var_find("opal", "opal_common", "ofi", "provider_exclude");
+    if (0 > param) {
+        if (NULL == opal_common_ofi.prov_exclude) {
+            opal_common_ofi.prov_exclude = (char **) malloc(sizeof(char *));
+            assert(NULL != opal_common_ofi.prov_exclude);
+        }
         *opal_common_ofi.prov_exclude = strdup(default_prov_exclude_list);
         exclude_index = mca_base_var_register(
             "opal", "opal_common", "ofi", "provider_exclude",
@@ -88,12 +102,16 @@ OPAL_DECLSPEC int opal_common_ofi_register_mca_variables(const mca_base_componen
             "exclusive with mtl_ofi_provider_include.",
             MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_1, MCA_BASE_VAR_SCOPE_READONLY,
             opal_common_ofi.prov_exclude);
+    }
+
+    param = mca_base_var_find("opal", "opal_common", "ofi", "verbose");
+    if (0 > param) {
         verbose_index = mca_base_var_register("opal", "opal_common", "ofi", "verbose",
                                               "Verbose level of the OFI components",
                                               MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                               MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
-                                              MCA_BASE_VAR_SCOPE_LOCAL, &opal_common_ofi.verbose);
-        registered = 1;
+                                              MCA_BASE_VAR_SCOPE_LOCAL,
+                                              &opal_common_ofi.verbose);
     }
 
     if (component) {
@@ -108,6 +126,8 @@ OPAL_DECLSPEC int opal_common_ofi_register_mca_variables(const mca_base_componen
                                       "verbose", 0);
     }
 
+    OPAL_THREAD_UNLOCK(&opal_common_ofi_mutex);
+
     return OPAL_SUCCESS;
 }
 
diff --git a/opal/mca/common/ucx/common_ucx.c b/opal/mca/common/ucx/common_ucx.c
@@ -3,6 +3,9 @@
  * Copyright (c) 2019      Intel, Inc.  All rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ *
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -33,6 +36,8 @@ opal_common_ucx_module_t opal_common_ucx = {.verbose = 0,
                                             .opal_mem_hooks = 0,
                                             .tls = NULL};
 
+static opal_mutex_t opal_common_ucx_mutex = OPAL_MUTEX_STATIC_INIT;
+
 static void opal_common_ucx_mem_release_cb(void *buf, size_t length, void *cbdata, bool from_alloc)
 {
     ucm_vm_munmap(buf, length);
@@ -42,34 +47,58 @@ OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *
 {
     static const char *default_tls = "rc_verbs,ud_verbs,rc_mlx5,dc_mlx5,cuda_ipc,rocm_ipc";
     static const char *default_devices = "mlx*";
-    static int registered = 0;
     static int hook_index;
     static int verbose_index;
     static int progress_index;
     static int tls_index;
     static int devices_index;
+    int param;
 
-    if (!registered) {
+    OPAL_THREAD_LOCK(&opal_common_ucx_mutex);
+
+    param = mca_base_var_find("opal", "opal_common", "ucx", "verbose");
+    if (0 > param) {
         verbose_index = mca_base_var_register("opal", "opal_common", "ucx", "verbose",
                                               "Verbose level of the UCX components",
                                               MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                               MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
                                               MCA_BASE_VAR_SCOPE_LOCAL, &opal_common_ucx.verbose);
+    }
+
+    param = mca_base_var_find("opal", "opal_common", "ucx", "progress_iterations");
+    if (0 > param) {
         progress_index = mca_base_var_register("opal", "opal_common", "ucx", "progress_iterations",
                                                "Set number of calls of internal UCX progress "
                                                "calls per opal_progress call",
                                                MCA_BASE_VAR_TYPE_INT, NULL, 0,
                                                MCA_BASE_VAR_FLAG_SETTABLE, OPAL_INFO_LVL_3,
                                                MCA_BASE_VAR_SCOPE_LOCAL,
                                                &opal_common_ucx.progress_iterations);
+    }
+
+    param = mca_base_var_find("opal", "opal_common", "ucx", "opal_mem_hooks");
+    if (0 > param) {
         hook_index = mca_base_var_register("opal", "opal_common", "ucx", "opal_mem_hooks",
                                            "Use OPAL memory hooks, instead of UCX internal "
                                            "memory hooks",
                                            MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, OPAL_INFO_LVL_3,
                                            MCA_BASE_VAR_SCOPE_LOCAL,
                                            &opal_common_ucx.opal_mem_hooks);
+    }
 
-        opal_common_ucx.tls = malloc(sizeof(*opal_common_ucx.tls));
+    param = mca_base_var_find("opal", "opal_common", "ucx", "tls");
+    if (0 > param) {
+
+        /*
+         * this monkey business is needed because of the way the MCA VARs framework tries to handle pointers to strings
+         * when destructing the MCA var database.  If you don't do something like this,the MCA var framework will try
+         * to dereference a pointer which itself is no longer a valid address owing to having been previously dlclosed.
+         * Same for the devices pointer below.
+         */
+        if (NULL == opal_common_ucx.tls) {
+            opal_common_ucx.tls  = malloc(sizeof(*opal_common_ucx.tls));
+            assert(NULL != opal_common_ucx.tls);
+        }
         *opal_common_ucx.tls = strdup(default_tls);
         tls_index = mca_base_var_register(
             "opal", "opal_common", "ucx", "tls",
@@ -80,17 +109,24 @@ OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *
             "please set to '^posix,sysv,self,tcp,cma,knem,xpmem'.",
             MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL,
             opal_common_ucx.tls);
+    }
+
+    param = mca_base_var_find("opal", "opal_common", "ucx", "devices");
+    if (0 > param) {
 
-        opal_common_ucx.devices = malloc(sizeof(*opal_common_ucx.devices));
+        if (NULL == opal_common_ucx.devices) {
+            opal_common_ucx.devices = malloc(sizeof(*opal_common_ucx.devices));
+            assert(NULL != opal_common_ucx.devices);
+        }
         *opal_common_ucx.devices = strdup(default_devices);
         devices_index = mca_base_var_register(
             "opal", "opal_common", "ucx", "devices",
             "List of device driver pattern names, which, if supported by UCX, will "
             "bump its priority above ob1. Special values: any (any available)",
             MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_3, MCA_BASE_VAR_SCOPE_LOCAL,
             opal_common_ucx.devices);
-        registered = 1;
     }
+
     if (component) {
         mca_base_var_register_synonym(verbose_index, component->mca_project_name,
                                       component->mca_type_name, component->mca_component_name,
@@ -108,6 +144,8 @@ OPAL_DECLSPEC void opal_common_ucx_mca_var_register(const mca_base_component_t *
                                       component->mca_type_name, component->mca_component_name,
                                       "devices", 0);
     }
+
+    OPAL_THREAD_UNLOCK(&opal_common_ucx_mutex);
 }
 
 OPAL_DECLSPEC void opal_common_ucx_mca_register(void)
