Merge pull request #11064 from hppritcha/move_singleton_to_opal

singleton:  add field to opal_process_info

Author: hppritcha

ompi/dpm/dpm.c

@@ -1610,7 +1610,7 @@ int ompi_dpm_spawn(int count, const char *array_of_commands[],
     }
     OPAL_LIST_DESTRUCT(&job_info);
 
-    if (ompi_singleton) {
+    if (opal_process_info.is_singleton) {
         /* The GDS 'hash' component is known to work for singleton, so
          * recommend it. The user may set this envar to override the setting.
          */
@@ -2154,7 +2154,7 @@ static int start_dvm(char **hostfiles, char **dash_host)
     PMIx_Commit();
 
     /* we are no longer a singleton */
-    ompi_singleton = false;
+     opal_process_info.is_singleton = false;
 
     return OMPI_SUCCESS;
 }

ompi/instance/instance.c

@@ -523,7 +523,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
    OMPI_TIMING_NEXT("commit");
 #if (OPAL_ENABLE_TIMING)
     if (OMPI_TIMING_ENABLED && !opal_pmix_base_async_modex &&
-            opal_pmix_collect_all_data && !ompi_singleton) {
+            opal_pmix_collect_all_data && !opal_process_info.is_singleton) {
         if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, NULL, 0))) {
             ret = opal_pmix_convert_status(rc);
             return ompi_instance_print_error ("timing: pmix-barrier-1 failed", ret);
@@ -536,7 +536,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
     }
 #endif
 
-   if (!ompi_singleton) {
+   if (! opal_process_info.is_singleton) {
         if (opal_pmix_base_async_modex) {
             /* if we are doing an async modex, but we are collecting all
              * data, then execute the non-blocking modex in the background.
@@ -715,7 +715,7 @@ static int ompi_mpi_instance_init_common (int argc, char **argv)
     /* Next timing measurement */
     OMPI_TIMING_NEXT("modex-barrier");
 
-    if (!ompi_singleton) {
+    if (!opal_process_info.is_singleton) {
         /* if we executed the above fence in the background, then
          * we have to wait here for it to complete. However, there
          * is no reason to do two barriers! */

ompi/interlib/interlib.c

@@ -17,6 +17,8 @@
  * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -29,6 +31,7 @@
 #include <string.h>
 
 #include "opal/mca/pmix/pmix-internal.h"
+#include "opal/util/proc.h"
 #include "ompi/runtime/ompi_rte.h"
 #include "ompi/interlib/interlib.h"
 
@@ -122,7 +125,7 @@ int ompi_interlib_declare(int threadlevel, char *version)
     PMIX_INFO_DESTRUCT(&info[3]);
     /* account for our refcount on pmix_init */
     PMIx_Finalize(NULL, 0);
-    if (ompi_singleton && PMIX_ERR_UNREACH == rc) {
+    if (opal_process_info.is_singleton && PMIX_ERR_UNREACH == rc) {
         ret = OMPI_SUCCESS;
     } else {
         ret = opal_pmix_convert_status(rc);

ompi/runtime/ompi_mpi_finalize.c

@@ -20,7 +20,7 @@
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
- * Copyright (c) 2019      Triad National Security, LLC. All rights
+ * Copyright (c) 2019-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
@@ -272,7 +272,7 @@ int ompi_mpi_finalize(void)
        del_procs behavior around May of 2014 (see
        https://svn.open-mpi.org/trac/ompi/ticket/4669#comment:4 for
        more details). */
-    if (!ompi_async_mpi_finalize && !ompi_singleton) {
+    if (!ompi_async_mpi_finalize && !opal_process_info.is_singleton) {
         active = true;
         OPAL_POST_OBJECT(&active);
         /* Note that use of the non-blocking PMIx fence will

ompi/runtime/ompi_mpi_init.c

@@ -27,7 +27,7 @@
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.
  *                         All Rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
- * Copyright (c) 2021      Triad National Security, LLC. All rights
+ * Copyright (c) 2021-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -394,7 +394,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
 
 #if (OPAL_ENABLE_TIMING)
     if (OMPI_TIMING_ENABLED && !opal_pmix_base_async_modex &&
-            opal_pmix_collect_all_data && !ompi_singleton) {
+            opal_pmix_collect_all_data && !opal_process_info.is_singleton) {
         if (PMIX_SUCCESS != (rc = PMIx_Fence(NULL, 0, NULL, 0))) {
             ret = opal_pmix_convert_status(rc);
             error = "timing: pmix-barrier-1 failed";
@@ -410,7 +410,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
     }
 #endif
 
-    if (!ompi_singleton) {
+    if (!opal_process_info.is_singleton) {
         if (opal_pmix_base_async_modex) {
             /* if we are doing an async modex, but we are collecting all
              * data, then execute the non-blocking modex in the background.
@@ -494,7 +494,7 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
     /* Next timing measurement */
     OMPI_TIMING_NEXT("modex-barrier");
 
-    if (!ompi_singleton) {
+    if (!opal_process_info.is_singleton) {
         /* if we executed the above fence in the background, then
          * we have to wait here for it to complete. However, there
          * is no reason to do two barriers! */

ompi/runtime/ompi_rte.c

@@ -8,7 +8,7 @@
  *                         reserved.
  * Copyright (c) 2014-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2018      Amazon.com, Inc. or its affiliates.  All Rights reserved.
- * Copyright (c) 2018-2020 Triad National Security, LLC. All rights
+ * Copyright (c) 2018-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
@@ -61,7 +61,6 @@
 /* storage to support OMPI */
 opal_process_name_t pmix_name_wildcard = {UINT32_MAX-1, UINT32_MAX-1};
 opal_process_name_t pmix_name_invalid = {UINT32_MAX, UINT32_MAX};
-bool ompi_singleton = false;
 
 static int _setup_top_session_dir(char **sdir);
 static int _setup_job_session_dir(char **sdir);
@@ -541,6 +540,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     pmix_value_t pval;
     pmix_status_t rc;
     char **tmp;
+    bool singleton = false;
 
     u32ptr = &u32;
     u16ptr = &u16;
@@ -586,7 +586,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
                     opal_show_help("help-mpi-runtime.txt", "no-pmix-but", false, size);
                 }
             }
-            ompi_singleton = true;
+            singleton = true;
         } else {
             /* we cannot run - this could be due to being direct launched
              * without the required PMI support being built, so print
@@ -605,6 +605,11 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_PROC_MY_NAME.vpid = pname.vpid;
     opal_process_info.my_name.jobid = OPAL_PROC_MY_NAME.jobid;
     opal_process_info.my_name.vpid = OPAL_PROC_MY_NAME.vpid;
+    if (singleton) {
+        opal_process_info.is_singleton = true;
+    } else {
+        opal_process_info.is_singleton = false;
+    }
 
 
     /* set our hostname */
@@ -623,7 +628,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCAL_RANK,
                                    &opal_process_info.my_name, &u16ptr, PMIX_UINT16);
     if (PMIX_SUCCESS != rc) {
-        if (ompi_singleton) {
+        if (opal_process_info.is_singleton) {
             /* just assume 0 */
             u16 = 0;
         } else {
@@ -638,7 +643,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_NODE_RANK,
                                    &opal_process_info.my_name, &u16ptr, PMIX_UINT16);
     if (PMIX_SUCCESS != rc) {
-        if (ompi_singleton) {
+        if (opal_process_info.is_singleton) {
             /* just assume 0 */
             u16 = 0;
         } else {
@@ -657,7 +662,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_JOB_SIZE,
                                    &pname, &u32ptr, PMIX_UINT32);
     if (PMIX_SUCCESS != rc) {
-        if (ompi_singleton) {
+        if ( opal_process_info.is_singleton) {
             /* just assume 1 */
             u32 = 1;
         } else {
@@ -672,7 +677,7 @@ int ompi_rte_init(int *pargc, char ***pargv)
     OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_UNIV_SIZE,
                                    &pname, &u32ptr, PMIX_UINT32);
     if (PMIX_SUCCESS != rc) {
-        if (ompi_singleton) {
+        if (opal_process_info.is_singleton) {
             /* just assume 1 */
             u32 = 1;
         } else {

ompi/runtime/ompi_rte.h

@@ -6,7 +6,7 @@
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2019      Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2022  Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
@@ -259,7 +259,6 @@ OMPI_DECLSPEC void ompi_rte_abort_peers(opal_process_name_t *procs,
 OMPI_DECLSPEC int ompi_rte_init(int *argc, char ***argv);
 OMPI_DECLSPEC int ompi_rte_finalize(void);
 OMPI_DECLSPEC void ompi_rte_wait_for_debugger(void);
-OMPI_DECLSPEC extern bool ompi_singleton;
 
 /* In a few places, we need to barrier until something happens
  * that changes a flag to indicate we can release - e.g., waiting

opal/util/proc.c

@@ -11,6 +11,8 @@
  *                         reserved.
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -54,6 +56,7 @@ opal_process_info_t opal_process_info = {
     .reincarnation = 0,
     .proc_is_bound = false,
     .initial_errhandler = NULL,
+    .is_singleton = false,
 };
 
 static opal_proc_t opal_local_proc = {{.opal_list_next = NULL, .opal_list_prev = NULL},

opal/util/proc.h

@@ -10,6 +10,8 @@
  * Copyright (c) 2020      Amazon.com, Inc. or its affiliates.  All Rights
  *                         reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
+ * Copyright (c) 2022      Triad National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -130,6 +132,7 @@ typedef struct opal_process_info_t {
     uint32_t reincarnation;
     bool proc_is_bound;
     char *initial_errhandler;
+    bool is_singleton;         /**<note this value can transition from false to true in some cases */
 } opal_process_info_t;
 OPAL_DECLSPEC extern opal_process_info_t opal_process_info;