Pass oversubscribe status to MPI layer

Update OMPI to check for PMIx attribute and set
`ompi_mpi_oversubscribe` accordingly.  Move logic for setting
yield_when_idle to a place after the oversubscribe flag has been
checked.

- change logic of setting ompi_mpi_yield_when_idle
- nit: change `ompi_mpi_oversubscribe` to `ompi_mpi_oversubscribed`
- add comment in ompi/runtime/params.h

This is a cherry-pick of the Open MPI parts of 2b335ed.  The Open
PMIx / PRRTE git submodule updates are in a different commit on this
PR (because they're different than the git submodule updates on
master).

Signed-off-by: Ralph Castain <rhc@pmix.org>
Signed-off-by: Jeff Squyres <jsquyres@cisco.com>
(cherry picked from commit 2b335ed)

Author: rhc54

ompi/runtime/ompi_mpi_init.c

@@ -559,6 +559,11 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided,
     OMPI_TIMING_IMPORT_OPAL("rte_init");
 
     ompi_rte_initialized = true;
+    /* if we are oversubscribed, then set yield_when_idle
+     * accordingly */
+    if (ompi_mpi_oversubscribed) {
+        ompi_mpi_yield_when_idle = true;
+    }
 
     /* Register the default errhandler callback  */
     /* we want to go first */

ompi/runtime/ompi_mpi_params.c

@@ -18,8 +18,11 @@
  * Copyright (c) 2013-2019 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies, Inc.
  *                         All rights reserved.
- * Copyright (c) 2016-2019 Research Organization for Information Science
- *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2016-2021 Research Organization for Information Science
+ *                         and Technology (RIST).  All rights reserved.
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -88,7 +91,7 @@ static bool show_default_mca_params = false;
 static bool show_file_mca_params = false;
 static bool show_enviro_mca_params = false;
 static bool show_override_mca_params = false;
-static bool ompi_mpi_oversubscribe = false;
+bool ompi_mpi_oversubscribed = false;
 
 #if OPAL_ENABLE_FT_MPI
 int ompi_ftmpi_output_handle = 0;
@@ -144,20 +147,7 @@ int ompi_mpi_register_params(void)
         ompi_mpi_param_check = false;
     }
 
-    /*
-     * opal_progress: decide whether to yield and the event library
-     * tick rate
-     */
-    ompi_mpi_oversubscribe = false;
-    (void) mca_base_var_register("ompi", "mpi", NULL, "oversubscribe",
-                                 "Internal MCA parameter set by the runtime environment when oversubscribing nodes",
-                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &ompi_mpi_oversubscribe);
-
     /* yield if the node is oversubscribed and allow users to override */
-    ompi_mpi_yield_when_idle |= ompi_mpi_oversubscribe;
     (void) mca_base_var_register("ompi", "mpi", NULL, "yield_when_idle",
                                  "Yield the processor when waiting for MPI communication (for MPI processes, will default to 1 when oversubscribing nodes)",
                                  MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,

ompi/runtime/ompi_rte.c

@@ -871,6 +871,16 @@ int ompi_rte_init(int *pargc, char ***pargv)
         }
     }
 
+#ifdef PMIX_NODE_OVERSUBSCRIBED
+    pname.jobid = opal_process_info.my_name.jobid;
+    pname.vpid = OPAL_VPID_WILDCARD;
+    OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, PMIX_NODE_OVERSUBSCRIBED, &pname,
+                                   NULL, PMIX_BOOL);
+    if (PMIX_SUCCESS == ret) {
+        ompi_mpi_oversubscribed = true;
+    }
+#endif
+
     return OPAL_SUCCESS;
 
   error:

ompi/runtime/params.h

@@ -16,6 +16,9 @@
  * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2013      Intel, Inc. All rights reserved
+ * Copyright (c) 2021      Triad National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -192,6 +195,12 @@ OMPI_DECLSPEC int ompi_mpi_register_params(void);
  */
 int ompi_show_all_mca_params(int32_t, int, char *);
 
+/**
+ * Set by checking PMIx to see if we are running in an oversubscribed
+ * environment or not.
+ */
+OMPI_DECLSPEC extern bool ompi_mpi_oversubscribed;
+
 END_C_DECLS
 
 #endif /* OMPI_RUNTIME_PARAMS_H */

opal/mca/pmix/pmix-internal.h

@@ -298,7 +298,7 @@ typedef struct {
             (r) = PMIX_ERR_NOT_FOUND;                                                  \
         } else if (_kv->type != (t)) {                                                 \
             (r) = PMIX_ERR_TYPE_MISMATCH;                                              \
-        } else if (PMIX_SUCCESS == (r)) {                                              \
+        } else if (PMIX_SUCCESS == (r) && NULL != (d)) {                               \
             PMIX_VALUE_UNLOAD((r), _kv, (void **) (d), &_sz);                          \
         }                                                                              \
         if (NULL != _kv) {                                                             \