MPI 4: Add MPI_COMM_TYPE_HW_UNGUIDED and MPI_COMM_TYPE_HW_GUIDED

jjhursey · jjhursey · commit 19e24fa10d5d · 2022-08-05T17:08:27.000-04:00
* `MPI_COMM_TYPE_HW_GUIDED` supports all of the existing `OMPI_COMM_TYPE_`
   options.
 * `MPI_COMM_TYPE_HW_UNGUIDED` is recognized, but not supported so it returns
   `MPI_COMM_NULL` indidicating that the MPI library cannot split the
   communicator any further.

Signed-off-by: Joshua Hursey &lt;jhursey@us.ibm.com&gt;
diff --git a/ompi/communicator/comm.c b/ompi/communicator/comm.c
@@ -22,7 +22,7 @@
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2015      Mellanox Technologies. All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
  * Copyright (c) 2018-2022 Triad National Security, LLC. All rights
  *                         reserved.
@@ -57,6 +57,28 @@
 
 #include "ompi/runtime/params.h"
 
+struct ompi_comm_split_type_hw_guided_t {
+    const char *info_value;
+    int split_type;
+};
+typedef struct ompi_comm_split_type_hw_guided_t ompi_comm_split_type_hw_guided_t;
+
+static const ompi_comm_split_type_hw_guided_t ompi_comm_split_type_hw_guided_support[] = {
+    {.info_value = "mpi_shared_memory", .split_type = MPI_COMM_TYPE_SHARED},
+    {.info_value = "hwthread", .split_type = OMPI_COMM_TYPE_HWTHREAD},
+    {.info_value = "core",     .split_type = OMPI_COMM_TYPE_CORE},
+    {.info_value = "l1cache",  .split_type = OMPI_COMM_TYPE_L1CACHE},
+    {.info_value = "l2cache",  .split_type = OMPI_COMM_TYPE_L2CACHE},
+    {.info_value = "l3cache",  .split_type = OMPI_COMM_TYPE_L3CACHE},
+    {.info_value = "socket",   .split_type = OMPI_COMM_TYPE_SOCKET},
+    {.info_value = "numanode", .split_type = OMPI_COMM_TYPE_NUMA},
+    {.info_value = "board",    .split_type = OMPI_COMM_TYPE_BOARD},
+    {.info_value = "host",     .split_type = OMPI_COMM_TYPE_HOST},
+    {.info_value = "cu",       .split_type = OMPI_COMM_TYPE_CU},
+    {.info_value = "cluster",  .split_type = OMPI_COMM_TYPE_CLUSTER},
+    {.info_value = NULL},
+};
+
 /*
 ** sort-function for MPI_Comm_split
 */
@@ -764,6 +786,15 @@ static int ompi_comm_split_type_get_part (ompi_group_t *group, const int split_t
         case OMPI_COMM_TYPE_CLUSTER:
             include = OPAL_PROC_ON_LOCAL_CLUSTER(locality);
             break;
+        case MPI_COMM_TYPE_HW_GUIDED:
+        case MPI_COMM_TYPE_HW_UNGUIDED:
+            /*
+             * MPI_COMM_TYPE_HW_(UN)GUIDED handled in calling function.
+             * We should not get here as the split type will be changed
+             * at a higher level.
+             */
+            opal_output(0, "Error: in ompi_comm_split_type_get_part() unexpected split_type=%d", split_type);
+            return OMPI_ERR_BAD_PARAM;
         }
 
         if (include) {
@@ -837,8 +868,9 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
     ompi_communicator_t *newcomp = MPI_COMM_NULL;
     int my_size, my_rsize = 0, mode, inter;
     int *lranks = NULL, *rranks = NULL;
-    int global_split_type, ok, tmp[4];
+    int global_split_type, ok, tmp[6];
     int rc;
+    int orig_split_type = split_type;
 
     /* silence clang warning. newcomm should never be NULL */
     if (OPAL_UNLIKELY(NULL == newcomm)) {
@@ -847,14 +879,58 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
 
     inter = OMPI_COMM_IS_INTER(comm);
 
+    /* Step 0: Convert MPI_COMM_TYPE_HW_GUIDED to the internal type */
+    if (MPI_COMM_TYPE_HW_GUIDED == split_type) {
+        int flag;
+        opal_cstring_t *value = NULL;
+
+        opal_info_get(info, "mpi_hw_resource_type", &value, &flag);
+        /* If key is not in the 'info', then return MPI_COMM_NULL.
+         * This is caught at the MPI interface level, but it doesn't hurt to
+         * check it again.
+         */
+        if (!flag) {
+            *newcomm = MPI_COMM_NULL;
+            return OMPI_SUCCESS;
+        }
+
+        /* Verify the value associated with the "mpi_hw_resource_type" key
+         * - is supported, and
+         * - is the same value at all ranks
+         *
+         * If not supported, then return MPI_COMM_NULL.
+         * If not the same at all ranks, throw an error.
+         */
+        flag = 0;
+        for (int i = 0; ompi_comm_split_type_hw_guided_support[i].info_value; ++i) {
+            if (0 == strncasecmp(value->string, ompi_comm_split_type_hw_guided_support[i].info_value, strlen(ompi_comm_split_type_hw_guided_support[i].info_value))) {
+                split_type = ompi_comm_split_type_hw_guided_support[i].split_type;
+                flag = 1;
+                break;
+            }
+        }
+        /* If not supported, then return MPI_COMM_NULL. */
+        if (0 == flag) {
+            *newcomm = MPI_COMM_NULL;
+            return OMPI_SUCCESS;
+        }
+    }
+
     /* Step 1: verify all ranks have supplied the same value for split type. All split types
      * must be the same or MPI_UNDEFINED (which is negative). */
-    tmp[0] = split_type;
-    tmp[1] = -split_type;
+    tmp[0] = orig_split_type;
+    tmp[1] = -orig_split_type;
     tmp[2] = key;
     tmp[3] = -key;
+    /* For MPI_COMM_TYPE_HW_GUIDED, verify all ranks have supplied the same
+     * split_type (represented by orig_split_type) and info 'value' (represented by split_type).
+     *
+     * For split_type != MPI_COMM_TYPE_HW_GUIDED then orig_split_type == split_type.
+     */
+    tmp[4] = split_type;
+    tmp[5] = -split_type;
 
-    rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &tmp, 4, MPI_INT, MPI_MAX, comm,
+    rc = comm->c_coll->coll_allreduce (MPI_IN_PLACE, &tmp, 6, MPI_INT, MPI_MAX, comm,
                                       comm->c_coll->coll_allreduce_module);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
         return rc;
@@ -899,6 +975,26 @@ int ompi_comm_split_type (ompi_communicator_t *comm, int split_type, int key,
         return OMPI_SUCCESS;
     }
 
+    /* MPI_COMM_TYPE_HW_GUIDED: Check if 'value' the same at all ranks */
+    if (tmp[4] != -tmp[5]) {
+        if (0 == ompi_comm_rank(comm)) {
+            opal_output(0, "Error: Mismatched info values for MPI_COMM_TYPE_HW_GUIDED");
+        }
+        return OMPI_ERR_BAD_PARAM;
+    }
+
+    /* TODO: Make this better...
+     *
+     * See Example 7.4 in the MPI 4.0 standard for example usage.
+     *
+     * Stage 0: Recognized, but not implemented.
+     * Stage 1: Do better than that
+     */
+    if (MPI_COMM_TYPE_HW_UNGUIDED == global_split_type) {
+        *newcomm = MPI_COMM_NULL;
+        return OMPI_SUCCESS;
+    }
+
     /* Step 2: Build potential communicator groups. If any ranks will not be part of
      * the ultimate communicator we will drop them later. This saves doing an extra
      * allgather on the whole communicator. By using ompi_comm_split() later only
diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in
@@ -19,7 +19,7 @@
  * Copyright (c) 2015      University of Houston. All rights reserved.
  * Copyright (c) 2015-2021 Research Organization for Information Science
  *                         and Technology (RIST).  All rights reserved.
- * Copyright (c) 2017-2019 IBM Corporation.  All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation.  All rights reserved.
  * Copyright (c) 2018      FUJITSU LIMITED.  All rights reserved.
  * Copyright (c) 2021-2022 Google, LLC. All rights reserved.
  * Copyright (c) 2021-2022 Amazon.com, Inc. or its affiliates.  All Rights
@@ -852,7 +852,9 @@ enum {
   OMPI_COMM_TYPE_BOARD,
   OMPI_COMM_TYPE_HOST,
   OMPI_COMM_TYPE_CU,
-  OMPI_COMM_TYPE_CLUSTER
+  OMPI_COMM_TYPE_CLUSTER,
+  MPI_COMM_TYPE_HW_UNGUIDED,
+  MPI_COMM_TYPE_HW_GUIDED
 };
 #define OMPI_COMM_TYPE_NODE MPI_COMM_TYPE_SHARED
 
diff --git a/ompi/include/mpif-values.pl b/ompi/include/mpif-values.pl
@@ -7,6 +7,7 @@
 # Copyright (c) 2020      The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
+# Copyright (c) 2022      IBM Corporation.  All rights reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -395,6 +396,8 @@ sub write_file {
 $constants->{OMPI_COMM_TYPE_HOST} = 9;
 $constants->{OMPI_COMM_TYPE_CU} = 10;
 $constants->{OMPI_COMM_TYPE_CLUSTER} = 11;
+$constants->{MPI_COMM_TYPE_HW_UNGUIDED} = 12;
+$constants->{MPI_COMM_TYPE_HW_GUIDED} = 13;
 
 #----------------------------------------------------------------------------
 
diff --git a/ompi/mpi/c/comm_split_type.c b/ompi/mpi/c/comm_split_type.c
@@ -14,7 +14,7 @@
  * Copyright (c) 2010-2012 Oak Ridge National Labs.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2017-2022 IBM Corporation.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -65,6 +65,8 @@ int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key,
         }
 
         if ( MPI_COMM_TYPE_SHARED != split_type && // Same as OMPI_COMM_TYPE_NODE
+             MPI_COMM_TYPE_HW_UNGUIDED != split_type &&
+             MPI_COMM_TYPE_HW_GUIDED != split_type &&
 	     OMPI_COMM_TYPE_CLUSTER != split_type &&
 	     OMPI_COMM_TYPE_CU != split_type &&
 	     OMPI_COMM_TYPE_HOST != split_type &&
@@ -99,6 +101,30 @@ int MPI_Comm_split_type(MPI_Comm comm, int split_type, int key,
     }
 #endif
 
+    if ( MPI_COMM_TYPE_HW_GUIDED == split_type ) {
+        int flag;
+        opal_cstring_t *value = NULL;
+
+        /* MPI_Info is required for this split_type.
+         * Not an error condition, per MPI 4.0.
+         */
+        if ( MPI_INFO_NULL == info ) {
+            *newcomm = MPI_COMM_NULL;
+            rc = MPI_SUCCESS;
+            OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+        }
+
+        /* MPI_Info with key "mpi_hw_resource_type" is required for this split_type.
+         * Not an error condition, per MPI 4.0.
+         */
+        ompi_info_get(info, "mpi_hw_resource_type", &value, &flag);
+        if ( !flag ) {
+            *newcomm = MPI_COMM_NULL;
+            rc = MPI_SUCCESS;
+            OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
+        }
+    }
+
     if( (MPI_COMM_SELF == comm) && (MPI_UNDEFINED == split_type) ) {
         *newcomm = MPI_COMM_NULL;
         rc = MPI_SUCCESS;
