coll/han: Add SMSC-based alltoall to HAN

Add Alltoall algorithm to coll/han.  Each rank on one host is assigned
a single partner on a remote host and vice versa.  Then the rank
collects all the data its partner will need to receive from it's host,
and sends it in one large send, and likewise receives it's data in one
large recv, then cycles to the next host.

This algorithm is only selected when SMSC component has ability to
direct-map peer memory, which only exists for XPMEM module.

Signed-off-by: Luke Robison <lrbison@amazon.com>

Author: Luke Robison

ompi/mca/coll/han/Makefile.am

@@ -2,7 +2,7 @@
 # Copyright (c) 2018-2020 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
-# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.  All Rights reserved.
+# Copyright (c) 2024      Amazon.com, Inc. or its affiliates.  All Rights reserved.
 # Copyright (c) 2022      BULL S.A.S. All rights reserved.
 # $COPYRIGHT$
 #
@@ -15,6 +15,7 @@ sources = \
 coll_han.h \
 coll_han_trigger.h \
 coll_han_algorithms.h \
+coll_han_alltoall.c \
 coll_han_dynamic.h \
 coll_han_dynamic_file.h \
 coll_han_barrier.c \

ompi/mca/coll/han/coll_han.h

@@ -198,6 +198,7 @@ typedef struct mca_coll_han_op_module_name_t {
     mca_coll_han_op_up_low_module_name_t gatherv;
     mca_coll_han_op_up_low_module_name_t scatter;
     mca_coll_han_op_up_low_module_name_t scatterv;
+    mca_coll_han_op_up_low_module_name_t alltoall;
 } mca_coll_han_op_module_name_t;
 
 /**
@@ -253,6 +254,13 @@ typedef struct mca_coll_han_component_t {
     uint32_t han_scatterv_up_module;
     /* low level module for scatterv */
     uint32_t han_scatterv_low_module;
+
+    /* low level module for alltoall */
+    uint32_t han_alltoall_low_module;
+    /* alltoall: parallel stages */
+    int32_t han_alltoall_pstages;
+
+
     /* name of the modules */
     mca_coll_han_op_module_name_t han_op_module_name;
     /* whether we need reproducible results
@@ -288,6 +296,7 @@ typedef struct mca_coll_han_single_collective_fallback_s
 {
     union
     {
+        mca_coll_base_module_alltoall_fn_t alltoall;
         mca_coll_base_module_allgather_fn_t allgather;
         mca_coll_base_module_allgatherv_fn_t allgatherv;
         mca_coll_base_module_allreduce_fn_t allreduce;
@@ -309,6 +318,7 @@ typedef struct mca_coll_han_single_collective_fallback_s
  */
 typedef struct mca_coll_han_collectives_fallback_s
 {
+    mca_coll_han_single_collective_fallback_t alltoall;
     mca_coll_han_single_collective_fallback_t allgather;
     mca_coll_han_single_collective_fallback_t allgatherv;
     mca_coll_han_single_collective_fallback_t allreduce;
@@ -371,6 +381,9 @@ OBJ_CLASS_DECLARATION(mca_coll_han_module_t);
  * Some defines to stick to the naming used in the other components in terms of
  * fallback routines
  */
+#define previous_alltoall           fallback.alltoall.alltoall
+#define previous_alltoall_module    fallback.alltoall.module
+
 #define previous_allgather          fallback.allgather.allgather
 #define previous_allgather_module   fallback.allgather.module
 
@@ -426,6 +439,7 @@ OBJ_CLASS_DECLARATION(mca_coll_han_module_t);
         HAN_UNINSTALL_COLL_API(COMM, HANM, allreduce);                 \
         HAN_UNINSTALL_COLL_API(COMM, HANM, allgather);                 \
         HAN_UNINSTALL_COLL_API(COMM, HANM, allgatherv);                \
+        HAN_UNINSTALL_COLL_API(COMM, HANM, alltoall);                  \
         han_module->enabled = false;  /* entire module set to pass-through from now on */ \
     } while(0)
 
@@ -486,6 +500,9 @@ mca_coll_han_get_all_coll_modules(struct ompi_communicator_t *comm,
                                   mca_coll_han_module_t *han_module);
 
 int
+mca_coll_han_alltoall_intra_dynamic(ALLTOALL_BASE_ARGS,
+                                    mca_coll_base_module_t *module);
+int
 mca_coll_han_allgather_intra_dynamic(ALLGATHER_BASE_ARGS,
                                      mca_coll_base_module_t *module);
 int

ompi/mca/coll/han/coll_han_algorithms.c

@@ -1,6 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * $COPYRIGHT$
  *
@@ -77,6 +78,10 @@ mca_coll_han_algorithm_value_t*  mca_coll_han_available_algorithms[COLLCOUNT] =
         {"simple", (fnptr_t)&mca_coll_han_allgather_intra_simple}, // 2-level
         { 0 }
     },
+    [ALLTOALL] = (mca_coll_han_algorithm_value_t[]){
+        {"smsc", (fnptr_t)&mca_coll_han_alltoall_using_smsc}, // 2-level
+        { 0 }
+    },
 };
 
 int

ompi/mca/coll/han/coll_han_algorithms.h

@@ -1,6 +1,7 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
+ * Copyright (c) 2024      Amazon.com, Inc. or its affiliates. All Rights Reserved.
  *
  * $COPYRIGHT$
  *
@@ -208,4 +209,10 @@ mca_coll_han_allgather_intra_simple(const void *sbuf, size_t scount,
                                     struct ompi_communicator_t *comm,
                                     mca_coll_base_module_t *module);
 
+/* Alltoall */
+int
+mca_coll_han_alltoall_using_smsc(ALLTOALL_BASE_ARGS,
+                                    mca_coll_base_module_t *module);
+
+
 #endif