Skip to content

Commit 6af5156

Browse files
author
Luke Robison
committed
coll/han: Add SMSC-based alltoall to HAN
Add Alltoall algorithm to coll/han. Each rank on one host is assigned a single partner on a remote host and vice versa. Then the rank collects all the data its partner will need to receive from it's host, and sends it in one large send, and likewise receives it's data in one large recv, then cycles to the next host. This algorithm is only selected when SMSC component has ability to direct-map peer memory, which only exists for XPMEM module. Signed-off-by: Luke Robison <lrbison@amazon.com>
1 parent 34c4a30 commit 6af5156

File tree

9 files changed

+572
-2
lines changed

9 files changed

+572
-2
lines changed

ompi/mca/coll/han/Makefile.am

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# Copyright (c) 2018-2020 The University of Tennessee and The University
33
# of Tennessee Research Foundation. All rights
44
# reserved.
5-
# Copyright (c) 2022 Amazon.com, Inc. or its affiliates. All Rights reserved.
5+
# Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights reserved.
66
# Copyright (c) 2022 BULL S.A.S. All rights reserved.
77
# $COPYRIGHT$
88
#
@@ -15,6 +15,7 @@ sources = \
1515
coll_han.h \
1616
coll_han_trigger.h \
1717
coll_han_algorithms.h \
18+
coll_han_alltoall.c \
1819
coll_han_dynamic.h \
1920
coll_han_dynamic_file.h \
2021
coll_han_barrier.c \

ompi/mca/coll/han/coll_han.h

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ typedef struct mca_coll_han_op_module_name_t {
198198
mca_coll_han_op_up_low_module_name_t gatherv;
199199
mca_coll_han_op_up_low_module_name_t scatter;
200200
mca_coll_han_op_up_low_module_name_t scatterv;
201+
mca_coll_han_op_up_low_module_name_t alltoall;
201202
} mca_coll_han_op_module_name_t;
202203

203204
/**
@@ -253,6 +254,13 @@ typedef struct mca_coll_han_component_t {
253254
uint32_t han_scatterv_up_module;
254255
/* low level module for scatterv */
255256
uint32_t han_scatterv_low_module;
257+
258+
/* low level module for alltoall */
259+
uint32_t han_alltoall_low_module;
260+
/* alltoall: parallel stages */
261+
int32_t han_alltoall_pstages;
262+
263+
256264
/* name of the modules */
257265
mca_coll_han_op_module_name_t han_op_module_name;
258266
/* whether we need reproducible results
@@ -288,6 +296,7 @@ typedef struct mca_coll_han_single_collective_fallback_s
288296
{
289297
union
290298
{
299+
mca_coll_base_module_alltoall_fn_t alltoall;
291300
mca_coll_base_module_allgather_fn_t allgather;
292301
mca_coll_base_module_allgatherv_fn_t allgatherv;
293302
mca_coll_base_module_allreduce_fn_t allreduce;
@@ -309,6 +318,7 @@ typedef struct mca_coll_han_single_collective_fallback_s
309318
*/
310319
typedef struct mca_coll_han_collectives_fallback_s
311320
{
321+
mca_coll_han_single_collective_fallback_t alltoall;
312322
mca_coll_han_single_collective_fallback_t allgather;
313323
mca_coll_han_single_collective_fallback_t allgatherv;
314324
mca_coll_han_single_collective_fallback_t allreduce;
@@ -371,6 +381,9 @@ OBJ_CLASS_DECLARATION(mca_coll_han_module_t);
371381
* Some defines to stick to the naming used in the other components in terms of
372382
* fallback routines
373383
*/
384+
#define previous_alltoall fallback.alltoall.alltoall
385+
#define previous_alltoall_module fallback.alltoall.module
386+
374387
#define previous_allgather fallback.allgather.allgather
375388
#define previous_allgather_module fallback.allgather.module
376389

@@ -426,6 +439,7 @@ OBJ_CLASS_DECLARATION(mca_coll_han_module_t);
426439
HAN_UNINSTALL_COLL_API(COMM, HANM, allreduce); \
427440
HAN_UNINSTALL_COLL_API(COMM, HANM, allgather); \
428441
HAN_UNINSTALL_COLL_API(COMM, HANM, allgatherv); \
442+
HAN_UNINSTALL_COLL_API(COMM, HANM, alltoall); \
429443
han_module->enabled = false; /* entire module set to pass-through from now on */ \
430444
} while(0)
431445

@@ -486,6 +500,9 @@ mca_coll_han_get_all_coll_modules(struct ompi_communicator_t *comm,
486500
mca_coll_han_module_t *han_module);
487501

488502
int
503+
mca_coll_han_alltoall_intra_dynamic(ALLTOALL_BASE_ARGS,
504+
mca_coll_base_module_t *module);
505+
int
489506
mca_coll_han_allgather_intra_dynamic(ALLGATHER_BASE_ARGS,
490507
mca_coll_base_module_t *module);
491508
int

ompi/mca/coll/han/coll_han_algorithms.c

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
22
/*
33
* Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
4+
* Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
45
*
56
* $COPYRIGHT$
67
*
@@ -77,6 +78,10 @@ mca_coll_han_algorithm_value_t* mca_coll_han_available_algorithms[COLLCOUNT] =
7778
{"simple", (fnptr_t)&mca_coll_han_allgather_intra_simple}, // 2-level
7879
{ 0 }
7980
},
81+
[ALLTOALL] = (mca_coll_han_algorithm_value_t[]){
82+
{"smsc", (fnptr_t)&mca_coll_han_alltoall_using_smsc}, // 2-level
83+
{ 0 }
84+
},
8085
};
8186

8287
int

ompi/mca/coll/han/coll_han_algorithms.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
22
/*
33
* Copyright (c) 2020-2022 Bull S.A.S. All rights reserved.
4+
* Copyright (c) 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved.
45
*
56
* $COPYRIGHT$
67
*
@@ -208,4 +209,10 @@ mca_coll_han_allgather_intra_simple(const void *sbuf, size_t scount,
208209
struct ompi_communicator_t *comm,
209210
mca_coll_base_module_t *module);
210211

212+
/* Alltoall */
213+
int
214+
mca_coll_han_alltoall_using_smsc(ALLTOALL_BASE_ARGS,
215+
mca_coll_base_module_t *module);
216+
217+
211218
#endif

0 commit comments

Comments
 (0)