Skip to content

Commit 9180628

Browse files
author
ferrol aderholdt
committed
OSHMEM/SCOLL: Add support for UCC collectives to OSHMEM
(cherry picked from commit a6efd82) Signed-off-by: ferrol aderholdt <faderholdt@nvidia.com>
1 parent e6cb338 commit 9180628

14 files changed

+1306
-0
lines changed

‎oshmem/mca/scoll/ucc/Makefile.am

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Copyright (c) 2021 Mellanox Technologies, Inc.
2+
# All rights reserved.
3+
# $COPYRIGHT$
4+
#
5+
# Additional copyrights may follow
6+
#
7+
# $HEADER$
8+
#
9+
#
10+
11+
dist_oshmemdata_DATA = \
12+
help-oshmem-scoll-ucc.txt
13+
14+
AM_CPPFLAGS = $(scoll_ucc_CPPFLAGS)
15+
16+
scoll_ucc_sources = \
17+
scoll_ucc.h \
18+
scoll_ucc_debug.h \
19+
scoll_ucc_dtypes.h \
20+
scoll_ucc_common.h \
21+
scoll_ucc_module.c \
22+
scoll_ucc_component.c \
23+
scoll_ucc_barrier.c \
24+
scoll_ucc_broadcast.c \
25+
scoll_ucc_reduce.c \
26+
scoll_ucc_collect.c \
27+
scoll_ucc_alltoall.c
28+
if MCA_BUILD_oshmem_scoll_ucc_DSO
29+
component_noinst =
30+
component_install = mca_scoll_ucc.la
31+
else
32+
component_noinst = libmca_scoll_ucc.la
33+
component_install =
34+
endif
35+
36+
mcacomponentdir = $(pkglibdir)
37+
mcacomponent_LTLIBRARIES = $(component_install)
38+
mca_scoll_ucc_la_SOURCES = $(scoll_ucc_sources)
39+
mca_scoll_ucc_la_LIBADD = $(top_builddir)/oshmem/liboshmem.la \
40+
$(scoll_ucc_LIBS)
41+
mca_scoll_ucc_la_LDFLAGS = -module -avoid-version $(scoll_ucc_LDFLAGS)
42+
43+
noinst_LTLIBRARIES = $(component_noinst)
44+
libmca_scoll_ucc_la_SOURCES =$(scoll_ucc_sources)
45+
libmca_scoll_ucc_la_LIBADD = $(scoll_ucc_LIBS)
46+
libmca_scoll_ucc_la_LDFLAGS = -module -avoid-version $(scoll_ucc_LDFLAGS)

‎oshmem/mca/scoll/ucc/configure.m4

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# -*- shell-script -*-
2+
#
3+
#
4+
# Copyright (c) 2021 Mellanox Technologies. All rights reserved.
5+
# Copyright (c) 2015 Research Organization for Information Science
6+
# and Technology (RIST). All rights reserved.
7+
# $COPYRIGHT$
8+
#
9+
# Additional copyrights may follow
10+
#
11+
# $HEADER$
12+
#
13+
14+
15+
# MCA_scoll_ucc_CONFIG([action-if-can-compile],
16+
# [action-if-cant-compile])
17+
# ------------------------------------------------
18+
AC_DEFUN([MCA_oshmem_scoll_ucc_CONFIG],[
19+
AC_CONFIG_FILES([oshmem/mca/scoll/ucc/Makefile])
20+
21+
OMPI_CHECK_UCC([scoll_ucc],
22+
[scoll_ucc_happy="yes"],
23+
[scoll_ucc_happy="no"])
24+
25+
AS_IF([test "$scoll_ucc_happy" = "yes"],
26+
[scoll_ucc_WRAPPER_EXTRA_LDFLAGS="$scoll_ucc_LDFLAGS"
27+
scoll_ucc_CPPFLAGS="$scoll_ucc_CPPFLAGS"
28+
scoll_ucc_WRAPPER_EXTRA_LIBS="$scoll_ucc_LIBS"
29+
$1],
30+
[$2])
31+
32+
# substitute in the things needed to build ucc
33+
AC_SUBST([scoll_ucc_CFLAGS])
34+
AC_SUBST([scoll_ucc_CPPFLAGS])
35+
AC_SUBST([scoll_ucc_LDFLAGS])
36+
AC_SUBST([scoll_ucc_LIBS])
37+
])dnl
38+

‎oshmem/mca/scoll/ucc/help-oshmem-scoll-ucc.txt

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#
2+
# Copyright (c) 2021 Mellanox Technologies, Inc.
3+
# All rights reserved.
4+
# $COPYRIGHT$
5+
#
6+
# Additional copyrights may follow
7+
#
8+
# $HEADER$
9+
#
10+
[module_enable:fatal]
11+
scoll:ucc module reports issue during module enabling phase.
12+
Try to use scoll:ucc component with anoter one
13+
for example scoll:basic
14+
15+
Error: %s
16+
#

‎oshmem/mca/scoll/ucc/scoll_ucc.h

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
/*
2+
* Copyright (c) 2021 Mellanox Technologies. All rights reserved.
3+
* $COPYRIGHT$
4+
*
5+
* Additional copyrights may follow
6+
*
7+
* $HEADER$
8+
*/
9+
10+
#ifndef MCA_SCOLL_UCC_H
11+
#define MCA_SCOLL_UCC_H
12+
13+
#include "oshmem_config.h"
14+
15+
#include "shmem.h"
16+
#include "oshmem/mca/mca.h"
17+
#include "oshmem/mca/scoll/scoll.h"
18+
#include "oshmem/proc/proc.h"
19+
20+
#include "scoll_ucc_debug.h"
21+
22+
#include <ucc/api/ucc.h>
23+
24+
BEGIN_C_DECLS
25+
26+
#define SCOLL_UCC_CTS (UCC_COLL_TYPE_BARRIER | UCC_COLL_TYPE_BCAST | \
27+
UCC_COLL_TYPE_ALLREDUCE | UCC_COLL_TYPE_ALLGATHER | \
28+
UCC_COLL_TYPE_ALLTOALL)
29+
30+
#define SCOLL_UCC_CTS_STR "barrier,broadcast,reduce,collect,alltoall"
31+
32+
int mca_scoll_ucc_progress(void);
33+
34+
/**
35+
* Globally exported structure
36+
*/
37+
struct mca_scoll_ucc_component_t {
38+
mca_scoll_base_component_1_0_0_t super;
39+
int ucc_priority;
40+
int ucc_verbose;
41+
int ucc_enable;
42+
int ucc_np;
43+
char * cls;
44+
char * cts;
45+
bool libucc_initialized;
46+
ucc_lib_h ucc_lib;
47+
ucc_lib_attr_t ucc_lib_attr;
48+
ucc_coll_type_t cts_requested;
49+
ucc_context_h ucc_context;
50+
};
51+
typedef struct mca_scoll_ucc_component_t mca_scoll_ucc_component_t;
52+
53+
OMPI_MODULE_DECLSPEC extern mca_scoll_ucc_component_t mca_scoll_ucc_component;
54+
55+
/**
56+
* UCC enabled team
57+
*/
58+
struct mca_scoll_ucc_module_t {
59+
mca_scoll_base_module_t super;
60+
61+
oshmem_group_t *group;
62+
ucc_team_h ucc_team;
63+
64+
/* Saved handlers - for fallback */
65+
mca_scoll_base_module_reduce_fn_t previous_reduce;
66+
mca_scoll_base_module_t *previous_reduce_module;
67+
mca_scoll_base_module_broadcast_fn_t previous_broadcast;
68+
mca_scoll_base_module_t *previous_broadcast_module;
69+
mca_scoll_base_module_barrier_fn_t previous_barrier;
70+
mca_scoll_base_module_t *previous_barrier_module;
71+
mca_scoll_base_module_collect_fn_t previous_collect;
72+
mca_scoll_base_module_t *previous_collect_module;
73+
mca_scoll_base_module_alltoall_fn_t previous_alltoall;
74+
mca_scoll_base_module_t *previous_alltoall_module;
75+
};
76+
typedef struct mca_scoll_ucc_module_t mca_scoll_ucc_module_t;
77+
78+
OBJ_CLASS_DECLARATION(mca_scoll_ucc_module_t);
79+
80+
/* API functions */
81+
int mca_scoll_ucc_init_query(bool enable_progress_threads, bool enable_mpi_threads);
82+
83+
mca_scoll_base_module_t* mca_scoll_ucc_comm_query(oshmem_group_t *osh_group, int *priority);
84+
85+
int mca_scoll_ucc_barrier(struct oshmem_group_t *group, long *pSync, int alg);
86+
87+
int mca_scoll_ucc_broadcast(struct oshmem_group_t *group,
88+
int PE_root,
89+
void *target,
90+
const void *source,
91+
size_t nlong,
92+
long *pSync,
93+
bool nlong_type,
94+
int alg);
95+
96+
int mca_scoll_ucc_collect(struct oshmem_group_t *group,
97+
void *target,
98+
const void *source,
99+
size_t nlong,
100+
long *pSync,
101+
bool nlong_type,
102+
int alg);
103+
104+
int mca_scoll_ucc_reduce(struct oshmem_group_t *group,
105+
struct oshmem_op_t *op,
106+
void *target,
107+
const void *source,
108+
size_t nlong,
109+
long *pSync,
110+
void *pWrk,
111+
int alg);
112+
113+
int mca_scoll_ucc_alltoall(struct oshmem_group_t *group,
114+
void *target,
115+
const void *source,
116+
ptrdiff_t dst, ptrdiff_t sst,
117+
size_t nelems,
118+
size_t element_size,
119+
long *pSync,
120+
int alg);
121+
122+
END_C_DECLS
123+
124+
#endif

‎oshmem/mca/scoll/ucc/scoll_ucc_alltoall.c

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
/**
2+
Copyright (c) 2021 Mellanox Technologies. All rights reserved.
3+
$COPYRIGHT$
4+
5+
Additional copyrights may follow
6+
7+
$HEADER$
8+
*/
9+
#include "scoll_ucc.h"
10+
#include "scoll_ucc_dtypes.h"
11+
#include "scoll_ucc_common.h"
12+
13+
#include <ucc/api/ucc.h>
14+
15+
static inline ucc_status_t mca_scoll_ucc_alltoall_init(const void *sbuf, void *rbuf,
16+
int count,
17+
mca_scoll_ucc_module_t * ucc_module,
18+
ucc_coll_req_h * req)
19+
{
20+
ucc_coll_args_t coll = {
21+
.mask = 0,
22+
.coll_type = UCC_COLL_TYPE_ALLTOALL,
23+
.src.info = {
24+
.buffer = (void *)sbuf,
25+
.count = count,
26+
.datatype = UCC_DT_UINT8,
27+
.mem_type = UCC_MEMORY_TYPE_UNKNOWN
28+
},
29+
.dst.info = {
30+
.buffer = rbuf,
31+
.count = count,
32+
.datatype = UCC_DT_UINT8,
33+
.mem_type = UCC_MEMORY_TYPE_UNKNOWN
34+
},
35+
};
36+
37+
SCOLL_UCC_REQ_INIT(req, coll, ucc_module);
38+
return UCC_OK;
39+
fallback:
40+
return UCC_ERR_NOT_SUPPORTED;
41+
}
42+
43+
44+
int mca_scoll_ucc_alltoall(struct oshmem_group_t *group,
45+
void *target,
46+
const void *source,
47+
ptrdiff_t dst, ptrdiff_t sst,
48+
size_t nelems,
49+
size_t element_size,
50+
long *pSync,
51+
int alg)
52+
{
53+
mca_scoll_ucc_module_t *ucc_module;
54+
size_t count;
55+
ucc_coll_req_h req;
56+
57+
UCC_VERBOSE(3, "running ucc alltoall");
58+
ucc_module = (mca_scoll_ucc_module_t *) group->g_scoll.scoll_alltoall_module;
59+
count = nelems * element_size;
60+
61+
/* Do nothing on zero-length request */
62+
if (OPAL_UNLIKELY(!nelems)) {
63+
return OSHMEM_SUCCESS;
64+
}
65+
66+
SCOLL_UCC_CHECK(mca_scoll_ucc_alltoall_init(source, target, count, ucc_module, &req));
67+
SCOLL_UCC_CHECK(ucc_collective_post(req));
68+
SCOLL_UCC_CHECK(scoll_ucc_req_wait(req));
69+
return OSHMEM_SUCCESS;
70+
fallback:
71+
UCC_VERBOSE(3, "running fallback alltoall");
72+
return ucc_module->previous_alltoall(group, target, source, dst, sst, nelems,
73+
element_size, pSync, alg);
74+
}
75+

‎oshmem/mca/scoll/ucc/scoll_ucc_barrier.c

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/**
2+
Copyright (c) 2021 Mellanox Technologies. All rights reserved.
3+
$COPYRIGHT$
4+
5+
Additional copyrights may follow
6+
7+
$HEADER$
8+
*/
9+
#include "scoll_ucc.h"
10+
#include "scoll_ucc_dtypes.h"
11+
#include "scoll_ucc_common.h"
12+
13+
#include <ucc/api/ucc.h>
14+
15+
static inline ucc_status_t mca_scoll_ucc_barrier_init(mca_scoll_ucc_module_t * ucc_module,
16+
ucc_coll_req_h * req)
17+
{
18+
ucc_coll_args_t coll = {
19+
.mask = 0,
20+
.coll_type = UCC_COLL_TYPE_BARRIER
21+
};
22+
SCOLL_UCC_REQ_INIT(req, coll, ucc_module);
23+
return UCC_OK;
24+
fallback:
25+
return UCC_ERR_NOT_SUPPORTED;
26+
}
27+
28+
int mca_scoll_ucc_barrier(struct oshmem_group_t *group, long *pSync, int alg)
29+
{
30+
mca_scoll_ucc_module_t *ucc_module;
31+
ucc_coll_req_h req;
32+
33+
UCC_VERBOSE(3, "running ucc barrier");
34+
ucc_module = (mca_scoll_ucc_module_t *) group->g_scoll.scoll_barrier_module;
35+
36+
SCOLL_UCC_CHECK(mca_scoll_ucc_barrier_init(ucc_module, &req));
37+
SCOLL_UCC_CHECK(ucc_collective_post(req));
38+
SCOLL_UCC_CHECK(scoll_ucc_req_wait(req));
39+
return OSHMEM_SUCCESS;
40+
fallback:
41+
UCC_VERBOSE(3, "running fallback barrier");
42+
return ucc_module->previous_barrier(group, pSync, alg);
43+
}
44+

0 commit comments

Comments
 (0)