Skip to content

Commit 560ebc5

Browse files
jsquyresjsquyres
authored
Merge pull request #7716 from bosilca/coll/adapt
ADAPT: Event-driven collective implementation
2 parents 01dcc39 + ee592f3 commit 560ebc5

25 files changed

+2473
-34
lines changed

‎ompi/communicator/comm_init.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
#include "ompi/constants.h"
4141
#include "ompi/mca/pml/pml.h"
4242
#include "ompi/mca/coll/base/base.h"
43+
#include "ompi/mca/coll/base/coll_tags.h"
4344
#include "ompi/mca/topo/base/base.h"
4445
#include "ompi/runtime/params.h"
4546
#include "ompi/communicator/communicator.h"
@@ -382,6 +383,7 @@ static void ompi_comm_construct(ompi_communicator_t* comm)
382383
comm->c_pml_comm = NULL;
383384
comm->c_topo = NULL;
384385
comm->c_coll = NULL;
386+
comm->c_nbc_tag = MCA_COLL_BASE_TAG_NONBLOCKING_BASE;
385387

386388
/* A keyhash will be created if/when an attribute is cached on
387389
this communicator */

‎ompi/communicator/communicator.h

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
44
* University Research and Technology
55
* Corporation. All rights reserved.
6-
* Copyright (c) 2004-2017 The University of Tennessee and The University
6+
* Copyright (c) 2004-2020 The University of Tennessee and The University
77
* of Tennessee Research Foundation. All rights
88
* reserved.
99
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -187,6 +187,13 @@ struct ompi_communicator_t {
187187

188188
/* Collectives module interface and data */
189189
mca_coll_base_comm_coll_t *c_coll;
190+
191+
/* Non-blocking collective tag. These tags might be shared between
192+
* all non-blocking collective modules (to avoid message collision
193+
* between them in the case where multiple outstanding non-blocking
194+
* collective coexists using multiple backends).
195+
*/
196+
opal_atomic_int32_t c_nbc_tag;
190197
};
191198
typedef struct ompi_communicator_t ompi_communicator_t;
192199

‎ompi/mca/coll/adapt/Makefile.am

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#
2+
# Copyright (c) 2014-2020 The University of Tennessee and The University
3+
# of Tennessee Research Foundation. All rights
4+
# reserved.
5+
# $COPYRIGHT$
6+
#
7+
# Additional copyrights may follow
8+
#
9+
# $HEADER$
10+
#
11+
12+
13+
sources = \
14+
coll_adapt_component.c \
15+
coll_adapt_module.c \
16+
coll_adapt_bcast.c \
17+
coll_adapt_ibcast.c \
18+
coll_adapt_reduce.c \
19+
coll_adapt_ireduce.c \
20+
coll_adapt.h \
21+
coll_adapt_algorithms.h \
22+
coll_adapt_context.h \
23+
coll_adapt_context.c \
24+
coll_adapt_inbuf.c \
25+
coll_adapt_inbuf.h \
26+
coll_adapt_item.c \
27+
coll_adapt_item.h
28+
29+
# Make the output library in this directory, and name it either
30+
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
31+
# (for static builds).
32+
33+
component_noinst =
34+
component_install =
35+
if MCA_BUILD_ompi_coll_adapt_DSO
36+
component_install += mca_coll_adapt.la
37+
else
38+
component_noinst += libmca_coll_adapt.la
39+
endif
40+
41+
mcacomponentdir = $(ompilibdir)
42+
mcacomponent_LTLIBRARIES = $(component_install)
43+
mca_coll_adapt_la_SOURCES = $(sources)
44+
mca_coll_adapt_la_LDFLAGS = -module -avoid-version
45+
mca_coll_adapt_la_LIBADD =
46+
47+
noinst_LTLIBRARIES = $(component_noinst)
48+
libmca_coll_adapt_la_SOURCES =$(sources)
49+
libmca_coll_adapt_la_LDFLAGS = -module -avoid-version

‎ompi/mca/coll/adapt/coll_adapt.h

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Copyright (c) 2014-2020 The University of Tennessee and The University
3+
* of Tennessee Research Foundation. All rights
4+
* reserved.
5+
* $COPYRIGHT$
6+
*
7+
* Additional copyrights may follow
8+
*
9+
* $HEADER$
10+
*/
11+
12+
13+
#ifndef MCA_COLL_ADAPT_EXPORT_H
14+
#define MCA_COLL_ADAPT_EXPORT_H
15+
16+
#include "ompi_config.h"
17+
18+
#include "mpi.h"
19+
#include "opal/mca/mca.h"
20+
#include "opal/datatype/opal_convertor.h"
21+
#include "ompi/mca/coll/coll.h"
22+
#include "ompi/mca/coll/base/coll_base_topo.h"
23+
24+
BEGIN_C_DECLS
25+
26+
typedef struct mca_coll_adapt_module_t mca_coll_adapt_module_t;
27+
28+
/*
29+
* Structure to hold the adapt coll component. First it holds the
30+
* base coll component, and then holds a bunch of
31+
* adapt-coll-component-specific stuff (e.g., current MCA param
32+
* values).
33+
*/
34+
typedef struct mca_coll_adapt_component_t {
35+
/* Base coll component */
36+
mca_coll_base_component_2_0_0_t super;
37+
38+
/* MCA parameter: Priority of this component */
39+
int adapt_priority;
40+
41+
/* MCA parameter: Output stream and verbose level */
42+
int adapt_output;
43+
int adapt_verbose;
44+
45+
/* MCA parameter: Maximum number of segment in context free list */
46+
int adapt_context_free_list_max;
47+
48+
/* MCA parameter: Minimum number of segment in context free list */
49+
int adapt_context_free_list_min;
50+
51+
/* MCA parameter: Increasement number of segment in context free list */
52+
int adapt_context_free_list_inc;
53+
54+
/* Bcast MCA parameter */
55+
int adapt_ibcast_algorithm;
56+
size_t adapt_ibcast_segment_size;
57+
int adapt_ibcast_max_send_requests;
58+
int adapt_ibcast_max_recv_requests;
59+
/* Bcast free list */
60+
opal_free_list_t *adapt_ibcast_context_free_list;
61+
62+
/* Reduce MCA parameter */
63+
int adapt_ireduce_algorithm;
64+
size_t adapt_ireduce_segment_size;
65+
int adapt_ireduce_max_send_requests;
66+
int adapt_ireduce_max_recv_requests;
67+
int adapt_inbuf_free_list_min;
68+
int adapt_inbuf_free_list_max;
69+
int adapt_inbuf_free_list_inc;
70+
71+
/* Reduce free list */
72+
opal_free_list_t *adapt_ireduce_context_free_list;
73+
74+
} mca_coll_adapt_component_t;
75+
76+
/* Coll adapt module per communicator*/
77+
struct mca_coll_adapt_module_t {
78+
/* Base module */
79+
mca_coll_base_module_t super;
80+
81+
/* Whether this module has been lazily initialized or not yet */
82+
bool adapt_enabled;
83+
};
84+
OBJ_CLASS_DECLARATION(mca_coll_adapt_module_t);
85+
86+
/* Global component instance */
87+
OMPI_MODULE_DECLSPEC extern mca_coll_adapt_component_t mca_coll_adapt_component;
88+
89+
/* ADAPT module functions */
90+
int ompi_coll_adapt_init_query(bool enable_progress_threads, bool enable_mpi_threads);
91+
mca_coll_base_module_t * ompi_coll_adapt_comm_query(struct ompi_communicator_t *comm, int *priority);
92+
93+
/* ADAPT request free */
94+
int ompi_coll_adapt_request_free(ompi_request_t **request);
95+
96+
#endif /* MCA_COLL_ADAPT_EXPORT_H */

‎ompi/mca/coll/adapt/coll_adapt_algorithms.h

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Copyright (c) 2014-2020 The University of Tennessee and The University
3+
* of Tennessee Research Foundation. All rights
4+
* reserved.
5+
* $COPYRIGHT$
6+
*
7+
* Additional copyrights may follow
8+
*
9+
* $HEADER$
10+
*/
11+
12+
#include "ompi/mca/coll/coll.h"
13+
#include "ompi/mca/coll/base/coll_base_topo.h"
14+
#include "ompi/mca/coll/base/coll_base_functions.h"
15+
#include <math.h>
16+
17+
typedef int (*ompi_mca_coll_adapt_ibcast_function_t)(IBCAST_ARGS);
18+
typedef int (*ompi_mca_coll_adapt_ireduce_function_t)(IREDUCE_ARGS);
19+
20+
typedef struct ompi_coll_adapt_algorithm_index_s {
21+
int algorithm_index;
22+
union {
23+
ompi_mca_coll_adapt_ibcast_function_t ibcast_fn_ptr;
24+
ompi_mca_coll_adapt_ireduce_function_t ireduce_fn_ptr;
25+
};
26+
} ompi_coll_adapt_algorithm_index_t;
27+
28+
/* Bcast */
29+
int ompi_coll_adapt_ibcast_register(void);
30+
int ompi_coll_adapt_ibcast_fini(void);
31+
int ompi_coll_adapt_bcast(BCAST_ARGS);
32+
int ompi_coll_adapt_ibcast(IBCAST_ARGS);
33+
34+
/* Reduce */
35+
int ompi_coll_adapt_ireduce_register(void);
36+
int ompi_coll_adapt_ireduce_fini(void);
37+
int ompi_coll_adapt_reduce(REDUCE_ARGS);
38+
int ompi_coll_adapt_ireduce(IREDUCE_ARGS);
39+

‎ompi/mca/coll/adapt/coll_adapt_bcast.c

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Copyright (c) 2014-2020 The University of Tennessee and The University
3+
* of Tennessee Research Foundation. All rights
4+
* reserved.
5+
* $COPYRIGHT$
6+
*
7+
* Additional copyrights may follow
8+
*
9+
* $HEADER$
10+
*/
11+
12+
#include "coll_adapt.h"
13+
#include "coll_adapt_algorithms.h"
14+
15+
int ompi_coll_adapt_bcast(void *buff, int count, struct ompi_datatype_t *datatype, int root,
16+
struct ompi_communicator_t *comm, mca_coll_base_module_t * module)
17+
{
18+
ompi_request_t *request = NULL;
19+
int err = ompi_coll_adapt_ibcast(buff, count, datatype, root, comm, &request, module);
20+
if( MPI_SUCCESS != err ) {
21+
if( NULL == request )
22+
return err;
23+
}
24+
ompi_request_wait(&request, MPI_STATUS_IGNORE);
25+
return err;
26+
}

0 commit comments

Comments
 (0)