Skip to content

Commit 374e5c9

Browse files
authored
Merge pull request #9441 from bwbarrett/feature/common-ofi-delay-initializing-patcher
Clean up OFI common code and delay patcher initialization until needed
2 parents d6d033f + 2349f6b commit 374e5c9

File tree

5 files changed

+301
-205
lines changed

5 files changed

+301
-205
lines changed

config/opal_check_ofi.m4

Lines changed: 16 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -126,19 +126,26 @@ AC_DEFUN([_OPAL_CHECK_OFI],[
126126
CPPFLAGS="$CPPFLAGS $opal_ofi_CPPFLAGS"
127127

128128
AS_IF([test $opal_ofi_happy = yes],
129-
[AC_CHECK_MEMBER([struct fi_info.nic],
129+
[AC_CHECK_HEADERS([rdma/fi_ext.h])
130+
131+
AC_CHECK_MEMBER([struct fi_info.nic],
130132
[opal_check_fi_info_pci=1],
131133
[opal_check_fi_info_pci=0],
132-
[[#include <rdma/fabric.h>]])])
134+
[[#include <rdma/fabric.h>]])
135+
136+
AC_DEFINE_UNQUOTED([OPAL_OFI_PCI_DATA_AVAILABLE],
137+
[$opal_check_fi_info_pci],
138+
[check if pci data is available in ofi])
133139

134-
AC_DEFINE_UNQUOTED([OPAL_OFI_PCI_DATA_AVAILABLE],
135-
[$opal_check_fi_info_pci],
136-
[check if pci data is available in ofi])
140+
AC_CHECK_DECLS([PMIX_PACKAGE_RANK],
141+
[],
142+
[],
143+
[#include <pmix.h>])
137144

138-
AC_CHECK_DECLS([PMIX_PACKAGE_RANK],
139-
[],
140-
[],
141-
[#include <pmix.h>])
145+
AC_CHECK_TYPES([struct fi_ops_mem_monitor], [], [],
146+
[#ifdef HAVE_RDMA_FI_EXT_H
147+
#include <rdma/fi_ext.h>
148+
#endif])])
142149

143150
CPPFLAGS=$opal_check_ofi_save_CPPFLAGS
144151
LDFLAGS=$opal_check_ofi_save_LDFLAGS
@@ -157,18 +164,6 @@ AC_DEFUN([_OPAL_CHECK_OFI],[
157164
[AC_MSG_WARN([OFI libfabric support requested (via --with-ofi or --with-libfabric), but not found.])
158165
AC_MSG_ERROR([Cannot continue.])])
159166
])
160-
opal_ofi_import_monitor=no
161-
AS_IF([test $opal_ofi_happy = "yes"],
162-
[OPAL_CHECK_OFI_VERSION_GE([1,14],
163-
[opal_ofi_import_monitor=yes],
164-
[opal_ofi_import_monitor=no])])
165-
166-
167-
if test "$opal_ofi_import_monitor" = "yes"; then
168-
AC_DEFINE_UNQUOTED([OPAL_OFI_IMPORT_MONITOR_SUPPORT],1,
169-
[Whether libfabric supports monitor import])
170-
fi
171-
172167
])dnl
173168

174169

ompi/mca/mtl/ofi/mtl_ofi_component.c

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -254,9 +254,7 @@ ompi_mtl_ofi_component_register(void)
254254
MCA_BASE_VAR_SCOPE_READONLY,
255255
&ompi_mtl_ofi.num_ofi_contexts);
256256

257-
opal_common_ofi_register_mca_variables(&mca_mtl_ofi_component.super.mtl_version);
258-
259-
return OMPI_SUCCESS;
257+
return opal_common_ofi_mca_register(&mca_mtl_ofi_component.super.mtl_version);
260258
}
261259

262260

@@ -285,7 +283,7 @@ ompi_mtl_ofi_component_open(void)
285283
"provider_exclude")) {
286284
return OMPI_ERR_NOT_AVAILABLE;
287285
}
288-
return opal_common_ofi_init();
286+
return opal_common_ofi_open();
289287
}
290288

291289
static int
@@ -302,9 +300,7 @@ ompi_mtl_ofi_component_close(void)
302300
#if OPAL_CUDA_SUPPORT
303301
mca_common_cuda_fini();
304302
#endif
305-
opal_common_ofi_mca_deregister();
306-
opal_common_ofi_fini();
307-
return OMPI_SUCCESS;
303+
return opal_common_ofi_close();
308304
}
309305

310306
int
@@ -582,8 +578,6 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
582578
int universe_size;
583579
char *univ_size_str;
584580

585-
opal_common_ofi_mca_register();
586-
587581
opal_output_verbose(1, opal_common_ofi.output,
588582
"%s:%d: mtl:ofi:provider_include = \"%s\"\n",
589583
__FILE__, __LINE__, *opal_common_ofi.prov_include);
@@ -893,6 +887,20 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
893887
}
894888
}
895889

890+
/* this must be called during single threaded part of the code and
891+
* before Libfabric configures its memory monitors. Easiest to do
892+
* that before domain open. Silently ignore not-supported errors,
893+
* as they are not critical to program correctness, but only
894+
* indicate that LIbfabric will have to pick a different, possibly
895+
* less optimial, monitor. */
896+
ret = opal_common_ofi_export_memory_monitor();
897+
if (0 != ret && -FI_ENOSYS != ret) {
898+
opal_output_verbose(1, opal_common_ofi.output,
899+
"Failed to inject Libfabric memory monitor: %s",
900+
fi_strerror(-ret));
901+
}
902+
903+
896904
/**
897905
* Open fabric
898906
* The getinfo struct returns a fabric attribute struct that can be used to

opal/mca/btl/ofi/btl_ofi_component.c

Lines changed: 22 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ static int validate_info(struct fi_info *info, uint64_t required_caps, char **in
123123
/* Register the MCA parameters */
124124
static int mca_btl_ofi_component_register(void)
125125
{
126+
int ret;
126127
char *msg;
127128
mca_btl_ofi_module_t *module = &mca_btl_ofi_module_template;
128129

@@ -191,27 +192,30 @@ static int mca_btl_ofi_component_register(void)
191192
/* for now we want this component to lose to the MTL. */
192193
module->super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_HIGH - 50;
193194

194-
opal_common_ofi_register_mca_variables(&mca_btl_ofi_component.super.btl_version);
195+
ret = opal_common_ofi_mca_register(&mca_btl_ofi_component.super.btl_version);
196+
if (OPAL_SUCCESS != ret) {
197+
return ret;
198+
}
195199

196200
return mca_btl_base_param_register(&mca_btl_ofi_component.super.btl_version, &module->super);
197201
}
198202

199203
static int mca_btl_ofi_component_open(void)
200204
{
201205
mca_btl_ofi_component.module_count = 0;
202-
return opal_common_ofi_init();
206+
return opal_common_ofi_open();
203207
}
204208

205209
/*
206210
* component cleanup - sanity checking of queue lengths
207211
*/
208212
static int mca_btl_ofi_component_close(void)
209213
{
210-
opal_common_ofi_mca_deregister();
211-
opal_common_ofi_fini();
214+
int ret;
215+
ret = opal_common_ofi_close();
212216
/* If we don't sleep, sockets provider freaks out. Ummm this is a scary comment */
213217
sleep(1);
214-
return OPAL_SUCCESS;
218+
return ret;
215219
}
216220

217221
void mca_btl_ofi_exit(void)
@@ -259,8 +263,6 @@ static mca_btl_base_module_t **mca_btl_ofi_component_init(int *num_btl_modules,
259263
struct fi_domain_attr domain_attr = {0};
260264
uint64_t required_caps;
261265

262-
opal_common_ofi_mca_register();
263-
264266
switch (mca_btl_ofi_component.mode) {
265267

266268
case MCA_BTL_OFI_MODE_TWO_SIDED:
@@ -444,6 +446,19 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
444446
* to prevent races. */
445447
mca_btl_ofi_rcache_init(module);
446448

449+
/* for similar reasons to the rcache call, this must be called
450+
* during single threaded part of the code and before Libfabric
451+
* configures its memory monitors. Easiest to do that before
452+
* domain open. Silently ignore not-supported errors, as they
453+
* are not critical to program correctness, but only indicate
454+
* that LIbfabric will have to pick a different, possibly less
455+
* optimial, monitor. */
456+
rc = opal_common_ofi_export_memory_monitor();
457+
if (0 != rc && -FI_ENOSYS != rc) {
458+
BTL_VERBOSE(("Failed to inject Libfabric memory monitor: %s",
459+
fi_strerror(-rc)));
460+
}
461+
447462
linux_device_name = info->domain_attr->name;
448463
BTL_VERBOSE(
449464
("initializing dev:%s provider:%s", linux_device_name, info->fabric_attr->prov_name));

0 commit comments

Comments
 (0)