Skip to content

Commit b821e8e

Browse files
authored
Merge pull request #9694 from wzamazon/btl_remote_completion
btl: introduce flag MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION
2 parents 2eb5df4 + 8f4cda3 commit b821e8e

File tree

5 files changed

+39
-1
lines changed

5 files changed

+39
-1
lines changed

opal/mca/btl/base/btl_base_am_rdma.c

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1138,5 +1138,28 @@ int mca_btl_base_am_rdma_init(mca_btl_base_module_t *btl)
11381138
OBJ_CONSTRUCT(&default_module, mca_btl_base_am_rdma_module_t);
11391139
}
11401140

1141+
/* This section check whether we can claim support of remote completion.
1142+
*
1143+
* In terms of remote completion, we are mainly interested in put and atomic ops,
1144+
* because get, atomics fops and atomic cswap support remote completion by their nature.
1145+
*
1146+
* For active message put (AM put), the target side will send a response, and the initiator
1147+
* side will wait for the response to complete the put operation. Thus if AM put is based on send,
1148+
* it support remote completion. (If AM put is based on get, it does not support remote
1149+
* completion because the target side does not wait for get's completion to send response).
1150+
*
1151+
* active message RDMA/atomics does not implement atomic ops. User was suppose to
1152+
* use atomic fops (unless the btl support atomic ops natively).
1153+
*
1154+
* In all, the conditions for AM rdma to claim support of remote completion are:
1155+
* 1. AM put is enabled (which means the btl does not support put)
1156+
* 2. AM put does not use get (so it must use send)
1157+
* 3. btl does not have native atomics ops support.
1158+
*/
1159+
if ((btl->btl_flags & MCA_BTL_FLAGS_PUT_AM) && !mca_btl_base_rdma_use_rdma_get(btl) &&
1160+
!(btl->btl_flags & MCA_BTL_FLAGS_ATOMIC_OPS)) {
1161+
btl->btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION;
1162+
}
1163+
11411164
return OPAL_SUCCESS;
11421165
}

opal/mca/btl/btl.h

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,16 @@ typedef uint8_t mca_btl_base_tag_t;
263263
/* The BTL has active-message based atomics */
264264
#define MCA_BTL_FLAGS_ATOMIC_AM_FOP 0x400000
265265

266+
/** Ths BTL's RDMA/atomics operation supports remote completion.
267+
* When the BTL reported the completion of a RDMA/atomic operation
268+
* on the initator side, the operation also finished on the target side.
269+
*
270+
* Note, this flag is for put and atomic write operations. Operations
271+
* like get, atomic fetch and atomic swap support remote
272+
* completion by nature.
273+
*/
274+
#define MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION 0x800000
275+
266276
/* Default exclusivity levels */
267277
#define MCA_BTL_EXCLUSIVITY_HIGH (64 * 1024) /* internal loopback */
268278
#define MCA_BTL_EXCLUSIVITY_DEFAULT 1024 /* GM/IB/etc. */

opal/mca/btl/ofi/btl_ofi_module.c

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -390,8 +390,10 @@ mca_btl_ofi_module_t *mca_btl_ofi_module_alloc(int mode)
390390
module->super.btl_register_mem = mca_btl_ofi_register_mem;
391391
module->super.btl_deregister_mem = mca_btl_ofi_deregister_mem;
392392

393+
/* btl/ofi support remote completion because it required FI_DELIVERY_COMPLETE capability
394+
*/
393395
module->super.btl_flags |= MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS
394-
| MCA_BTL_FLAGS_RDMA;
396+
| MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION;
395397

396398
module->super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_SWAP
397399
| MCA_BTL_ATOMIC_SUPPORTS_CSWAP

opal/mca/btl/self/btl_self_component.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ static int mca_btl_self_component_register(void)
107107
mca_btl_self.btl_rdma_pipeline_frag_size = INT_MAX;
108108
mca_btl_self.btl_min_rdma_pipeline_size = 0;
109109
mca_btl_self.btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND;
110+
/* for self, remote completion is local completion */
111+
mca_btl_self.btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION;
110112
mca_btl_self.btl_bandwidth = 100;
111113
mca_btl_self.btl_latency = 0;
112114
mca_btl_base_param_register(&mca_btl_self_component.super.btl_version, &mca_btl_self);

opal/mca/btl/ugni/btl_ugni_component.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,7 @@ static int btl_ugni_component_register(void)
469469
mca_btl_ugni_module.super.btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA
470470
| MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_ATOMIC_OPS
471471
| MCA_BTL_FLAGS_ATOMIC_FOPS;
472+
mca_btl_ugni_module.super.btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION;
472473
mca_btl_ugni_module.super.btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD
473474
| MCA_BTL_ATOMIC_SUPPORTS_AND
474475
| MCA_BTL_ATOMIC_SUPPORTS_OR

0 commit comments

Comments
 (0)