File tree Expand file tree Collapse file tree 5 files changed +39
-1
lines changed Expand file tree Collapse file tree 5 files changed +39
-1
lines changed Original file line number Diff line number Diff line change @@ -1138,5 +1138,28 @@ int mca_btl_base_am_rdma_init(mca_btl_base_module_t *btl)
1138
1138
OBJ_CONSTRUCT (& default_module , mca_btl_base_am_rdma_module_t );
1139
1139
}
1140
1140
1141
+ /* This section check whether we can claim support of remote completion.
1142
+ *
1143
+ * In terms of remote completion, we are mainly interested in put and atomic ops,
1144
+ * because get, atomics fops and atomic cswap support remote completion by their nature.
1145
+ *
1146
+ * For active message put (AM put), the target side will send a response, and the initiator
1147
+ * side will wait for the response to complete the put operation. Thus if AM put is based on send,
1148
+ * it support remote completion. (If AM put is based on get, it does not support remote
1149
+ * completion because the target side does not wait for get's completion to send response).
1150
+ *
1151
+ * active message RDMA/atomics does not implement atomic ops. User was suppose to
1152
+ * use atomic fops (unless the btl support atomic ops natively).
1153
+ *
1154
+ * In all, the conditions for AM rdma to claim support of remote completion are:
1155
+ * 1. AM put is enabled (which means the btl does not support put)
1156
+ * 2. AM put does not use get (so it must use send)
1157
+ * 3. btl does not have native atomics ops support.
1158
+ */
1159
+ if ((btl -> btl_flags & MCA_BTL_FLAGS_PUT_AM ) && !mca_btl_base_rdma_use_rdma_get (btl ) &&
1160
+ !(btl -> btl_flags & MCA_BTL_FLAGS_ATOMIC_OPS )) {
1161
+ btl -> btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION ;
1162
+ }
1163
+
1141
1164
return OPAL_SUCCESS ;
1142
1165
}
Original file line number Diff line number Diff line change @@ -263,6 +263,16 @@ typedef uint8_t mca_btl_base_tag_t;
263
263
/* The BTL has active-message based atomics */
264
264
#define MCA_BTL_FLAGS_ATOMIC_AM_FOP 0x400000
265
265
266
+ /** Ths BTL's RDMA/atomics operation supports remote completion.
267
+ * When the BTL reported the completion of a RDMA/atomic operation
268
+ * on the initator side, the operation also finished on the target side.
269
+ *
270
+ * Note, this flag is for put and atomic write operations. Operations
271
+ * like get, atomic fetch and atomic swap support remote
272
+ * completion by nature.
273
+ */
274
+ #define MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION 0x800000
275
+
266
276
/* Default exclusivity levels */
267
277
#define MCA_BTL_EXCLUSIVITY_HIGH (64 * 1024) /* internal loopback */
268
278
#define MCA_BTL_EXCLUSIVITY_DEFAULT 1024 /* GM/IB/etc. */
Original file line number Diff line number Diff line change @@ -390,8 +390,10 @@ mca_btl_ofi_module_t *mca_btl_ofi_module_alloc(int mode)
390
390
module -> super .btl_register_mem = mca_btl_ofi_register_mem ;
391
391
module -> super .btl_deregister_mem = mca_btl_ofi_deregister_mem ;
392
392
393
+ /* btl/ofi support remote completion because it required FI_DELIVERY_COMPLETE capability
394
+ */
393
395
module -> super .btl_flags |= MCA_BTL_FLAGS_ATOMIC_FOPS | MCA_BTL_FLAGS_ATOMIC_OPS
394
- | MCA_BTL_FLAGS_RDMA ;
396
+ | MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION ;
395
397
396
398
module -> super .btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD | MCA_BTL_ATOMIC_SUPPORTS_SWAP
397
399
| MCA_BTL_ATOMIC_SUPPORTS_CSWAP
Original file line number Diff line number Diff line change @@ -107,6 +107,8 @@ static int mca_btl_self_component_register(void)
107
107
mca_btl_self .btl_rdma_pipeline_frag_size = INT_MAX ;
108
108
mca_btl_self .btl_min_rdma_pipeline_size = 0 ;
109
109
mca_btl_self .btl_flags = MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_SEND ;
110
+ /* for self, remote completion is local completion */
111
+ mca_btl_self .btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION ;
110
112
mca_btl_self .btl_bandwidth = 100 ;
111
113
mca_btl_self .btl_latency = 0 ;
112
114
mca_btl_base_param_register (& mca_btl_self_component .super .btl_version , & mca_btl_self );
Original file line number Diff line number Diff line change @@ -469,6 +469,7 @@ static int btl_ugni_component_register(void)
469
469
mca_btl_ugni_module .super .btl_flags = MCA_BTL_FLAGS_SEND | MCA_BTL_FLAGS_RDMA
470
470
| MCA_BTL_FLAGS_SEND_INPLACE | MCA_BTL_FLAGS_ATOMIC_OPS
471
471
| MCA_BTL_FLAGS_ATOMIC_FOPS ;
472
+ mca_btl_ugni_module .super .btl_flags |= MCA_BTL_FLAGS_RDMA_REMOTE_COMPLETION ;
472
473
mca_btl_ugni_module .super .btl_atomic_flags = MCA_BTL_ATOMIC_SUPPORTS_ADD
473
474
| MCA_BTL_ATOMIC_SUPPORTS_AND
474
475
| MCA_BTL_ATOMIC_SUPPORTS_OR
You can’t perform that action at this time.
0 commit comments