Merge pull request #5274 from thananon/ofi_sep

btl/ofi: add scalable endpoint support.

Author: Thananon Patinyasakdikul

opal/mca/btl/ofi/btl_ofi.h

@@ -46,9 +46,7 @@
 #include <rdma/fi_rma.h>
 
 BEGIN_C_DECLS
-
-#define MCA_BTL_OFI_MAX_MODULES 16
-#define MCA_BTL_OFI_MAX_WORKERS 1
+#define MCA_BTL_OFI_MAX_MODULES         16
 #define MCA_BTL_OFI_MAX_CQ_READ_ENTRIES 128
 
 #define MCA_BTL_OFI_ABORT(args)     mca_btl_ofi_exit(args)
@@ -62,6 +60,26 @@ enum mca_btl_ofi_type {
     MCA_BTL_OFI_TYPE_TOTAL
 };
 
+struct mca_btl_ofi_context_t {
+    int32_t context_id;
+
+    /* transmit context */
+    struct fid_ep *tx_ctx;
+    struct fid_ep *rx_ctx;
+
+    /* completion queue */
+    struct fid_cq *cq;
+
+    /* completion info freelist */
+    /* We have it per context to reduce the thread contention
+     * on the freelist. Things can get really slow. */
+    opal_free_list_t comp_list;
+
+    /* for thread locking */
+    volatile int32_t lock;
+};
+typedef struct mca_btl_ofi_context_t mca_btl_ofi_context_t;
+
 /**
  * @brief OFI BTL module
  */
@@ -74,26 +92,25 @@ struct mca_btl_ofi_module_t {
     struct fid_fabric *fabric;
     struct fid_domain *domain;
     struct fid_ep *ofi_endpoint;
-    struct fid_cq *cq;
     struct fid_av *av;
 
+    int num_contexts;
+    mca_btl_ofi_context_t *contexts;
+
     char *linux_device_name;
 
     /** whether the module has been fully initialized or not */
     bool initialized;
     bool use_virt_addr;
-
-    /** spin-lock to protect the module */
-    volatile int32_t lock;
+    bool is_scalable_ep;
 
     int64_t outstanding_rdma;
 
     /** linked list of BTL endpoints. this list is never searched so
      * there is no need for a complicated structure here at this time*/
     opal_list_t endpoints;
 
-    /* free lists */
-    opal_free_list_t comp_list;
+    opal_mutex_t module_lock;
 
     /** registration cache */
     mca_rcache_base_module_t *rcache;
@@ -110,17 +127,14 @@ struct mca_btl_ofi_component_t {
 
     /** number of TL modules */
     int module_count;
+    int num_contexts_per_module;
     int num_cqe_read;
 
     size_t namelen;
 
     /** All BTL OFI modules (1 per tl) */
     mca_btl_ofi_module_t *modules[MCA_BTL_OFI_MAX_MODULES];
 
-#if OPAL_C_HAVE__THREAD_LOCAL
-    /** bind threads to contexts */
-    bool bind_threads_to_contexts;
-#endif
 };
 typedef struct mca_btl_ofi_component_t mca_btl_ofi_component_t;
 
@@ -151,6 +165,7 @@ struct mca_btl_ofi_completion_t {
 
     struct mca_btl_base_module_t *btl;
     struct mca_btl_base_endpoint_t *endpoint;
+    struct mca_btl_ofi_context_t *my_context;
     uint32_t type;
 
     void *local_address;
@@ -269,7 +284,25 @@ int mca_btl_ofi_reg_mem (void *reg_data, void *base, size_t size,
                          mca_rcache_base_registration_t *reg);
 int mca_btl_ofi_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg);
 
+int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context);
 void mca_btl_ofi_exit(void);
 
+/* thread atomics */
+static inline bool mca_btl_ofi_context_trylock (mca_btl_ofi_context_t *context)
+{
+    return (context->lock || OPAL_ATOMIC_SWAP_32(&context->lock, 1));
+}
+
+static inline void mca_btl_ofi_context_lock(mca_btl_ofi_context_t *context)
+{
+    while (mca_btl_ofi_context_trylock(context));
+}
+
+static inline void mca_btl_ofi_context_unlock(mca_btl_ofi_context_t *context)
+{
+    opal_atomic_mb();
+    context->lock = 0;
+}
+
 END_C_DECLS
 #endif

opal/mca/btl/ofi/btl_ofi_atomics.c

@@ -43,6 +43,9 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
     mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
     mca_btl_ofi_completion_t *comp = NULL;
+    mca_btl_ofi_context_t *ofi_context;
+
+    ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
         fi_datatype = FI_UINT32;
@@ -51,6 +54,7 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
     fi_op = to_fi_op(op);
 
     comp = mca_btl_ofi_completion_alloc(btl, endpoint,
+                                        ofi_context,
                                         local_address,
                                         local_handle,
                                         cbfunc, cbcontext, cbdata,
@@ -61,7 +65,7 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
 
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
-    rc = fi_fetch_atomic(ofi_btl->ofi_endpoint,
+    rc = fi_fetch_atomic(ofi_context->tx_ctx,
                          (void*) &comp->operand, 1, NULL,       /* operand */
                          local_address, local_handle->desc,     /* results */
                          btl_endpoint->peer_addr,               /* remote addr */
@@ -77,6 +81,9 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
 
     MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
 
+    /* force a bit of progress. */
+    mca_btl_ofi_component.super.btl_progress();
+
     return OPAL_SUCCESS;
 }
 
@@ -92,6 +99,9 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
     mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
     mca_btl_ofi_completion_t *comp = NULL;
+    mca_btl_ofi_context_t *ofi_context;
+
+    ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
         fi_datatype = FI_UINT32;
@@ -100,6 +110,7 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
     fi_op = to_fi_op(op);
 
     comp = mca_btl_ofi_completion_alloc(btl, endpoint,
+                                        ofi_context,
                                         NULL,
                                         NULL,
                                         cbfunc, cbcontext, cbdata,
@@ -110,7 +121,7 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
 
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
-    rc = fi_atomic(ofi_btl->ofi_endpoint,
+    rc = fi_atomic(ofi_context->tx_ctx,
                    (void*) &comp->operand, 1, NULL,       /* operand */
                    btl_endpoint->peer_addr,               /* remote addr */
                    remote_address, remote_handle->rkey,   /* remote buffer */
@@ -124,6 +135,7 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
     }
 
     MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
+    mca_btl_ofi_component.super.btl_progress();
 
     return OPAL_SUCCESS;
 }
@@ -139,12 +151,16 @@ int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
     mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
     mca_btl_ofi_completion_t *comp = NULL;
+    mca_btl_ofi_context_t *ofi_context;
+
+    ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
         fi_datatype = FI_UINT32;
     }
 
     comp = mca_btl_ofi_completion_alloc(btl, endpoint,
+                                        ofi_context,
                                         local_address,
                                         local_handle,
                                         cbfunc, cbcontext, cbdata,
@@ -157,7 +173,7 @@ int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
     /* perform atomic */
-    rc = fi_compare_atomic(ofi_btl->ofi_endpoint,
+    rc = fi_compare_atomic(ofi_context->tx_ctx,
                            (void*) &comp->operand, 1, NULL,
                            (void*) &comp->compare, NULL,
                            local_address, local_handle->desc,
@@ -176,5 +192,8 @@ int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
 
     MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
 
+    /* force a bit of progress. */
+    mca_btl_ofi_component.super.btl_progress();
+
     return OPAL_SUCCESS;
 }