btl/ofi: add scalable endpoint support.

This commit add support for scalable endpoint to enhance multithreaded
application performance. The BTL will detect the support from ofi
provider and will fallback to normal usage of scalable endpoint is not
supported.

NEW MCA parameters:
- mca_btl_ofi_disable_sep: force the btl to not use scalable endpoint.
- mca_btl_ofi_num_contexts_per_module: number of communication context
  to create (should be the same as number of thread).

Signed-off-by: Thananon Patinyasakdikul <thananon.patinyasakdikul@intel.com>

Author: Thananon Patinyasakdikul

opal/mca/btl/ofi/btl_ofi.h

@@ -46,9 +46,7 @@
 #include <rdma/fi_rma.h>
 
 BEGIN_C_DECLS
-
-#define MCA_BTL_OFI_MAX_MODULES 16
-#define MCA_BTL_OFI_MAX_WORKERS 1
+#define MCA_BTL_OFI_MAX_MODULES         16
 #define MCA_BTL_OFI_MAX_CQ_READ_ENTRIES 128
 
 #define MCA_BTL_OFI_ABORT(args)     mca_btl_ofi_exit(args)
@@ -62,6 +60,26 @@ enum mca_btl_ofi_type {
     MCA_BTL_OFI_TYPE_TOTAL
 };
 
+struct mca_btl_ofi_context_t {
+    int32_t context_id;
+
+    /* transmit context */
+    struct fid_ep *tx_ctx;
+    struct fid_ep *rx_ctx;
+
+    /* completion queue */
+    struct fid_cq *cq;
+
+    /* completion info freelist */
+    /* We have it per context to reduce the thread contention
+     * on the freelist. Things can get really slow. */
+    opal_free_list_t comp_list;
+
+    /* for thread locking */
+    volatile int32_t lock;
+};
+typedef struct mca_btl_ofi_context_t mca_btl_ofi_context_t;
+
 /**
  * @brief OFI BTL module
  */
@@ -74,26 +92,25 @@ struct mca_btl_ofi_module_t {
     struct fid_fabric *fabric;
     struct fid_domain *domain;
     struct fid_ep *ofi_endpoint;
-    struct fid_cq *cq;
     struct fid_av *av;
 
+    int num_contexts;
+    mca_btl_ofi_context_t *contexts;
+
     char *linux_device_name;
 
     /** whether the module has been fully initialized or not */
     bool initialized;
     bool use_virt_addr;
-
-    /** spin-lock to protect the module */
-    volatile int32_t lock;
+    bool is_scalable_ep;
 
     int64_t outstanding_rdma;
 
     /** linked list of BTL endpoints. this list is never searched so
      * there is no need for a complicated structure here at this time*/
     opal_list_t endpoints;
 
-    /* free lists */
-    opal_free_list_t comp_list;
+    opal_mutex_t module_lock;
 
     /** registration cache */
     mca_rcache_base_module_t *rcache;
@@ -110,17 +127,14 @@ struct mca_btl_ofi_component_t {
 
     /** number of TL modules */
     int module_count;
+    int num_contexts_per_module;
     int num_cqe_read;
 
     size_t namelen;
 
     /** All BTL OFI modules (1 per tl) */
     mca_btl_ofi_module_t *modules[MCA_BTL_OFI_MAX_MODULES];
 
-#if OPAL_C_HAVE__THREAD_LOCAL
-    /** bind threads to contexts */
-    bool bind_threads_to_contexts;
-#endif
 };
 typedef struct mca_btl_ofi_component_t mca_btl_ofi_component_t;
 
@@ -151,6 +165,7 @@ struct mca_btl_ofi_completion_t {
 
     struct mca_btl_base_module_t *btl;
     struct mca_btl_base_endpoint_t *endpoint;
+    struct mca_btl_ofi_context_t *my_context;
     uint32_t type;
 
     void *local_address;
@@ -269,7 +284,25 @@ int mca_btl_ofi_reg_mem (void *reg_data, void *base, size_t size,
                          mca_rcache_base_registration_t *reg);
 int mca_btl_ofi_dereg_mem (void *reg_data, mca_rcache_base_registration_t *reg);
 
+int mca_btl_ofi_context_progress(mca_btl_ofi_context_t *context);
 void mca_btl_ofi_exit(void);
 
+/* thread atomics */
+static inline bool mca_btl_ofi_context_trylock (mca_btl_ofi_context_t *context)
+{
+    return (context->lock || OPAL_ATOMIC_SWAP_32(&context->lock, 1));
+}
+
+static inline void mca_btl_ofi_context_lock(mca_btl_ofi_context_t *context)
+{
+    while (mca_btl_ofi_context_trylock(context));
+}
+
+static inline void mca_btl_ofi_context_unlock(mca_btl_ofi_context_t *context)
+{
+    opal_atomic_mb();
+    context->lock = 0;
+}
+
 END_C_DECLS
 #endif

opal/mca/btl/ofi/btl_ofi_atomics.c

@@ -43,6 +43,9 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
     mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
     mca_btl_ofi_completion_t *comp = NULL;
+    mca_btl_ofi_context_t *ofi_context;
+
+    ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
         fi_datatype = FI_UINT32;
@@ -51,6 +54,7 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
     fi_op = to_fi_op(op);
 
     comp = mca_btl_ofi_completion_alloc(btl, endpoint,
+                                        ofi_context,
                                         local_address,
                                         local_handle,
                                         cbfunc, cbcontext, cbdata,
@@ -61,7 +65,7 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
 
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
-    rc = fi_fetch_atomic(ofi_btl->ofi_endpoint,
+    rc = fi_fetch_atomic(ofi_context->tx_ctx,
                          (void*) &comp->operand, 1, NULL,       /* operand */
                          local_address, local_handle->desc,     /* results */
                          btl_endpoint->peer_addr,               /* remote addr */
@@ -77,6 +81,9 @@ int mca_btl_ofi_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
 
     MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
 
+    /* force a bit of progress. */
+    mca_btl_ofi_component.super.btl_progress();
+
     return OPAL_SUCCESS;
 }
 
@@ -92,6 +99,9 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
     mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
     mca_btl_ofi_completion_t *comp = NULL;
+    mca_btl_ofi_context_t *ofi_context;
+
+    ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
         fi_datatype = FI_UINT32;
@@ -100,6 +110,7 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
     fi_op = to_fi_op(op);
 
     comp = mca_btl_ofi_completion_alloc(btl, endpoint,
+                                        ofi_context,
                                         NULL,
                                         NULL,
                                         cbfunc, cbcontext, cbdata,
@@ -110,7 +121,7 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
 
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
-    rc = fi_atomic(ofi_btl->ofi_endpoint,
+    rc = fi_atomic(ofi_context->tx_ctx,
                    (void*) &comp->operand, 1, NULL,       /* operand */
                    btl_endpoint->peer_addr,               /* remote addr */
                    remote_address, remote_handle->rkey,   /* remote buffer */
@@ -124,6 +135,7 @@ int mca_btl_ofi_aop (struct mca_btl_base_module_t *btl, mca_btl_base_endpoint_t
     }
 
     MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
+    mca_btl_ofi_component.super.btl_progress();
 
     return OPAL_SUCCESS;
 }
@@ -139,12 +151,16 @@ int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
     mca_btl_ofi_module_t *ofi_btl = (mca_btl_ofi_module_t *) btl;
     mca_btl_ofi_endpoint_t *btl_endpoint = (mca_btl_ofi_endpoint_t*) endpoint;
     mca_btl_ofi_completion_t *comp = NULL;
+    mca_btl_ofi_context_t *ofi_context;
+
+    ofi_context = get_ofi_context(ofi_btl);
 
     if (flags & MCA_BTL_ATOMIC_FLAG_32BIT) {
         fi_datatype = FI_UINT32;
     }
 
     comp = mca_btl_ofi_completion_alloc(btl, endpoint,
+                                        ofi_context,
                                         local_address,
                                         local_handle,
                                         cbfunc, cbcontext, cbdata,
@@ -157,7 +173,7 @@ int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
     remote_address = (remote_address - (uint64_t) remote_handle->base_addr);
 
     /* perform atomic */
-    rc = fi_compare_atomic(ofi_btl->ofi_endpoint,
+    rc = fi_compare_atomic(ofi_context->tx_ctx,
                            (void*) &comp->operand, 1, NULL,
                            (void*) &comp->compare, NULL,
                            local_address, local_handle->desc,
@@ -176,5 +192,8 @@ int mca_btl_ofi_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_e
 
     MCA_BTL_OFI_NUM_RDMA_INC(ofi_btl);
 
+    /* force a bit of progress. */
+    mca_btl_ofi_component.super.btl_progress();
+
     return OPAL_SUCCESS;
 }