btl/ugni: improve multi-threaded RDMA performance

This commit improves the injection rate and latency for RDMA
operations. This is done by the following improvements:

 - If C11's _Thread_local keyword is available then always use the
   same virtual device index for the same thread when using RDMA. If
   the keyword is not available then attempt to use any device that
   isn't already in use. The binding support is enabled by default but
   can be disabled via the btl_ugni_bind_devices MCA variable.

 - When posting FMA and RDMA operations always attempt to reap
   completions after posting the operation. This allows us to
   better balance the work of reaping completions across all
   application threads.

 - Limit the total number of outstanding BTE transactions. This
   fixes a performance bug when using many threads.

 - Split out RDMA and local SMSG completion queue sizes. The RDMA
   queue size is better tuned for performance with RMA-MT.

 - Split out put and get FMA limits. The old btl_ugni_fma_limit MCA
   variable is deprecated. The new variable names are:
   btl_ugni_fma_put_limit and btl_ugni_fma_get_limit.

 - Change how post descriptors are handled. They are no longer
   allocated seperately from the RDMA endpoints.

 - Some cleanup to move error code out of the critical path.

 - Disable the FMA sharing flag on the CDM when we detect that there
   should be enough FMA descriptors for the number of virtual devices
   we plan will create. If the user sets this flag we will not unset
   it. This change should improve the small-message RMA performance by
   ~ 10%.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>

Author: hjelmn

opal/mca/btl/ugni/btl_ugni.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2011-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
  * Copyright (c) 2014      Research Organization for Information Science
@@ -51,7 +51,7 @@
 #define MCA_BTL_UGNI_MAX_DEV_HANDLES 128
 
 /** number of rdma completion queue items to remove per progress loop */
-#define MCA_BTL_UGNI_COMPLETIONS_PER_LOOP 16
+#define MCA_BTL_UGNI_COMPLETIONS_PER_LOOP 32
 
 /** how often to check for connection requests */
 #define MCA_BTL_UGNI_CONNECT_USEC 10
@@ -96,7 +96,7 @@ struct mca_btl_ugni_cq_t {
     /** ugni CQ handle */
     gni_cq_handle_t gni_handle;
     /** number of completions expected on the CQ */
-    int32_t active_operations;
+    volatile int32_t active_operations;
 };
 typedef struct mca_btl_ugni_cq_t mca_btl_ugni_cq_t;
 
@@ -116,6 +116,9 @@ struct mca_btl_ugni_device_t {
     /** number of SMSG connections */
     volatile int32_t smsg_connections;
 
+    /** boolean indicating that the device was recently flushed */
+    volatile bool flushed;
+
     /** uGNI device handle */
     gni_nic_handle_t dev_handle;
 
@@ -132,10 +135,7 @@ struct mca_btl_ugni_device_t {
     gni_mem_handle_t smsg_irq_mhndl;
 
     /** RDMA endpoint free list */
-    opal_free_list_t endpoints;
-
-    /** post descriptors pending resources */
-    opal_list_t pending_post;
+    opal_free_list_t rdma_descs;
 };
 typedef struct mca_btl_ugni_device_t mca_btl_ugni_device_t;
 
@@ -162,8 +162,6 @@ typedef struct mca_btl_ugni_module_t {
     opal_mutex_t eager_get_pending_lock;
     opal_list_t eager_get_pending;
 
-    opal_free_list_t post_descriptors;
-
     mca_mpool_base_module_t *mpool;
     opal_free_list_t         smsg_mboxes;
 
@@ -196,9 +194,7 @@ typedef struct mca_btl_ugni_module_t {
      * this rank should be limited too */
     int nlocal_procs;
 
-    volatile int active_send_count;
-    volatile int64_t connected_peer_count;
-    volatile int64_t active_rdma_count;
+    volatile int32_t active_rdma_count;
 
     mca_rcache_base_module_t *rcache;
 } mca_btl_ugni_module_t;
@@ -212,6 +208,10 @@ typedef struct mca_btl_ugni_component_t {
     /* Maximum number of entries a completion queue can hold */
     uint32_t remote_cq_size;
     uint32_t local_cq_size;
+    uint32_t local_rdma_cq_size;
+    /* There is a hardware limitation that hurts BTE performance
+     * if we submit too many BTE requests. This acts as a throttle. */
+    int32_t active_rdma_threshold;
 
     /* number of ugni modules */
     uint32_t ugni_num_btls;
@@ -221,7 +221,16 @@ typedef struct mca_btl_ugni_component_t {
     size_t smsg_max_data;
 
     /* After this message size switch to BTE protocols */
-    size_t ugni_fma_limit;
+    long int ugni_fma_limit;
+    /** FMA switchover for get */
+    long int ugni_fma_get_limit;
+    /** FMA switchover for put */
+    long int ugni_fma_put_limit;
+
+#if OPAL_C_HAVE__THREAD_LOCAL
+    bool bind_threads_to_devices;
+#endif
+
     /* Switch to get when sending above this size */
     size_t ugni_smsg_limit;
 
@@ -282,25 +291,30 @@ typedef struct mca_btl_ugni_component_t {
 
     /** NIC address */
     uint32_t dev_addr;
+
+    /** MCA variable identifier for the cdm_flags variable */
+    int cdm_flags_id;
 } mca_btl_ugni_component_t;
 
 /* Global structures */
 
 OPAL_MODULE_DECLSPEC extern mca_btl_ugni_component_t mca_btl_ugni_component;
 OPAL_MODULE_DECLSPEC extern mca_btl_ugni_module_t mca_btl_ugni_module;
 
-/**
- * Get a virtual device for communication
- */
-static inline mca_btl_ugni_device_t *mca_btl_ugni_ep_get_device (mca_btl_ugni_module_t *ugni_module)
+static inline uint32_t mca_btl_ugni_ep_get_device_index (mca_btl_ugni_module_t *ugni_module)
 {
     static volatile uint32_t device_index = (uint32_t) 0;
-    uint32_t dev_index;
 
     /* don't really care if the device index is atomically updated */
-    dev_index = (device_index++) & (mca_btl_ugni_component.virtual_device_count - 1);
+    return opal_atomic_fetch_add_32 (&device_index, 1) % mca_btl_ugni_component.virtual_device_count;
+}
 
-    return ugni_module->devices + dev_index;
+/**
+ * Get a virtual device for communication
+ */
+static inline mca_btl_ugni_device_t *mca_btl_ugni_ep_get_device (mca_btl_ugni_module_t *ugni_module)
+{
+    return ugni_module->devices + mca_btl_ugni_ep_get_device_index (ugni_module);
 }
 
 static inline int mca_btl_rc_ugni_to_opal (gni_return_t rc)
@@ -322,6 +336,9 @@ static inline int mca_btl_rc_ugni_to_opal (gni_return_t rc)
     return codes[rc];
 }
 
+
+int mca_btl_ugni_flush (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint);
+
 /**
  * BML->BTL notification of change in the process list.
  *
@@ -481,6 +498,16 @@ static inline uint64_t mca_btl_ugni_proc_name_to_id (opal_process_name_t name) {
 int mca_btl_ugni_spawn_progress_thread(struct mca_btl_base_module_t* btl);
 int mca_btl_ugni_kill_progress_thread(void);
 
+struct mca_btl_ugni_post_descriptor_t;
+
+void btl_ugni_dump_post_desc (struct mca_btl_ugni_post_descriptor_t *desc);
+
+
+struct mca_btl_ugni_post_descriptor_t;
+
+void mca_btl_ugni_handle_rdma_completions (mca_btl_ugni_module_t *ugni_module, mca_btl_ugni_device_t *device,
+                                           struct mca_btl_ugni_post_descriptor_t *post_desc, const int count);
+
 /**
  * Try to lock a uGNI device for exclusive access
  */
@@ -531,6 +558,58 @@ static inline intptr_t mca_btl_ugni_device_serialize (mca_btl_ugni_device_t *dev
     return rc;
 }
 
+static inline intptr_t mca_btl_ugni_device_serialize_any (mca_btl_ugni_module_t *ugni_module,
+                                                          mca_btl_ugni_device_serialize_fn_t fn, void *arg)
+{
+    mca_btl_ugni_device_t *device;
+    intptr_t rc;
+
+    if (!opal_using_threads ()) {
+        return fn (ugni_module->devices, arg);
+    }
+
+#if OPAL_C_HAVE__THREAD_LOCAL
+    if (mca_btl_ugni_component.bind_threads_to_devices) {
+        /* NTH: if we have C11 _Thread_local just go ahead and assign the devices round-robin to each
+         * thread. in testing this give much better performance than just picking any device */
+        static _Thread_local mca_btl_ugni_device_t *device_local = NULL;
+
+        device = device_local;
+        if (OPAL_UNLIKELY(NULL == device)) {
+            /* assign device contexts round-robin */
+            device_local = device = mca_btl_ugni_ep_get_device (ugni_module);
+        }
+
+        mca_btl_ugni_device_lock (device);
+    } else {
+#endif
+        /* get the next starting index */
+        uint32_t device_index = mca_btl_ugni_ep_get_device_index (ugni_module);
+        const int device_count = mca_btl_ugni_component.virtual_device_count;
+
+        for (int i = 0 ; i < device_count ; ++i) {
+            device = ugni_module->devices + ((device_index + i) % device_count);
+            if (!mca_btl_ugni_device_trylock (device)) {
+                break;
+            }
+
+            device = NULL;
+        }
+
+        if (NULL == device) {
+            device = mca_btl_ugni_ep_get_device (ugni_module);
+            mca_btl_ugni_device_lock (device);
+        }
+#if OPAL_C_HAVE__THREAD_LOCAL
+    }
+#endif
+
+    rc = fn (device, arg);
+    mca_btl_ugni_device_unlock (device);
+
+    return rc;
+}
+
 
 /** Number of times the progress thread has woken up */
 extern unsigned int mca_btl_ugni_progress_thread_wakeups;

opal/mca/btl/ugni/btl_ugni_add_procs.c

@@ -79,7 +79,7 @@ int mca_btl_ugni_add_procs (struct mca_btl_base_module_t* btl, size_t nprocs,
     if (false == ugni_module->initialized) {
         for (int i = 0 ; i < mca_btl_ugni_component.virtual_device_count ; ++i) {
             mca_btl_ugni_device_t *device = ugni_module->devices + i;
-            rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_cq_size, 0,
+            rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_rdma_cq_size, 0,
                                GNI_CQ_NOBLOCK, NULL, NULL, &device->dev_rdma_local_cq.gni_handle);
             if (GNI_RC_SUCCESS != rc) {
                 BTL_ERROR(("error creating local BTE/FMA CQ"));
@@ -94,7 +94,7 @@ int mca_btl_ugni_add_procs (struct mca_btl_base_module_t* btl, size_t nprocs,
             }
 
             if (mca_btl_ugni_component.progress_thread_enabled) {
-                rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_cq_size,
+                rc = GNI_CqCreate (device->dev_handle, mca_btl_ugni_component.local_rdma_cq_size,
                                    0, GNI_CQ_BLOCKING, NULL, NULL, &device->dev_rdma_local_irq_cq.gni_handle);
                 if (GNI_RC_SUCCESS != rc) {
                     BTL_ERROR(("error creating local BTE/FMA CQ"));
@@ -448,15 +448,6 @@ mca_btl_ugni_setup_mpools (mca_btl_ugni_module_t *ugni_module)
         return rc;
     }
 
-    rc = opal_free_list_init (&ugni_module->post_descriptors,
-                              sizeof (mca_btl_ugni_post_descriptor_t),
-                              8, OBJ_CLASS(mca_btl_ugni_post_descriptor_t),
-                              0, 0, 0, -1, 256, NULL, 0, NULL, NULL, NULL);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        BTL_ERROR(("error creating post descriptor free list"));
-        return rc;
-    }
-
     return OPAL_SUCCESS;
 }
 

opal/mca/btl/ugni/btl_ugni_atomic.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * $COPYRIGHT$
  *
@@ -79,8 +79,8 @@ int mca_btl_ugni_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
                       mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
     gni_mem_handle_t dummy = {0, 0};
-    mca_btl_ugni_post_descriptor_t *post_desc;
-    int gni_op, rc, type;
+    mca_btl_ugni_post_descriptor_t post_desc;
+    int gni_op, type;
     size_t size;
 
     size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
@@ -95,23 +95,13 @@ int mca_btl_ugni_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_end
         return OPAL_ERR_NOT_SUPPORTED;
     }
 
-    post_desc = mca_btl_ugni_alloc_post_descriptor (endpoint, NULL, cbfunc, cbcontext, cbdata);
-    if (OPAL_UNLIKELY(NULL == post_desc)) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-    init_gni_post_desc (post_desc, order, GNI_POST_AMO, 0, dummy, remote_address,
-                        remote_handle->gni_handle, size, 0);
-    post_desc->desc.amo_cmd = gni_op;
-
-    post_desc->desc.first_operand = operand;
-
-    rc = mca_btl_ugni_endpoint_post_fma (endpoint, post_desc);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        mca_btl_ugni_return_post_descriptor (post_desc);
-    }
+    init_post_desc (&post_desc, endpoint, order, GNI_POST_AMO, 0, dummy, remote_address,
+                    remote_handle->gni_handle, size, 0, cbfunc, cbcontext, cbdata,
+                    NULL);
+    post_desc.gni_desc.amo_cmd = gni_op;
+    post_desc.gni_desc.first_operand = operand;
 
-    return rc;
+    return mca_btl_ugni_endpoint_post_fma (endpoint, &post_desc);
 }
 
 int mca_btl_ugni_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
@@ -120,8 +110,8 @@ int mca_btl_ugni_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_en
                        uint64_t operand, int flags, int order, mca_btl_base_rdma_completion_fn_t cbfunc,
                        void *cbcontext, void *cbdata)
 {
-    mca_btl_ugni_post_descriptor_t *post_desc;
-    int gni_op, rc, type;
+    mca_btl_ugni_post_descriptor_t post_desc;
+    int gni_op, type;
     size_t size;
 
     size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
@@ -136,55 +126,33 @@ int mca_btl_ugni_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_en
         return OPAL_ERR_NOT_SUPPORTED;
     }
 
-    post_desc = mca_btl_ugni_alloc_post_descriptor (endpoint, local_handle, cbfunc, cbcontext, cbdata);
-    if (OPAL_UNLIKELY(NULL == post_desc)) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-
-    init_gni_post_desc (post_desc, order, GNI_POST_AMO, (intptr_t) local_address, local_handle->gni_handle,
-                        remote_address, remote_handle->gni_handle, size, 0);
-    post_desc->desc.amo_cmd = gni_op;
-
-    post_desc->desc.first_operand = operand;
+    init_post_desc (&post_desc, endpoint, order, GNI_POST_AMO, (intptr_t) local_address,
+                    local_handle->gni_handle, remote_address, remote_handle->gni_handle,
+                    size, 0, cbfunc, cbcontext, cbdata, local_handle);
+    post_desc.gni_desc.amo_cmd = gni_op;
+    post_desc.gni_desc.first_operand = operand;
 
-    rc = mca_btl_ugni_endpoint_post_fma (endpoint, post_desc);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        mca_btl_ugni_return_post_descriptor (post_desc);
-    }
-
-    return rc;
+    return mca_btl_ugni_endpoint_post_fma (endpoint, &post_desc);
 }
 
 int mca_btl_ugni_acswap (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint,
                          void *local_address, uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
                          mca_btl_base_registration_handle_t *remote_handle, uint64_t compare, uint64_t value, int flags,
                          int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
 {
-    mca_btl_ugni_post_descriptor_t *post_desc;
-    int gni_op, rc;
+    mca_btl_ugni_post_descriptor_t post_desc;
     size_t size;
+    int gni_op;
 
     gni_op = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? GNI_FMA_ATOMIC2_CSWAP_S : GNI_FMA_ATOMIC_CSWAP;
     size = (MCA_BTL_ATOMIC_FLAG_32BIT & flags) ? 4 : 8;
 
-    post_desc = mca_btl_ugni_alloc_post_descriptor (endpoint, local_handle, cbfunc, cbcontext, cbdata);
-    if (OPAL_UNLIKELY(NULL == post_desc)) {
-        return OPAL_ERR_OUT_OF_RESOURCE;
-    }
-
-
-    init_gni_post_desc (post_desc, order, GNI_POST_AMO, (intptr_t) local_address, local_handle->gni_handle,
-                        remote_address, remote_handle->gni_handle, size, 0);
-    post_desc->desc.amo_cmd = gni_op;
-
-    post_desc->desc.first_operand = compare;
-    post_desc->desc.second_operand = value;
-
-    rc = mca_btl_ugni_endpoint_post_fma (endpoint, post_desc);
-    if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
-        mca_btl_ugni_return_post_descriptor (post_desc);
-    }
+    init_post_desc (&post_desc, endpoint, order, GNI_POST_AMO, (intptr_t) local_address,
+                    local_handle->gni_handle, remote_address, remote_handle->gni_handle, size, 0,
+                    cbfunc, cbcontext, cbdata, local_handle);
+    post_desc.gni_desc.amo_cmd = gni_op;
+    post_desc.gni_desc.first_operand = compare;
+    post_desc.gni_desc.second_operand = value;
 
-    return rc;
+    return mca_btl_ugni_endpoint_post_fma (endpoint, &post_desc);
 }