SPML/UCX: provide backward dependency compatibility

- due to changes in SPML there was dependency inconsistent
  with 4.x branches
- unified dependency with 4.x branches

Signed-off-by: Sergey Oblomov <sergeyo@nvidia.com>
(cherry picked from commit dcf122d)

Author: Sergey Oblomov

oshmem/mca/spml/ucx/spml_ucx.c

@@ -78,15 +78,15 @@ mca_spml_ucx_t mca_spml_ucx = {
     .num_disconnect         = 1,
     .heap_reg_nb            = 0,
     .enabled                = 0,
-    .get_mkey_slow          = NULL,
-    .synchronized_quiet     = false,
-    .strong_sync            = SPML_UCX_STRONG_ORDERING_NONE
+    .get_mkey_slow          = NULL
 };
 
 mca_spml_ucx_ctx_t mca_spml_ucx_ctx_default = {
     .ucp_worker         = NULL,
     .ucp_peers          = NULL,
-    .options            = 0
+    .options            = 0,
+    .synchronized_quiet = false,
+    .strong_sync        = SPML_UCX_STRONG_ORDERING_NONE
 };
 
 #if HAVE_DECL_UCP_ATOMIC_OP_NBX
@@ -405,7 +405,7 @@ int mca_spml_ucx_init_put_op_mask(mca_spml_ucx_ctx_t *ctx, size_t nprocs)
 {
     int res;
 
-    if (mca_spml_ucx_is_strong_ordering()) {
+    if (mca_spml_ucx_is_strong_ordering(ctx)) {
         ctx->put_proc_indexes = malloc(nprocs * sizeof(*ctx->put_proc_indexes));
         if (NULL == ctx->put_proc_indexes) {
             return OSHMEM_ERR_OUT_OF_RESOURCE;
@@ -427,7 +427,7 @@ int mca_spml_ucx_init_put_op_mask(mca_spml_ucx_ctx_t *ctx, size_t nprocs)
 
 int mca_spml_ucx_clear_put_op_mask(mca_spml_ucx_ctx_t *ctx)
 {
-    if (mca_spml_ucx_is_strong_ordering() && ctx->put_proc_indexes) {
+    if (mca_spml_ucx_is_strong_ordering(ctx) && ctx->put_proc_indexes) {
         OBJ_DESTRUCT(&ctx->put_op_bitmap);
         free(ctx->put_proc_indexes);
     }
@@ -844,6 +844,8 @@ static int mca_spml_ucx_ctx_create_common(long options, mca_spml_ucx_ctx_t **ucx
     ucx_ctx->options = options;
     ucx_ctx->ucp_worker = calloc(1, sizeof(ucp_worker_h));
     ucx_ctx->ucp_workers = 1;
+    ucx_ctx->synchronized_quiet = mca_spml_ucx_ctx_default.synchronized_quiet;
+    ucx_ctx->strong_sync = mca_spml_ucx_ctx_default.strong_sync;      
 
     params.field_mask  = UCP_WORKER_PARAM_FIELD_THREAD_MODE;
     if (oshmem_mpi_thread_provided == SHMEM_THREAD_SINGLE || options & SHMEM_CTX_PRIVATE || options & SHMEM_CTX_SERIALIZED) {
@@ -1193,7 +1195,7 @@ static int mca_spml_ucx_strong_sync(shmem_ctx_t ctx)
     for (i = 0; i < ucx_ctx->put_proc_count; i++) {
         idx = ucx_ctx->put_proc_indexes[i];
 
-        switch (mca_spml_ucx.strong_sync) {
+        switch (ucx_ctx->strong_sync) {
         case SPML_UCX_STRONG_ORDERING_NONE:
         case SPML_UCX_STRONG_ORDERING_GETNB:
             ret = mca_spml_ucx_get_nb(ctx,
@@ -1245,7 +1247,7 @@ int mca_spml_ucx_fence(shmem_ctx_t ctx)
 
     opal_atomic_wmb();
 
-    if (mca_spml_ucx.strong_sync != SPML_UCX_STRONG_ORDERING_NONE) {
+    if (ucx_ctx->strong_sync != SPML_UCX_STRONG_ORDERING_NONE) {
         ret = mca_spml_ucx_strong_sync(ctx);
         if (ret != OSHMEM_SUCCESS) {
             oshmem_shmem_abort(-1);
@@ -1272,7 +1274,7 @@ int mca_spml_ucx_quiet(shmem_ctx_t ctx)
     unsigned i;
     mca_spml_ucx_ctx_t *ucx_ctx = (mca_spml_ucx_ctx_t *)ctx;
 
-    if (mca_spml_ucx.synchronized_quiet) {
+    if (ucx_ctx->synchronized_quiet) {
         ret = mca_spml_ucx_strong_sync(ctx);
         if (ret != OSHMEM_SUCCESS) {
             oshmem_shmem_abort(-1);

oshmem/mca/spml/ucx/spml_ucx.h

@@ -86,6 +86,8 @@ struct mca_spml_ucx_ctx {
     unsigned int             ucp_workers;
     int                     *put_proc_indexes;
     unsigned                 put_proc_count;
+    bool                     synchronized_quiet;
+    int                      strong_sync;
 };
 typedef struct mca_spml_ucx_ctx mca_spml_ucx_ctx_t;
 
@@ -120,8 +122,6 @@ struct mca_spml_ucx {
     mca_spml_ucx_ctx_t       *aux_ctx;
     pthread_spinlock_t       async_lock;
     int                      aux_refcnt;
-    bool                     synchronized_quiet;
-    int                      strong_sync;
     unsigned long            nb_progress_thresh_global;
     unsigned long            nb_put_progress_thresh;
     unsigned long            nb_get_progress_thresh;
@@ -301,15 +301,15 @@ static inline int ucx_status_to_oshmem_nb(ucs_status_t status)
 #endif
 }
 
-static inline int mca_spml_ucx_is_strong_ordering(void)
+static inline int mca_spml_ucx_is_strong_ordering(mca_spml_ucx_ctx_t *ctx)
 {
-    return (mca_spml_ucx.strong_sync != SPML_UCX_STRONG_ORDERING_NONE) ||
-           mca_spml_ucx.synchronized_quiet;
+    return (ctx->strong_sync != SPML_UCX_STRONG_ORDERING_NONE) ||
+           ctx->synchronized_quiet;
 }
 
 static inline void mca_spml_ucx_remote_op_posted(mca_spml_ucx_ctx_t *ctx, int dst)
 {
-    if (OPAL_UNLIKELY(mca_spml_ucx_is_strong_ordering())) {
+    if (OPAL_UNLIKELY(mca_spml_ucx_is_strong_ordering(ctx))) {
         if (!opal_bitmap_is_set_bit(&ctx->put_op_bitmap, dst)) {
             ctx->put_proc_indexes[ctx->put_proc_count++] = dst;
             opal_bitmap_set_bit(&ctx->put_op_bitmap, dst);

oshmem/mca/spml/ucx/spml_ucx_component.c

@@ -159,12 +159,12 @@ static int mca_spml_ucx_component_register(void)
 
     mca_spml_ucx_param_register_bool("synchronized_quiet", 0,
                                      "Use synchronized quiet on shmem_quiet or shmem_barrier_all operations",
-                                     &mca_spml_ucx.synchronized_quiet);
+                                     &mca_spml_ucx_ctx_default.synchronized_quiet);
 
     mca_spml_ucx_param_register_int("strong_sync", 0,
                                     "Use strong synchronization on shmem_quiet, shmem_fence or shmem_barrier_all operations: "
                                     "0 - don't do strong synchronization, 1 - use non blocking get, 2 - use blocking get, 3 - use flush operation",
-                                    &mca_spml_ucx.strong_sync);
+                                    &mca_spml_ucx_ctx_default.strong_sync);
 
     mca_spml_ucx_param_register_ulong("nb_progress_thresh_global", 0,
                                     "Number of nb_put or nb_get operations before ucx progress is triggered. Disabled by default (0). Setting this value will override nb_put/get_progress_thresh.",
@@ -388,14 +388,14 @@ mca_spml_ucx_component_init(int* priority,
     if (OSHMEM_SUCCESS != spml_ucx_init())
         return NULL ;
 
-    SPML_UCX_VERBOSE(50, "*** ucx initialized ****");
-
-    if ((mca_spml_ucx.strong_sync < SPML_UCX_STRONG_ORDERING_NONE) ||
-        (mca_spml_ucx.strong_sync > SPML_UCX_STRONG_ORDERING_FLUSH)) {
+    if ((mca_spml_ucx_ctx_default.strong_sync < SPML_UCX_STRONG_ORDERING_NONE) ||
+        (mca_spml_ucx_ctx_default.strong_sync > SPML_UCX_STRONG_ORDERING_FLUSH)) {
         SPML_UCX_ERROR("incorrect value of strong_sync parameter: %d",
-                       mca_spml_ucx.strong_sync);
+                       mca_spml_ucx_ctx_default.strong_sync);
     }
 
+    SPML_UCX_VERBOSE(50, "*** ucx initialized ****");
+
     return &mca_spml_ucx.super;
 }