Merge pull request #5425 from hoopoepg/topic/atomic-mxm-abstraction-viola-master

ATOMIC/MXM: fixed abstraction violation

Author: yosefe

oshmem/mca/atomic/mxm/Makefile.am

@@ -35,7 +35,7 @@ mcacomponentdir = $(oshmemlibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_atomic_mxm_la_SOURCES = $(mxm_sources)
 mca_atomic_mxm_la_LIBADD = $(top_builddir)/oshmem/liboshmem.la \
-	$(atomic_mxm_LIBS)
+	$(atomic_mxm_LIBS) $(top_builddir)/oshmem/mca/spml/libmca_spml.la
 mca_atomic_mxm_la_LDFLAGS = -module -avoid-version $(atomic_mxm_LDFLAGS)
 
 noinst_LTLIBRARIES = $(component_noinst)

oshmem/mca/atomic/mxm/atomic_mxm.h

@@ -101,7 +101,7 @@ static inline void mca_atomic_mxm_req_init(mxm_send_req_t *sreq, int pe, void *t
 
     nlong_order = mca_atomic_mxm_order(nlong);
 
-    mkey = mca_spml_ikrit_get_mkey(pe, target, MXM_PTL_RDMA, &remote_addr);
+    mkey = mca_spml_ikrit_get_mkey(pe, target, MXM_PTL_RDMA, &remote_addr, mca_atomic_mxm_spml_self);
 
     /* mxm request init */
     sreq->base.state        = MXM_REQ_NEW;

oshmem/mca/spml/ikrit/spml_ikrit.c

@@ -151,6 +151,8 @@ int mca_spml_ikrit_put_simple(void* dst_addr,
 
 static void mca_spml_ikrit_cache_mkeys(sshmem_mkey_t *, uint32_t seg, int remote_pe, int tr_id);
 
+static mxm_mem_key_t *mca_spml_ikrit_get_mkey_slow(int pe, void *va, int ptl_id, void **rva);
+
 mca_spml_ikrit_t mca_spml_ikrit = {
     {
         /* Init mca_spml_base_module_t */
@@ -176,7 +178,8 @@ mca_spml_ikrit_t mca_spml_ikrit = {
         mca_spml_base_memuse_hook,
 
         (void*)&mca_spml_ikrit
-    }
+    },
+    mca_spml_ikrit_get_mkey_slow
 };
 
 static void mca_spml_ikrit_cache_mkeys(sshmem_mkey_t *mkey, uint32_t seg, int dst_pe, int tr_id)
@@ -197,6 +200,7 @@ static void mca_spml_ikrit_cache_mkeys(sshmem_mkey_t *mkey, uint32_t seg, int ds
     }
 }
 
+static
 mxm_mem_key_t *mca_spml_ikrit_get_mkey_slow(int pe, void *va, int ptl_id, void **rva)
 {
     sshmem_mkey_t *mkey;
@@ -578,7 +582,7 @@ static inline int mca_spml_ikrit_get_helper(mxm_send_req_t *sreq,
     void *rva;
     mxm_mem_key_t *mkey;
 
-    mkey = mca_spml_ikrit_get_mkey(src, src_addr, MXM_PTL_RDMA, &rva);
+    mkey = mca_spml_ikrit_get_mkey(src, src_addr, MXM_PTL_RDMA, &rva, &mca_spml_ikrit);
 
     SPML_VERBOSE_FASTPATH(100,
                           "get: pe:%d ptl=%d src=%p -> dst: %p sz=%d. src_rva=%p",
@@ -613,7 +617,7 @@ static inline int mca_spml_ikrit_get_shm(void *src_addr,
     if (ptl_id != MXM_PTL_SHM)
         return OSHMEM_ERROR;
 
-    if (NULL != mca_spml_ikrit_get_mkey(src, src_addr, MXM_PTL_SHM, &rva))
+    if (NULL != mca_spml_ikrit_get_mkey(src, src_addr, MXM_PTL_SHM, &rva, &mca_spml_ikrit))
         return OSHMEM_ERROR;
 
     SPML_VERBOSE_FASTPATH(100,
@@ -798,7 +802,7 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
     }
 
     ptl_id = get_ptl_id(dst);
-    mkey = mca_spml_ikrit_get_mkey(dst, dst_addr, ptl_id, &rva);
+    mkey = mca_spml_ikrit_get_mkey(dst, dst_addr, ptl_id, &rva, &mca_spml_ikrit);
 
     if (OPAL_UNLIKELY(NULL == mkey)) {
         memcpy((void *) (unsigned long) rva, src_addr, size);
@@ -885,7 +889,7 @@ int mca_spml_ikrit_put_simple(void* dst_addr,
     static int count;
 
     ptl_id = get_ptl_id(dst);
-    mkey = mca_spml_ikrit_get_mkey(dst, dst_addr, ptl_id, &rva);
+    mkey = mca_spml_ikrit_get_mkey(dst, dst_addr, ptl_id, &rva, &mca_spml_ikrit);
 
     SPML_VERBOSE_FASTPATH(100, "put: pe:%d ptl=%d dst=%p <- src: %p sz=%d. dst_rva=%p, %s",
                           dst, ptl_id, dst_addr, src_addr, (int)size, (void *)rva);

oshmem/mca/spml/ikrit/spml_ikrit.h

@@ -81,9 +81,13 @@ struct mxm_peer {
 
 typedef struct mxm_peer mxm_peer_t;
 
+typedef mxm_mem_key_t *(*mca_spml_ikrit_get_mkey_slow_fn_t)(int pe, void *va, int ptl_id, void **rva);
+
 struct mca_spml_ikrit_t {
     mca_spml_base_module_t super;
 
+    mca_spml_ikrit_get_mkey_slow_fn_t get_mkey_slow;
+
     mxm_context_opts_t *mxm_ctx_opts;
     mxm_ep_opts_t *mxm_ep_opts;
     mxm_ep_opts_t *mxm_ep_hw_rdma_opts;
@@ -173,25 +177,26 @@ extern int mca_spml_ikrit_del_procs(ompi_proc_t** procs, size_t nprocs);
 extern int mca_spml_ikrit_fence(void);
 extern int spml_ikrit_progress(void);
 
-mxm_mem_key_t *mca_spml_ikrit_get_mkey_slow(int pe, void *va, int ptl_id, void **rva);
-
 /* the functionreturns NULL if data can be directly copied via shared memory 
  * else it returns mxm mem key
  *
  * the function will abort() if va is not symmetric var address.
  */
-static inline mxm_mem_key_t *mca_spml_ikrit_get_mkey(int pe, void *va, int ptl_id, void **rva) 
+static inline mxm_mem_key_t *mca_spml_ikrit_get_mkey(int pe, void *va, int ptl_id, void **rva,
+                                                     mca_spml_ikrit_t *module)
 {
     spml_ikrit_mkey_t *mkey;
 
     if (OPAL_UNLIKELY(MXM_PTL_RDMA != ptl_id)) {
-        return mca_spml_ikrit_get_mkey_slow(pe, va, ptl_id, rva);
+        assert(module->get_mkey_slow);
+        return module->get_mkey_slow(pe, va, ptl_id, rva);
     }
 
-    mkey = mca_spml_ikrit.mxm_peers[pe].mkeys;
+    mkey = module->mxm_peers[pe].mkeys;
     mkey = (spml_ikrit_mkey_t *)map_segment_find_va(&mkey->super.super, sizeof(*mkey), va);
     if (OPAL_UNLIKELY(NULL == mkey)) {
-        return mca_spml_ikrit_get_mkey_slow(pe, va, ptl_id, rva);
+        assert(module->get_mkey_slow);
+        return module->get_mkey_slow(pe, va, ptl_id, rva);
     }
     *rva = map_segment_va2rva(&mkey->super, va);
     return &mkey->key;