pml/cm: Add CUDA detection to the PML/CM send paths

PML/CM recv paths already have CUDA detection, both recv and irecv make
calls to opal_convertor_copy_and_prepare_for_recv, which does detection.

This patch and a subsequent MTL datapack patch addresses an error using
heterogenous device send buffers (The code will malloc a temporary send
buffer and then due to CONVERTOR_CUDA never being set, the subsequent
opal_cuda_memcpy will attempt a mempcy from device to host buffer, which
will cause an error). Preparing the send buffers and setting
CONVERTOR_CUDA will allow for using the correct cuMemcpy.

Signed-off-by: William Zhang <wilzhang@amazon.com>
Co-authored-by: William Zhang <wilzhang@amazon.com>
Co-authored-by: Palmer Stolly <pstolly@amazon.com>

Author: wckzhang

ompi/mca/mtl/portals4/mtl_portals4_component.c

@@ -428,6 +428,12 @@ ompi_mtl_portals4_component_init(bool enable_progress_threads,
                          id.phys.nid, id.phys.pid));
 
     ompi_mtl_portals4.base.mtl_max_tag = MTL_PORTALS4_MAX_TAG;
+
+    /* Disable opal from checking if buffer being sent is cuda */
+#if OPAL_CUDA_SUPPORT
+    ompi_mtl_portals4.base.mtl_flags |= MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE;
+#endif /* OPAL_CUDA_SUPPORT */
+
     return &ompi_mtl_portals4.base;
 
  error:

ompi/mca/pml/cm/pml_cm.h

@@ -379,7 +379,16 @@ mca_pml_cm_send(const void *buf,
 		convertor.pBaseBuf   = (unsigned char*)buf + datatype->super.true_lb;
 		convertor.count      = count;
 		convertor.pDesc      = &datatype->super;
-	} else
+
+#if OPAL_CUDA_SUPPORT
+        /* Switches off CUDA detection if
+           MTL set MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE during init */
+        MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);
+        convertor.flags      |= flags;
+        /* Sets CONVERTOR_CUDA flag if CUDA buffer */
+        opal_convertor_prepare_for_send( &convertor, &datatype->super, count, buf );
+#endif
+    } else
 #endif
 	{
 		ompi_proc = ompi_comm_peer_lookup(comm, dst);

ompi/mca/pml/cm/pml_cm_sendreq.h

@@ -242,6 +242,14 @@ do {                                                                    \
             (unsigned char*)buf + datatype->super.true_lb;              \
         (req_send)->req_base.req_convertor.count      = count;          \
         (req_send)->req_base.req_convertor.pDesc      = &datatype->super; \
+        /* Switches off CUDA detection if                               \
+         MTL set MCA_MTL_BASE_FLAG_CUDA_INIT_DISABLE during init */     \
+        MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);   \
+        (req_send)->req_base.req_convertor.flags |= flags;              \
+        /* Sets CONVERTOR_CUDA flag if CUDA buffer */                   \
+        opal_convertor_prepare_for_send(                                \
+            &req_send->req_base.req_convertor,                          \
+            &datatype->super, count, buf );                             \
     } else {                                                            \
         MCA_PML_CM_SWITCH_CUDA_CONVERTOR_OFF(flags, datatype, count);   \
         opal_convertor_copy_and_prepare_for_send(                       \