mtl/ofi: call fi_setopt to state MPI p2p requirements for CUDA

Call fi_setopt after endpoint initialization to tell Libfabric that while
device peer to peer support should be enabled when possible for network
transfers, it is not required and copies may be used instead.

An MCA parameter needs to be added for this in the future so that users can
toggle this option.

Signed-off-by: Robert Wespetal <wesper@amazon.com>

Author: rwespetal

config/opal_check_ofi.m4

@@ -142,6 +142,10 @@ AC_DEFUN([_OPAL_CHECK_OFI],[
                           [],
                           [#include <pmix.h>])
 
+           AC_CHECK_DECLS([FI_OPT_FI_HMEM_P2P],
+                          [], [],
+                          [#include <rdma/fi_endpoint.h>])
+
            AC_CHECK_TYPES([struct fi_ops_mem_monitor], [], [],
                           [#ifdef HAVE_RDMA_FI_EXT_H
 #include <rdma/fi_ext.h>

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -514,6 +514,37 @@ static int ompi_mtl_ofi_init_regular_ep(struct fi_info * prov, int universe_size
         return ret;
     }
 
+#if OPAL_CUDA_SUPPORT && HAVE_DECL_FI_OPT_FI_HMEM_P2P
+    /*
+     * Set the FI_HMEM peer to peer option to ENABLED. This notifies Libfabric
+     * that the provider can decide whether to use device peer to peer support
+     * for network transfers, and allows copies if p2p is not supported.
+     *
+     * Note that this option may not be supported by the provider, so continue
+     * if FI_HMEM is supported by the provider but it does not support this
+     * setopt option. This setopt parameter was introduced in Libfabric 1.14.
+     *
+     * The version check is needed as one of the Libfabric setopt handlers
+     * incorrectly assumed all option values are size_t, which was also fixed
+     * in 1.14.
+     */
+    int setopt_val = FI_HMEM_P2P_ENABLED;
+
+    if (FI_VERSION_GE(fi_version(), FI_VERSION(1, 14))) {
+            ret = fi_setopt(&ompi_mtl_ofi.sep->fid,
+                            FI_OPT_ENDPOINT, FI_OPT_FI_HMEM_P2P,
+                            &setopt_val, sizeof(setopt_val));
+
+            if (!(0 == ret || -FI_ENOPROTOOPT == ret)) {
+                opal_show_help("help-mtl-ofi.txt", "OFI call fail", true,
+                               "fi_setopt",
+                               ompi_process_info.nodename, __FILE__, __LINE__,
+                               fi_strerror(-ret), -ret);
+                return ret;
+            }
+    }
+#endif /* OPAL_CUDA_SUPPORT && FI_OPT_FI_HMEM_P2P */
+
     /**
      * Create the objects that will be bound to the endpoint.
      * The objects include: