Merge pull request #10620 from hppritcha/remove_fi_name_max

OFI: call fi_getname twice

Author: hppritcha

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -604,7 +604,7 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     struct fi_info *providers = NULL;
     struct fi_info *prov = NULL;
     struct fi_info *prov_cq_data = NULL;
-    char ep_name[FI_NAME_MAX] = {0};
+    void *ep_name;
     size_t namelen;
     int universe_size;
     char *univ_size_str;
@@ -1070,15 +1070,11 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     fi_freeinfo(providers);
     providers = NULL;
 
-    /**
-     * Get our address and publish it with modex.
-     */
-    namelen = sizeof(ep_name);
-    ret = fi_getname((fid_t)ompi_mtl_ofi.sep,
-                     &ep_name[0],
-                     &namelen);
-    if (ret) {
-        MTL_OFI_LOG_FI_ERR(ret, "fi_getname failed");
+    ret = opal_common_ofi_fi_getname((fid_t)ompi_mtl_ofi.sep,
+                                     &ep_name,
+                                     &namelen);
+    if (OMPI_SUCCESS != ret) {
+        MTL_OFI_LOG_FI_ERR(ret, "opal_common_ofi_fi_getname failed");
         goto error;
     }
 
@@ -1094,6 +1090,7 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     }
 
     ompi_mtl_ofi.epnamelen = namelen;
+    free(ep_name);
 
     /**
      * Set the ANY_SRC address.
@@ -1153,6 +1150,9 @@ ompi_mtl_ofi_component_init(bool enable_progress_threads,
     if (ompi_mtl_ofi.ofi_ctxt) {
         free(ompi_mtl_ofi.ofi_ctxt);
     }
+    if (ep_name) {
+        free(ep_name);
+    }
 
     return NULL;
 }

opal/mca/btl/ofi/btl_ofi_component.c

@@ -419,7 +419,7 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     size_t num_contexts_to_create;
 
     char *linux_device_name;
-    char ep_name[FI_NAME_MAX];
+    void *ep_name;
 
     struct fi_info *ofi_info;
     struct fi_ep_attr *ep_attr;
@@ -596,15 +596,14 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
         goto fail;
     }
 
-    /* create and send the modex for this device */
-    namelen = sizeof(ep_name);
-    rc = fi_getname((fid_t) ep, &ep_name[0], &namelen);
-    if (0 != rc) {
-        BTL_VERBOSE(("%s failed fi_getname with err=%s", linux_device_name, fi_strerror(-rc)));
+    rc = opal_common_ofi_fi_getname((fid_t)ep,
+                                     &ep_name,
+                                     &namelen);
+    if (OPAL_SUCCESS != rc) {
+        BTL_VERBOSE(("%s failed opal_common_ofi_fi_getname  with err=%d", linux_device_name, rc));
         goto fail;
     }
 
-    /* If we have two-sided support. */
     if (TWO_SIDED_ENABLED) {
 
         /* post wildcard recvs */
@@ -618,8 +617,9 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     }
 
     /* post our endpoint name so peer can use it to connect to us */
-    OPAL_MODEX_SEND(rc, PMIX_GLOBAL, &mca_btl_ofi_component.super.btl_version, &ep_name, namelen);
+    OPAL_MODEX_SEND(rc, PMIX_GLOBAL, &mca_btl_ofi_component.super.btl_version, ep_name, namelen);
     mca_btl_ofi_component.namelen = namelen;
+    free(ep_name);
 
     /* add this module to the list */
     mca_btl_ofi_component.modules[(*module_count)++] = module;
@@ -663,6 +663,10 @@ static int mca_btl_ofi_init_device(struct fi_info *info)
     }
     free(module);
 
+    if (NULL != ep_name) {
+        free(ep_name);
+    }
+
     /* not really a failure. just skip this device. */
     return OPAL_ERR_OUT_OF_RESOURCE;
 }

opal/mca/common/ofi/common_ofi.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2015-2020 Intel, Inc.  All rights reserved.
  * Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2020-2021 Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2020-2021 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2021      Nanook Consulting.  All rights reserved.
@@ -22,6 +22,7 @@
 #include <unistd.h>
 #include <rdma/fabric.h>
 #include <rdma/fi_errno.h>
+#include <rdma/fi_cm.h>
 #ifdef HAVE_RDMA_FI_EXT_H
 #include <rdma/fi_ext.h>
 #endif
@@ -685,3 +686,71 @@ struct fi_info *opal_mca_common_ofi_select_provider(struct fi_info *provider_lis
     free(provider_table);
     return provider;
 }
+
+/**
+ * Obtain EP endpoint name
+ *
+ * Obtain the EP endpoint name and length for the supplied endpoint fid.
+ *
+ * @param fid (IN)     fid of (S)EP endpoint
+ * @param addr (OUT)   buffer containing endpoint name 
+ * @param addrlen (OUT) length of allocated buffer in bytes
+ *
+ * @return             OPAL_SUCCESS or OPAL error code
+ *
+ * The caller is responsible for freeing the buffer allocated to
+ * contain the endpoint name.
+ *
+ */
+OPAL_DECLSPEC int opal_common_ofi_fi_getname(fid_t fid, void **addr, size_t *addrlen)
+{
+    int ret=OPAL_SUCCESS;
+    size_t namelen = 0;
+    char *ep_name = NULL;
+
+    /**
+     * Get our address and publish it with modex.
+     * Use the two step process of first getting the required
+     * buffer size, then allocating the memory and calling
+     * fi_getname again.
+     */
+    namelen = 0;
+    ret = fi_getname(fid,
+                     NULL,
+                     &namelen);
+    if ((FI_SUCCESS != ret) && (-FI_ETOOSMALL != ret)) {
+        opal_output_verbose(1, opal_common_ofi.output, "%s:%d:fi_endpoint (namelen) returned %s\n",
+                            __FILE__, __LINE__, fi_strerror(-ret));
+        ret = OPAL_ERROR;
+        goto error;
+    }
+
+    ep_name = (char *)malloc(namelen);
+    if (NULL == ep_name) {
+        ret = OPAL_ERR_OUT_OF_RESOURCE;
+        goto error;
+    }
+
+    ret = fi_getname(fid,
+                     ep_name,
+                     &namelen);
+    if (ret) {
+        opal_output_verbose(1, opal_common_ofi.output, "%s:%d:fi_endpoint (ep_name) returned %s\n",
+                            __FILE__, __LINE__, fi_strerror(-ret));
+        ret = OPAL_ERROR;
+        goto error;
+    }
+
+    *addr = ep_name;
+    *addrlen = namelen;
+
+    return ret;
+
+error:
+    if (NULL != ep_name) {
+       free(ep_name); 
+    }
+    return ret;
+}
+
+

opal/mca/common/ofi/common_ofi.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2015      Intel, Inc. All rights reserved.
  * Copyright (c) 2017      Los Alamos National Security, LLC.  All rights
  *                         reserved.
- * Copyright (c) 2020      Triad National Security, LLC. All rights
+ * Copyright (c) 2020-2022 Triad National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2021      Amazon.com, Inc. or its affiliates. All rights
  *                         reserved.
@@ -21,6 +21,9 @@
 #include "opal/util/proc.h"
 #include "opal/memoryhooks/memory.h"
 
+#include <rdma/fabric.h>
+#include <rdma/fi_cm.h>
+
 BEGIN_C_DECLS
 
 typedef struct opal_common_ofi_module {
@@ -151,6 +154,23 @@ OPAL_DECLSPEC int opal_common_ofi_is_in_list(char **list, char *item);
 OPAL_DECLSPEC struct fi_info *opal_mca_common_ofi_select_provider(struct fi_info *provider_list,
                                                                   opal_process_info_t *process_info);
 
+/**
+ * Obtain EP endpoint name
+ *
+ * Obtain the EP endpoint name and length for the supplied endpoint fid.
+ *
+ * @param fid (IN)     fid of (S)EP endpoint
+ * @param addr (OUT)   buffer containing endpoint name 
+ * @param addrlen (OUT) length of allocated buffer in bytes
+ *
+ * @return             OPAL_SUCCESS or OPAL error code
+ *
+ * The caller is responsible for freeing the buffer allocated to
+ * contain the endpoint name.
+ *
+ */
+OPAL_DECLSPEC int opal_common_ofi_fi_getname(fid_t fid, void **addr, size_t *addrlen);
+
 END_C_DECLS
 
 #endif /* OPAL_MCA_COMMON_OFI_H */