Merge pull request #10874 from edgargabriel/topic/accelerator-rocm-updates

edgargabriel · web-flow · commit 5f4fdd53f8a2 · 2022-09-29T16:06:47.000-05:00
accelerator/rocm: updates to the component
diff --git a/config/opal_check_rocm.m4 b/config/opal_check_rocm.m4
@@ -20,11 +20,12 @@ dnl
 #
 AC_DEFUN([OPAL_CHECK_ROCM],[
 
-     OPAL_VAR_SCOPE_PUSH([opal_check_rocm_happy rocm_save_CPPFLAGS rocm_save_LDFLAGS rocm_CPPFLAGS rocm_LDFLAGS])
+     OPAL_VAR_SCOPE_PUSH([opal_check_rocm_happy rocm_save_CPPFLAGS rocm_save_LDFLAGS rocm_save_LIBS rocm_CPPFLAGS rocm_LDFLAGS rocm_LIBS])
 
      rocm_save_CPPFLAGS="$CPPFLAGS"
      rocm_save_LDFLAGS="$LDFLAGS"
-     
+     rocm_save_LIBS="$LIBS"
+
      # Get some configuration information
      AC_ARG_WITH([rocm],
         [AS_HELP_STRING([--with-rocm(=DIR)],
@@ -51,7 +52,8 @@ AC_DEFUN([OPAL_CHECK_ROCM],[
 
      LDFLAGS="$rocm_save_LDFLAGS"
      OPAL_APPEND([CPPFLAGS], [${$1_CPPFLAGS}] )
-     
+     LIBS="$rocm_save_LIBS"
+
      AS_IF([ test "$opal_check_rocm_happy" = "no" ],
            [ CPPFLAGS="$rocm_save_CPPFLAGS"])
 
@@ -70,13 +72,3 @@ AC_DEFUN([OPAL_CHECK_ROCM],[
      AM_CONDITIONAL([OPAL_rocm_support], [test "$opal_check_rocm_happy" = "yes"])
      OPAL_VAR_SCOPE_POP
 ])
-
-AC_DEFUN([OPAL_CHECK_ROCM_AFTER_OPAL_DL],[
-    # We cannot have ROCm support without OPAL DL support.  Error out
-    # if the user wants Rocm but we do not have OPAL DL support.
-     AS_IF([test $OPAL_HAVE_DL_SUPPORT -eq 0 && test "$opal_check_rocm_happy" = "yes"],
-          [AC_MSG_WARN([--with-rocm was specified, but dlopen support is disabled.])
-           AC_MSG_WARN([You must reconfigure Open MPI with dlopen ("dl") support.])
-           AC_MSG_ERROR([Cannot continue.])])
-
-])
diff --git a/configure.ac b/configure.ac
@@ -1222,18 +1222,6 @@ fi
 # checkpoint results
 AC_CACHE_SAVE
 
-##################################
-# CUDA: part two
-##################################
-
-# This is somewhat gross to have a configure check for a common MCA
-# component outside of the normal MCA checks, but this check must come
-# after the opal DL MCA checks have done.  Someday this could perhaps
-# be done better by having some kind of "run this check at the end of
-# all other MCA checks" hook...?
-
-OPAL_CHECK_ROCM_AFTER_OPAL_DL
-
 ##################################
 # MPI Extended Interfaces
 ##################################
diff --git a/opal/mca/accelerator/rocm/Makefile.am b/opal/mca/accelerator/rocm/Makefile.am
@@ -35,9 +35,11 @@ endif
 mcacomponentdir = $(opallibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_accelerator_rocm_la_SOURCES = $(sources)
-mca_accelerator_rocm_la_LDFLAGS = -module -avoid-version
-mca_accelerator_rocm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
+mca_accelerator_rocm_la_LDFLAGS = -module -avoid-version $(opal_rocm_LDFLAGS)
+mca_accelerator_rocm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la \
+	$(opal_rocm_LIBS)
 
 noinst_LTLIBRARIES = $(component_noinst)
 libmca_accelerator_rocm_la_SOURCES =$(sources)
-libmca_accelerator_rocm_la_LDFLAGS = -module -avoid-version
+libmca_accelerator_rocm_la_LDFLAGS = -module -avoid-version $(opal_rocm_LDFLAGS)
+libmca_accelerator_rocm_la_LIBADD = $(opal_rocm_LIBS)
diff --git a/opal/mca/accelerator/rocm/accelerator_rocm_component.c b/opal/mca/accelerator/rocm/accelerator_rocm_component.c
@@ -23,17 +23,12 @@
 #include "opal/runtime/opal_params.h"
 #include "accelerator_rocm.h"
 
-static struct opal_dl_handle_t* hip_handle = NULL;
-opal_accelerator_rocm_hipFunctionTable_t opal_accelerator_hip_funcs={};
-
 int opal_accelerator_rocm_memcpy_async = 1;
 int opal_accelerator_rocm_verbose = 0;
 size_t opal_accelerator_rocm_memcpyD2H_limit=1024;
 size_t opal_accelerator_rocm_memcpyH2D_limit=1048576;
 
 hipStream_t opal_accelerator_rocm_MemcpyStream = NULL;
-static opal_mutex_t opal_accelerator_rocm_init_lock = OPAL_MUTEX_STATIC_INIT;
-
 
 /*
  * Public string showing the accelerator rocm component version number
@@ -46,7 +41,7 @@ const char *opal_accelerator_rocm_component_version_string
 {                                                                            \
     hipError_t error = condition;                                            \
     if (hipSuccess != error){                                                \
-        const char* msg = HIP_FUNCS.hipGetErrorString(error);                \
+        const char* msg = hipGetErrorString(error);                \
         opal_output(0, "HIP error: %d %s file: %s line: %d\n", error, msg, __FILE__, __LINE__); \
         return error;                                                        \
     }                                                                        \
@@ -56,7 +51,7 @@ const char *opal_accelerator_rocm_component_version_string
 {                                                                            \
     hipError_t error = condition;                                            \
     if (hipSuccess != error){                                                \
-        const char* msg = HIP_FUNCS.hipGetErrorString(error);                \
+        const char* msg = hipGetErrorString(error);                \
         opal_output(0, "HIP error: %d %s file: %s line: %d\n", error, msg, __FILE__, __LINE__); \
         return NULL;                                                         \
     }                                                                        \
@@ -159,221 +154,6 @@ static int accelerator_rocm_component_register(void)
     return OPAL_SUCCESS;
 }
 
-
-static int hip_dl_init(void)
-{
-    char *str;
-    void *ptr;
-    int ret  = opal_dl_open("libamdhip64.so", false, false, &hip_handle, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output(0, "Unable to open libamdhip64.so\n");
-        return OPAL_ERROR;
-    }
-
-    ret = opal_dl_lookup(hip_handle, "hipMalloc", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output(0, "Failed to find hipMalloc\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMalloc = (hipMalloc_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipFree", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipFree\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipFree = (hipFree_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemcpy", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipMemcpy\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemcpy = (hipMemcpy_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemcpyAsync", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipMemcpyAsync\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemcpyAsync = (hipMemcpyAsync_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemcpy2D", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipMemcpy2D\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemcpy2D = (hipMemcpy2D_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemcpy2DAsync", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipMemcpy2DAsync\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemcpy2DAsync = (hipMemcpy2DAsync_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipMemGetAddressRange", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipMemGetAddressRange\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipMemGetAddressRange  = (hipMemGetAddressRange_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipHostRegister", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipHostRegister\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipHostRegister = (hipHostRegister_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipHostUnregister", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipHostUnregister\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipHostUnregister = (hipHostUnregister_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipStreamCreate", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipStreamCreate\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipStreamCreate = (hipStreamCreate_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipStreamDestroy", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipStreamDestroy\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipStreamDestroy = (hipStreamDestroy_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipStreamSynchronize", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipStreamSynchronize\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipStreamSynchronize = (hipStreamSynchronize_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipGetErrorString", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipGetErrorString\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipGetErrorString = (hipGetErrorString_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipPointerGetAttributes", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipPointerGetAttributes\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipPointerGetAttributes = (hipPointerGetAttributes_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventCreateWithFlags", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipEventCreateWithFlags\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventCreateWithFlags = (hipEventCreateWithFlags_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventDestroy", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipEventDestroy\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventDestroy = (hipEventDestroy_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventRecord", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipEventRecord\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventRecord = (hipEventRecord_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventQuery", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipEventQuery\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventQuery = (hipEventQuery_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipEventSynchronize", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-      opal_output_verbose(10, 0, "Failed to find hipEventSynchronize\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipEventSynchronize = (hipEventSynchronize_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipIpcGetMemHandle", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipIpcGetMemHandle\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipIpcGetMemHandle = (hipIpcGetMemHandle_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipIpcOpenMemHandle", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipIpcOpenMemHandle\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipIpcOpenMemHandle = (hipIpcOpenMemHandle_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipIpcCloseMemHandle", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipIpcCloseMemHandle\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipIpcCloseMemHandle = (hipIpcCloseMemHandle_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipGetDevice", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipGetDevice\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipGetDevice = (hipGetDevice_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipGetDeviceCount", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipGetDeviceCount\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipGetDeviceCount = (hipGetDeviceCount_t)ptr;
-
-    ret = opal_dl_lookup(hip_handle, "hipDeviceCanAccessPeer", &ptr, &str);
-    if (OPAL_SUCCESS != ret) {
-        opal_output_verbose(10, 0, "Failed to find hipDeviceCanAccessPeer\n");
-        dlclose(hip_handle);
-        return OPAL_ERROR;
-    }
-    HIP_FUNCS.hipDeviceCanAccessPeer = (hipDeviceCanAccessPeer_t)ptr;
-
-    return OPAL_SUCCESS;
-}
-
-
 static opal_accelerator_base_module_t* accelerator_rocm_init(void)
 {
     hipError_t err;
@@ -382,57 +162,35 @@ static opal_accelerator_base_module_t* accelerator_rocm_init(void)
         return NULL;
     }
 
-    OPAL_THREAD_LOCK(&opal_accelerator_rocm_init_lock);
-
-    if (opal_rocm_runtime_initialized) {
-        OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
-        return NULL;
-    }
-
-    if (OPAL_SUCCESS != hip_dl_init()) {
-        opal_output(0, "Could not open libamdhip64.so. Please check your LD_LIBRARY_PATH\n");
-        OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
-        return NULL;
-    }
-
     int count=0;
-    err = HIP_FUNCS.hipGetDeviceCount(&count);
+    err = hipGetDeviceCount(&count);
     if (hipSuccess != err || 0 == count) {
         opal_output(0, "No HIP capabale device found. Disabling component.\n");
-        OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
         return NULL;
     }
 
-    err = HIP_FUNCS.hipStreamCreate(&opal_accelerator_rocm_MemcpyStream);
+    err = hipStreamCreate(&opal_accelerator_rocm_MemcpyStream);
     if (hipSuccess != err) {
         opal_output(0, "Could not create hipStream, err=%d %s\n",
-                err, HIP_FUNCS.hipGetErrorString(err));
-        OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
+                err, hipGetErrorString(err));
         return NULL;
     }
 
     opal_atomic_mb();
     opal_rocm_runtime_initialized = true;
-    OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
 
     return &opal_accelerator_rocm_module;
 }
 
 static void accelerator_rocm_finalize(opal_accelerator_base_module_t* module)
 {
-    OPAL_THREAD_LOCK(&opal_accelerator_rocm_init_lock);
     if (NULL != (void*)opal_accelerator_rocm_MemcpyStream) {
-        hipError_t err = HIP_FUNCS.hipStreamDestroy(opal_accelerator_rocm_MemcpyStream);
+        hipError_t err = hipStreamDestroy(opal_accelerator_rocm_MemcpyStream);
         if (hipSuccess != err) {
             opal_output_verbose(10, 0, "hip_dl_finalize: error while destroying the hipStream\n");
         }
         opal_accelerator_rocm_MemcpyStream = NULL;
     }
-    if (NULL != hip_handle) {
-        opal_dl_close(hip_handle);
-        hip_handle = NULL;
-    }
-    OPAL_THREAD_UNLOCK(&opal_accelerator_rocm_init_lock);
 
     return;
 }
diff --git a/opal/mca/accelerator/rocm/accelerator_rocm_module.c b/opal/mca/accelerator/rocm/accelerator_rocm_module.c
diff --git a/opal/mca/accelerator/rocm/configure.m4 b/opal/mca/accelerator/rocm/configure.m4
