accelerator/rocm: add rocm component

code compiles and all tests that passed in the non-framework version seem to be
passing again.

Signed-off-by: Edgar Gabriel <edgar.gabriel@amd.com>
Co-authored-by: Edgar Gabriel <edgar.gabriel@amd.com>
Co-authored-by: William Zhang <wilzhang@amazon.com>

Author: edgargabriel

config/opal_config_files.m4

@@ -18,7 +18,6 @@ AC_DEFUN([OPAL_CONFIG_FILES],[
     AC_CONFIG_FILES([
         opal/Makefile
         opal/cuda/Makefile
-        opal/rocm/Makefile
         opal/etc/Makefile
         opal/include/Makefile
         opal/datatype/Makefile

configure.ac

@@ -988,19 +988,6 @@ AC_CACHE_SAVE
 opal_show_title "System-specific tests"
 
 OPAL_CHECK_CUDA
-##################################
-# ROCm support
-##################################
-OPAL_CHECK_ROCM([opal_rocm],
-	        [opal_rocm_happy="yes"],
-	        [opal_rocm_happy="no"])
-OPAL_SUMMARY_ADD([Miscellaneous], [ROCm support], [], [$opal_rocm_happy])
-
-AS_IF([test "$OPAL_CUDA_SUPPORT" = "1" && test "$OPAL_ROCM_SUPPORT" = "1"],
-          [AC_MSG_WARN([Cannot support both CUDA and ROCm.])
-           AC_MSG_WARN([You must reconfigure Open MPI choosing either CUDA or ROCm .])
-           AC_MSG_ERROR([Cannot continue.])])
-
 ##################################
 OPAL_CHECK_OS_FLAVORS
 

ompi/mca/mtl/ofi/mtl_ofi.h

@@ -323,6 +323,9 @@ int ompi_mtl_ofi_register_buffer(struct opal_convertor_t *convertor,
         } else if (0 == strcmp(accelerator_base_selected_component.base_version.mca_component_name, "cuda")) {
             attr.iface = FI_HMEM_CUDA;
             opal_accelerator.get_device(&attr.device.cuda);
+        } else if (0 == strcmp(accelerator_base_selected_component.base_version.mca_component_name, "rocm")) {
+            attr.iface = FI_HMEM_ROCR;
+            opal_accelerator.get_device(&attr.device.cuda);
         } else {
             return OPAL_ERROR;
         }

ompi/mpiext/rocm/c/mpiext_rocm.c

@@ -25,29 +25,8 @@
 #include "opal/runtime/opal_params.h"
 #include "ompi/mpiext/rocm/c/mpiext_rocm_c.h"
 
-#if OPAL_ROCM_SUPPORT
-#include "opal/rocm/common_rocm_prototypes.h"
-#endif
 
 int MPIX_Query_rocm_support(void)
 {
-
-    if (!opal_built_with_rocm_support) {
-        return 0;
-    } else {
-        if ( opal_rocm_runtime_initialized ) {
-            return 1;
-        }
-#if OPAL_ROCM_SUPPORT
-        // There is a chance that the rocm runtime has simply not
-        // been initialized yet, since that is done during the first convertor creation
-        // Invoke a function that will trigger the rocm runtime initialized and
-        // check the value again after that.
-
-        int val1, val2;
-        mca_common_rocm_check_bufs((char *)&val1, (char *)&val2);
-#endif
-    }
-
-    return opal_rocm_runtime_initialized;
+    return opal_built_with_rocm_support && opal_rocm_runtime_initialized;
 }

opal/Makefile.am

@@ -25,11 +25,6 @@
 if OPAL_cuda_support
 LIBOPAL_GPU_SUBDIR = cuda
 LIBOPAL_GPU_LA = cuda/libopalcuda.la
-else
-if OPAL_rocm_support
-LIBOPAL_GPU_SUBDIR = rocm
-LIBOPAL_GPU_LA = rocm/libopalrocm.la
-endif
 endif
 
 
@@ -47,7 +42,6 @@ SUBDIRS = \
 DIST_SUBDIRS = \
 	include \
 	cuda \
-	rocm \
         datatype \
         etc \
 	util \

opal/datatype/opal_convertor.c

@@ -70,9 +70,6 @@ static void opal_convertor_construct(opal_convertor_t *convertor)
     convertor->remoteArch = opal_local_arch;
     convertor->flags = OPAL_DATATYPE_FLAG_NO_GAPS | CONVERTOR_COMPLETED;
     convertor->cbmemcpy = &opal_convertor_accelerator_memcpy;
-#if OPAL_ROCM_SUPPORT
-    convertor->cbmemcpy = &mca_common_rocm_memcpy;
-#endif
 }
 
 static void opal_convertor_destruct(opal_convertor_t *convertor)
@@ -579,11 +576,6 @@ int32_t opal_convertor_prepare_for_recv(opal_convertor_t *convertor,
     if (!(convertor->flags & CONVERTOR_SKIP_ACCELERATOR_INIT)) {
         opal_convertor_accelerator_init(convertor, pUserBuf);
     }
-#if OPAL_ROCM_SUPPORT
-    if (!(convertor->flags & CONVERTOR_SKIP_ACCELERATOR_INIT)) {
-        mca_common_rocm_convertor_init(convertor, pUserBuf);
-    }
-#endif
 
     assert(!(convertor->flags & CONVERTOR_SEND));
     OPAL_CONVERTOR_PREPARE(convertor, datatype, count, pUserBuf);
@@ -624,11 +616,6 @@ int32_t opal_convertor_prepare_for_send(opal_convertor_t *convertor,
     if (!(convertor->flags & CONVERTOR_SKIP_ACCELERATOR_INIT)) {
         opal_convertor_accelerator_init(convertor, pUserBuf);
     }
-#if OPAL_ROCM_SUPPORT
-    if (!(convertor->flags & CONVERTOR_SKIP_ACCELERATOR_INIT)) {
-        mca_common_rocm_convertor_init(convertor, pUserBuf);
-    }
-#endif
 
     OPAL_CONVERTOR_PREPARE(convertor, datatype, count, pUserBuf);
 

opal/datatype/opal_convertor.h

@@ -50,7 +50,6 @@ BEGIN_C_DECLS
 #define CONVERTOR_NO_OP                  0x00100000
 #define CONVERTOR_WITH_CHECKSUM          0x00200000
 #define CONVERTOR_ACCELERATOR            0x00400000
-#define CONVERTOR_ROCM                   0x00400000 //same as CUDA on purpose
 #define CONVERTOR_ACCELERATOR_ASYNC      0x00800000
 #define CONVERTOR_TYPE_MASK              0x10FF0000
 #define CONVERTOR_STATE_START            0x01000000

opal/datatype/opal_datatype_pack.h

@@ -106,8 +106,7 @@ static inline void pack_predefined_data(opal_convertor_t *CONVERTOR, const dt_el
     *(COUNT) -= cando_count;
 
     if (_elem->blocklen < 9) {
-        if (!(CONVERTOR->flags & CONVERTOR_ACCELERATOR) &&
-            !(CONVERTOR->flags & CONVERTOR_ROCM)
+        if (!(CONVERTOR->flags & CONVERTOR_ACCELERATOR)
             && OPAL_LIKELY(
                 OPAL_SUCCESS
                 == opal_datatype_pack_predefined_element(&_memory, &_packed, cando_count, _elem))) {

opal/datatype/opal_datatype_unpack.h

@@ -103,8 +103,7 @@ static inline void unpack_predefined_data(opal_convertor_t *CONVERTOR, const dt_
     *(COUNT) -= cando_count;
 
     if (_elem->blocklen < 9) {
-        if (!(CONVERTOR->flags & CONVERTOR_ACCELERATOR) &&
-            !(CONVERTOR->flags & CONVERTOR_ROCM)
+        if (!(CONVERTOR->flags & CONVERTOR_ACCELERATOR)
             && OPAL_LIKELY(OPAL_SUCCESS
                            == opal_datatype_unpack_predefined_element(&_packed, &_memory,
                                                                       cando_count, _elem))) {

opal/mca/accelerator/rocm/Makefile.am

@@ -0,0 +1,43 @@
+#
+# Copyright (c) 2014      Intel, Inc.  All rights reserved.
+# Copyright (c) 2017      IBM Corporation.  All rights reserved.
+# Copyright (c) 2022      Amazon.com, Inc. or its affiliates.
+#                         All Rights reserved.
+# Copyright (c) 2022      Advanced Micro Devices, Inc.
+#                         All Rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+
+AM_CPPFLAGS = $(common_rocm_CPPFLAGS)
+
+sources = \
+        accelerator_rocm.h \
+        accelerator_rocm_component.c \
+        accelerator_rocm_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_opal_accelerator_rocm_DSO
+component_noinst =
+component_install = mca_accelerator_rocm.la
+else
+component_noinst = libmca_accelerator_rocm.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_accelerator_rocm_la_SOURCES = $(sources)
+mca_accelerator_rocm_la_LDFLAGS = -module -avoid-version
+mca_accelerator_rocm_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_NAME@.la
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_accelerator_rocm_la_SOURCES =$(sources)
+libmca_accelerator_rocm_la_LDFLAGS = -module -avoid-version