accelerator/rocm: add buffer_id

add functionality to retrieve a buffer id. This is required for the
rcache module to identify whether a buffer pointer is still the
one that has originally been registered.

In addition, it seems that I forgot to remove the function declaration
when removing the dlopen part of the code.

Signed-off-by: Edgar Gabriel <Edgar.Gabriel@amd.com>

Author: edgargabriel

opal/mca/accelerator/rocm/accelerator_rocm.h

@@ -15,79 +15,9 @@
 
 #include <stdio.h>
 #include <hip/hip_runtime_api.h>
+#include <hip/hip_version.h>
 
-#include "opal/mca/btl/base/base.h"
-
-typedef hipError_t(*hipMalloc_t)(void **, size_t);
-typedef hipError_t(*hipFree_t)(void*);
-typedef hipError_t(*hipMemcpy_t)(void*, const void*, size_t, hipMemcpyKind);
-typedef hipError_t(*hipMemcpyAsync_t)(void*, const void*, size_t, hipMemcpyKind, hipStream_t);
-typedef hipError_t(*hipMemcpy2D_t)(void*, size_t, const void*, size_t, size_t, size_t, hipMemcpyKind);
-typedef hipError_t(*hipMemcpy2DAsync_t)(void*, size_t, const void*, size_t, size_t, size_t,
-                                       hipMemcpyKind, hipStream_t);
-typedef hipError_t(*hipMemGetAddressRange_t)(hipDeviceptr_t*, size_t*, hipDeviceptr_t);
-typedef hipError_t(*hipHostRegister_t)(void*, size_t, unsigned int);
-typedef hipError_t(*hipHostUnregister_t)(void*);
-
-typedef hipError_t(*hipStreamCreate_t)(hipStream_t*);
-typedef hipError_t(*hipStreamDestroy_t)(hipStream_t);
-typedef hipError_t(*hipStreamSynchronize_t)(hipStream_t);
-
-typedef const char*(*hipGetErrorString_t)(hipError_t);
-typedef hipError_t(*hipPointerGetAttributes_t)(hipPointerAttribute_t *, const void *);
-
-typedef hipError_t(*hipEventCreateWithFlags_t)(hipEvent_t*, unsigned);
-typedef hipError_t(*hipEventDestroy_t)(hipEvent_t);
-typedef hipError_t(*hipEventRecord_t)(hipEvent_t, hipStream_t);
-typedef hipError_t(*hipEventQuery_t)(hipEvent_t);
-typedef hipError_t(*hipEventSynchronize_t)(hipEvent_t);
-
-typedef hipError_t(*hipIpcGetMemHandle_t)(hipIpcMemHandle_t*, void*);
-typedef hipError_t(*hipIpcOpenMemHandle_t)(void **, hipIpcMemHandle_t, unsigned int);
-typedef hipError_t(*hipIpcCloseMemHandle_t)(void*);
-
-typedef hipError_t(*hipGetDevice_t)(int*);
-typedef hipError_t(*hipGetDeviceCount_t)(int*);
-typedef hipError_t(*hipDeviceCanAccessPeer_t)(int*, int, int);
-
-struct opal_accelerator_rocm_hipFunctionTable_s {
-    hipError_t (*hipMalloc)(void** pts, size_t size);
-    hipError_t (*hipFree)(void* ptr);
-    hipError_t (*hipMemcpy)(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind);
-    hipError_t (*hipMemcpyAsync)(void* dst, const void* src, size_t sizeBytes, hipMemcpyKind kind,
-                                 hipStream_t stream);
-    hipError_t(*hipMemcpy2D)(void* dst, size_t dpitch, const void* src, size_t spitch,
-                             size_t width, size_t height, hipMemcpyKind kind);
-    hipError_t(*hipMemcpy2DAsync)(void* dst, size_t dpitch, const void* src, size_t spitch,
-                                  size_t width, size_t height, hipMemcpyKind kind,
-                                  hipStream_t stream);
-    hipError_t(*hipMemGetAddressRange)(hipDeviceptr_t* pbase, size_t* psize, hipDeviceptr_t dptr);
-
-    hipError_t(*hipHostRegister)(void* ptr, size_t size, unsigned int flags);
-    hipError_t(*hipHostUnregister)(void* ptr);
-
-    hipError_t (*hipStreamCreate)(hipStream_t* stream);
-    hipError_t (*hipStreamDestroy)(hipStream_t stream);
-    hipError_t (*hipStreamSynchronize)(hipStream_t stream);
-    const char* (*hipGetErrorString)(hipError_t hipError);
-    hipError_t (*hipPointerGetAttributes)(hipPointerAttribute_t *attributes, const void *ptr);
-
-    hipError_t(*hipEventCreateWithFlags)(hipEvent_t* event, unsigned int flags);
-    hipError_t(*hipEventDestroy)(hipEvent_t event);
-    hipError_t(*hipEventRecord)(hipEvent_t event, hipStream_t stream);
-    hipError_t(*hipEventQuery)(hipEvent_t event);
-    hipError_t(*hipEventSynchronize)(hipEvent_t event);
-
-    hipError_t(*hipIpcGetMemHandle)(hipIpcMemHandle_t* handle, void* devPtr);
-    hipError_t(*hipIpcOpenMemHandle)(void **devPtr, hipIpcMemHandle_t handle, unsigned int flags);
-    hipError_t(*hipIpcCloseMemHandle)(void* devPtr);
-
-    hipError_t(*hipGetDevice)(int* dev_id);
-    hipError_t(*hipGetDeviceCount)(int* count);
-    hipError_t(*hipDeviceCanAccessPeer)(int* canAccess, int dev1, int dev2);
-};
-typedef struct opal_accelerator_rocm_hipFunctionTable_s opal_accelerator_rocm_hipFunctionTable_t;
-
+#include "opal/mca/accelerator/accelerator.h"
 
 typedef struct {
     opal_accelerator_base_component_t super;
@@ -96,8 +26,6 @@ typedef struct {
 OPAL_DECLSPEC extern opal_accelerator_rocm_component_t mca_accelerator_rocm_component;
 OPAL_DECLSPEC extern opal_accelerator_base_module_t opal_accelerator_rocm_module;
 
-OPAL_DECLSPEC extern opal_accelerator_rocm_hipFunctionTable_t opal_accelerator_hip_funcs;
-
 struct opal_accelerator_rocm_stream_t {
     opal_accelerator_stream_t base;
 };
@@ -110,8 +38,6 @@ struct opal_accelerator_rocm_event_t {
 typedef struct opal_accelerator_rocm_event_t opal_accelerator_rocm_event_t;
 OBJ_CLASS_DECLARATION(opal_accelerator_rocm_event_t);
 
-#define HIP_FUNCS opal_accelerator_hip_funcs
-
 OPAL_DECLSPEC extern hipStream_t opal_accelerator_rocm_MemcpyStream;
 OPAL_DECLSPEC extern int opal_accelerator_rocm_memcpy_async;
 OPAL_DECLSPEC extern int opal_accelerator_rocm_verbose;

opal/mca/accelerator/rocm/accelerator_rocm_module.c

@@ -11,6 +11,8 @@
 
 #include "accelerator_rocm.h"
 #include "opal/mca/accelerator/base/base.h"
+#include "opal/constants.h"
+#include "opal/util/output.h"
 
 /* Accelerator API's */
 static int mca_accelerator_rocm_check_addr(const void *addr, int *dev_id, uint64_t *flags);
@@ -491,5 +493,16 @@ static int mca_accelerator_rocm_device_can_access_peer(int *access, int dev1, in
 static int mca_accelerator_rocm_get_buffer_id(int dev_id, const void *addr, opal_accelerator_buffer_id_t *buf_id)
 {
     *buf_id = 0;
+
+#if HIP_VERSION >= 50120531
+    hipError_t result = hipPointerGetAttribute((unsigned long long *)&buf_id, HIP_POINTER_ATTRIBUTE_BUFFER_ID,
+                                               (hipDeviceptr_t)addr);
+    if (hipSuccess != result) {
+        opal_output_verbose(10, opal_accelerator_base_framework.framework_output,
+                            "error in hipPointerGetAttribute, could not retrieve buffer_id");
+        return OPAL_ERROR;
+    }
+#endif
+
     return OPAL_SUCCESS;
 }