Skip to content

Commit 8cec2a2

Browse files
wenduwanwenduwan
committed
opal/accelerator: introduce accelerator ipc APIs and implement stubs
Signed-off-by: Wenduo Wang <wenduwan@amazon.com>
1 parent 7e092c3 commit 8cec2a2

File tree

5 files changed

+271
-0
lines changed

5 files changed

+271
-0
lines changed

‎opal/mca/accelerator/accelerator.h

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,19 @@ struct opal_accelerator_stream_t {
111111
};
112112
typedef struct opal_accelerator_stream_t opal_accelerator_stream_t;
113113

114+
#define IPC_MAX_HANDLE_SIZE 64
115+
struct opal_accelerator_ipc_handle_t {
116+
size_t size;
117+
uint8_t handle[IPC_MAX_HANDLE_SIZE];
118+
};
119+
typedef struct opal_accelerator_ipc_handle_t opal_accelerator_ipc_handle_t;
120+
121+
struct opal_accelerator_ipc_event_handle_t {
122+
size_t size;
123+
uint8_t handle[IPC_MAX_HANDLE_SIZE];
124+
};
125+
typedef struct opal_accelerator_ipc_event_handle_t opal_accelerator_ipc_event_handle_t;
126+
114127
struct opal_accelerator_pci_attr_t {
115128
uint16_t domain_id;
116129
uint8_t bus_id;
@@ -318,6 +331,78 @@ typedef int (*opal_accelerator_base_module_mem_release_fn_t)(
318331
typedef int (*opal_accelerator_base_module_get_address_range_fn_t)(
319332
int dev_id, const void *ptr, void **base, size_t *size);
320333

334+
/*********************************************************/
335+
/**** Inter Process Communication (IPC) Functions ****/
336+
/*********************************************************/
337+
338+
/**
339+
* Queries whether the device supports IPC or not.
340+
*
341+
* If true, the functions:
342+
*
343+
* opal_accelerator_base_module_get_ipc_handle_fn_t()
344+
* opal_accelerator_base_module_open_ipc_handle_fn_t()
345+
* opal_accelerator_base_module_get_ipc_event_handle_fn_t()
346+
* opal_accelerator_base_module_open_ipc_event_handle_fn_t()
347+
*
348+
* must be implemented.
349+
*
350+
* @return true IPC supported
351+
* @return false IPC not supported
352+
*/
353+
typedef bool (*opal_accelerator_base_module_is_ipc_enabled_fn_t)(void);
354+
355+
/**
356+
* Gets an IPC memory handle for an existing device memory allocation.
357+
*
358+
* @param[IN] dev_id Associated device for the IPC memory handle or
359+
* MCA_ACCELERATOR_NO_DEVICE_ID
360+
* @param[IN] dev_ptr Device memory address
361+
* @param[OUT] handle Pointer to IPC handle object
362+
*
363+
* @return OPAL_SUCCESS or error status on failure
364+
*
365+
*/
366+
typedef int (*opal_accelerator_base_module_get_ipc_handle_fn_t)(
367+
int dev_id, void *dev_ptr, opal_accelerator_ipc_handle_t *handle);
368+
369+
/**
370+
* Opens an IPC memory handle from another process and returns
371+
* a device pointer usable in the local process.
372+
*
373+
* @param[IN] dev_id Associated device for the IPC memory handle or
374+
* MCA_ACCELERATOR_NO_DEVICE_ID
375+
* @param[IN] handle IPC handle object from another process
376+
* @param[OUT] dev_ptr Returned device pointer
377+
*
378+
* @return OPAL_SUCCESS or error status on failure
379+
*/
380+
typedef int (*opal_accelerator_base_module_open_ipc_handle_fn_t)(
381+
int dev_id, opal_accelerator_ipc_handle_t *handle, void **dev_ptr);
382+
383+
/**
384+
* Gets an IPC event handle for an event created by opal_accelerator_base_module_create_event_fn_t.
385+
*
386+
* @param[IN] event Event created previously
387+
* @param[OUT] handle Pointer to IPC event handle object
388+
*
389+
* @return OPAL_SUCCESS or error status on failure
390+
*/
391+
typedef int (*opal_accelerator_base_module_get_ipc_event_handle_fn_t)(
392+
opal_accelerator_event_t *event, opal_accelerator_ipc_event_handle_t *handle);
393+
394+
/**
395+
* Opens an IPC event handle from another process opened by
396+
* opal_accelerator_base_module_get_ipc_event_handle_fn_t.
397+
*
398+
* @param[IN] handle IPC event handle from another process
399+
* @param[OUT] event Pointer to store the opened event
400+
*
401+
* @return OPAL_SUCCESS or error status on failure
402+
*/
403+
typedef int (*opal_accelerator_base_module_open_ipc_event_handle_fn_t)(
404+
opal_accelerator_ipc_event_handle_t *handle, opal_accelerator_event_t *event);
405+
321406
/**
322407
* Page-locks the memory range specified by ptr and size
323408
*
@@ -414,6 +499,12 @@ typedef struct {
414499
opal_accelerator_base_module_mem_release_fn_t mem_release;
415500
opal_accelerator_base_module_get_address_range_fn_t get_address_range;
416501

502+
opal_accelerator_base_module_is_ipc_enabled_fn_t is_ipc_enabled;
503+
opal_accelerator_base_module_get_ipc_handle_fn_t get_ipc_handle;
504+
opal_accelerator_base_module_open_ipc_handle_fn_t open_ipc_handle;
505+
opal_accelerator_base_module_get_ipc_event_handle_fn_t get_ipc_event_handle;
506+
opal_accelerator_base_module_open_ipc_event_handle_fn_t open_ipc_event_handle;
507+
417508
opal_accelerator_base_module_host_register_fn_t host_register;
418509
opal_accelerator_base_module_host_unregister_fn_t host_unregister;
419510

‎opal/mca/accelerator/cuda/accelerator_cuda.c

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,16 @@ static int accelerator_cuda_mem_release(int dev_id, void *ptr);
4141
static int accelerator_cuda_get_address_range(int dev_id, const void *ptr, void **base,
4242
size_t *size);
4343

44+
static bool accelerator_cuda_is_ipc_enabled(void);
45+
static int accelerator_cuda_get_ipc_handle(int dev_id, void *dev_ptr,
46+
opal_accelerator_ipc_handle_t *handle);
47+
static int accelerator_cuda_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
48+
void **dev_ptr);
49+
static int accelerator_cuda_get_ipc_event_handle(opal_accelerator_event_t *event,
50+
opal_accelerator_ipc_event_handle_t *handle);
51+
static int accelerator_cuda_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
52+
opal_accelerator_event_t *event);
53+
4454
static int accelerator_cuda_host_register(int dev_id, void *ptr, size_t size);
4555
static int accelerator_cuda_host_unregister(int dev_id, void *ptr);
4656

@@ -67,6 +77,12 @@ opal_accelerator_base_module_t opal_accelerator_cuda_module =
6777
accelerator_cuda_mem_release,
6878
accelerator_cuda_get_address_range,
6979

80+
accelerator_cuda_is_ipc_enabled,
81+
accelerator_cuda_get_ipc_handle,
82+
accelerator_cuda_open_ipc_handle,
83+
accelerator_cuda_get_ipc_event_handle,
84+
accelerator_cuda_open_ipc_event_handle,
85+
7086
accelerator_cuda_host_register,
7187
accelerator_cuda_host_unregister,
7288

@@ -520,6 +536,35 @@ static int accelerator_cuda_get_address_range(int dev_id, const void *ptr, void
520536
return 0;
521537
}
522538

539+
static bool accelerator_cuda_is_ipc_enabled(void)
540+
{
541+
return false;
542+
}
543+
544+
static int accelerator_cuda_get_ipc_handle(int dev_id, void *dev_ptr,
545+
opal_accelerator_ipc_handle_t *handle)
546+
{
547+
return OPAL_ERR_NOT_IMPLEMENTED;
548+
}
549+
550+
static int accelerator_cuda_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
551+
void **dev_ptr)
552+
{
553+
return OPAL_ERR_NOT_IMPLEMENTED;
554+
}
555+
556+
static int accelerator_cuda_get_ipc_event_handle(opal_accelerator_event_t *event,
557+
opal_accelerator_ipc_event_handle_t *handle)
558+
{
559+
return OPAL_ERR_NOT_IMPLEMENTED;
560+
}
561+
562+
static int accelerator_cuda_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
563+
opal_accelerator_event_t *event)
564+
{
565+
return OPAL_ERR_NOT_IMPLEMENTED;
566+
}
567+
523568
static int accelerator_cuda_host_register(int dev_id, void *ptr, size_t size)
524569
{
525570
CUresult result;

‎opal/mca/accelerator/null/accelerator_null_component.c

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,16 @@ static int accelerator_null_mem_alloc(int dev_id, void **ptr, size_t size);
5555
static int accelerator_null_mem_release(int dev_id, void *ptr);
5656
static int accelerator_null_get_address_range(int dev_id, const void *ptr, void **base, size_t *size);
5757

58+
static bool accelerator_null_is_ipc_enabled(void);
59+
static int accelerator_null_get_ipc_handle(int dev_id, void *dev_ptr,
60+
opal_accelerator_ipc_handle_t *handle);
61+
static int accelerator_null_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
62+
void **dev_ptr);
63+
static int accelerator_null_get_ipc_event_handle(opal_accelerator_event_t *event,
64+
opal_accelerator_ipc_event_handle_t *handle);
65+
static int accelerator_null_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
66+
opal_accelerator_event_t *event);
67+
5868
static int accelerator_null_host_register(int dev_id, void *ptr, size_t size);
5969
static int accelerator_null_host_unregister(int dev_id, void *ptr);
6070

@@ -119,6 +129,12 @@ opal_accelerator_base_module_t opal_accelerator_null_module =
119129
accelerator_null_mem_release,
120130
accelerator_null_get_address_range,
121131

132+
accelerator_null_is_ipc_enabled,
133+
accelerator_null_get_ipc_handle,
134+
accelerator_null_open_ipc_handle,
135+
accelerator_null_get_ipc_event_handle,
136+
accelerator_null_open_ipc_event_handle,
137+
122138
accelerator_null_host_register,
123139
accelerator_null_host_unregister,
124140

@@ -222,6 +238,35 @@ static int accelerator_null_get_address_range(int dev_id, const void *ptr, void
222238
return OPAL_ERR_NOT_IMPLEMENTED;
223239
}
224240

241+
static bool accelerator_null_is_ipc_enabled(void)
242+
{
243+
return false;
244+
}
245+
246+
static int accelerator_null_get_ipc_handle(int dev_id, void *dev_ptr,
247+
opal_accelerator_ipc_handle_t *handle)
248+
{
249+
return OPAL_ERR_NOT_IMPLEMENTED;
250+
}
251+
252+
static int accelerator_null_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
253+
void **dev_ptr)
254+
{
255+
return OPAL_ERR_NOT_IMPLEMENTED;
256+
}
257+
258+
static int accelerator_null_get_ipc_event_handle(opal_accelerator_event_t *event,
259+
opal_accelerator_ipc_event_handle_t *handle)
260+
{
261+
return OPAL_ERR_NOT_IMPLEMENTED;
262+
}
263+
264+
static int accelerator_null_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
265+
opal_accelerator_event_t *event)
266+
{
267+
return OPAL_ERR_NOT_IMPLEMENTED;
268+
}
269+
225270
static int accelerator_null_host_register(int dev_id, void *ptr, size_t size)
226271
{
227272
return OPAL_ERR_NOT_IMPLEMENTED;

‎opal/mca/accelerator/rocm/accelerator_rocm_module.c

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,16 @@ static int mca_accelerator_rocm_mem_release(int dev_id, void *ptr);
3333
static int mca_accelerator_rocm_get_address_range(int dev_id, const void *ptr, void **base,
3434
size_t *size);
3535

36+
static bool mca_accelerator_rocm_is_ipc_enabled(void);
37+
static int mca_accelerator_rocm_get_ipc_handle(int dev_id, void *dev_ptr,
38+
opal_accelerator_ipc_handle_t *handle);
39+
static int mca_accelerator_rocm_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
40+
void **dev_ptr);
41+
static int mca_accelerator_rocm_get_ipc_event_handle(opal_accelerator_event_t *event,
42+
opal_accelerator_ipc_event_handle_t *handle);
43+
static int mca_accelerator_rocm_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
44+
opal_accelerator_event_t *event);
45+
3646
static int mca_accelerator_rocm_host_register(int dev_id, void *ptr, size_t size);
3747
static int mca_accelerator_rocm_host_unregister(int dev_id, void *ptr);
3848

@@ -59,6 +69,12 @@ opal_accelerator_base_module_t opal_accelerator_rocm_module =
5969
mca_accelerator_rocm_mem_release,
6070
mca_accelerator_rocm_get_address_range,
6171

72+
mca_accelerator_rocm_is_ipc_enabled,
73+
mca_accelerator_rocm_get_ipc_handle,
74+
mca_accelerator_rocm_open_ipc_handle,
75+
mca_accelerator_rocm_get_ipc_event_handle,
76+
mca_accelerator_rocm_open_ipc_event_handle,
77+
6278
mca_accelerator_rocm_host_register,
6379
mca_accelerator_rocm_host_unregister,
6480

@@ -438,6 +454,35 @@ static int mca_accelerator_rocm_get_address_range(int dev_id, const void *ptr, v
438454
return OPAL_SUCCESS;
439455
}
440456

457+
static bool mca_accelerator_rocm_is_ipc_enabled(void)
458+
{
459+
return false;
460+
}
461+
462+
static int mca_accelerator_rocm_get_ipc_handle(int dev_id, void *dev_ptr,
463+
opal_accelerator_ipc_handle_t *handle)
464+
{
465+
return OPAL_ERR_NOT_IMPLEMENTED;
466+
}
467+
468+
static int mca_accelerator_rocm_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
469+
void **dev_ptr)
470+
{
471+
return OPAL_ERR_NOT_IMPLEMENTED;
472+
}
473+
474+
static int mca_accelerator_rocm_get_ipc_event_handle(opal_accelerator_event_t *event,
475+
opal_accelerator_ipc_event_handle_t *handle)
476+
{
477+
return OPAL_ERR_NOT_IMPLEMENTED;
478+
}
479+
480+
static int mca_accelerator_rocm_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
481+
opal_accelerator_event_t *event)
482+
{
483+
return OPAL_ERR_NOT_IMPLEMENTED;
484+
}
485+
441486
static int mca_accelerator_rocm_host_register(int dev_id, void *ptr, size_t size)
442487
{
443488
if (NULL == ptr && size > 0) {

‎opal/mca/accelerator/ze/accelerator_ze_module.c

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,16 @@ static int mca_accelerator_ze_mem_release(int dev_id, void *ptr);
3838
static int mca_accelerator_ze_get_address_range(int dev_id, const void *ptr, void **base,
3939
size_t *size);
4040

41+
static bool mca_accelerator_ze_is_ipc_enabled(void);
42+
static int mca_accelerator_ze_get_ipc_handle(int dev_id, void *dev_ptr,
43+
opal_accelerator_ipc_handle_t *handle);
44+
static int mca_accelerator_ze_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
45+
void **dev_ptr);
46+
static int mca_accelerator_ze_get_ipc_event_handle(opal_accelerator_event_t *event,
47+
opal_accelerator_ipc_event_handle_t *handle);
48+
static int mca_accelerator_ze_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
49+
opal_accelerator_event_t *event);
50+
4151
static int mca_accelerator_ze_host_register(int dev_id, void *ptr, size_t size);
4252
static int mca_accelerator_ze_host_unregister(int dev_id, void *ptr);
4353

@@ -65,6 +75,12 @@ opal_accelerator_base_module_t opal_accelerator_ze_module =
6575
.mem_release = mca_accelerator_ze_mem_release,
6676
.get_address_range = mca_accelerator_ze_get_address_range,
6777

78+
.is_ipc_enabled = mca_accelerator_ze_is_ipc_enabled,
79+
.get_ipc_handle = mca_accelerator_ze_get_ipc_handle,
80+
.open_ipc_handle = mca_accelerator_ze_open_ipc_handle,
81+
.get_ipc_event_handle = mca_accelerator_ze_get_ipc_event_handle,
82+
.open_ipc_event_handle = mca_accelerator_ze_open_ipc_event_handle,
83+
6884
.host_register = mca_accelerator_ze_host_register,
6985
.host_unregister = mca_accelerator_ze_host_unregister,
7086

@@ -571,6 +587,35 @@ static int mca_accelerator_ze_get_address_range(int dev_id, const void *ptr, voi
571587
return OPAL_SUCCESS;
572588
}
573589

590+
static bool mca_accelerator_ze_is_ipc_enabled(void)
591+
{
592+
return false;
593+
}
594+
595+
static int mca_accelerator_ze_get_ipc_handle(int dev_id, void *dev_ptr,
596+
opal_accelerator_ipc_handle_t *handle)
597+
{
598+
return OPAL_ERR_NOT_IMPLEMENTED;
599+
}
600+
601+
static int mca_accelerator_ze_open_ipc_handle(int dev_id, opal_accelerator_ipc_handle_t *handle,
602+
void **dev_ptr)
603+
{
604+
return OPAL_ERR_NOT_IMPLEMENTED;
605+
}
606+
607+
static int mca_accelerator_ze_get_ipc_event_handle(opal_accelerator_event_t *event,
608+
opal_accelerator_ipc_event_handle_t *handle)
609+
{
610+
return OPAL_ERR_NOT_IMPLEMENTED;
611+
}
612+
613+
static int mca_accelerator_ze_open_ipc_event_handle(opal_accelerator_ipc_event_handle_t *handle,
614+
opal_accelerator_event_t *event)
615+
{
616+
return OPAL_ERR_NOT_IMPLEMENTED;
617+
}
618+
574619
/*
575620
* ZE doesn't have explicit host memory registration functions
576621
*/

0 commit comments

Comments
 (0)