Skip to content

Commit 35be18b

Browse files
edgargabrieledgargabriel
committed
common/ompio: rename ompio_cuda* to ompio_buffer*
the infrastructure put in place to manage cuda buffers is actually a lot more generic than just for cuda buffers. Specifically, we ca reuse much of the code to implement the external32 data representation. This commit converts the code from common_ompio_cuda* to common_ompio_buffer*. There are just very few places where we actually need to keep the OPAL_CUDA_SUPPORT ifdef in place. Signed-off-by: Edgar Gabriel <egabriel@central.uh.edu>
1 parent a96efb7 commit 35be18b

File tree

7 files changed

+47
-55
lines changed

7 files changed

+47
-55
lines changed

‎ompi/mca/common/ompio/Makefile.am

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ headers = \
2525
common_ompio_aggregators.h \
2626
common_ompio_print_queue.h \
2727
common_ompio_request.h \
28+
common_ompio_buffer.h \
2829
common_ompio.h
2930

3031
sources = \
@@ -34,6 +35,7 @@ sources = \
3435
common_ompio_file_open.c \
3536
common_ompio_file_view.c \
3637
common_ompio_file_read.c \
38+
common_ompio_buffer.c \
3739
common_ompio_file_write.c
3840

3941

@@ -74,10 +76,6 @@ else
7476
ompidir = $(includedir)
7577
endif
7678

77-
if OPAL_cuda_support
78-
headers += common_ompio_cuda.h
79-
sources += common_ompio_cuda.c
80-
endif
8179

8280
# These two rules will sym link the "noinst" libtool library filename
8381
# to the installable libtool library filename in the case where we are

‎ompi/mca/common/ompio/common_ompio_cuda.c renamed to ‎ompi/mca/common/ompio/common_ompio_buffer.c

Lines changed: 34 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
* University of Stuttgart. All rights reserved.
1010
* Copyright (c) 2004-2005 The Regents of the University of California.
1111
* All rights reserved.
12-
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
12+
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
1313
* $COPYRIGHT$
1414
*
1515
* Additional copyrights may follow
@@ -27,18 +27,19 @@
2727
#include "opal/mca/allocator/allocator.h"
2828
#include "opal/mca/allocator/base/base.h"
2929
#include "common_ompio.h"
30-
#include "common_ompio_cuda.h"
30+
#include "common_ompio_buffer.h"
3131

3232

33-
static opal_mutex_t mca_common_ompio_cuda_mutex; /* lock for thread safety */
33+
static opal_mutex_t mca_common_ompio_buffer_mutex; /* lock for thread safety */
3434
static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL;
3535
static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL;
3636

37-
static opal_atomic_int32_t mca_common_ompio_cuda_init = 0;
37+
static opal_atomic_int32_t mca_common_ompio_buffer_init = 0;
3838
static int32_t mca_common_ompio_pagesize=4096;
39-
static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size );
40-
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf );
39+
static void* mca_common_ompio_buffer_alloc_seg ( void *ctx, size_t *size );
40+
static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf );
4141

42+
#if OPAL_CUDA_SUPPORT
4243
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is_gpu,
4344
int *is_managed)
4445
{
@@ -57,8 +58,9 @@ void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf, int *is
5758

5859
return;
5960
}
61+
#endif
6062

61-
static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
63+
static void* mca_common_ompio_buffer_alloc_seg ( void*ctx, size_t *size )
6264
{
6365
char *buf=NULL;
6466
size_t realsize, numpages;
@@ -67,64 +69,67 @@ static void* mca_common_ompio_cuda_alloc_seg ( void*ctx, size_t *size )
6769
realsize = numpages * mca_common_ompio_pagesize;
6870

6971
buf = malloc ( realsize);
72+
#if OPAL_CUDA_SUPPORT
7073
if ( NULL != buf ) {
7174
mca_common_cuda_register ( ( char *)buf, realsize, NULL );
7275
}
76+
#endif
7377
*size = realsize;
7478
return buf;
7579
}
7680

77-
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf )
81+
static void mca_common_ompio_buffer_free_seg ( void *ctx, void *buf )
7882
{
7983
if ( NULL != buf ) {
84+
#if OPAL_CUDA_SUPPORT
8085
mca_common_cuda_unregister ( (char *) buf, NULL );
86+
#endif
8187
free ( buf );
8288
}
8389
return;
8490
}
8591

86-
int mca_common_ompio_cuda_alloc_init ( void )
92+
int mca_common_ompio_buffer_alloc_init ( void )
8793
{
8894
bool thread_safe=true;
8995

90-
if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_cuda_init, 1) > 1)
96+
if(OPAL_THREAD_ADD_FETCH32(&mca_common_ompio_buffer_init, 1) > 1)
9197
return OMPI_SUCCESS;
9298

9399
/* initialize static objects */
94-
OBJ_CONSTRUCT(&mca_common_ompio_cuda_mutex, opal_mutex_t);
100+
OBJ_CONSTRUCT(&mca_common_ompio_buffer_mutex, opal_mutex_t);
95101

96-
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex );
102+
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex );
97103
/* lookup name of the allocator to use */
98104
if(NULL == (mca_common_ompio_allocator_component = mca_allocator_component_lookup("basic"))) {
99-
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
105+
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
100106
return OMPI_ERR_BUFFER;
101107
}
102108

103109
/* create an instance of the allocator */
104110
mca_common_ompio_allocator = mca_common_ompio_allocator_component->allocator_init(thread_safe,
105-
mca_common_ompio_cuda_alloc_seg,
106-
mca_common_ompio_cuda_free_seg,
111+
mca_common_ompio_buffer_alloc_seg,
112+
mca_common_ompio_buffer_free_seg,
107113
NULL);
108114
if(NULL == mca_common_ompio_allocator) {
109-
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
115+
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
110116
return OMPI_ERR_BUFFER;
111117
}
112118

113-
// mca_common_ompio_pagesize = sysconf(_SC_PAGESIZE);
114119
mca_common_ompio_pagesize = opal_getpagesize();
115120

116-
OPAL_THREAD_UNLOCK(&mca_common_ompio_cuda_mutex);
121+
OPAL_THREAD_UNLOCK(&mca_common_ompio_buffer_mutex);
117122
return OMPI_SUCCESS;
118123
}
119124

120-
int mca_common_ompio_cuda_alloc_fini ( void )
125+
int mca_common_ompio_buffer_alloc_fini ( void )
121126
{
122127
if ( NULL != mca_common_ompio_allocator ) {
123-
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
128+
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
124129
mca_common_ompio_allocator->alc_finalize(mca_common_ompio_allocator);
125130
mca_common_ompio_allocator=NULL;
126-
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
127-
OBJ_DESTRUCT (&mca_common_ompio_cuda_mutex);
131+
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
132+
OBJ_DESTRUCT (&mca_common_ompio_buffer_mutex);
128133
}
129134

130135
return OMPI_SUCCESS;
@@ -134,31 +139,31 @@ void *mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize )
134139
{
135140
char *tmp=NULL;
136141

137-
if ( !mca_common_ompio_cuda_init ){
138-
mca_common_ompio_cuda_alloc_init ();
142+
if ( !mca_common_ompio_buffer_init ){
143+
mca_common_ompio_buffer_alloc_init ();
139144
}
140145

141-
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
146+
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
142147
tmp = mca_common_ompio_allocator->alc_alloc (mca_common_ompio_allocator,
143148
bufsize, 0 );
144-
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
149+
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
145150
return tmp;
146151
}
147152

148153
void mca_common_ompio_release_buf ( ompio_file_t *fh, void *buf )
149154
{
150155

151-
if ( !mca_common_ompio_cuda_init ){
156+
if ( !mca_common_ompio_buffer_init ){
152157
/* Should not happen. You can not release a buf without
153158
** having it allocated first.
154159
*/
155160
opal_output (1, "error in mca_common_ompio_release_buf: allocator not initialized\n");
156161
}
157162

158-
OPAL_THREAD_LOCK (&mca_common_ompio_cuda_mutex);
163+
OPAL_THREAD_LOCK (&mca_common_ompio_buffer_mutex);
159164
mca_common_ompio_allocator->alc_free (mca_common_ompio_allocator,
160165
buf);
161-
OPAL_THREAD_UNLOCK (&mca_common_ompio_cuda_mutex);
166+
OPAL_THREAD_UNLOCK (&mca_common_ompio_buffer_mutex);
162167

163168
return;
164169
}

‎ompi/mca/common/ompio/common_ompio_cuda.h renamed to ‎ompi/mca/common/ompio/common_ompio_buffer.h

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* University of Stuttgart. All rights reserved.
1111
* Copyright (c) 2004-2005 The Regents of the University of California.
1212
* All rights reserved.
13-
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
13+
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
1414
* $COPYRIGHT$
1515
*
1616
* Additional copyrights may follow
@@ -22,7 +22,7 @@
2222
#define MCA_COMMON_OMPIO_CUDA_H
2323

2424

25-
#define OMPIO_CUDA_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
25+
#define OMPIO_PREPARE_BUF(_fh,_buf,_count,_datatype,_tbuf,_convertor,_max_data,_decoded_iov,_iov_count){ \
2626
opal_convertor_clone ( _fh->f_convertor, _convertor, 0); \
2727
opal_convertor_prepare_for_send ( _convertor, &(_datatype->super), _count, _buf );\
2828
opal_convertor_get_packed_size( _convertor, &_max_data ); \
@@ -40,11 +40,12 @@
4040
_decoded_iov->iov_len = _max_data; \
4141
_iov_count=1;}
4242

43-
43+
#if OPAL_CUDA_SUPPORT
4444
void mca_common_ompio_check_gpu_buf ( ompio_file_t *fh, const void *buf,
4545
int *is_gpu, int *is_managed);
46-
int mca_common_ompio_cuda_alloc_init ( void );
47-
int mca_common_ompio_cuda_alloc_fini ( void );
46+
#endif
47+
int mca_common_ompio_buffer_alloc_init ( void );
48+
int mca_common_ompio_buffer_alloc_fini ( void );
4849

4950

5051
void* mca_common_ompio_alloc_buf ( ompio_file_t *fh, size_t bufsize);

‎ompi/mca/common/ompio/common_ompio_file_read.c

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,10 @@
3333

3434
#include "common_ompio.h"
3535
#include "common_ompio_request.h"
36+
#include "common_ompio_buffer.h"
3637
#include <unistd.h>
3738
#include <math.h>
3839

39-
#if OPAL_CUDA_SUPPORT
40-
#include "common_ompio_cuda.h"
41-
#endif
4240

4341
/* Read and write routines are split into two interfaces.
4442
** The

‎ompi/mca/common/ompio/common_ompio_file_write.c

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,12 +31,10 @@
3131

3232
#include "common_ompio.h"
3333
#include "common_ompio_request.h"
34+
#include "common_ompio_buffer.h"
3435
#include <unistd.h>
3536
#include <math.h>
3637

37-
#if OPAL_CUDA_SUPPORT
38-
#include "common_ompio_cuda.h"
39-
#endif
4038

4139
int mca_common_ompio_file_write (ompio_file_t *fh,
4240
const void *buf,

‎ompi/mca/common/ompio/common_ompio_request.c

Lines changed: 2 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* University of Stuttgart. All rights reserved.
1111
* Copyright (c) 2004-2005 The Regents of the University of California.
1212
* All rights reserved.
13-
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
13+
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
1414
* $COPYRIGHT$
1515
*
1616
* Additional copyrights may follow
@@ -19,9 +19,7 @@
1919
*/
2020

2121
#include "common_ompio_request.h"
22-
#if OPAL_CUDA_SUPPORT
23-
#include "common_ompio_cuda.h"
24-
#endif
22+
#include "common_ompio_buffer.h"
2523

2624
static void mca_common_ompio_request_construct(mca_ompio_request_t* req);
2725
static void mca_common_ompio_request_destruct(mca_ompio_request_t *req);
@@ -37,7 +35,6 @@ opal_list_t mca_common_ompio_pending_requests = {{0}};
3735
static int mca_common_ompio_request_free ( struct ompi_request_t **req)
3836
{
3937
mca_ompio_request_t *ompio_req = ( mca_ompio_request_t *)*req;
40-
#if OPAL_CUDA_SUPPORT
4138
if ( NULL != ompio_req->req_tbuf ) {
4239
if ( MCA_OMPIO_REQUEST_READ == ompio_req->req_type ){
4340
struct iovec decoded_iov;
@@ -50,7 +47,6 @@ static int mca_common_ompio_request_free ( struct ompi_request_t **req)
5047
}
5148
mca_common_ompio_release_buf ( NULL, ompio_req->req_tbuf );
5249
}
53-
#endif
5450
if ( NULL != ompio_req->req_free_fn ) {
5551
ompio_req->req_free_fn (ompio_req );
5652
}
@@ -77,10 +73,8 @@ void mca_common_ompio_request_construct(mca_ompio_request_t* req)
7773
req->req_ompi.req_cancel = mca_common_ompio_request_cancel;
7874
req->req_ompi.req_type = OMPI_REQUEST_IO;
7975
req->req_data = NULL;
80-
#if OPAL_CUDA_SUPPORT
8176
req->req_tbuf = NULL;
8277
req->req_size = 0;
83-
#endif
8478
req->req_progress_fn = NULL;
8579
req->req_free_fn = NULL;
8680

‎ompi/mca/common/ompio/common_ompio_request.h

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
* University of Stuttgart. All rights reserved.
1111
* Copyright (c) 2004-2005 The Regents of the University of California.
1212
* All rights reserved.
13-
* Copyright (c) 2008-2018 University of Houston. All rights reserved.
13+
* Copyright (c) 2008-2019 University of Houston. All rights reserved.
1414
* Copyright (c) 2018 Research Organization for Information Science
1515
* and Technology (RIST). All rights reserved.
1616
* $COPYRIGHT$
@@ -52,11 +52,9 @@ struct mca_ompio_request_t {
5252
mca_ompio_request_type_t req_type;
5353
void *req_data;
5454
opal_list_item_t req_item;
55-
#if OPAL_CUDA_SUPPORT
5655
void *req_tbuf;
5756
size_t req_size;
5857
opal_convertor_t req_convertor;
59-
#endif
6058
mca_fbtl_base_module_progress_fn_t req_progress_fn;
6159
mca_fbtl_base_module_request_free_fn_t req_free_fn;
6260
};

0 commit comments

Comments
 (0)