Skip to content

Commit 8500fc4

Browse files
shintaro-iwasakishintaro-iwasaki
committed
opal/mca/threads/argobots: use static mutex/cond objects
Argobots 1.1 started to support static mutex/cond initialization. This patch introduces them to clean up the current implementation. Signed-off-by: Shintaro Iwasaki <siwasaki@anl.gov>
1 parent 43c21f4 commit 8500fc4

File tree

2 files changed

+52
-122
lines changed

2 files changed

+52
-122
lines changed

‎opal/mca/threads/argobots/threads_argobots_mutex.c

Lines changed: 18 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
* Copyright (c) 2015 Research Organization for Information Science
1616
* and Technology (RIST). All rights reserved.
1717
* Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
18+
* Copyright (c) 2021 Argonne National Laboratory. All rights reserved.
1819
*
1920
* $COPYRIGHT$
2021
*
@@ -27,6 +28,7 @@
2728
#include "opal/mca/threads/argobots/threads_argobots.h"
2829

2930
#include <errno.h>
31+
#include <string.h>
3032

3133
#include "opal/constants.h"
3234
#include "opal/mca/threads/argobots/threads_argobots_mutex.h"
@@ -41,7 +43,8 @@ bool opal_uses_threads = false;
4143
static void mca_threads_argobots_mutex_constructor(opal_mutex_t *p_mutex)
4244
{
4345
opal_threads_argobots_ensure_init();
44-
p_mutex->m_lock_argobots = OPAL_ABT_MUTEX_NULL;
46+
const ABT_mutex_memory init_mutex = ABT_MUTEX_INITIALIZER;
47+
memcpy(&p_mutex->m_lock_argobots, &init_mutex, sizeof(ABT_mutex_memory));
4548
p_mutex->m_recursive = 0;
4649
#if OPAL_ENABLE_DEBUG
4750
p_mutex->m_lock_debug = 0;
@@ -51,17 +54,11 @@ static void mca_threads_argobots_mutex_constructor(opal_mutex_t *p_mutex)
5154
opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
5255
}
5356

54-
static void mca_threads_argobots_mutex_destructor(opal_mutex_t *p_mutex)
55-
{
56-
if (OPAL_ABT_MUTEX_NULL != p_mutex->m_lock_argobots) {
57-
ABT_mutex_free(&p_mutex->m_lock_argobots);
58-
}
59-
}
60-
6157
static void mca_threads_argobots_recursive_mutex_constructor(opal_recursive_mutex_t *p_mutex)
6258
{
6359
opal_threads_argobots_ensure_init();
64-
p_mutex->m_lock_argobots = OPAL_ABT_MUTEX_NULL;
60+
const ABT_mutex_memory init_mutex = ABT_RECURSIVE_MUTEX_INITIALIZER;
61+
memcpy(&p_mutex->m_lock_argobots, &init_mutex, sizeof(ABT_mutex_memory));
6562
p_mutex->m_recursive = 1;
6663
#if OPAL_ENABLE_DEBUG
6764
p_mutex->m_lock_debug = 0;
@@ -71,97 +68,40 @@ static void mca_threads_argobots_recursive_mutex_constructor(opal_recursive_mute
7168
opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
7269
}
7370

74-
static void mca_threads_argobots_recursive_mutex_destructor(opal_recursive_mutex_t *p_mutex)
75-
{
76-
if (OPAL_ABT_MUTEX_NULL != p_mutex->m_lock_argobots) {
77-
ABT_mutex_free(&p_mutex->m_lock_argobots);
78-
}
79-
}
80-
81-
OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, mca_threads_argobots_mutex_constructor,
82-
mca_threads_argobots_mutex_destructor);
71+
OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, mca_threads_argobots_mutex_constructor, NULL);
8372
OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, opal_object_t,
84-
mca_threads_argobots_recursive_mutex_constructor,
85-
mca_threads_argobots_recursive_mutex_destructor);
86-
87-
void opal_mutex_create(struct opal_mutex_t *m)
88-
{
89-
opal_threads_argobots_ensure_init();
90-
while (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
91-
ABT_mutex abt_mutex;
92-
if (m->m_recursive) {
93-
ABT_mutex_attr abt_mutex_attr;
94-
ABT_mutex_attr_create(&abt_mutex_attr);
95-
ABT_mutex_attr_set_recursive(abt_mutex_attr, ABT_TRUE);
96-
ABT_mutex_create_with_attr(abt_mutex_attr, &abt_mutex);
97-
ABT_mutex_attr_free(&abt_mutex_attr);
98-
} else {
99-
ABT_mutex_create(&abt_mutex);
100-
}
101-
void *null_ptr = OPAL_ABT_MUTEX_NULL;
102-
if (opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) &m->m_lock_argobots,
103-
(intptr_t *) &null_ptr, (intptr_t) abt_mutex)) {
104-
/* mutex is successfully created and substituted. */
105-
return;
106-
}
107-
ABT_mutex_free(&abt_mutex);
108-
}
109-
}
110-
111-
static void opal_cond_create(opal_cond_t *cond)
112-
{
113-
opal_threads_argobots_ensure_init();
114-
while (OPAL_ABT_COND_NULL == *cond) {
115-
ABT_cond new_cond;
116-
ABT_cond_create(&new_cond);
117-
void *null_ptr = OPAL_ABT_COND_NULL;
118-
if (opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) cond,
119-
(intptr_t *) &null_ptr, (intptr_t) new_cond)) {
120-
/* cond is successfully created and substituted. */
121-
return;
122-
}
123-
ABT_cond_free(&new_cond);
124-
}
125-
}
73+
mca_threads_argobots_recursive_mutex_constructor, NULL);
12674

12775
int opal_cond_init(opal_cond_t *cond)
12876
{
129-
*cond = OPAL_ABT_COND_NULL;
77+
const ABT_cond_memory init_cond = ABT_COND_INITIALIZER;
78+
memcpy(cond, &init_cond, sizeof(ABT_cond_memory));
13079
return OPAL_SUCCESS;
13180
}
13281

13382
int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock)
13483
{
135-
if (OPAL_ABT_COND_NULL == *cond) {
136-
opal_cond_create(cond);
137-
}
138-
int ret = ABT_cond_wait(*cond, lock->m_lock_argobots);
84+
ABT_mutex abt_mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&lock->m_lock_argobots);
85+
ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond);
86+
int ret = ABT_cond_wait(abt_cond, abt_mutex);
13987
return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
14088
}
14189

14290
int opal_cond_broadcast(opal_cond_t *cond)
14391
{
144-
if (OPAL_ABT_COND_NULL == *cond) {
145-
opal_cond_create(cond);
146-
}
147-
int ret = ABT_cond_broadcast(*cond);
92+
ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond);
93+
int ret = ABT_cond_broadcast(abt_cond);
14894
return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
14995
}
15096

15197
int opal_cond_signal(opal_cond_t *cond)
15298
{
153-
if (OPAL_ABT_COND_NULL == *cond) {
154-
opal_cond_create(cond);
155-
}
156-
int ret = ABT_cond_signal(*cond);
99+
ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond);
100+
int ret = ABT_cond_signal(abt_cond);
157101
return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
158102
}
159103

160104
int opal_cond_destroy(opal_cond_t *cond)
161105
{
162-
int ret = ABT_SUCCESS;
163-
if (OPAL_ABT_COND_NULL != *cond) {
164-
ret = ABT_cond_free(cond);
165-
}
166-
return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
106+
return OPAL_SUCCESS;
167107
}

‎opal/mca/threads/argobots/threads_argobots_mutex.h

Lines changed: 34 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
* Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
1818
* Copyright (c) 2020 Triad National Security, LLC. All rights
1919
* reserved.
20-
*
2120
* Copyright (c) 2020 Cisco Systems, Inc. All rights reserved.
21+
* Copyright (c) 2021 Argonne National Laboratory. All rights reserved.
2222
* $COPYRIGHT$
2323
*
2424
* Additional copyrights may follow
@@ -34,21 +34,18 @@
3434
#include <errno.h>
3535
#include <stdio.h>
3636

37-
#include "opal/mca/threads/argobots/threads_argobots.h"
38-
3937
#include "opal/class/opal_object.h"
38+
#include "opal/mca/threads/argobots/threads_argobots.h"
39+
#include "opal/mca/threads/mutex.h"
4040
#include "opal/sys/atomic.h"
4141
#include "opal/util/output.h"
4242

4343
BEGIN_C_DECLS
4444

45-
/* Don't use ABT_MUTEX_NULL, since it might be not NULL. */
46-
#define OPAL_ABT_MUTEX_NULL 0
47-
4845
struct opal_mutex_t {
4946
opal_object_t super;
5047

51-
ABT_mutex m_lock_argobots;
48+
ABT_mutex_memory m_lock_argobots;
5249
int m_recursive;
5350

5451
#if OPAL_ENABLE_DEBUG
@@ -64,32 +61,34 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t);
6461
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
6562

6663
#if OPAL_ENABLE_DEBUG
67-
# define OPAL_MUTEX_STATIC_INIT \
68-
{ \
69-
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = OPAL_ABT_MUTEX_NULL, \
70-
.m_recursive = 0, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \
71-
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
64+
# define OPAL_MUTEX_STATIC_INIT \
65+
{ \
66+
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = ABT_MUTEX_INITIALIZER, \
67+
.m_recursive = 0, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \
68+
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
7269
}
7370
#else
74-
# define OPAL_MUTEX_STATIC_INIT \
75-
{ \
76-
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = OPAL_ABT_MUTEX_NULL, \
77-
.m_recursive = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
71+
# define OPAL_MUTEX_STATIC_INIT \
72+
{ \
73+
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = ABT_MUTEX_INITIALIZER, \
74+
.m_recursive = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
7875
}
7976
#endif
8077

8178
#if OPAL_ENABLE_DEBUG
82-
# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
83-
{ \
84-
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = OPAL_ABT_MUTEX_NULL, \
85-
.m_recursive = 1, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \
86-
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
79+
# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
80+
{ \
81+
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \
82+
.m_lock_argobots = ABT_RECURSIVE_MUTEX_INITIALIZER, .m_recursive = 1, \
83+
.m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \
84+
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
8785
}
8886
#else
89-
# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
90-
{ \
91-
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = OPAL_ABT_MUTEX_NULL, \
92-
.m_recursive = 1, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
87+
# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
88+
{ \
89+
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \
90+
.m_lock_argobots = ABT_RECURSIVE_MUTEX_INITIALIZER, .m_recursive = 1, \
91+
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
9392
}
9493
#endif
9594

@@ -99,14 +98,10 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
9998
*
10099
************************************************************************/
101100

102-
void opal_mutex_create(struct opal_mutex_t *m);
103-
104101
static inline int opal_mutex_trylock(opal_mutex_t *m)
105102
{
106-
if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
107-
opal_mutex_create(m);
108-
}
109-
int ret = ABT_mutex_trylock(m->m_lock_argobots);
103+
ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots);
104+
int ret = ABT_mutex_trylock(mutex);
110105
if (ABT_ERR_MUTEX_LOCKED == ret) {
111106
return 1;
112107
} else if (ABT_SUCCESS != ret) {
@@ -120,31 +115,27 @@ static inline int opal_mutex_trylock(opal_mutex_t *m)
120115

121116
static inline void opal_mutex_lock(opal_mutex_t *m)
122117
{
123-
if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
124-
opal_mutex_create(m);
125-
}
118+
ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots);
126119
#if OPAL_ENABLE_DEBUG
127-
int ret = ABT_mutex_lock(m->m_lock_argobots);
120+
int ret = ABT_mutex_lock(mutex);
128121
if (ABT_SUCCESS != ret) {
129122
opal_output(0, "opal_mutex_lock()");
130123
}
131124
#else
132-
ABT_mutex_lock(m->m_lock_argobots);
125+
ABT_mutex_lock(mutex);
133126
#endif
134127
}
135128

136129
static inline void opal_mutex_unlock(opal_mutex_t *m)
137130
{
138-
if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
139-
opal_mutex_create(m);
140-
}
131+
ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots);
141132
#if OPAL_ENABLE_DEBUG
142-
int ret = ABT_mutex_unlock(m->m_lock_argobots);
133+
int ret = ABT_mutex_unlock(mutex);
143134
if (ABT_SUCCESS != ret) {
144135
opal_output(0, "opal_mutex_unlock()");
145136
}
146137
#else
147-
ABT_mutex_unlock(m->m_lock_argobots);
138+
ABT_mutex_unlock(mutex);
148139
#endif
149140
/* For fairness of locking. */
150141
ABT_thread_yield();
@@ -200,9 +191,8 @@ static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
200191

201192
#endif
202193

203-
#define OPAL_ABT_COND_NULL NULL
204-
typedef ABT_cond opal_cond_t;
205-
#define OPAL_CONDITION_STATIC_INIT OPAL_ABT_COND_NULL
194+
typedef ABT_cond_memory opal_cond_t;
195+
#define OPAL_CONDITION_STATIC_INIT ABT_COND_INITIALIZER
206196

207197
int opal_cond_init(opal_cond_t *cond);
208198
int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock);

0 commit comments

Comments
 (0)