Skip to content

Commit e82406f

Browse files
devrealdevreal
authored
Merge pull request #8767 from shintaro-iwasaki/topic/fix_argobots
opal/mca/threads: fix Argobots support
2 parents ee2a557 + 0be6865 commit e82406f

File tree

7 files changed

+106
-134
lines changed

7 files changed

+106
-134
lines changed

‎opal/mca/pmix/pmix-internal.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@
1010
* Copyright (c) 2020 Amazon.com, Inc. or its affiliates.
1111
* All Rights reserved.
1212
* Copyright (c) 2021 Nanook Consulting. All rights reserved.
13+
* Copyright (c) 2021 Argonne National Laboratory. All rights
14+
* reserved.
1315
* $COPYRIGHT$
1416
*
1517
* Additional copyrights may follow
@@ -37,6 +39,7 @@
3739

3840
#include "opal/hash_string.h"
3941
#include "opal/mca/mca.h"
42+
#include "opal/class/opal_list.h"
4043
#include "opal/mca/threads/threads.h"
4144
#include "opal/util/error.h"
4245
#include "opal/util/event.h"

‎opal/mca/threads/argobots/configure.m4

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
# Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
1717
# Copyright (c) 2019 Triad National Security, LLC. All rights
1818
# Reserved.
19+
# Copyright (c) 2021 Argonne National Laboratory. All rights reserved.
1920
# $COPYRIGHT$
2021
#
2122
# Additional copyrights may follow
@@ -38,6 +39,7 @@ AC_DEFUN([OPAL_CONFIG_ARGOBOTS_THREADS],[
3839
opal_check_argo_save_LIBS=$LIBS
3940

4041
opal_argo_happy=yes
42+
opal_argo11_happy=yes
4143
AS_IF([test "$with_argo" = "no"],
4244
[opal_argo_happy=no])
4345

@@ -63,7 +65,15 @@ AC_DEFUN([OPAL_CONFIG_ARGOBOTS_THREADS],[
6365
[],
6466
[opal_argo_happy=no])])
6567

66-
AS_IF([test $opal_argo_happy = yes && test -n "$opal_argo_dir"],
68+
69+
# ABT_unit_get_thread() is a new Argobots 1.1 API.
70+
# It was introduced after static mutex/cond initializers.
71+
AS_IF([test $opal_argo_happy = yes],
72+
[AC_CHECK_FUNCS([ABT_unit_get_thread], [], [opal_argo11_happy="yes"])])
73+
74+
AS_IF([test $opal_argo_happy = yes && test $opal_argo11_happy = no],
75+
[AC_MSG_ERROR([Open MPI requires Argobots 1.1 or newer.])])
76+
AS_IF([test $opal_argo_happy = yes && test $opal_argo11_happy = yes && test -n "$opal_argo_dir"],
6777
[OPAL_ARGO_INCLUDE_PATH="$opal_argo_dir/include/"],
6878
[OPAL_ARGO_INCLUDE_PATH=""])
6979

‎opal/mca/threads/argobots/threads_argobots_module.c

Lines changed: 2 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@
1111
* Copyright (c) 2004-2005 The Regents of the University of California.
1212
* All rights reserved.
1313
* Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
14-
*
1514
* Copyright (c) 2020 Cisco Systems, Inc. All rights reserved.
15+
* Copyright (c) 2021 Argonne National Laboratory. All rights reserved.
16+
*
1617
* $COPYRIGHT$
1718
*
1819
* Additional copyrights may follow
@@ -32,16 +33,6 @@
3233
#include "opal/util/output.h"
3334
#include "opal/util/sys_limits.h"
3435

35-
struct opal_tsd_key_value {
36-
opal_tsd_key_t key;
37-
opal_tsd_destructor_t destructor;
38-
};
39-
40-
static opal_mutex_t opal_tsd_lock = OPAL_MUTEX_STATIC_INIT;
41-
static struct opal_tsd_key_value *opal_tsd_key_values = NULL;
42-
static int opal_tsd_key_values_count = 0;
43-
static int opal_tsd_key_values_size = 0;
44-
4536
/*
4637
* Constructor
4738
*/

‎opal/mca/threads/argobots/threads_argobots_mutex.c

Lines changed: 18 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
* Copyright (c) 2015 Research Organization for Information Science
1616
* and Technology (RIST). All rights reserved.
1717
* Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
18+
* Copyright (c) 2021 Argonne National Laboratory. All rights reserved.
1819
*
1920
* $COPYRIGHT$
2021
*
@@ -27,6 +28,7 @@
2728
#include "opal/mca/threads/argobots/threads_argobots.h"
2829

2930
#include <errno.h>
31+
#include <string.h>
3032

3133
#include "opal/constants.h"
3234
#include "opal/mca/threads/argobots/threads_argobots_mutex.h"
@@ -41,7 +43,8 @@ bool opal_uses_threads = false;
4143
static void mca_threads_argobots_mutex_constructor(opal_mutex_t *p_mutex)
4244
{
4345
opal_threads_argobots_ensure_init();
44-
p_mutex->m_lock_argobots = OPAL_ABT_MUTEX_NULL;
46+
const ABT_mutex_memory init_mutex = ABT_MUTEX_INITIALIZER;
47+
memcpy(&p_mutex->m_lock_argobots, &init_mutex, sizeof(ABT_mutex_memory));
4548
p_mutex->m_recursive = 0;
4649
#if OPAL_ENABLE_DEBUG
4750
p_mutex->m_lock_debug = 0;
@@ -51,17 +54,11 @@ static void mca_threads_argobots_mutex_constructor(opal_mutex_t *p_mutex)
5154
opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
5255
}
5356

54-
static void mca_threads_argobots_mutex_destructor(opal_mutex_t *p_mutex)
55-
{
56-
if (OPAL_ABT_MUTEX_NULL != p_mutex->m_lock_argobots) {
57-
ABT_mutex_free(&p_mutex->m_lock_argobots);
58-
}
59-
}
60-
6157
static void mca_threads_argobots_recursive_mutex_constructor(opal_recursive_mutex_t *p_mutex)
6258
{
6359
opal_threads_argobots_ensure_init();
64-
p_mutex->m_lock_argobots = OPAL_ABT_MUTEX_NULL;
60+
const ABT_mutex_memory init_mutex = ABT_RECURSIVE_MUTEX_INITIALIZER;
61+
memcpy(&p_mutex->m_lock_argobots, &init_mutex, sizeof(ABT_mutex_memory));
6562
p_mutex->m_recursive = 1;
6663
#if OPAL_ENABLE_DEBUG
6764
p_mutex->m_lock_debug = 0;
@@ -71,97 +68,40 @@ static void mca_threads_argobots_recursive_mutex_constructor(opal_recursive_mute
7168
opal_atomic_lock_init(&p_mutex->m_lock_atomic, 0);
7269
}
7370

74-
static void mca_threads_argobots_recursive_mutex_destructor(opal_recursive_mutex_t *p_mutex)
75-
{
76-
if (OPAL_ABT_MUTEX_NULL != p_mutex->m_lock_argobots) {
77-
ABT_mutex_free(&p_mutex->m_lock_argobots);
78-
}
79-
}
80-
81-
OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, mca_threads_argobots_mutex_constructor,
82-
mca_threads_argobots_mutex_destructor);
71+
OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, mca_threads_argobots_mutex_constructor, NULL);
8372
OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, opal_object_t,
84-
mca_threads_argobots_recursive_mutex_constructor,
85-
mca_threads_argobots_recursive_mutex_destructor);
86-
87-
void opal_mutex_create(struct opal_mutex_t *m)
88-
{
89-
opal_threads_argobots_ensure_init();
90-
while (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
91-
ABT_mutex abt_mutex;
92-
if (m->m_recursive) {
93-
ABT_mutex_attr abt_mutex_attr;
94-
ABT_mutex_attr_create(&abt_mutex_attr);
95-
ABT_mutex_attr_set_recursive(abt_mutex_attr, ABT_TRUE);
96-
ABT_mutex_create_with_attr(abt_mutex_attr, &abt_mutex);
97-
ABT_mutex_attr_free(&abt_mutex_attr);
98-
} else {
99-
ABT_mutex_create(&abt_mutex);
100-
}
101-
void *null_ptr = OPAL_ABT_MUTEX_NULL;
102-
if (opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) &m->m_lock_argobots,
103-
(intptr_t *) &null_ptr, (intptr_t) abt_mutex)) {
104-
/* mutex is successfully created and substituted. */
105-
return;
106-
}
107-
ABT_mutex_free(&abt_mutex);
108-
}
109-
}
110-
111-
static void opal_cond_create(opal_cond_t *cond)
112-
{
113-
opal_threads_argobots_ensure_init();
114-
while (OPAL_ABT_COND_NULL == *cond) {
115-
ABT_cond new_cond;
116-
ABT_cond_create(&new_cond);
117-
void *null_ptr = OPAL_ABT_COND_NULL;
118-
if (opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) cond,
119-
(intptr_t *) &null_ptr, (intptr_t) new_cond)) {
120-
/* cond is successfully created and substituted. */
121-
return;
122-
}
123-
ABT_cond_free(&new_cond);
124-
}
125-
}
73+
mca_threads_argobots_recursive_mutex_constructor, NULL);
12674

12775
int opal_cond_init(opal_cond_t *cond)
12876
{
129-
*cond = OPAL_ABT_COND_NULL;
77+
const ABT_cond_memory init_cond = ABT_COND_INITIALIZER;
78+
memcpy(cond, &init_cond, sizeof(ABT_cond_memory));
13079
return OPAL_SUCCESS;
13180
}
13281

13382
int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock)
13483
{
135-
if (OPAL_ABT_COND_NULL == *cond) {
136-
opal_cond_create(cond);
137-
}
138-
int ret = ABT_cond_wait(*cond, lock->m_lock_argobots);
84+
ABT_mutex abt_mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&lock->m_lock_argobots);
85+
ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond);
86+
int ret = ABT_cond_wait(abt_cond, abt_mutex);
13987
return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
14088
}
14189

14290
int opal_cond_broadcast(opal_cond_t *cond)
14391
{
144-
if (OPAL_ABT_COND_NULL == *cond) {
145-
opal_cond_create(cond);
146-
}
147-
int ret = ABT_cond_broadcast(*cond);
92+
ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond);
93+
int ret = ABT_cond_broadcast(abt_cond);
14894
return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
14995
}
15096

15197
int opal_cond_signal(opal_cond_t *cond)
15298
{
153-
if (OPAL_ABT_COND_NULL == *cond) {
154-
opal_cond_create(cond);
155-
}
156-
int ret = ABT_cond_signal(*cond);
99+
ABT_cond abt_cond = ABT_COND_MEMORY_GET_HANDLE(cond);
100+
int ret = ABT_cond_signal(abt_cond);
157101
return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
158102
}
159103

160104
int opal_cond_destroy(opal_cond_t *cond)
161105
{
162-
int ret = ABT_SUCCESS;
163-
if (OPAL_ABT_COND_NULL != *cond) {
164-
ret = ABT_cond_free(cond);
165-
}
166-
return ABT_SUCCESS == ret ? OPAL_SUCCESS : OPAL_ERROR;
106+
return OPAL_SUCCESS;
167107
}

‎opal/mca/threads/argobots/threads_argobots_mutex.h

Lines changed: 34 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
* Copyright (c) 2019 Sandia National Laboratories. All rights reserved.
1818
* Copyright (c) 2020 Triad National Security, LLC. All rights
1919
* reserved.
20-
*
2120
* Copyright (c) 2020 Cisco Systems, Inc. All rights reserved.
21+
* Copyright (c) 2021 Argonne National Laboratory. All rights reserved.
2222
* $COPYRIGHT$
2323
*
2424
* Additional copyrights may follow
@@ -34,21 +34,18 @@
3434
#include <errno.h>
3535
#include <stdio.h>
3636

37-
#include "opal/mca/threads/argobots/threads_argobots.h"
38-
3937
#include "opal/class/opal_object.h"
38+
#include "opal/mca/threads/argobots/threads_argobots.h"
39+
#include "opal/mca/threads/mutex.h"
4040
#include "opal/sys/atomic.h"
4141
#include "opal/util/output.h"
4242

4343
BEGIN_C_DECLS
4444

45-
/* Don't use ABT_MUTEX_NULL, since it might be not NULL. */
46-
#define OPAL_ABT_MUTEX_NULL 0
47-
4845
struct opal_mutex_t {
4946
opal_object_t super;
5047

51-
ABT_mutex m_lock_argobots;
48+
ABT_mutex_memory m_lock_argobots;
5249
int m_recursive;
5350

5451
#if OPAL_ENABLE_DEBUG
@@ -64,32 +61,34 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t);
6461
OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
6562

6663
#if OPAL_ENABLE_DEBUG
67-
# define OPAL_MUTEX_STATIC_INIT \
68-
{ \
69-
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = OPAL_ABT_MUTEX_NULL, \
70-
.m_recursive = 0, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \
71-
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
64+
# define OPAL_MUTEX_STATIC_INIT \
65+
{ \
66+
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = ABT_MUTEX_INITIALIZER, \
67+
.m_recursive = 0, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \
68+
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
7269
}
7370
#else
74-
# define OPAL_MUTEX_STATIC_INIT \
75-
{ \
76-
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = OPAL_ABT_MUTEX_NULL, \
77-
.m_recursive = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
71+
# define OPAL_MUTEX_STATIC_INIT \
72+
{ \
73+
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = ABT_MUTEX_INITIALIZER, \
74+
.m_recursive = 0, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
7875
}
7976
#endif
8077

8178
#if OPAL_ENABLE_DEBUG
82-
# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
83-
{ \
84-
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = OPAL_ABT_MUTEX_NULL, \
85-
.m_recursive = 1, .m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \
86-
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
79+
# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
80+
{ \
81+
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \
82+
.m_lock_argobots = ABT_RECURSIVE_MUTEX_INITIALIZER, .m_recursive = 1, \
83+
.m_lock_debug = 0, .m_lock_file = NULL, .m_lock_line = 0, \
84+
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
8785
}
8886
#else
89-
# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
90-
{ \
91-
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), .m_lock_argobots = OPAL_ABT_MUTEX_NULL, \
92-
.m_recursive = 1, .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
87+
# define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
88+
{ \
89+
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \
90+
.m_lock_argobots = ABT_RECURSIVE_MUTEX_INITIALIZER, .m_recursive = 1, \
91+
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
9392
}
9493
#endif
9594

@@ -99,14 +98,10 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
9998
*
10099
************************************************************************/
101100

102-
void opal_mutex_create(struct opal_mutex_t *m);
103-
104101
static inline int opal_mutex_trylock(opal_mutex_t *m)
105102
{
106-
if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
107-
opal_mutex_create(m);
108-
}
109-
int ret = ABT_mutex_trylock(m->m_lock_argobots);
103+
ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots);
104+
int ret = ABT_mutex_trylock(mutex);
110105
if (ABT_ERR_MUTEX_LOCKED == ret) {
111106
return 1;
112107
} else if (ABT_SUCCESS != ret) {
@@ -120,31 +115,27 @@ static inline int opal_mutex_trylock(opal_mutex_t *m)
120115

121116
static inline void opal_mutex_lock(opal_mutex_t *m)
122117
{
123-
if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
124-
opal_mutex_create(m);
125-
}
118+
ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots);
126119
#if OPAL_ENABLE_DEBUG
127-
int ret = ABT_mutex_lock(m->m_lock_argobots);
120+
int ret = ABT_mutex_lock(mutex);
128121
if (ABT_SUCCESS != ret) {
129122
opal_output(0, "opal_mutex_lock()");
130123
}
131124
#else
132-
ABT_mutex_lock(m->m_lock_argobots);
125+
ABT_mutex_lock(mutex);
133126
#endif
134127
}
135128

136129
static inline void opal_mutex_unlock(opal_mutex_t *m)
137130
{
138-
if (OPAL_ABT_MUTEX_NULL == m->m_lock_argobots) {
139-
opal_mutex_create(m);
140-
}
131+
ABT_mutex mutex = ABT_MUTEX_MEMORY_GET_HANDLE(&m->m_lock_argobots);
141132
#if OPAL_ENABLE_DEBUG
142-
int ret = ABT_mutex_unlock(m->m_lock_argobots);
133+
int ret = ABT_mutex_unlock(mutex);
143134
if (ABT_SUCCESS != ret) {
144135
opal_output(0, "opal_mutex_unlock()");
145136
}
146137
#else
147-
ABT_mutex_unlock(m->m_lock_argobots);
138+
ABT_mutex_unlock(mutex);
148139
#endif
149140
/* For fairness of locking. */
150141
ABT_thread_yield();
@@ -200,9 +191,8 @@ static inline void opal_mutex_atomic_unlock(opal_mutex_t *m)
200191

201192
#endif
202193

203-
#define OPAL_ABT_COND_NULL NULL
204-
typedef ABT_cond opal_cond_t;
205-
#define OPAL_CONDITION_STATIC_INIT OPAL_ABT_COND_NULL
194+
typedef ABT_cond_memory opal_cond_t;
195+
#define OPAL_CONDITION_STATIC_INIT ABT_COND_INITIALIZER
206196

207197
int opal_cond_init(opal_cond_t *cond);
208198
int opal_cond_wait(opal_cond_t *cond, opal_mutex_t *lock);

0 commit comments

Comments
 (0)