Skip to content

opal/atomic: always inline load-link store-conditional #3988

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

opal/atomic: always inline load-link store-conditional #3988

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 17 additions & 12 deletions opal/class/opal_fifo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
* All rights reserved.
* Copyright (c) 2007 Voltaire All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
* reseved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -215,28 +215,33 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
*/
static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
{
opal_list_item_t *item, *next;

#if OPAL_HAVE_ATOMIC_LLSC_PTR
register opal_list_item_t *ghost = &fifo->opal_fifo_ghost;
register opal_list_item_t *item, *next;
register opal_list_item_t volatile * volatile * head = &fifo->opal_fifo_head.data.item;
int ret;

/* use load-linked store-conditional to avoid ABA issues */
do {
item = opal_atomic_ll_ptr (&fifo->opal_fifo_head.data.item);
if (&fifo->opal_fifo_ghost == item) {
if (&fifo->opal_fifo_ghost == fifo->opal_fifo_tail.data.item) {
opal_atomic_ll_ptr(head, item);
next = (opal_list_item_t *) item->opal_list_next;
if (ghost != item) {
opal_atomic_sc_ptr(head, next, ret);
if (ret) {
break;
}
} else {
if (ghost == fifo->opal_fifo_tail.data.item) {
return NULL;
}

/* fifo does not appear empty. wait for the fifo to be made
* consistent by conflicting thread. */
continue;
}

next = (opal_list_item_t *) item->opal_list_next;
if (opal_atomic_sc_ptr (&fifo->opal_fifo_head.data.item, next)) {
break;
}
} while (1);
#else
opal_list_item_t *item, *next;

/* protect against ABA issues by "locking" the head */
do {
if (opal_atomic_cmpset_32 ((int32_t *) &fifo->opal_fifo_head.data.counter, 0, 1)) {
Expand Down
17 changes: 10 additions & 7 deletions opal/class/opal_lifo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
* All rights reserved.
* Copyright (c) 2007 Voltaire All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2014-2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
* reseved.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
Expand Down Expand Up @@ -204,8 +204,10 @@ static inline void _opal_lifo_release_cpu (void)
*/
static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
{
opal_list_item_t *item, *next;
int attempt = 0;
register opal_list_item_t volatile * volatile * head = &lifo->opal_lifo_head.data.item;
register opal_list_item_t *ghost = &lifo->opal_lifo_ghost;
register opal_list_item_t *item, *next;
int attempt = 0, ret;

do {
if (++attempt == 5) {
Expand All @@ -215,13 +217,14 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
attempt = 0;
}

item = (opal_list_item_t *) opal_atomic_ll_ptr (&lifo->opal_lifo_head.data.item);
if (&lifo->opal_lifo_ghost == item) {
opal_atomic_ll_ptr(head, item);
next = (opal_list_item_t *) item->opal_list_next;
if (ghost == item) {
return NULL;
}

next = (opal_list_item_t *) item->opal_list_next;
} while (!opal_atomic_sc_ptr (&lifo->opal_lifo_head.data.item, next));
opal_atomic_sc_ptr(head, next, ret);
} while (!ret);

opal_atomic_wmb ();

Expand Down
70 changes: 26 additions & 44 deletions opal/include/opal/sys/arm64/atomic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
* All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2010 ARM ltd. All rights reserved.
* Copyright (c) 2016 Los Alamos National Security, LLC. All rights
* Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -150,28 +150,19 @@ static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr,
return (ret == oldval);
}

static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr)
{
int32_t ret;

__asm__ __volatile__ ("ldaxr %w0, [%1] \n"
: "=&r" (ret)
: "r" (addr));

return ret;
}

static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
{
int ret;

__asm__ __volatile__ ("stlxr %w0, %w2, [%1] \n"
: "=&r" (ret)
: "r" (addr), "r" (newval)
: "cc", "memory");
#define opal_atomic_ll_32(addr, value) \
__asm__ __volatile__ ("ldaxr %w0, [%1] \n" \
: "=&r" ((int32_t) (value)) \
: "r" ((volatile int32_t *) (addr)))

return ret == 0;
}
#define opal_atomic_sc_32(addr, newval, ret) \
do { \
__asm__ __volatile__ ("stlxr %w0, %w2, [%1] \n" \
: "=&r" ((int) (ret)) \
: "r" ((volatile int32_t *) (addr)), "r" ((int32_t) (newval)) \
: "cc", "memory"); \
(ret) = ((ret) == 0); \
} while (0)

static inline int opal_atomic_cmpset_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
Expand Down Expand Up @@ -251,28 +242,19 @@ static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr,
return (ret == oldval);
}

static inline int64_t opal_atomic_ll_64 (volatile int64_t *addr)
{
int64_t ret;

__asm__ __volatile__ ("ldaxr %0, [%1] \n"
: "=&r" (ret)
: "r" (addr));

return ret;
}

static inline int opal_atomic_sc_64 (volatile int64_t *addr, int64_t newval)
{
int ret;

__asm__ __volatile__ ("stlxr %w0, %2, [%1] \n"
: "=&r" (ret)
: "r" (addr), "r" (newval)
: "cc", "memory");

return ret == 0;
}
#define opal_atomic_ll_64(addr, value) \
__asm__ __volatile__ ("ldaxr %0, [%1] \n" \
: "=&r" ((int64_t) (value)) \
: "r" ((volatile int64_t *) (addr)))

#define opal_atomic_sc_64(addr, newval, ret) \
do { \
__asm__ __volatile__ ("stlxr %w0, %2, [%1] \n" \
: "=&r" ((int) (ret)) \
: "r" ((volatile int64_t *) (addr)), "r" ((int64_t) (newval)) \
: "cc", "memory"); \
(ret) = ((ret) == 0); \
} while (0)

#define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \
static inline type opal_atomic_ ## name ## _ ## bits (volatile type *addr, type value) \
Expand Down
10 changes: 5 additions & 5 deletions opal/include/opal/sys/atomic_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
* Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -278,15 +278,15 @@ static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr,

#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32

#define opal_atomic_ll_ptr(addr) (void *) opal_atomic_ll_32((int32_t *) addr)
#define opal_atomic_sc_ptr(addr, newval) opal_atomic_sc_32((int32_t *) addr, (int32_t) newval)
#define opal_atomic_ll_ptr opal_atomic_ll_32
#define opal_atomic_sc_ptr opal_atomic_sc_32

#define OPAL_HAVE_ATOMIC_LLSC_PTR 1

#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64

#define opal_atomic_ll_ptr(addr) (void *) opal_atomic_ll_64((int64_t *) addr)
#define opal_atomic_sc_ptr(addr, newval) opal_atomic_sc_64((int64_t *) addr, (int64_t) newval)
#define opal_atomic_ll_ptr opal_atomic_ll_64
#define opal_atomic_sc_ptr opal_atomic_sc_64

#define OPAL_HAVE_ATOMIC_LLSC_PTR 1

Expand Down
84 changes: 32 additions & 52 deletions opal/include/opal/sys/powerpc/atomic.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2010-2017 IBM Corporation. All rights reserved.
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
* Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
Expand Down Expand Up @@ -135,7 +135,7 @@ void opal_atomic_isync(void)
* it will emit lwz instead of ld to load the 64-bit operand. */
#define OPAL_ASM_VALUE64(x) (void *)(intptr_t) (x)
#else
#define OPAL_ASM_VALUE64(x) x
#define OPAL_ASM_VALUE64(x) (int64_t) (x)
#endif


Expand All @@ -158,31 +158,21 @@ static inline int opal_atomic_cmpset_32(volatile int32_t *addr,
return (ret == oldval);
}

static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr)
{
int32_t ret;

__asm__ __volatile__ ("lwarx %0, 0, %1 \n\t"
: "=&r" (ret)
: "r" (addr)
);
return ret;
}

static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
{
int32_t ret, foo;

__asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t"
" li %0,0 \n\t"
" bne- 1f \n\t"
" ori %0,%0,1 \n\t"
"1:"
: "=r" (ret), "=m" (*addr), "=r" (foo)
: "r" (addr), "r" (newval)
: "cc", "memory");
return ret;
}
#define opal_atomic_ll_32(addr, value) \
__asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \
: "=&r" ((int32_t) (value)) \
: "r" ((volatile int32_t *) (addr)))

#define opal_atomic_sc_32(addr, newval, ret) \
__asm__ __volatile__ (" stwcx. %2, 0, %1 \n\t" \
" li %0,0 \n\t" \
" bne- 1f \n\t" \
" ori %0,%0,1 \n\t" \
"1:" \
: "=r" ((int32_t) ret) \
: "r" ((volatile int32_t *) (addr)), \
"r" ((int32_t) newval) \
: "cc", "memory")

/* these two functions aren't inlined in the non-gcc case because then
there would be two function calls (since neither cmpset_32 nor
Expand Down Expand Up @@ -280,31 +270,21 @@ static inline int opal_atomic_cmpset_64(volatile int64_t *addr,
return (ret == oldval);
}

static inline int64_t opal_atomic_ll_64(volatile int64_t *addr)
{
int64_t ret;

__asm__ __volatile__ ("ldarx %0, 0, %1 \n\t"
: "=&r" (ret)
: "r" (addr)
);
return ret;
}

static inline int opal_atomic_sc_64(volatile int64_t *addr, int64_t newval)
{
int32_t ret;

__asm__ __volatile__ (" stdcx. %2, 0, %1 \n\t"
" li %0,0 \n\t"
" bne- 1f \n\t"
" ori %0,%0,1 \n\t"
"1:"
: "=r" (ret)
: "r" (addr), "r" (OPAL_ASM_VALUE64(newval))
: "cc", "memory");
return ret;
}
#define opal_atomic_ll_64(addr, value) \
__asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \
: "=&r" (OPAL_ASM_VALUE64(value)) \
: "r" ((volatile int64_t *) (addr)))

#define opal_atomic_sc_64(addr, newval, ret) \
__asm__ __volatile__ (" stdcx. %2, 0, %1 \n\t" \
" li %0,0 \n\t" \
" bne- 1f \n\t" \
" ori %0,%0,1 \n\t" \
"1:" \
: "=r" ((ret)) \
: "r" ((volatile int64_t *) (addr)), \
"r" (OPAL_ASM_VALUE64(newval)) \
: "cc", "memory")

/* these two functions aren't inlined in the non-gcc case because then
there would be two function calls (since neither cmpset_64 nor
Expand Down