opal/mca/threads/argobots: implement wait_sync_global_wakeup()

Signed-off-by: Shintaro Iwasaki <siwasaki@anl.gov>

Author: shintaro-iwasaki

opal/mca/threads/argobots/threads_argobots_wait_sync.c

@@ -7,6 +7,7 @@
  *                         reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
  * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -21,6 +22,32 @@
 static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT;
 static ompi_wait_sync_t *wait_sync_list = NULL;
 
+void wait_sync_global_wakeup_st(int status)
+{
+    ompi_wait_sync_t *sync;
+    for (sync = wait_sync_list; sync != NULL; sync = sync->next) {
+        wait_sync_update(sync, 0, status);
+    }
+}
+
+void wait_sync_global_wakeup_mt(int status)
+{
+    ompi_wait_sync_t *sync;
+    opal_mutex_lock(&wait_sync_lock);
+    for (sync = wait_sync_list; sync != NULL; sync = sync->next) {
+        /* sync_update is going to  take the sync->lock from within
+         * the wait_sync_lock. Thread lightly here: Idealy we should
+         * find a way to not take a lock in a lock as this is deadlock prone,
+         * but as of today we are the only place doing this so it is safe.
+         */
+        wait_sync_update(sync, 0, status);
+        if (sync->next == wait_sync_list) {
+            break; /* special case for rings */
+        }
+    }
+    opal_mutex_unlock(&wait_sync_lock);
+}
+
 static opal_atomic_int32_t num_thread_in_progress = 0;
 
 #define WAIT_SYNC_PASS_OWNERSHIP(who)      \

opal/mca/threads/argobots/threads_argobots_wait_sync.h

@@ -15,6 +15,7 @@
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2019      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2021      Argonne National Laboratory.  All rights reserved.
  *
  * $COPYRIGHT$
  *
@@ -100,4 +101,14 @@ static inline int sync_wait_st(ompi_wait_sync_t *sync)
         }                                        \
     } while (0)
 
+/**
+ * Wake up all syncs with a particular status. If status is OMPI_SUCCESS this
+ * operation is a NO-OP. Otherwise it will trigger the "error condition" from
+ * all registered sync.
+ */
+OPAL_DECLSPEC void wait_sync_global_wakeup_st(int status);
+OPAL_DECLSPEC void wait_sync_global_wakeup_mt(int status);
+#define wait_sync_global_wakeup(st) \
+    (opal_using_threads() ? wait_sync_global_wakeup_mt(st) : wait_sync_global_wakeup_st(st))
+
 #endif /* OPAL_MCA_THREADS_ARGOBOTS_THREADS_ARGOBOTS_WAIT_SYNC_H */