Merge pull request #8053 from shintaro-iwasaki/topic/fix_issue_8036

opal/mca/threads/qthreads: Fix #8036

Author: jsquyres

.gitignore

@@ -688,3 +688,4 @@ test/util/bipartite_graph
 opal/test/reachable/reachable_netlink
 opal/test/reachable/reachable_weighted
 opal/mca/threads/argobots/threads_argobots.h
+opal/mca/threads/qthreads/threads_qthreads.h

opal/mca/threads/argobots/configure.m4

@@ -123,12 +123,6 @@ AC_DEFUN([MCA_opal_threads_argobots_POST_CONFIG],[
            THREAD_CXXCPPFLAGS="$TPKG_CXXCPPFLAGS"
            THREAD_LDFLAGS="$TPKG_LDFLAGS"
            THREAD_LIBS="$TPKG_LIBS"
-           AC_SUBST(THREAD_CFLAGS)
-           AC_SUBST(THREAD_FCFLAGS)
-           AC_SUBST(THREAD_CXXFLAGS)
-           AC_SUBST(THREAD_CPPFLAGS)
-           AC_SUBST(THREAD_LDFLAGS)
-           AC_SUBST(THREAD_LIBS)
            LIBS="$LIBS $THREAD_LIBS"
            LDFLAGS="$LDFLAGS $THREAD_LDFLAGS"
           ])

opal/mca/threads/configure.m4

@@ -56,5 +56,12 @@ AS_IF([test x"$opal_thread_type_found" = x""],
 
 AC_MSG_RESULT([Found thread type $opal_thread_type_found])
 
+AC_SUBST(THREAD_CFLAGS)
+AC_SUBST(THREAD_FCFLAGS)
+AC_SUBST(THREAD_CXXFLAGS)
+AC_SUBST(THREAD_CPPFLAGS)
+AC_SUBST(THREAD_LDFLAGS)
+AC_SUBST(THREAD_LIBS)
+
 OPAL_SUMMARY_ADD([[Miscellaneous]],[[Threading Package]],[], [$opal_thread_type_found])
 ])dnl

opal/mca/threads/pthreads/configure.m4

@@ -824,12 +824,6 @@ AC_DEFUN([MCA_opal_threads_pthreads_POST_CONFIG],[
            THREAD_CXXCPPFLAGS="$TPKG_CXXCPPFLAGS"
            THREAD_LDFLAGS="$TPKG_LDFLAGS"
            THREAD_LIBS="$TPKG_LIBS"
-           AC_SUBST(THREAD_CFLAGS)
-           AC_SUBST(THREAD_FCFLAGS)
-           AC_SUBST(THREAD_CXXFLAGS)
-           AC_SUBST(THREAD_CPPFLAGS)
-           AC_SUBST(THREAD_LDFLAGS)
-           AC_SUBST(THREAD_LIBS)
           ])
 ])dnl
 

opal/mca/threads/qthreads/Makefile.am

@@ -19,12 +19,38 @@
 # $HEADER$
 #
 
-noinst_LTLIBRARIES = libmca_threads_qthreads.la
+AM_CPPFLAGS = $(opal_qthreads_CPPFLAGS)
 
-libmca_threads_qthreads_la_SOURCES = \
+sources = \
+    threads_qthreads.h \
     threads_qthreads_component.c \
-    threads_qthreads_mutex.c \
     threads_qthreads_condition.c \
-    threads_qthreads_module.c
+    threads_qthreads_module.c \
+    threads_qthreads_mutex.c \
+    threads_qthreads_wait_sync.c \
+    threads_qthreads_mutex.h \
+    threads_qthreads_threads.h \
+    threads_qthreads_tsd.h \
+    threads_qthreads_wait_sync.h
+
+lib_sources = $(sources)
+
+if MCA_BUILD_opal_threads_qthreads_DSO
+component_noinst =
+component_install = mca_threads_qthreads.la
+else
+component_noinst = libmca_threads_qthreads.la
+component_install =
+endif
+
+mcacomponentdir = $(opallibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_threads_qthreads_la_SOURCES = $(sources)
+mca_threads_qthreads_la_LDFLAGS = -module -avoid-version
+mca_threads_qthreads_la_LIBADD = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \
+        $(opal_qthreads_LIBS)
 
-AM_LDFLAGS = -lqthread
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_threads_qthreads_la_SOURCES = $(sources)
+libmca_threads_qthreads_la_LDFLAGS = -module -avoid-version $(opal_qthreads_LDFLAGS)
+libmca_threads_qthreads_la_LIBADD = $(opal_qthreads_LIBS)

opal/mca/threads/qthreads/configure.m4

@@ -25,13 +25,68 @@
 
 AC_DEFUN([OPAL_CONFIG_QTHREADS],[
 
-    AC_CHECK_HEADERS([qthread/qthread.h],
-                     [AC_CHECK_LIB([qthread],[qthread_initialize],
-                                    [threads_qthreads_happy="yes"],
-                                    [threads_qthreads_happy="no"])],
-                     [threads_qthreads_happy="no"])
+    AC_ARG_WITH([qthreads],
+                [AC_HELP_STRING([--with-qthreads=DIR],
+                                [Specify location of qthreads installation.  Error if qthreads support cannot be found.])])
 
-    AS_IF([test "$threads_qthreads_happy" = "yes"],
+    AC_ARG_WITH([qthreads-libdir],
+                [AC_HELP_STRING([--with-qthreads-libdir=DIR],
+                                [Search for qthreads libraries in DIR])])
+
+    opal_check_qthreads_save_CPPFLAGS=$CPPFLAGS
+    opal_check_qthreads_save_LDFLAGS=$LDFLAGS
+    opal_check_qthreads_save_LIBS=$LIBS
+
+    opal_qthreads_happy=yes
+    AS_IF([test "$with_qthreads" = "no"],
+          [opal_qthreads_happy=no])
+
+    AS_IF([test $opal_qthreads_happy = yes],
+          [AC_MSG_CHECKING([looking for qthreads in])
+           AS_IF([test "$with_qthreads" != "yes"],
+                 [opal_qthreads_dir=$with_qthreads
+                  AC_MSG_RESULT([($opal_qthreads_dir)])],
+                 [AC_MSG_RESULT([(default search paths)])])
+           AS_IF([test ! -z "$with_qthreads_libdir" && \
+                         test "$with_qthreads_libdir" != "yes"],
+                 [opal_qthreads_libdir=$with_qthreads_libdir])
+          ])
+
+    AS_IF([test $opal_qthreads_happy = yes],
+          [OPAL_CHECK_PACKAGE([opal_qthreads],
+                              [qthread.h],
+                              [qthread],
+                              [qthread_initialize],
+                              [],
+                              [$opal_qthreads_dir],
+                              [$opal_qthreads_libdir],
+                              [],
+                              [opal_qthreads_happy=no])])
+
+    AS_IF([test $opal_qthreads_happy = yes && test -n "$opal_qthreads_dir"],
+          [OPAL_QTHREADS_INCLUDE_PATH="$opal_qthreads_dir/include/"],
+          [OPAL_QTHREADS_INCLUDE_PATH=""])
+
+    AS_IF([test $opal_qthreads_happy = yes],
+          [TPKG_CFLAGS="$opal_qthreads_CPPFLAGS"
+           TPKG_FCFLAGS="$opal_qthreads_CPPFLAGS"
+           TPKG_CXXFLAGS="$opal_qthreads_CPPFLAGS"
+           TPKG_CPPFLAGS="$opal_qthreads_CPPFLAGS"
+           TPKG_CXXCPPFLAGS="$opal_qthreads_CPPFLAGS"
+           TPKG_LDFLAGS="$opal_qthreads_LDFLAGS"
+           TPKG_LIBS="$opal_qthreads_LIBS"])
+
+    AC_CONFIG_FILES([opal/mca/threads/qthreads/threads_qthreads.h])
+    AC_SUBST([OPAL_QTHREADS_INCLUDE_PATH])
+    AC_SUBST([opal_qthreads_CPPFLAGS])
+    AC_SUBST([opal_qthreads_LDFLAGS])
+    AC_SUBST([opal_qthreads_LIBS])
+
+    CPPFLAGS=$opal_check_qthreads_save_CPPFLAGS
+    LDFLAGS=$opal_check_qthreads_save_LDFLAGS
+    LIBS=$opal_check_qthreads_save_LIBS
+
+    AS_IF([test "$opal_qthreads_happy" = "yes"],
           [$1],
           [$2])
 ])dnl
@@ -60,6 +115,15 @@ AC_DEFUN([MCA_opal_threads_qthreads_POST_CONFIG],[
            AC_DEFINE_UNQUOTED([MCA_threads_wait_sync_base_include_HEADER],
                               ["opal/mca/threads/qthreads/threads_qthreads_wait_sync.h"],
                               [Header to include for wait_sync implementation])
+           THREAD_CFLAGS="$TPKG_CFLAGS"
+           THREAD_FCFLAGS="$TPKG_FCFLAGS"
+           THREAD_CXXFLAGS="$TPKG_CXXFLAGS"
+           THREAD_CPPFLAGS="$TPKG_CPPFLAGS"
+           THREAD_CXXCPPFLAGS="$TPKG_CXXCPPFLAGS"
+           THREAD_LDFLAGS="$TPKG_LDFLAGS"
+           THREAD_LIBS="$TPKG_LIBS"
+           LIBS="$LIBS $THREAD_LIBS"
+           LDFLAGS="$LDFLAGS $THREAD_LDFLAGS"
          ])
 ])dnl
 

opal/mca/threads/qthreads/threads_qthreads.h renamed to opal/mca/threads/qthreads/threads_qthreads.h.in

@@ -26,9 +26,10 @@
 #ifndef OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_H
 #define OPAL_MCA_THREADS_QTHREADS_THREADS_QTHREADS_H 1
 
-#include <qthread/qthread.h>
+#include "@OPAL_QTHREADS_INCLUDE_PATH@qthread.h"
+#include "@OPAL_QTHREADS_INCLUDE_PATH@qthread/tls.h"
 
-static inline void ensure_init_qthreads(void)
+static inline void opal_threads_ensure_init_qthreads(void)
 {
     qthread_initialize();
 }

opal/mca/threads/qthreads/threads_qthreads_component.c

@@ -23,6 +23,7 @@
 
 #include "opal_config.h"
 
+#include "opal/mca/threads/qthreads/threads_qthreads.h"
 #include "opal/mca/threads/thread.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/constants.h"

opal/mca/threads/qthreads/threads_qthreads_module.c

@@ -19,13 +19,8 @@
  * $HEADER$
  */
 
-#include <unistd.h>
-
 #include "opal/mca/threads/qthreads/threads_qthreads.h"
 #include "opal/constants.h"
-#include "opal/util/sys_limits.h"
-#include "opal/util/output.h"
-#include "opal/prefetch.h"
 #include "opal/mca/threads/threads.h"
 #include "opal/mca/threads/tsd.h"
 
@@ -34,40 +29,82 @@ struct opal_tsd_key_value {
     opal_tsd_destructor_t destructor;
 };
 
-static int opal_main_thread;
-struct opal_tsd_key_value *opal_tsd_key_values = NULL;
-static int opal_tsd_key_values_count = 0;
+/* false: uninitialized, true: initialized. */
+static opal_atomic_lock_t opal_thread_self_key_lock = OPAL_ATOMIC_LOCK_INIT;
+static bool opal_thread_self_key_init = false;
+static opal_tsd_key_t opal_thread_self_key;
+
+static inline void self_key_ensure_init(void)
+{
+    if (false == opal_thread_self_key_init) {
+        /* not initialized yet. */
+        opal_atomic_lock(&opal_thread_self_key_lock);
+        /* check again. */
+        if (false == opal_thread_self_key_init) {
+            /* This thread is responsible for initializing this key. */
+            qthread_key_create(&opal_thread_self_key, NULL);
+            opal_atomic_mb();
+            opal_thread_self_key_init = true;
+        }
+        opal_atomic_unlock(&opal_thread_self_key_lock);
+    }
+    /* opal_thread_self_key has been already initialized. */
+}
 
 /*
  * Constructor
  */
 static void opal_thread_construct(opal_thread_t *t)
 {
+    t->t_run = 0;
+    t->t_thread_ret = 0;
 }
 
 OBJ_CLASS_INSTANCE(opal_thread_t,
                    opal_object_t,
                    opal_thread_construct, NULL);
 
+static inline aligned_t *opal_thread_get_qthreads_self(void)
+{
+    self_key_ensure_init();
+    void *ptr = qthread_getspecific(opal_thread_self_key);
+    return (aligned_t *)ptr;
+}
 
-opal_thread_t *opal_thread_get_self(void)
+static aligned_t opal_thread_qthreads_wrapper(void *arg)
 {
-    return NULL;
+    opal_thread_t *t = (opal_thread_t *)arg;
+
+    /* Register itself. */
+    self_key_ensure_init();
+    qthread_setspecific(opal_thread_self_key, t->t_thread_ret_ptr);
+
+    t->t_ret = ((void *(*)(void *))t->t_run)(t);
+    return 0;
 }
 
-bool opal_thread_self_compare(opal_thread_t *t)
+opal_thread_t *opal_thread_get_self(void)
 {
-    return OPAL_ERR_NOT_IMPLEMENTED;
+    opal_threads_ensure_init_qthreads();
+    opal_thread_t *t = OBJ_NEW(opal_thread_t);
+    t->t_thread_ret_ptr = opal_thread_get_qthreads_self();
+    return t;
 }
 
-int sync_wait_mt(void *p)
+bool opal_thread_self_compare(opal_thread_t *t)
 {
-    return OPAL_ERR_NOT_IMPLEMENTED;
+    opal_threads_ensure_init_qthreads();
+    return opal_thread_get_qthreads_self() == &t->t_thread_ret;
 }
 
 int opal_thread_join(opal_thread_t *t, void **thr_return)
 {
-    return OPAL_ERR_NOT_IMPLEMENTED;
+    qthread_readFF(NULL, t->t_thread_ret_ptr);
+    if (thr_return) {
+        *thr_return = t->t_ret;
+    }
+    t->t_thread_ret = 0;
+    return OPAL_SUCCESS;
 }
 
 void opal_thread_set_main(void)
@@ -76,12 +113,17 @@ void opal_thread_set_main(void)
 
 int opal_thread_start(opal_thread_t *t)
 {
-    return OPAL_ERR_NOT_IMPLEMENTED;
+    opal_threads_ensure_init_qthreads();
+    t->t_thread_ret_ptr = &t->t_thread_ret;
+    qthread_fork(opal_thread_qthreads_wrapper, t, &t->t_thread_ret);
+    return OPAL_SUCCESS;
 }
 
 OBJ_CLASS_DECLARATION(opal_thread_t);
 
 int opal_tsd_key_create(opal_tsd_key_t *key, opal_tsd_destructor_t destructor)
 {
-    return OPAL_ERR_NOT_IMPLEMENTED;
+    opal_threads_ensure_init_qthreads();
+    qthread_key_create(key, destructor);
+    return OPAL_SUCCESS;
 }