Avoid allocating TLS for the first thread that loads. NFC (#17018)

This change depend on an LLVM-side change in order to actually
do anything: https://reviews.llvm.org/D126107. It can land on its
own but won't have any effect since __tls_base will always be zero
on startup until that change lands.

The linker will load the TLS data into a static location that the
loading thread can use directly rather than dynamically allocating it.
Secondary threads still require dynamic allocation but I'm hoping to
avoid that too (at least for load-time dynamic libraries and especially
for the main module).

See: #16948

Author: sbc100

src/postamble.js

@@ -425,7 +425,11 @@ function exit(status, implicit) {
       throw 'unwind';
     } else {
 #if PTHREADS_DEBUG
+#if EXIT_RUNTIME
       err('main thread called exit: keepRuntimeAlive=' + keepRuntimeAlive() + ' (counter=' + runtimeKeepaliveCounter + ')');
+#else
+      err('main thread called exit: keepRuntimeAlive=' + keepRuntimeAlive());
+#endif
 #endif
     }
   }

system/lib/pthread/emscripten_tls_init.c

@@ -8,7 +8,9 @@
 // Included for emscripten_builtin_free / emscripten_builtin_malloc
 // TODO(sbc): Should these be in their own header to avoid emmalloc here?
 #include <emscripten/emmalloc.h>
-#include <pthread.h>
+#include <emscripten/threading.h>
+
+#include "pthread_impl.h"
 
 // Uncomment to trace TLS allocations.
 // #define DEBUG_TLS
@@ -21,15 +23,49 @@ extern void __wasm_init_tls(void *memory);
 
 extern int __dso_handle;
 
+// Create a thread-local wasm global which signal that the current thread is non
+// to use the primary/static TLS region.  Once this gets set it forces that all
+// future calls to emscripten_tls_init to dynamically allocate TLS.
+// This is needed because emscripten re-uses module instances owned by the
+// worker for new pthreads.  This in turn means that stale values of __tls_base
+// from a previous pthreads need to be ignored.
+// If this global is true then TLS needs to be dyanically allocated, if its
+// flase we are free to use the existing/global __tls_base.
+__asm__(".globaltype is_not_primary_tls, i32\n"
+        "is_not_primary_tls:\n");
+
+static void set_non_primary(void) {
+  __asm__("i32.const 1\n"
+          "global.set is_not_primary_tls\n");
+}
+
+static int is_non_primary(void) {
+  int val;
+  __asm__("global.get is_not_primary_tls\n"
+          "local.set %0" : "=r" (val));
+  return val;
+}
+
+void emscripten_tls_free() {
+  void* tls_block = __builtin_wasm_tls_base();
+  if (tls_block && is_non_primary()) {
+#ifdef DEBUG_TLS
+    printf("tls free: thread=%p dso=%p <- %p\n", pthread_self(), &__dso_handle, tls_block);
+#endif
+    emscripten_builtin_free(tls_block);
+  }
+}
+
 void* emscripten_tls_init(void) {
   size_t tls_size = __builtin_wasm_tls_size();
-  size_t tls_align = __builtin_wasm_tls_align();
-  if (!tls_size) {
-    return NULL;
+  void* tls_block = __builtin_wasm_tls_base();
+  if (is_non_primary() || (!tls_block && tls_size)) {
+    set_non_primary();
+    size_t tls_align = __builtin_wasm_tls_align();
+    tls_block = emscripten_builtin_memalign(tls_align, tls_size);
   }
-  void *tls_block = emscripten_builtin_memalign(tls_align, tls_size);
 #ifdef DEBUG_TLS
-  printf("tls init: thread[%p] dso[%p] -> %p\n", pthread_self(), &__dso_handle, tls_block);
+  printf("tls init: size=%zu thread=%p dso=%p -> %p\n", tls_size, pthread_self(), &__dso_handle, tls_block);
 #endif
   __wasm_init_tls(tls_block);
   return tls_block;

system/lib/pthread/pthread_create.c

@@ -26,7 +26,6 @@ extern int __pthread_create_js(struct pthread *thread, const pthread_attr_t *att
 extern void __set_thread_state(pthread_t ptr, int is_main, int is_runtime, int can_block);
 extern int _emscripten_default_pthread_stack_size();
 extern void __emscripten_thread_cleanup(pthread_t thread);
-extern void* _emscripten_tls_base();
 extern int8_t __dso_handle;
 
 static void dummy_0()
@@ -197,15 +196,7 @@ void _emscripten_thread_free_data(pthread_t t) {
   emscripten_builtin_free(t);
 }
 
-static void free_tls_data() {
-  void* tls_block = _emscripten_tls_base();
-  if (tls_block) {
-#ifdef DEBUG_TLS
-    printf("tls free: thread[%p] dso[%p] <- %p\n", pthread_self(), &__dso_handle, tls_block);
-#endif
-    emscripten_builtin_free(tls_block);
-  }
-}
+void emscripten_tls_free(void);
 
 void _emscripten_thread_exit(void* result) {
   struct pthread *self = __pthread_self();
@@ -221,7 +212,7 @@ void _emscripten_thread_exit(void* result) {
   // Call into the musl function that runs destructors of all thread-specific data.
   __pthread_tsd_run_dtors();
 
-  free_tls_data();
+  emscripten_tls_free();
 
   if (!--libc.threads_minus_1) libc.need_locks = 0;
 

tests/test_other.py

@@ -11349,7 +11349,7 @@ def test_standard_library_mapping(self):
     self.run_process([EMBUILDER, 'build', 'libc-mt-debug', 'libcompiler_rt-mt', 'libdlmalloc-mt'])
 
     libs = ['-lc', '-lcompiler_rt', '-lmalloc']
-    err = self.run_process([EMCC, test_file('hello_world.c'), '-pthread', '-nostdlib', '-v'] + libs, stderr=PIPE).stderr
+    err = self.run_process([EMCC, test_file('hello_world.c'), '-pthread', '-nodefaultlibs', '-v'] + libs, stderr=PIPE).stderr
 
     # Check the the linker was run with `-mt` variants because `-pthread` was passed.
     self.assertContained(' -lc-mt-debug ', err)

tools/system_libs.py

@@ -1146,7 +1146,7 @@ def can_use(self):
     return super().can_use() and settings.STANDALONE_WASM
 
 
-class crtbegin(Library):
+class crtbegin(MuslInternalLibrary):
   name = 'crtbegin'
   cflags = ['-sUSE_PTHREADS']
   src_dir = 'system/lib/pthread'