[NativeAOT-LLVM] Wasm managed threads - build new nupkg

docs/workflow/building/coreclr/nativeaot.md

@@ -76,6 +76,26 @@ You should now be able to publish the project for Wasm: `dotnet publish -r brows
 
 Once you build the repo, you can use the produced binaries in one of four ways specified below ("Using built binaries", "Building packages", "Convenience Visual Studio "repro" project", "Running tests").
 
+## Building for Multithreaded packages
+
+This is a work in progress and far from functional.  Currentlty there exists just enough infrastucture to build packages for mutlithreaded runtime and libs, but they are not functional in the sense that they support multithreaded programs yet.  To build the WASI packages:
+```
+build clr.aot+libs+nativeaot.packages -c Debug -a wasm -os wasi '/p:WasmEnableThreads=true'
+```
+To build the browser multithreaded packages:
+```
+build clr.aot+libs+nativeaot.packages -c Debug -a wasm -os browser '/p:WasmEnableThreads=true'
+```
+To build the runtime tests for WASI
+```
+src\tests\build nativeaot Debug wasm tree nativeaot wasi /p:LibrariesConfiguration=debug /p:TestWrapperTargetsWindows=true /p:WasmEnableThreads=true
+```
+To build the runtime tests for browser
+```
+src\tests\build nativeaot Debug wasm tree nativeaot browser /p:LibrariesConfiguration=debug /p:TestWrapperTargetsWindows=true /p:WasmEnableThreads=true
+```
+
+
 ### Using built binaries
 
 In this workflow, you have a project file that you want to `dotnet publish`, but you want to use your own build of the compiler/runtime/framework. You need to be using a daily build of the .NET SDK downloaded from the dotnet/sdk repo. It's typically enough to just download the daily build ZIP file, unpack it, and make sure the unpacked directory is the first thing in your PATH. Don't forget to add a NuGet.config as specified by the dotnet/sdk repo- you'll hit restore issues otherwise.

eng/native/configurecompiler.cmake

@@ -148,6 +148,10 @@ elseif (CLR_CMAKE_HOST_UNIX)
     add_compile_options(-Wno-alloca)
     add_compile_options(-Wno-implicit-int-float-conversion)
   endif()
+
+  if (CLR_CMAKE_TARGET_OS_SUBGROUP STREQUAL multithread AND CLR_CMAKE_HOST_BROWSER)
+    add_compile_options(-pthread)
+  endif(CLR_CMAKE_TARGET_OS_SUBGROUP STREQUAL multithread AND CLR_CMAKE_HOST_BROWSER)
 endif(MSVC)
 
 if (CLR_CMAKE_ENABLE_SANITIZERS)

eng/pipelines/common/platform-matrix.yml

@@ -489,6 +489,7 @@ jobs:
       helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
       variables: ${{ parameters.variables }}
       osGroup: wasi
+      osSubGroup: ${{ parameters.osSubGroup }}
       archType: wasm
       targetRid: wasi-wasm
       platform: wasi_wasm_win
@@ -611,6 +612,7 @@ jobs:
       helixQueuesTemplate: ${{ parameters.helixQueuesTemplate }}
       variables: ${{ parameters.variables }}
       osGroup: browser
+      osSubGroup: ${{ parameters.osSubGroup }}
       archType: wasm
       targetRid: browser-wasm
       platform: browser_wasm_win

eng/pipelines/runtimelab.yml

@@ -71,6 +71,28 @@ extends:
                   parameters:
                     librariesConfiguration: Debug
 
+        #
+        # Build and test Wasm Debug multithreaded libraries and Debug runtime
+        #
+        - template: /eng/pipelines/common/platform-matrix.yml
+          parameters:
+            jobTemplate: /eng/pipelines/common/global-build-job.yml
+            helixQueuesTemplate: /eng/pipelines/coreclr/templates/helix-queues-setup.yml
+            buildConfig: debug
+            osSubgroup: multithread
+            platforms:
+            - browser_wasm_win
+            - wasi_wasm_win
+            jobParameters:
+              timeoutInMinutes: 300
+              buildArgs: -s clr.aot+libs -c debug -rc $(_BuildConfig) '/p:WasmEnableThreads=true'
+              nameSuffix: Multithreaded
+              postBuildSteps:
+                - template: /eng/pipelines/runtimelab/runtimelab-post-build-steps.yml
+                  parameters:
+                    librariesConfiguration: Debug
+                    wasmEnableThreadsArg: /p:WasmEnableThreads=true
+
         #
         # Build and test with Debug libraries and Checked runtime
         #

eng/pipelines/runtimelab/runtimelab-post-build-steps.yml

@@ -8,6 +8,7 @@ parameters:
   runtimeVariant: ''
   isOfficialBuild: false
   librariesConfiguration: Debug
+  wasmEnableThreadsArg: ''
 
 steps:
   # For NativeAOT-LLVM, we have just built the Wasm-targeting native artifacts (the runtime and libraries).
@@ -17,7 +18,7 @@ steps:
       displayName: Build the ILC and RyuJit cross-compilers
 
     # Build target packages (note: target libs already built).
-    - script: $(Build.SourcesDirectory)/build$(scriptExt) nativeaot.packages -os ${{ parameters.osGroup }} -a wasm -c $(buildConfigUpper) $(_officialBuildParameter) -ci
+    - script: $(Build.SourcesDirectory)/build$(scriptExt) nativeaot.packages -os ${{ parameters.osGroup }} -a wasm -c $(buildConfigUpper) $(_officialBuildParameter) -ci $(wasmEnableThreadsArg)
       displayName: Build target packages
 
     # Build host packages.
@@ -26,13 +27,14 @@ steps:
 
   # Build coreclr native test output outside of official build
   - ${{ if ne(parameters.isOfficialBuild, true) }}:
-    - ${{ if and(eq(parameters.archType, 'wasm'), ne(parameters.nameSuffix, '')) }}:
+    - ${{ if and(eq(parameters.archType, 'wasm'), and(eq(parameters.wasmEnableThreadsArg, ''))) }}:
       - script: pwsh $(Build.SourcesDirectory)/eng/pipelines/runtimelab/set-ilc-emulation-environment.ps1 -Arch $(hostedTargetArch)
         displayName: Set up ILC emulation environment
 
     - ${{ if eq(parameters.archType, 'wasm') }}:
-      - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot $(buildConfigUpper) ${{ parameters.osGroup }} $(crossArg) $(_officialBuildParameter) ci tree nativeaot /p:LibrariesConfiguration=${{ parameters.librariesConfiguration }}
-        displayName: Build runtime tests
+      - ${{ if not(and(eq(parameters.osGroup, 'browser'), ne(parameters.wasmEnableThreadsArg, ''))) }}:
+        - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot $(buildConfigUpper) ${{ parameters.osGroup }} $(crossArg) $(_officialBuildParameter) ci tree nativeaot /p:LibrariesConfiguration=${{ parameters.librariesConfiguration }} ${{ parameters.wasmEnableThreadsArg }}
+          displayName: Build runtime tests
     - ${{ else }}:
       - ${{ if eq(parameters.osGroup, 'windows') }}:
         - script: $(Build.SourcesDirectory)/src/tests/build$(scriptExt) nativeaot $(buildConfigUpper) ${{ parameters.archType }} $(crossArg) $(_officialBuildParameter) ci tree nativeaot /p:LibrariesConfiguration=${{ parameters.librariesConfiguration }}
@@ -45,11 +47,12 @@ steps:
       - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) runnativeaottests $(buildConfigUpper) ${{ parameters.archType }} ${{ parameters.osGroup }}
         displayName: Run runtime tests
     - ${{ else }}:
-      - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) --runnativeaottests $(buildConfigUpper) ${{ parameters.archType }} ${{ parameters.osGroup }}
-        displayName: Run runtime tests
+      - ${{ if not(and(eq(parameters.osGroup, 'browser'), ne(parameters.wasmEnableThreadsArg, ''))) }}:
+        - script: $(Build.SourcesDirectory)/src/tests/run$(scriptExt) --runnativeaottests $(buildConfigUpper) ${{ parameters.archType }} ${{ parameters.osGroup }}
+          displayName: Run runtime tests
 
     # Don't compile/run the libraries tests with emulated ILC to save CI time/resources.
-    - ${{ if and(eq(parameters.archType, 'wasm'), eq(parameters.nameSuffix, '')) }}:
+    - ${{ if and(eq(parameters.archType, 'wasm'), eq(parameters.wasmEnableThreadsArg, '')) }}:
       - script: $(Build.SourcesDirectory)/build$(scriptExt) libs.tests -test -a ${{ parameters.archType }} -os ${{ parameters.osGroup }} -lc ${{ parameters.librariesConfiguration }} -rc $(buildConfigUpper) /p:TestNativeAot=true /p:RunSmokeTestsOnly=true
         displayName: Build and run WebAssembly libraries tests
 
@@ -61,3 +64,4 @@ steps:
     - template: /eng/pipelines/common/upload-intermediate-artifacts-step.yml
       parameters:
         name: ${{ parameters.platform }}${{ parameters.nameSuffix }}
+

src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.targets

@@ -438,6 +438,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <CompileWasmArgs Condition="'$(NativeDebugSymbols)' == 'true'">$(CompileWasmArgs) -g3</CompileWasmArgs>
       <CompileWasmArgs Condition="'$(WasmEnableNonTrappingFloatToIntConversions)' == 'true'">$(CompileWasmArgs) -mnontrapping-fptoint</CompileWasmArgs>
       <CompileWasmArgs Condition="'$(IlcLlvmExceptionHandlingModel)' == 'wasm'">$(CompileWasmArgs) -fwasm-exceptions</CompileWasmArgs>
+      <CompileWasmArgs Condition="'$(WasmEnableThreads)' == 'true'">$(CompileWasmArgs) -pthread -matomics -mbulk-memory</CompileWasmArgs>
     </PropertyGroup>
 
     <PropertyGroup Condition="'$(_targetOS)' == 'browser'">
@@ -608,6 +609,7 @@ The .NET Foundation licenses this file to you under the MIT license.
       <CustomLinkerArg Include="-s MAXIMUM_MEMORY=$(EmccMaximumHeapSize)" Condition="'$(EmccMaximumHeapSize)' != ''" />
       <CustomLinkerArg Include="-s INITIAL_MEMORY=$(EmccInitialHeapSize)" Condition="'$(EmccInitialHeapSize)' != ''" />
       <CustomLinkerArg Condition="'$(EmccEnableAssertions)' == 'true'" Include="-s ASSERTIONS=1" />
+      <CustomLinkerArg Condition="'$(WasmEnableThreads)' == 'true'" Include="-pthread" />
     </ItemGroup>
 
     <!-- wasm-ld only supports listing exports on the command line -->

src/coreclr/nativeaot/CMakeLists.txt

@@ -32,6 +32,12 @@ if(CLR_CMAKE_TARGET_OS STREQUAL wasi)
   add_definitions(-DTARGET_UNIX)
 endif(CLR_CMAKE_TARGET_OS STREQUAL wasi)
 
+if((CLR_CMAKE_TARGET_OS STREQUAL wasi OR CLR_CMAKE_TARGET_OS STREQUAL emscripten) 
+  AND CLR_CMAKE_TARGET_OS_SUBGROUP STREQUAL multithread)
+  add_definitions(-DFEATURE_WASM_MANAGED_THREADS)
+endif((CLR_CMAKE_TARGET_OS STREQUAL wasi OR CLR_CMAKE_TARGET_OS STREQUAL emscripten) 
+  AND CLR_CMAKE_TARGET_OS_SUBGROUP STREQUAL multithread)
+
 if(CLR_CMAKE_TARGET_ANDROID)
     add_definitions(-DFEATURE_EMULATED_TLS)
 endif()

src/coreclr/nativeaot/Runtime/unix/PalRedhawkUnix.cpp

@@ -1169,6 +1169,8 @@ REDHAWK_PALEXPORT bool PalGetMaximumStackBounds(_Out_ void** ppStackLowOut, _Out
         status = pthread_attr_get_np(thread, &attr);
 #elif HAVE_PTHREAD_GETATTR_NP
         status = pthread_getattr_np(thread, &attr);
+#elif defined(HOST_WASM) && defined(FEATURE_WASM_MANAGED_THREADS)
+        // We dont have a pthread_getattr_np, but so far we don't need it.
 #else
 #error Dont know how to get thread attributes on this platform!
 #endif

src/coreclr/nativeaot/Runtime/wasm/PalRedhawkWasm.cpp

@@ -191,6 +191,186 @@ extern "C" int __cxa_thread_atexit(Dtor dtor, void* obj, void*)
 #endif // TARGET_WASI
 #endif // !FEATURE_WASM_MANAGED_THREADS
 
+// TODO-LLVM: For now, a copy of the single threaded implementation.
+#ifdef FEATURE_WASM_MANAGED_THREADS
+int __cxa_thread_atexit(void (*func)(), void *obj, void *dso_symbol)
+{
+    return 0;
+}
+
+//
+// Note that we return the native stack bounds here, not shadow stack ones. Currently this functionality is mainly
+// used for RuntimeHelpers.TryEnsureSufficientExecutionStack, and we do use the native stack in codegen, so this
+// is an acceptable approximation.
+//
+extern "C" unsigned char __stack_low;
+extern "C" unsigned char __stack_high;
+
+#ifdef TARGET_WASI
+// TODO-LLVM: No-op stubs, maybe when threads are implemented in WASI, we wont have to provide all of these.
+int pthread_mutex_init(pthread_mutex_t *, const pthread_mutexattr_t *)
+{
+    return 0;
+}
+
+int pthread_mutexattr_init(pthread_mutexattr_t *)
+{
+    return 0;
+}
+
+int pthread_mutexattr_settype(pthread_mutexattr_t *, int)
+{
+    return 0;
+}
+
+int pthread_mutex_destroy(pthread_mutex_t *)
+{
+    return 0;
+}
+
+int pthread_mutexattr_destroy(pthread_mutexattr_t *)
+{
+    return 0;
+}
+
+int pthread_cond_init(pthread_cond_t *, const pthread_condattr_t *)
+{
+    return 0;
+}
+
+int pthread_cond_destroy(pthread_cond_t *)
+{
+    return 0;
+}
+
+int pthread_cond_wait(pthread_cond_t *, pthread_mutex_t *)
+{
+    return 0;
+}
+
+int pthread_cond_timedwait(pthread_cond_t *, pthread_mutex_t *, const struct timespec *)
+{
+    return 0;
+}
+
+int pthread_condattr_init(pthread_condattr_t *)
+{
+    return 0;
+}
+
+int pthread_mutex_lock(pthread_mutex_t *)
+{
+    return 0;
+}
+
+int pthread_mutex_unlock(pthread_mutex_t *)
+{
+    return 0;
+}
+
+pthread_t pthread_self(void)
+{
+    return (pthread_t)0;
+}
+
+int pthread_equal(pthread_t, pthread_t)
+{
+    return 1;
+}
+
+int pthread_attr_init(pthread_attr_t *)
+{
+    // See https://github.com/emscripten-core/emscripten/pull/18057 and https://reviews.llvm.org/D135910.
+    unsigned char* pStackLow = &__stack_low;
+    unsigned char* pStackHigh = &__stack_high;
+
+    // Sanity check that we have the expected memory layout.
+    ASSERT((pStackHigh - pStackLow) >= 64 * 1024);
+    if (pStackLow >= pStackHigh)
+    {
+        PalPrintFatalError("\nFatal error. Unexpected stack layout.\n");
+        RhFailFast();
+    }
+
+    return 0;
+}
+
+int pthread_attr_getstack(pthread_attr_t *, void **stackaddr, size_t *stacksize)
+{
+    unsigned char* pStackLow = &__stack_low;
+    unsigned char* pStackHigh = &__stack_high;
+
+    *stackaddr = pStackLow;
+    *stacksize = pStackHigh - pStackLow;
+    return 0;
+}
+
+int pthread_attr_destroy(pthread_attr_t *)
+{
+    return 0;
+}
+
+int pthread_condattr_destroy(pthread_condattr_t *)
+{
+    return 0;
+}
+
+int pthread_cond_broadcast(pthread_cond_t *)
+{
+    return 0;
+}
+
+int pthread_attr_setdetachstate(pthread_attr_t *, int)
+{
+    return 0;
+}
+
+using Dtor = void(*)(void*);
+
+// TODO=LLVM: This is a copy of the single threaded implementation.
+extern "C" int __cxa_thread_atexit(Dtor dtor, void* obj, void*)
+{
+    struct DtorList
+    {
+        Dtor dtor;
+        void* obj;
+        DtorList* next;
+    };
+
+    struct DtorsManager
+    {
+        DtorList* m_dtors = nullptr;
+
+        ~DtorsManager()
+        {
+            while (DtorList* head = m_dtors)
+            {
+                m_dtors = head->next;
+                head->dtor(head->obj);
+                free(head);
+            }
+        }
+    };
+
+    // The linked list of "thread-local" destructors to run.
+    static DtorsManager s_dtorsManager;
+
+    DtorList* head = static_cast<DtorList*>(malloc(sizeof(DtorList)));
+    if (head == nullptr)
+    {
+        return -1;
+    }
+
+    head->dtor = dtor;
+    head->obj = obj;
+    head->next = s_dtorsManager.m_dtors;
+    s_dtorsManager.m_dtors = head;
+
+    return 0;
+}
+#endif // TARGET_WASI
+#endif // FEATURE_WASM_MANAGED_THREADS
+
 // Recall that WASM's model is extremely simple: we have one linear memory, which can only be grown, in chunks
 // of 64K pages. Thus, "mmap"/"munmap" fundamentally cannot be faithfully recreated and the Unix emulators we
 // layer on top of (Emscripten/WASI libc) reflect this by not supporting the scenario. Fortunately, the current

src/coreclr/nativeaot/Runtime/wasm/PalRedhawkWasm.h

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
-#ifndef FEATURE_WASM_MANAGED_THREADS
+#ifdef FEATURE_WASM_MANAGED_THREADS
+int __cxa_thread_atexit(void (*func)(), void *obj, void *dso_symbol);
+#else
 void PalGetMaximumStackBounds_SingleThreadedWasm(void** ppStackLowOut, void** ppStackHighOut);
 #endif // !FEATURE_WASM_MANAGED_THREADS

src/coreclr/nativeaot/System.Private.CoreLib/src/System.Private.CoreLib.csproj

@@ -35,6 +35,11 @@
   <PropertyGroup Condition="'$(TargetsSingleThreadedWasm)' != 'true'">
     <FeaturePortableThreadPool>true</FeaturePortableThreadPool>
     <FeaturePortableTimer>true</FeaturePortableTimer>
+    <!-- LLVM-TODO: Remove FeatureWasmManagedThreads in favor of WasmEnableThreads. -->
+    <FeatureWasmManagedThreads Condition="'$(WasmEnableThreads)' == 'true'">true</FeatureWasmManagedThreads>
+    <FeatureWasmPerfTracing Condition="('$(TargetsBrowser)' == 'true' or '$(TargetsWasi)' == 'true') and ('$(WasmEnableThreads)' == 'true')">true</FeatureWasmPerfTracing>
+    <DefineConstants Condition="'$(WasmEnableThreads)' == 'true'">$(DefineConstants);FEATURE_WASM_MANAGED_THREADS</DefineConstants>
+    <DefineConstants Condition="'$(FeatureWasmPerfTracing)' == 'true'">$(DefineConstants);FEATURE_WASM_PERFTRACING</DefineConstants>
   </PropertyGroup>
   <PropertyGroup>
     <FeatureHardwareIntrinsics>true</FeatureHardwareIntrinsics>
@@ -347,6 +352,8 @@
     <Compile Include="System\Diagnostics\StackFrame.NativeAot.Wasm.cs" />
     <Compile Include="System\Diagnostics\StackFrame.NativeAot.Wasi.cs" Condition="'$(TargetsWasi)' == 'true'" />
     <Compile Include="System\Diagnostics\StackFrame.NativeAot.Browser.cs" Condition="'$(TargetsBrowser)' == 'true'" />
+    <Compile Include="System\Threading\PortableThreadPool.NativeAot.Browser.cs" Condition="'$(TargetsBrowser)' == 'true' and '$(WasmEnableThreads)' == 'true'" />
+    <Compile Include="System\Threading\ThreadPool.NativeAot.Browser.cs" Condition="'$(TargetsBrowser)' == 'true' and '$(WasmEnableThreads)' == 'true'" />
   </ItemGroup>
 
   <!-- WASM-specific things. Keep in sync with Mono. -->