From 7d270903eff26c9bc979560ce9a765d9dadd7dca Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 4 Oct 2025 15:20:26 +0200 Subject: [PATCH 01/48] Initial --- src/coreclr/vm/ceeload.cpp | 76 ++++++++++++++++++++++++++++++++++++++ src/coreclr/vm/ceeload.h | 2 + 2 files changed, 78 insertions(+) diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index bb0f2037816852..f3e9810b1f6692 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -2296,6 +2296,82 @@ BYTE *Module::GetProfilerBase() } } +template +static void EnumerateRunsOfObjRefs(unsigned size, bool* objRefs, TFunc func) +{ + bool* objRefsEnd = objRefs + (size + (TARGET_POINTER_SIZE - 1)) / TARGET_POINTER_SIZE; + bool* start = objRefs; + while (start < objRefsEnd) + { + while (start < objRefsEnd && !*start) + start++; + + if (start >= objRefsEnd) + return; + + bool* end = start; + while (end < objRefsEnd && *end) + end++; + + func((start - objRefs) * TARGET_POINTER_SIZE, (end - start) * TARGET_POINTER_SIZE); + } +} + +MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objRefs, class AllocMemTracker* pamTracker) +{ + CONTRACTL { + STANDARD_VM_CHECK; + PRECONDITION(dataSize > 0); + } CONTRACTL_END; + + MethodTable* pParentClass = CoreLibBinder::GetClass(CLASS__CONTINUATION); + unsigned parentInstanceSize = pParentClass->GetNumInstanceFieldBytes(); + DWORD numVirtuals = pParentClass->GetNumVirtuals(); + + size_t cbMT = sizeof(MethodTable); + cbMT += MethodTable::GetNumVtableIndirections(numVirtuals) * sizeof(MethodTable::VTableIndir_t); + + unsigned numSeries = 0; + EnumerateRunsOfObjRefs(dataSize, objRefs, [&](unsigned start, unsigned count) + { + numSeries++; + }); + + _ASSERTE(!pParentClass->ContainsGCPointers()); + DWORD cbGC = numSeries == 0 ? 0 : CGCDesc::ComputeSize(numSeries); + + LoaderAllocator* pAllocator = GetLoaderAllocator(); + BYTE* pMemory = (BYTE*)pamTracker->Track(pAllocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); + memset(pMemory, 0, cbGC + cbMT); + + MethodTable* pMT = (MethodTable*)(pMemory + cbGC); + pMT->AllocateAuxiliaryData(pAllocator, this, pamTracker, MethodTableStaticsFlags::None); + pMT->SetLoaderAllocator(pAllocator); + pMT->SetModule(this); + pMT->SetNumVirtuals(numVirtuals); + pMT->SetParentMethodTable(pParentClass); + pMT->SetClass(pParentClass->GetClass()); // TODO: needs its own? + pMT->SetBaseSize(OBJECT_BASESIZE + parentInstanceSize + dataSize); + + if (numSeries > 0) + { + pMT->SetContainsGCPointers(); + CGCDesc::Init(pMT, numSeries); + CGCDescSeries* pSeries = CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries(); + auto writeSeries = [&](unsigned start, unsigned length) { + pSeries->SetSeriesSize(length - pMT->GetBaseSize()); + pSeries->SetSeriesOffset(OBJECT_SIZE + parentInstanceSize + start); + pSeries++; + }; + EnumerateRunsOfObjRefs(size, objRefs, writeSeries); + _ASSERTE(pSeries == CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries() + numSeries); + } + + pMT->SetClassInited(); + + return pMT; +} + #endif //!DACCESS_COMPILE Assembly * diff --git a/src/coreclr/vm/ceeload.h b/src/coreclr/vm/ceeload.h index c4d02b6e02dbb6..31c5aaa6e2b8b7 100644 --- a/src/coreclr/vm/ceeload.h +++ b/src/coreclr/vm/ceeload.h @@ -1163,6 +1163,8 @@ class Module : public ModuleBase // the class load, which avoids the need for a 'being loaded' list MethodTable* CreateArrayMethodTable(TypeHandle elemType, CorElementType kind, unsigned rank, class AllocMemTracker *pamTracker); + MethodTable* CreateContinuationMethodTable(unsigned dataSize, bool* objRefs, class AllocMemTracker* pamTracker); + // Module/Assembly traversal Assembly * GetAssemblyIfLoaded( mdAssemblyRef kAssemblyRef, From 6743534bcbc96f3adfa721d1d19994bdcef80f09 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 4 Oct 2025 20:55:47 +0200 Subject: [PATCH 02/48] Fix build, make tests run --- src/coreclr/vm/ceeload.cpp | 10 +++++----- src/tests/async/Directory.Build.props | 2 +- src/tests/async/Directory.Build.targets | 13 ------------- 3 files changed, 6 insertions(+), 19 deletions(-) delete mode 100644 src/tests/async/Directory.Build.targets diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index f3e9810b1f6692..d10522a9985469 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -2332,13 +2332,13 @@ MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objR cbMT += MethodTable::GetNumVtableIndirections(numVirtuals) * sizeof(MethodTable::VTableIndir_t); unsigned numSeries = 0; - EnumerateRunsOfObjRefs(dataSize, objRefs, [&](unsigned start, unsigned count) + EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) { numSeries++; }); _ASSERTE(!pParentClass->ContainsGCPointers()); - DWORD cbGC = numSeries == 0 ? 0 : CGCDesc::ComputeSize(numSeries); + size_t cbGC = numSeries == 0 ? 0 : CGCDesc::ComputeSize(numSeries); LoaderAllocator* pAllocator = GetLoaderAllocator(); BYTE* pMemory = (BYTE*)pamTracker->Track(pAllocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); @@ -2348,7 +2348,7 @@ MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objR pMT->AllocateAuxiliaryData(pAllocator, this, pamTracker, MethodTableStaticsFlags::None); pMT->SetLoaderAllocator(pAllocator); pMT->SetModule(this); - pMT->SetNumVirtuals(numVirtuals); + pMT->SetNumVirtuals(static_cast(numVirtuals)); pMT->SetParentMethodTable(pParentClass); pMT->SetClass(pParentClass->GetClass()); // TODO: needs its own? pMT->SetBaseSize(OBJECT_BASESIZE + parentInstanceSize + dataSize); @@ -2358,12 +2358,12 @@ MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objR pMT->SetContainsGCPointers(); CGCDesc::Init(pMT, numSeries); CGCDescSeries* pSeries = CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries(); - auto writeSeries = [&](unsigned start, unsigned length) { + auto writeSeries = [&](size_t start, size_t length) { pSeries->SetSeriesSize(length - pMT->GetBaseSize()); pSeries->SetSeriesOffset(OBJECT_SIZE + parentInstanceSize + start); pSeries++; }; - EnumerateRunsOfObjRefs(size, objRefs, writeSeries); + EnumerateRunsOfObjRefs(dataSize, objRefs, writeSeries); _ASSERTE(pSeries == CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries() + numSeries); } diff --git a/src/tests/async/Directory.Build.props b/src/tests/async/Directory.Build.props index a1f7d48cf60a7d..f505092384b1b1 100644 --- a/src/tests/async/Directory.Build.props +++ b/src/tests/async/Directory.Build.props @@ -5,7 +5,7 @@ true - $(NoWarn);xUnit1013;CS1998 + $(NoWarn);xUnit1013;CS1998;SYSLIB5007 false $(Features);runtime-async=on diff --git a/src/tests/async/Directory.Build.targets b/src/tests/async/Directory.Build.targets deleted file mode 100644 index 5a4d413a9e1f62..00000000000000 --- a/src/tests/async/Directory.Build.targets +++ /dev/null @@ -1,13 +0,0 @@ - - - - true - - - - - true - - - - From 0f8cd9cc58e937b2f2a96933be01530a1c2b37b6 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Oct 2025 01:11:07 +0200 Subject: [PATCH 03/48] Tailored continuation layouts Add ability for the VM to dynamically create continuation layout types and for the JIT to request such types to be created. --- .github/prompts/add-new-jit-ee-api.prompt.md | 16 +- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 204 +++--- .../RuntimeHelpers.CoreCLR.cs | 3 + src/coreclr/inc/corinfo.h | 36 +- src/coreclr/inc/icorjitinfoimpl_generated.h | 5 + src/coreclr/inc/jiteeversionguid.h | 10 +- src/coreclr/jit/ICorJitInfo_names_generated.h | 1 + .../jit/ICorJitInfo_wrapper_generated.hpp | 11 + src/coreclr/jit/async.cpp | 688 +++++------------- src/coreclr/jit/async.h | 76 +- .../tools/Common/JitInterface/CorInfoImpl.cs | 7 + .../JitInterface/CorInfoImpl_generated.cs | 18 +- .../tools/Common/JitInterface/CorInfoTypes.cs | 6 + .../ThunkGenerator/ThunkInput.txt | 2 + .../aot/jitinterface/jitinterface_generated.h | 12 + .../tools/superpmi/superpmi-shared/agnostic.h | 11 +- .../tools/superpmi/superpmi-shared/lwmlist.h | 1 + .../superpmi-shared/methodcontext.cpp | 59 +- .../superpmi/superpmi-shared/methodcontext.h | 5 + .../superpmi-shim-collector/icorjitinfo.cpp | 10 +- .../icorjitinfo_generated.cpp | 9 + .../icorjitinfo_generated.cpp | 8 + .../tools/superpmi/superpmi/icorjitinfo.cpp | 9 +- src/coreclr/vm/ceeload.cpp | 58 +- src/coreclr/vm/ceeload.h | 2 +- src/coreclr/vm/corelib.h | 2 - src/coreclr/vm/jitinterface.cpp | 122 +--- src/coreclr/vm/methodtable.h | 25 +- 28 files changed, 600 insertions(+), 816 deletions(-) diff --git a/.github/prompts/add-new-jit-ee-api.prompt.md b/.github/prompts/add-new-jit-ee-api.prompt.md index ce3f56b937f6f5..81c5de02328e65 100644 --- a/.github/prompts/add-new-jit-ee-api.prompt.md +++ b/.github/prompts/add-new-jit-ee-api.prompt.md @@ -1,23 +1,23 @@ --- mode: 'agent' -tools: ['githubRepo', 'codebase', 'terminalLastCommand'] +tools: ['fetch', 'codebase', 'runCommands', 'usages', 'search', 'think'] description: 'Add a new API to the JIT-VM (aka JIT-EE) interface in the codebase.' --- #### 1 — Goal -Implement **one** new JIT-VM (also known as JIT-EE) API and all supporting glue. +Implement **one** new JIT-VM (also known as JIT-EE) API and all supporting glue. The JIT-VM interface defines the APIs through which the JIT compiler communicates with the runtime (VM). #### 2 — Prerequisites for the model * You have full repo access -* You may run scripts (e.g., `.sh` or `.bat`) +* You may run scripts (e.g., `.sh` or `.bat`) * Ask **clarifying questions** before the first code change if anything (signature, types, platform constraints) is unclear. #### 3 — Required user inputs -Ask the user for a C-like signature of the new API if it's not provided. +Ask the user for a C-like signature of the new API if it's not provided. Suggest `/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt` file as a reference. Example: ``` @@ -83,8 +83,8 @@ Use the correct directory for the script to run. +} ``` -6. Now implement the most complex part - SuperPMI. SuperPMI acts as a (de)serializer for JIT-VM queries in order -to then replay them without the actual VM to speed up jit-diffs and other scenarios. All parameters and return +6. Now implement the most complex part - SuperPMI. SuperPMI acts as a (de)serializer for JIT-VM queries in order +to then replay them without the actual VM to speed up jit-diffs and other scenarios. All parameters and return values recorded/restored using special primitve types and helpers. We need to update the following files: * `/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h`: @@ -96,7 +96,7 @@ Go through each of them one by one. * `/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h`: Define two `Agnostic_*` types for input arguments and another one for output parameters (return value, output arguments). - Do not create them if one of the generics ones can be re-used such as `DLD`, `DD`, `DLDL`, etc. Use `DWORD*` + Do not create them if one of the generics ones can be re-used such as `DLD`, `DD`, `DLDL`, etc. Use `DWORD*` like types for integers. Inspect the whole file to see how other APIs are defined. * `/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h`: @@ -126,7 +126,7 @@ Now add a new element to `enum mcPackets` enum in the same file. Example: ``` * `/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp`: -Add the implementation of the 3 methods to `methodcontext.cpp` at the end of it. +Add the implementation of the 3 methods to `methodcontext.cpp` at the end of it. Consider other similar methods in the file for reference. Do not change implementations of other methods in the file. Example: ```diff diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index ba703738d3c7c3..f35ee277ece488 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -49,81 +49,85 @@ public void Pop() [Flags] internal enum CorInfoContinuationFlags { - // Whether or not the continuation expects the result to be boxed and - // placed in the GCData array at index 0. Not set if the callee is void. - CORINFO_CONTINUATION_RESULT_IN_GCDATA = 1, // If this bit is set the continuation resumes inside a try block and thus // if an exception is being propagated, needs to be resumed. The exception // should be placed at index 0 or 1 depending on whether the continuation // also expects a result. - CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2, - // If this bit is set the continuation has the IL offset that inspired the - // OSR method saved in the beginning of 'Data', or -1 if the continuation - // belongs to a tier 0 method. - CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA = 4, + CORINFO_CONTINUATION_NEEDS_EXCEPTION = 1, // If this bit is set the continuation should continue on the thread // pool. - CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 8, + CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 2, // If this bit is set the continuation has a SynchronizationContext // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 16, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 4, // If this bit is set the continuation has a TaskScheduler // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 32, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 8, } - internal sealed unsafe class Continuation + internal struct CORINFO_CONTINUATION_DATA_OFFSETS + { + public uint Result; + public uint Exception; + public uint ContinuationContext; + public uint KeepAlive; + } + +#pragma warning disable CA1852 // "Type can be sealed" -- no it cannot because the runtime constructs subtypes dynamically + internal unsafe class Continuation { public Continuation? Next; - public delegate* Resume; - public uint State; + public delegate* Resume; public CorInfoContinuationFlags Flags; + public int State; - // Data and GCData contain the state of the continuation. - // Note: The JIT is ultimately responsible for laying out these arrays. - // However, other parts of the system depend on the layout to - // know where to locate or place various pieces of data: - // - // 1. Resumption stubs need to know where to place the return value - // inside the next continuation. If the return value has GC references - // then it is boxed and placed at GCData[0]; otherwise, it is placed - // inside Data at offset 0 if - // CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA is NOT set and otherwise - // at offset 4. - // - // 2. Likewise, Finalize[Value]TaskReturningThunk needs to know from - // where to extract the return value. - // - // 3. The dispatcher needs to know where to place the exception inside - // the next continuation with a handler. Continuations with handlers - // have CORINFO_CONTINUATION_NEEDS_EXCEPTION set. The exception is - // placed at GCData[0] if CORINFO_CONTINUATION_RESULT_IN_GCDATA is NOT - // set, and otherwise at GCData[1]. - // - public byte[]? Data; - public object?[]? GCData; - - public object GetContinuationContext() + public unsafe object GetContinuationContext() { - int index = 0; - if ((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_RESULT_IN_GCDATA) != 0) - index++; - if ((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION) != 0) - index++; - Debug.Assert(GCData != null && GCData.Length > index); - object? continuationContext = GCData[index]; - Debug.Assert(continuationContext != null); - return continuationContext; + // Only the special continuation sub types are expected. + Debug.Assert(GetType() != typeof(Continuation)); + + MethodTable* mt = RuntimeHelpers.GetMethodTable(this); + Debug.Assert(mt->ContinuationOffsets->ContinuationContext != uint.MaxValue); + + ref byte data = ref RuntimeHelpers.GetRawData(this); + return Unsafe.As(ref Unsafe.Add(ref data, mt->ContinuationOffsets->ContinuationContext)); } public void SetException(Exception ex) { - int index = 0; - if ((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_RESULT_IN_GCDATA) != 0) - index++; + // Only the special continuation sub types are expected. + Debug.Assert(GetType() != typeof(Continuation)); + + MethodTable* mt = RuntimeHelpers.GetMethodTable(this); + Debug.Assert(mt->ContinuationOffsets->Exception != uint.MaxValue); - Debug.Assert(GCData != null && GCData.Length > index); - GCData[index] = ex; + ref byte data = ref RuntimeHelpers.GetRawData(this); + Unsafe.As(ref Unsafe.Add(ref data, mt->ContinuationOffsets->Exception)) = ex; + } + + public ref byte GetResultStorageOrNull() + { + // Only the special continuation sub types are expected. + Debug.Assert(GetType() != typeof(Continuation)); + + MethodTable* mt = RuntimeHelpers.GetMethodTable(this); + if (mt->ContinuationOffsets->Result == uint.MaxValue) + return ref Unsafe.NullRef(); + + ref byte data = ref RuntimeHelpers.GetRawData(this); + return ref Unsafe.Add(ref data, mt->ContinuationOffsets->Result); + } + + public void SetKeepAlive(object? obj) + { + // Only the special continuation sub types are expected. + Debug.Assert(GetType() != typeof(Continuation)); + + MethodTable* mt = RuntimeHelpers.GetMethodTable(this); + Debug.Assert(mt->ContinuationOffsets->KeepAlive != uint.MaxValue); + + ref byte data = ref RuntimeHelpers.GetRawData(this); + Unsafe.As(ref Unsafe.Add(ref data, mt->ContinuationOffsets->KeepAlive)) = obj; } } @@ -146,48 +150,32 @@ private struct RuntimeAsyncAwaitState [ThreadStatic] private static RuntimeAsyncAwaitState t_runtimeAsyncAwaitState; - private static Continuation AllocContinuation(Continuation prevContinuation, nuint numGCRefs, nuint dataSize) + private static unsafe Continuation AllocContinuation(Continuation prevContinuation, MethodTable* contMT) { - Continuation newContinuation = new Continuation { Data = new byte[dataSize], GCData = new object[numGCRefs] }; + Continuation newContinuation = (Continuation)RuntimeTypeHandle.InternalAllocNoChecks(contMT); prevContinuation.Next = newContinuation; return newContinuation; } - private static unsafe Continuation AllocContinuationMethod(Continuation prevContinuation, nuint numGCRefs, nuint dataSize, MethodDesc* method) + private static unsafe Continuation AllocContinuationMethod(Continuation prevContinuation, MethodTable* contMT, MethodDesc* method) { LoaderAllocator loaderAllocator = RuntimeMethodHandle.GetLoaderAllocator(new RuntimeMethodHandleInternal((IntPtr)method)); - object?[] gcData; - if (loaderAllocator != null) - { - gcData = new object[numGCRefs + 1]; - gcData[numGCRefs] = loaderAllocator; - } - else - { - gcData = new object[numGCRefs]; - } - - Continuation newContinuation = new Continuation { Data = new byte[dataSize], GCData = gcData }; + Continuation newContinuation = (Continuation)RuntimeTypeHandle.InternalAllocNoChecks(contMT); + newContinuation.SetKeepAlive(loaderAllocator); prevContinuation.Next = newContinuation; return newContinuation; } - private static unsafe Continuation AllocContinuationClass(Continuation prevContinuation, nuint numGCRefs, nuint dataSize, MethodTable* methodTable) + private static unsafe Continuation AllocContinuationClass(Continuation prevContinuation, MethodTable* contMT, MethodTable* methodTable) { IntPtr loaderAllocatorHandle = methodTable->GetLoaderAllocatorHandle(); - object?[] gcData; + + Continuation newContinuation = (Continuation)RuntimeTypeHandle.InternalAllocNoChecks(contMT); + prevContinuation.Next = newContinuation; if (loaderAllocatorHandle != IntPtr.Zero) { - gcData = new object[numGCRefs + 1]; - gcData[numGCRefs] = GCHandle.FromIntPtr(loaderAllocatorHandle).Target; - } - else - { - gcData = new object[numGCRefs]; + newContinuation.SetKeepAlive(GCHandle.FromIntPtr(loaderAllocatorHandle).Target); } - - Continuation newContinuation = new Continuation { Data = new byte[dataSize], GCData = gcData }; - prevContinuation.Next = newContinuation; return newContinuation; } @@ -208,8 +196,9 @@ private interface IThunkTaskOps static abstract Action GetContinuationAction(T task); static abstract Continuation GetContinuationState(T task); static abstract void SetContinuationState(T task, Continuation value); - static abstract bool SetCompleted(T task, Continuation continuation); + static abstract bool SetCompleted(T task); static abstract void PostToSyncContext(T task, SynchronizationContext syncCtx); + static abstract ref byte GetResultStorage(T task); } private sealed class ThunkTask : Task @@ -248,39 +237,23 @@ public void HandleSuspended() private struct Ops : IThunkTaskOps> { public static Action GetContinuationAction(ThunkTask task) => (Action)task.m_action!; - public static void MoveNext(ThunkTask task) => task.MoveNext(); public static Continuation GetContinuationState(ThunkTask task) => (Continuation)task.m_stateObject!; public static void SetContinuationState(ThunkTask task, Continuation value) { task.m_stateObject = value; } - public static bool SetCompleted(ThunkTask task, Continuation continuation) + public static bool SetCompleted(ThunkTask task) { - T result; - if (RuntimeHelpers.IsReferenceOrContainsReferences()) - { - if (typeof(T).IsValueType) - { - result = Unsafe.As(ref continuation.GCData![0]!.GetRawData()); - } - else - { - result = Unsafe.As(ref continuation.GCData![0]!); - } - } - else - { - result = Unsafe.As(ref continuation.Data![0]); - } - - return task.TrySetResult(result); + return task.TrySetResult(task.m_result); } public static void PostToSyncContext(ThunkTask task, SynchronizationContext syncContext) { syncContext.Post(s_postCallback, task); } + + public static ref byte GetResultStorage(ThunkTask task) => ref Unsafe.As(ref task.m_result); } } @@ -320,14 +293,13 @@ public void HandleSuspended() private struct Ops : IThunkTaskOps { public static Action GetContinuationAction(ThunkTask task) => (Action)task.m_action!; - public static void MoveNext(ThunkTask task) => task.MoveNext(); public static Continuation GetContinuationState(ThunkTask task) => (Continuation)task.m_stateObject!; public static void SetContinuationState(ThunkTask task, Continuation value) { task.m_stateObject = value; } - public static bool SetCompleted(ThunkTask task, Continuation continuation) + public static bool SetCompleted(ThunkTask task) { return task.TrySetResult(); } @@ -336,6 +308,8 @@ public static void PostToSyncContext(ThunkTask task, SynchronizationContext sync { syncContext.Post(s_postCallback, task); } + + public static ref byte GetResultStorage(ThunkTask task) => ref Unsafe.NullRef(); } } @@ -351,7 +325,10 @@ public static unsafe void MoveNext(T task) where T : Task where TOps : { try { - Continuation? newContinuation = continuation.Resume(continuation); + Debug.Assert(continuation.Next != null); + + ref byte resultLoc = ref continuation.Next.Resume != null ? ref continuation.Next.GetResultStorageOrNull() : ref TOps.GetResultStorage(task); + Continuation? newContinuation = continuation.Resume(continuation, ref resultLoc); if (newContinuation != null) { @@ -361,7 +338,6 @@ public static unsafe void MoveNext(T task) where T : Task where TOps : return; } - Debug.Assert(continuation.Next != null); continuation = continuation.Next; } catch (Exception ex) @@ -391,7 +367,7 @@ public static unsafe void MoveNext(T task) where T : Task where TOps : if (continuation.Resume == null) { - bool successfullySet = TOps.SetCompleted(task, continuation); + bool successfullySet = TOps.SetCompleted(task); contexts.Pop(); @@ -536,22 +512,10 @@ private static bool QueueContinuationFollowUpActionIfNecessary(T task, // it calls here to make a Task that awaits on the current async state. private static Task FinalizeTaskReturningThunk(Continuation continuation) { - Continuation finalContinuation = new Continuation(); - - // Note that the exact location the return value is placed is tied - // into getAsyncResumptionStub in the VM, so do not change this - // without also changing that code (and the JIT). - if (RuntimeHelpers.IsReferenceOrContainsReferences()) - { - finalContinuation.Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_RESULT_IN_GCDATA | CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION; - finalContinuation.GCData = new object[1]; - } - else + Continuation finalContinuation = new Continuation { - finalContinuation.Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION; - finalContinuation.Data = new byte[Unsafe.SizeOf()]; - } - + Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION, + }; continuation.Next = finalContinuation; ThunkTask result = new(); diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 9fa1f1f004d9b5..67c35f122ee600 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -739,6 +739,9 @@ internal unsafe struct MethodTable [FieldOffset(ElementTypeOffset)] public MethodTable*** PerInstInfo; + [FieldOffset(ElementTypeOffset)] + public CORINFO_CONTINUATION_DATA_OFFSETS* ContinuationOffsets; + /// /// This interface map used to list out the set of interfaces. Only meaningful if InterfaceCount is non-zero. /// diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index b0d0108a31dffe..e1a6f1f0c3d7c5 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1700,27 +1700,20 @@ struct CORINFO_EE_INFO enum CorInfoContinuationFlags { - // Whether or not the continuation expects the result to be boxed and - // placed in the GCData array at index 0. Not set if the callee is void. - CORINFO_CONTINUATION_RESULT_IN_GCDATA = 1, // If this bit is set the continuation resumes inside a try block and thus // if an exception is being propagated, needs to be resumed. The exception // should be placed at index 0 or 1 depending on whether the continuation // also expects a result. - CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2, - // If this bit is set the continuation has the IL offset that inspired the - // OSR method saved in the beginning of 'Data', or -1 if the continuation - // belongs to a tier 0 method. - CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA = 4, + CORINFO_CONTINUATION_NEEDS_EXCEPTION = 1, // If this bit is set the continuation should continue on the thread // pool. - CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 8, + CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 2, // If this bit is set the continuation has a SynchronizationContext // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 16, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 4, // If this bit is set the continuation has a TaskScheduler // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 32, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 8, }; struct CORINFO_ASYNC_INFO @@ -1735,10 +1728,6 @@ struct CORINFO_ASYNC_INFO CORINFO_FIELD_HANDLE continuationStateFldHnd; // 'Flags' field CORINFO_FIELD_HANDLE continuationFlagsFldHnd; - // 'Data' field - CORINFO_FIELD_HANDLE continuationDataFldHnd; - // 'GCData' field - CORINFO_FIELD_HANDLE continuationGCDataFldHnd; // Whether or not the continuation needs to be allocated through the // helper that also takes a method handle bool continuationsNeedMethodHandle; @@ -1751,6 +1740,15 @@ struct CORINFO_ASYNC_INFO CORINFO_METHOD_HANDLE restoreContextsMethHnd; }; +// Offsets for continuation data layout +struct CORINFO_CONTINUATION_DATA_OFFSETS +{ + uint32_t Result; + uint32_t Exception; + uint32_t ContinuationContext; + uint32_t KeepAlive; +}; + // Flags passed from JIT to runtime. enum CORINFO_GET_TAILCALL_HELPERS_FLAGS { @@ -1960,6 +1958,8 @@ struct CORINFO_FPSTRUCT_LOWERING #define OFFSETOF__CORINFO_Span__reference 0 #define OFFSETOF__CORINFO_Span__length TARGET_POINTER_SIZE +#define OFFSETOF__CORINFO_Continuation__data (SIZEOF__CORINFO_Object + TARGET_POINTER_SIZE /* Next */ + TARGET_POINTER_SIZE /* Resume */ + TARGET_POINTER_SIZE /* Flags + maybe padding */) + /* data to optimize delegate construction */ struct DelegateCtorArgs @@ -3342,6 +3342,12 @@ class ICorDynamicInfo : public ICorStaticInfo virtual CORINFO_METHOD_HANDLE getAsyncResumptionStub() = 0; + virtual CORINFO_CLASS_HANDLE getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets + ) = 0; + // Optionally, convert calli to regular method call. This is for PInvoke argument marshalling. virtual bool convertPInvokeCalliToCall( CORINFO_RESOLVED_TOKEN * pResolvedToken, diff --git a/src/coreclr/inc/icorjitinfoimpl_generated.h b/src/coreclr/inc/icorjitinfoimpl_generated.h index 6f4908d9c05235..ad8b89da83b8bf 100644 --- a/src/coreclr/inc/icorjitinfoimpl_generated.h +++ b/src/coreclr/inc/icorjitinfoimpl_generated.h @@ -746,6 +746,11 @@ uint32_t getJitFlags( CORINFO_METHOD_HANDLE getSpecialCopyHelper( CORINFO_CLASS_HANDLE type) override; +CORINFO_CLASS_HANDLE getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) override; + /**********************************************************************************/ // clang-format on /**********************************************************************************/ diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index e1fda2eeb3cbb0..678c0095670c49 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* 3d2bdd20-eced-4a07-b9fb-227ce7f55fcd */ - 0x3d2bdd20, - 0xeced, - 0x4a07, - {0xb9, 0xfb, 0x22, 0x7c, 0xe7, 0xf5, 0x5f, 0xcd} +constexpr GUID JITEEVersionIdentifier = { /* 9bcc23e4-25da-470d-954f-ae08907e3b09 */ + 0x9bcc23e4, + 0x25da, + 0x470d, + {0x95, 0x4f, 0xae, 0x08, 0x90, 0x7e, 0x3b, 0x09} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/jit/ICorJitInfo_names_generated.h b/src/coreclr/jit/ICorJitInfo_names_generated.h index 1e455eb98bc74e..d23e331501cb57 100644 --- a/src/coreclr/jit/ICorJitInfo_names_generated.h +++ b/src/coreclr/jit/ICorJitInfo_names_generated.h @@ -180,5 +180,6 @@ DEF_CLR_API(getRelocTypeHint) DEF_CLR_API(getExpectedTargetArchitecture) DEF_CLR_API(getJitFlags) DEF_CLR_API(getSpecialCopyHelper) +DEF_CLR_API(getContinuationType) #undef DEF_CLR_API diff --git a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp index 71dc8eaf7119af..7fcd7644f88422 100644 --- a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp +++ b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp @@ -1744,6 +1744,17 @@ CORINFO_METHOD_HANDLE WrapICorJitInfo::getSpecialCopyHelper( return temp; } +CORINFO_CLASS_HANDLE WrapICorJitInfo::getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + API_ENTER(getContinuationType); + CORINFO_CLASS_HANDLE temp = wrapHnd->getContinuationType(dataSize, objRefs, dataOffsets); + API_LEAVE(getContinuationType); + return temp; +} + /**********************************************************************************/ // clang-format on /**********************************************************************************/ diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index ec92f9b23ca6a7..0013992925d439 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -624,11 +624,12 @@ PhaseStatus AsyncTransformation::Run() // ahead of time. for (BasicBlock* block : m_comp->Blocks()) { + unsigned suspPointsInBlock = 0; for (GenTree* tree : LIR::AsRange(block)) { if (tree->IsCall() && tree->AsCall()->IsAsync() && !tree->AsCall()->IsTailCall()) { - JITDUMP(FMT_BB " contains await(s)\n", block->bbNum); + JITDUMP(FMT_BB " contains await(s) that will turn into suspension points\n", block->bbNum); worklist.Push(block); break; } @@ -962,38 +963,26 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* if (layout->IsCustomLayout()) { - inf.Alignment = 1; - inf.DataSize = layout->GetSize(); - inf.GCDataCount = layout->GetGCPtrCount(); + inf.Alignment = 1; + inf.Size = layout->GetSize(); } else { inf.Alignment = m_comp->info.compCompHnd->getClassAlignmentRequirement(layout->GetClassHandle()); - if ((layout->GetGCPtrCount() * TARGET_POINTER_SIZE) == layout->GetSize()) - { - inf.DataSize = 0; - } - else - { - inf.DataSize = layout->GetSize(); - } - - inf.GCDataCount = layout->GetGCPtrCount(); + inf.Size = layout->GetSize(); } } else if (dsc->TypeIs(TYP_REF)) { - inf.Alignment = TARGET_POINTER_SIZE; - inf.DataSize = 0; - inf.GCDataCount = 1; + inf.Alignment = TARGET_POINTER_SIZE; + inf.Size = TARGET_POINTER_SIZE; } else { assert(!dsc->TypeIs(TYP_BYREF)); - inf.Alignment = genTypeAlignments[dsc->TypeGet()]; - inf.DataSize = genTypeSize(dsc); - inf.GCDataCount = 0; + inf.Alignment = genTypeAlignments[dsc->TypeGet()]; + inf.Size = genTypeSize(dsc); } } @@ -1008,108 +997,117 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* return lhs.Alignment > rhs.Alignment; }); - // For OSR, we store the IL offset that inspired the OSR method at the - // beginning of the data (-1 in the tier0 version): - if (m_comp->doesMethodHavePatchpoints() || m_comp->opts.IsOSR()) - { - JITDUMP(" Method %s; keeping IL offset that inspired OSR method at the beginning of non-GC data\n", - m_comp->doesMethodHavePatchpoints() ? "has patchpoints" : "is an OSR method"); - layout.DataSize += sizeof(int); - } - if (call->gtReturnType == TYP_STRUCT) { layout.ReturnStructLayout = m_comp->typGetObjLayout(call->gtRetClsHnd); layout.ReturnSize = layout.ReturnStructLayout->GetSize(); - layout.ReturnInGCData = layout.ReturnStructLayout->HasGCPtr(); + layout.ReturnAlignment = m_comp->info.compCompHnd->getClassAlignmentRequirement(call->gtRetClsHnd); } else { - layout.ReturnSize = genTypeSize(call->gtReturnType); - layout.ReturnInGCData = varTypeIsGC(call->gtReturnType); + layout.ReturnSize = genTypeSize(call->gtReturnType); + layout.ReturnAlignment = layout.ReturnSize; } assert((layout.ReturnSize > 0) == (call->gtReturnType != TYP_VOID)); - // The return value is always stored: - // 1. At index 0 in GCData if it is a TYP_REF or a struct with GC references - // 2. At index 0 in Data, for non OSR methods without GC ref returns - // 3. At index 4 in Data for OSR methods without GC ref returns. The - // continuation flags indicates this scenario with a flag. - if (layout.ReturnInGCData) - { - layout.GCRefsCount++; - } - else if (layout.ReturnSize > 0) + auto allocLayout = [&layout](unsigned align, unsigned size) { + layout.Size = roundUp(layout.Size, align); + unsigned offset = layout.Size; + layout.Size += size; + return offset; + }; + + // For OSR, we store the IL offset that inspired the OSR method at the + // beginning of the data (and store -1 in the tier0 version). This must be + // at the beginning because the tier0 and OSR versions need to agree on + // this. + if (m_comp->doesMethodHavePatchpoints() || m_comp->opts.IsOSR()) { - layout.ReturnValDataOffset = layout.DataSize; - layout.DataSize += layout.ReturnSize; + JITDUMP(" Method %s; keeping IL offset that inspired OSR method at the beginning of non-GC data\n", + m_comp->doesMethodHavePatchpoints() ? "has patchpoints" : "is an OSR method"); + allocLayout(4, 4); } -#ifdef DEBUG if (layout.ReturnSize > 0) { - JITDUMP(" Will store return of type %s, size %u in", + layout.ReturnValOffset = allocLayout(1, layout.ReturnSize); + + JITDUMP(" Will store return of type %s, size %u at offset %u\n", call->gtReturnType == TYP_STRUCT ? layout.ReturnStructLayout->GetClassName() : varTypeName(call->gtReturnType), - layout.ReturnSize); - - if (layout.ReturnInGCData) - { - JITDUMP(" GC data\n"); - } - else - { - JITDUMP(" non-GC data at offset %u\n", layout.ReturnValDataOffset); - } + layout.ReturnSize, layout.ReturnValOffset); } -#endif if (block->hasTryIndex()) { - layout.ExceptionGCDataIndex = layout.GCRefsCount++; - JITDUMP(" " FMT_BB " is in try region %u; exception will be at GC@+%02u in GC data\n", block->bbNum, - block->getTryIndex(), layout.ExceptionGCDataIndex); + layout.ExceptionOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); + JITDUMP(" " FMT_BB " is in try region %u; exception will be at offset %u\n", block->bbNum, + block->getTryIndex(), layout.ExceptionOffset); } if (call->GetAsyncInfo().ContinuationContextHandling == ContinuationContextHandling::ContinueOnCapturedContext) { - layout.ContinuationContextGCDataIndex = layout.GCRefsCount++; - JITDUMP(" Continuation continues on captured context; context will be at GC@+%02u in GC data\n", - layout.ContinuationContextGCDataIndex); + layout.ContinuationContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); + JITDUMP(" Continuation continues on captured context; context will be at offset %u\n", + layout.ContinuationContextOffset); } if (call->GetAsyncInfo().ExecutionContextHandling == ExecutionContextHandling::AsyncSaveAndRestore) { - layout.ExecContextGCDataIndex = layout.GCRefsCount++; - JITDUMP( - " Call has async-only save and restore of ExecutionContext; ExecutionContext will be at GC@+%02u in GC data\n", - layout.ExecContextGCDataIndex); + layout.ExecContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); + JITDUMP(" Call has async-only save and restore of ExecutionContext; ExecutionContext will be at offset %u\n", + layout.ExecContextOffset); } for (LiveLocalInfo& inf : liveLocals) { - layout.DataSize = roundUp(layout.DataSize, inf.Alignment); - - inf.DataOffset = layout.DataSize; - inf.GCDataIndex = layout.GCRefsCount; - - layout.DataSize += inf.DataSize; - layout.GCRefsCount += inf.GCDataCount; + inf.Offset = allocLayout(inf.Alignment, inf.Size); } + layout.Size = roundUp(layout.Size, TARGET_POINTER_SIZE); + #ifdef DEBUG if (m_comp->verbose) { - printf(" Continuation layout (%u bytes, %u GC pointers):\n", layout.DataSize, layout.GCRefsCount); + printf(" Continuation layout (%u bytes):\n", layout.Size); for (LiveLocalInfo& inf : liveLocals) { - printf(" +%03u (GC@+%02u) V%02u: %u bytes, %u GC pointers\n", inf.DataOffset, inf.GCDataIndex, - inf.LclNum, inf.DataSize, inf.GCDataCount); + printf(" +%03u V%02u: %u bytes\n", inf.Offset, inf.LclNum, inf.Size); } } #endif + // Now create continuation type. First create bitmap of object refs. + bool* objRefs = new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE]; + for (LiveLocalInfo& inf : liveLocals) + { + LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); + if (dsc->TypeIs(TYP_REF)) + { + objRefs[inf.Offset / TARGET_POINTER_SIZE] = true; + } + else if (dsc->TypeIs(TYP_STRUCT)) + { + ClassLayout* layout = dsc->GetLayout(); + + for (unsigned slot = 0; slot < layout->GetSlotCount(); slot++) + { + if (layout->GetGCPtrType(slot) == TYP_REF) + { + objRefs[inf.Offset / TARGET_POINTER_SIZE + slot] = true; + } + } + } + } + + CORINFO_CONTINUATION_DATA_OFFSETS offsets; + offsets.Result = layout.ReturnValOffset; + offsets.Exception = layout.ExceptionOffset; + offsets.ContinuationContext = layout.ContinuationContextOffset; + offsets.KeepAlive = UINT_MAX; + layout.ClassHnd = m_comp->info.compCompHnd->getContinuationType(layout.Size, objRefs, offsets); + return layout; } @@ -1287,8 +1285,7 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Allocate continuation GenTree* returnedContinuation = m_comp->gtNewLclvNode(m_returnedContinuationVar, TYP_REF); - GenTreeCall* allocContinuation = - CreateAllocContinuationCall(life, returnedContinuation, layout.GCRefsCount, layout.DataSize); + GenTreeCall* allocContinuation = CreateAllocContinuationCall(life, returnedContinuation, layout.ClassHnd); m_comp->compCurBB = suspendBB; m_comp->fgMorphTree(allocContinuation); @@ -1315,12 +1312,8 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Fill in 'flags' const AsyncCallInfo& callInfo = call->GetAsyncInfo(); unsigned continuationFlags = 0; - if (layout.ReturnInGCData) - continuationFlags |= CORINFO_CONTINUATION_RESULT_IN_GCDATA; if (block->hasTryIndex()) continuationFlags |= CORINFO_CONTINUATION_NEEDS_EXCEPTION; - if (m_comp->doesMethodHavePatchpoints() || m_comp->opts.IsOSR()) - continuationFlags |= CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA; if (callInfo.ContinuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) continuationFlags |= CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL; @@ -1330,14 +1323,9 @@ BasicBlock* AsyncTransformation::CreateSuspension( GenTree* storeFlags = StoreAtOffset(newContinuation, flagsOffset, flagsNode, TYP_INT); LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storeFlags)); - if (layout.GCRefsCount > 0) - { - FillInGCPointersOnSuspension(call, layout, suspendBB); - } - - if (layout.DataSize > 0) + if (layout.Size > 0) { - FillInDataOnSuspension(layout.Locals, suspendBB); + FillInDataOnSuspension(call, layout, suspendBB); } if (suspendBB->KindIs(BBJ_RETURN)) @@ -1363,13 +1351,11 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Returns: // IR node representing the allocation. // -GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& life, - GenTree* prevContinuation, - unsigned gcRefsCount, - unsigned dataSize) +GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& life, + GenTree* prevContinuation, + CORINFO_CLASS_HANDLE contClassHnd) { - GenTree* gcRefsCountNode = m_comp->gtNewIconNode((ssize_t)gcRefsCount, TYP_I_IMPL); - GenTree* dataSizeNode = m_comp->gtNewIconNode((ssize_t)dataSize, TYP_I_IMPL); + GenTree* contClassHndNode = m_comp->gtNewIconEmbClsHndNode(contClassHnd); // If VM requests that we report the method handle, or if we have a shared generic context method handle // that is live here, then we need to call a different helper to keep the loader alive. GenTree* methodHandleArg = nullptr; @@ -1392,120 +1378,82 @@ GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& lif if (methodHandleArg != nullptr) { return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION_METHOD, TYP_REF, prevContinuation, - gcRefsCountNode, dataSizeNode, methodHandleArg); + contClassHndNode, methodHandleArg); } if (classHandleArg != nullptr) { return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION_CLASS, TYP_REF, prevContinuation, - gcRefsCountNode, dataSizeNode, classHandleArg); + contClassHndNode, classHandleArg); } - return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION, TYP_REF, prevContinuation, gcRefsCountNode, - dataSizeNode); + return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION, TYP_REF, prevContinuation, contClassHndNode); } //------------------------------------------------------------------------ -// AsyncTransformation::FillInGCPointersOnSuspension: -// Create IR that fills the GC pointers of the continuation object. -// This also nulls out the GC pointers in the locals if the local has data -// parts that need to be stored. +// AsyncTransformation::FillInDataOnSuspension: +// Create IR that fills the data array of the continuation object. // // Parameters: -// call - The async call that is being transformed -// layout - Layout information +// call - The async call +// layout - Information about the continuation layout // suspendBB - Basic block to add IR to. // -void AsyncTransformation::FillInGCPointersOnSuspension(GenTreeCall* call, - const ContinuationLayout& layout, - BasicBlock* suspendBB) +void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, + const ContinuationLayout& layout, + BasicBlock* suspendBB) { - unsigned objectArrLclNum = GetGCDataArrayVar(); + if (m_comp->doesMethodHavePatchpoints() || m_comp->opts.IsOSR()) + { + GenTree* ilOffsetToStore; + if (m_comp->doesMethodHavePatchpoints()) + ilOffsetToStore = m_comp->gtNewIconNode(-1); + else + ilOffsetToStore = m_comp->gtNewIconNode((int)m_comp->info.compILEntry); - GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF); - unsigned gcDataOffset = m_comp->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationGCDataFldHnd); - GenTree* gcDataInd = LoadFromOffset(newContinuation, gcDataOffset, TYP_REF); - GenTree* storeAllocedObjectArr = m_comp->gtNewStoreLclVarNode(objectArrLclNum, gcDataInd); - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storeAllocedObjectArr)); + GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF); + unsigned offset = OFFSETOF__CORINFO_Continuation__data; + GenTree* storePatchpointOffset = StoreAtOffset(newContinuation, offset, ilOffsetToStore, TYP_INT); + LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storePatchpointOffset)); + } + // Fill in data for (const LiveLocalInfo& inf : layout.Locals) { - if (inf.GCDataCount <= 0) - { - continue; - } + assert(inf.Size > 0); LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); - if (dsc->TypeIs(TYP_REF)) + + GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF); + unsigned offset = OFFSETOF__CORINFO_Continuation__data + inf.Offset; + + GenTree* value; + if (dsc->IsImplicitByRef()) { - GenTree* value = m_comp->gtNewLclvNode(inf.LclNum, TYP_REF); - GenTree* objectArr = m_comp->gtNewLclvNode(objectArrLclNum, TYP_REF); - GenTree* store = - StoreAtOffset(objectArr, OFFSETOF__CORINFO_Array__data + (inf.GCDataIndex * TARGET_POINTER_SIZE), value, - TYP_REF); - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); + GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet()); + value = m_comp->gtNewLoadValueNode(dsc->GetLayout(), baseAddr, GTF_IND_NONFAULTING); } else { - assert(dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()); - ClassLayout* layout = dsc->GetLayout(); - unsigned numSlots = layout->GetSlotCount(); - unsigned gcRefIndex = 0; - for (unsigned i = 0; i < numSlots; i++) - { - var_types gcPtrType = layout->GetGCPtrType(i); - assert((gcPtrType == TYP_I_IMPL) || (gcPtrType == TYP_REF)); - if (gcPtrType != TYP_REF) - { - continue; - } - - GenTree* value; - if (dsc->IsImplicitByRef()) - { - GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet()); - value = LoadFromOffset(baseAddr, i * TARGET_POINTER_SIZE, TYP_REF); - } - else - { - value = m_comp->gtNewLclFldNode(inf.LclNum, TYP_REF, i * TARGET_POINTER_SIZE); - } - - GenTree* objectArr = m_comp->gtNewLclvNode(objectArrLclNum, TYP_REF); - unsigned offset = - OFFSETOF__CORINFO_Array__data + ((inf.GCDataIndex + gcRefIndex) * TARGET_POINTER_SIZE); - GenTree* store = StoreAtOffset(objectArr, offset, value, TYP_REF); - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); - - gcRefIndex++; - - if (inf.DataSize > 0) - { - // Null out the GC field in preparation of storing the rest. - GenTree* null = m_comp->gtNewNull(); - - if (dsc->IsImplicitByRef()) - { - GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet()); - store = StoreAtOffset(baseAddr, i * TARGET_POINTER_SIZE, null, TYP_REF); - } - else - { - store = m_comp->gtNewStoreLclFldNode(inf.LclNum, TYP_REF, i * TARGET_POINTER_SIZE, null); - } - - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); - } - } + value = m_comp->gtNewLclVarNode(inf.LclNum); + } - if (!dsc->IsImplicitByRef()) - { - m_comp->lvaSetVarDoNotEnregister(inf.LclNum DEBUGARG(DoNotEnregisterReason::LocalField)); - } + GenTree* store; + if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) + { + GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); + GenTree* addr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, newContinuation, cns); + store = m_comp->gtNewStoreValueNode(dsc->GetLayout(), addr, value, GTF_IND_NONFAULTING); } + else + { + store = StoreAtOffset(newContinuation, offset, value, dsc->TypeGet()); + } + + LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); } - if (layout.ContinuationContextGCDataIndex != UINT_MAX) + if (layout.ContinuationContextOffset != UINT_MAX) { const AsyncCallInfo& callInfo = call->GetAsyncInfo(); assert(callInfo.SaveAndRestoreSynchronizationContextField); @@ -1515,7 +1463,7 @@ void AsyncTransformation::FillInGCPointersOnSuspension(GenTreeCall* // Insert call // AsyncHelpers.CaptureContinuationContext( // syncContextFromBeforeCall, - // ref newContinuation.GCData[ContinuationContextGCDataIndex], + // ref newContinuation.ContinuationContext, // ref newContinuation.Flags). GenTree* syncContextPlaceholder = m_comp->gtNewNull(); GenTree* contextElementPlaceholder = m_comp->gtNewZeroConNode(TYP_BYREF); @@ -1545,10 +1493,10 @@ void AsyncTransformation::FillInGCPointersOnSuspension(GenTreeCall* gotUse = LIR::AsRange(suspendBB).TryGetUse(contextElementPlaceholder, &use); assert(gotUse); - GenTree* objectArr = m_comp->gtNewLclvNode(objectArrLclNum, TYP_REF); - unsigned offset = OFFSETOF__CORINFO_Array__data + (layout.ContinuationContextGCDataIndex * TARGET_POINTER_SIZE); - GenTree* contextElementOffset = - m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, objectArr, m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL)); + GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF); + unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ContinuationContextOffset; + GenTree* contextElementOffset = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, newContinuation, + m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL)); LIR::AsRange(suspendBB).InsertBefore(contextElementPlaceholder, LIR::SeqTree(m_comp, contextElementOffset)); use.ReplaceWith(contextElementOffset); @@ -1568,7 +1516,7 @@ void AsyncTransformation::FillInGCPointersOnSuspension(GenTreeCall* LIR::AsRange(suspendBB).Remove(flagsPlaceholder); } - if (layout.ExecContextGCDataIndex != UINT_MAX) + if (layout.ExecContextOffset != UINT_MAX) { GenTreeCall* captureExecContext = m_comp->gtNewCallNode(CT_USER_FUNC, m_asyncInfo->captureExecutionContextMethHnd, TYP_REF); @@ -1576,86 +1524,9 @@ void AsyncTransformation::FillInGCPointersOnSuspension(GenTreeCall* m_comp->compCurBB = suspendBB; m_comp->fgMorphTree(captureExecContext); - GenTree* objectArr = m_comp->gtNewLclvNode(objectArrLclNum, TYP_REF); - GenTree* store = - StoreAtOffset(objectArr, - OFFSETOF__CORINFO_Array__data + (layout.ExecContextGCDataIndex * TARGET_POINTER_SIZE), - captureExecContext, TYP_REF); - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); - } -} - -//------------------------------------------------------------------------ -// AsyncTransformation::FillInDataOnSuspension: -// Create IR that fills the data array of the continuation object. -// -// Parameters: -// liveLocals - Information about each live local. -// suspendBB - Basic block to add IR to. -// -void AsyncTransformation::FillInDataOnSuspension(const jitstd::vector& liveLocals, BasicBlock* suspendBB) -{ - unsigned byteArrLclNum = GetDataArrayVar(); - - GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF); - unsigned dataOffset = m_comp->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationDataFldHnd); - GenTree* dataInd = LoadFromOffset(newContinuation, dataOffset, TYP_REF); - GenTree* storeAllocedByteArr = m_comp->gtNewStoreLclVarNode(byteArrLclNum, dataInd); - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storeAllocedByteArr)); - - if (m_comp->doesMethodHavePatchpoints() || m_comp->opts.IsOSR()) - { - GenTree* ilOffsetToStore; - if (m_comp->doesMethodHavePatchpoints()) - ilOffsetToStore = m_comp->gtNewIconNode(-1); - else - ilOffsetToStore = m_comp->gtNewIconNode((int)m_comp->info.compILEntry); - - GenTree* byteArr = m_comp->gtNewLclvNode(byteArrLclNum, TYP_REF); - unsigned offset = OFFSETOF__CORINFO_Array__data; - GenTree* storePatchpointOffset = StoreAtOffset(byteArr, offset, ilOffsetToStore, TYP_INT); - LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, storePatchpointOffset)); - } - - // Fill in data - for (const LiveLocalInfo& inf : liveLocals) - { - if (inf.DataSize <= 0) - { - continue; - } - - LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); - - GenTree* byteArr = m_comp->gtNewLclvNode(byteArrLclNum, TYP_REF); - unsigned offset = OFFSETOF__CORINFO_Array__data + inf.DataOffset; - - GenTree* value; - if (dsc->IsImplicitByRef()) - { - GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet()); - value = m_comp->gtNewLoadValueNode(dsc->GetLayout(), baseAddr, GTF_IND_NONFAULTING); - } - else - { - value = m_comp->gtNewLclVarNode(inf.LclNum); - } - - GenTree* store; - if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) - { - GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); - GenTree* addr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, byteArr, cns); - // This is to heap, but all GC refs are nulled out already, so we can skip the write barrier. - // TODO-CQ: Backend does not care about GTF_IND_TGT_NOT_HEAP for STORE_BLK. - store = - m_comp->gtNewStoreValueNode(dsc->GetLayout(), addr, value, GTF_IND_NONFAULTING | GTF_IND_TGT_NOT_HEAP); - } - else - { - store = StoreAtOffset(byteArr, offset, value, dsc->TypeGet()); - } - + GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF); + unsigned offset = OFFSETOF__CORINFO_Continuation__data + layout.ExecContextOffset; + GenTree* store = StoreAtOffset(newContinuation, offset, captureExecContext, TYP_REF); LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); } } @@ -1745,48 +1616,21 @@ BasicBlock* AsyncTransformation::CreateResumption(BasicBlock* bloc SetSuspendedIndicator(resumeBB, block, call); - // We need to restore data before we restore GC pointers, since restoring - // the data may also write the GC pointer fields with nulls. - unsigned resumeByteArrLclNum = BAD_VAR_NUM; - if (layout.DataSize > 0) + if (layout.Size > 0) { - resumeByteArrLclNum = GetDataArrayVar(); - - GenTree* newContinuation = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF); - unsigned dataOffset = m_comp->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationDataFldHnd); - GenTree* dataInd = LoadFromOffset(newContinuation, dataOffset, TYP_REF); - GenTree* storeAllocedByteArr = m_comp->gtNewStoreLclVarNode(resumeByteArrLclNum, dataInd); - - LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, storeAllocedByteArr)); - - RestoreFromDataOnResumption(resumeByteArrLclNum, layout.Locals, resumeBB); + RestoreFromDataOnResumption(layout, resumeBB); } - unsigned resumeObjectArrLclNum = BAD_VAR_NUM; - BasicBlock* storeResultBB = resumeBB; + BasicBlock* storeResultBB = resumeBB; - if (layout.GCRefsCount > 0) + if (layout.ExceptionOffset != UINT_MAX) { - resumeObjectArrLclNum = GetGCDataArrayVar(); - - GenTree* newContinuation = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF); - unsigned gcDataOffset = m_comp->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationGCDataFldHnd); - GenTree* gcDataInd = LoadFromOffset(newContinuation, gcDataOffset, TYP_REF); - GenTree* storeAllocedObjectArr = m_comp->gtNewStoreLclVarNode(resumeObjectArrLclNum, gcDataInd); - LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, storeAllocedObjectArr)); - - RestoreFromGCPointersOnResumption(resumeObjectArrLclNum, layout, resumeBB); - - if (layout.ExceptionGCDataIndex != UINT_MAX) - { - storeResultBB = RethrowExceptionOnResumption(block, resumeObjectArrLclNum, layout, resumeBB); - } + storeResultBB = RethrowExceptionOnResumption(block, layout, resumeBB); } if ((layout.ReturnSize > 0) && (callDefInfo.DefinitionNode != nullptr)) { - CopyReturnValueOnResumption(call, callDefInfo, resumeByteArrLclNum, resumeObjectArrLclNum, layout, - storeResultBB); + CopyReturnValueOnResumption(call, callDefInfo, layout, storeResultBB); } return resumeBB; @@ -1798,70 +1642,12 @@ BasicBlock* AsyncTransformation::CreateResumption(BasicBlock* bloc // object. // // Parameters: -// resumeByteArrLclNum - Local that has the continuation object's data array -// liveLocals - Information about each live local. -// resumeBB - Basic block to append IR to -// -void AsyncTransformation::RestoreFromDataOnResumption(unsigned resumeByteArrLclNum, - const jitstd::vector& liveLocals, - BasicBlock* resumeBB) -{ - // Copy data - for (const LiveLocalInfo& inf : liveLocals) - { - if (inf.DataSize <= 0) - { - continue; - } - - LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); - - GenTree* byteArr = m_comp->gtNewLclvNode(resumeByteArrLclNum, TYP_REF); - unsigned offset = OFFSETOF__CORINFO_Array__data + inf.DataOffset; - GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); - GenTree* addr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, byteArr, cns); - - GenTree* value; - if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) - { - value = m_comp->gtNewLoadValueNode(dsc->GetLayout(), addr, GTF_IND_NONFAULTING); - } - else - { - value = m_comp->gtNewIndir(dsc->TypeGet(), addr, GTF_IND_NONFAULTING); - } - - GenTree* store; - if (dsc->IsImplicitByRef()) - { - GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet()); - store = m_comp->gtNewStoreValueNode(dsc->GetLayout(), baseAddr, value, - GTF_IND_NONFAULTING | GTF_IND_TGT_NOT_HEAP); - } - else - { - store = m_comp->gtNewStoreLclVarNode(inf.LclNum, value); - } - - LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); - } -} - -//------------------------------------------------------------------------ -// AsyncTransformation::RestoreFromGCPointersOnResumption: -// Create IR that restores locals from the GC pointers array of the -// continuation object. -// -// Parameters: -// resumeObjectArrLclNum - Local that has the continuation object's GC pointers array -// liveLocals - Information about each live local. -// resumeBB - Basic block to append IR to +// layout - Information about the continuation layout +// resumeBB - Basic block to append IR to // -void AsyncTransformation::RestoreFromGCPointersOnResumption(unsigned resumeObjectArrLclNum, - const ContinuationLayout& layout, - BasicBlock* resumeBB) +void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& layout, BasicBlock* resumeBB) { - if (layout.ExecContextGCDataIndex != BAD_VAR_NUM) + if (layout.ExecContextOffset != BAD_VAR_NUM) { GenTree* valuePlaceholder = m_comp->gtNewZeroConNode(TYP_REF); GenTreeCall* restoreCall = @@ -1877,10 +1663,9 @@ void AsyncTransformation::RestoreFromGCPointersOnResumption(unsigned bool gotUse = LIR::AsRange(resumeBB).TryGetUse(valuePlaceholder, &valueUse); assert(gotUse); - GenTree* objectArr = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF); - unsigned execContextOffset = - OFFSETOF__CORINFO_Array__data + (layout.ExecContextGCDataIndex * TARGET_POINTER_SIZE); - GenTree* execContextValue = LoadFromOffset(objectArr, execContextOffset, TYP_REF); + GenTree* continuation = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF); + unsigned execContextOffset = OFFSETOF__CORINFO_Continuation__data + layout.ExecContextOffset; + GenTree* execContextValue = LoadFromOffset(continuation, execContextOffset, TYP_REF); LIR::AsRange(resumeBB).InsertBefore(valuePlaceholder, LIR::SeqTree(m_comp, execContextValue)); valueUse.ReplaceWith(execContextValue); @@ -1888,60 +1673,39 @@ void AsyncTransformation::RestoreFromGCPointersOnResumption(unsigned LIR::AsRange(resumeBB).Remove(valuePlaceholder); } + // Copy data for (const LiveLocalInfo& inf : layout.Locals) { - if (inf.GCDataCount <= 0) + LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); + + GenTree* continuation = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF); + unsigned offset = OFFSETOF__CORINFO_Continuation__data + inf.Offset; + GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); + GenTree* addr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, continuation, cns); + + GenTree* value; + if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) { - continue; + value = m_comp->gtNewLoadValueNode(dsc->GetLayout(), addr, GTF_IND_NONFAULTING); } - - LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); - if (dsc->TypeIs(TYP_REF)) + else { - GenTree* objectArr = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF); - unsigned offset = OFFSETOF__CORINFO_Array__data + (inf.GCDataIndex * TARGET_POINTER_SIZE); - GenTree* value = LoadFromOffset(objectArr, offset, TYP_REF); - GenTree* store = m_comp->gtNewStoreLclVarNode(inf.LclNum, value); + value = m_comp->gtNewIndir(dsc->TypeGet(), addr, GTF_IND_NONFAULTING); + } - LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); + GenTree* store; + if (dsc->IsImplicitByRef()) + { + GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet()); + store = m_comp->gtNewStoreValueNode(dsc->GetLayout(), baseAddr, value, + GTF_IND_NONFAULTING | GTF_IND_TGT_NOT_HEAP); } else { - assert(dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()); - ClassLayout* layout = dsc->GetLayout(); - unsigned numSlots = layout->GetSlotCount(); - unsigned gcRefIndex = 0; - for (unsigned i = 0; i < numSlots; i++) - { - var_types gcPtrType = layout->GetGCPtrType(i); - assert((gcPtrType == TYP_I_IMPL) || (gcPtrType == TYP_REF)); - if (gcPtrType != TYP_REF) - { - continue; - } - - GenTree* objectArr = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF); - unsigned offset = - OFFSETOF__CORINFO_Array__data + ((inf.GCDataIndex + gcRefIndex) * TARGET_POINTER_SIZE); - GenTree* value = LoadFromOffset(objectArr, offset, TYP_REF); - GenTree* store; - if (dsc->IsImplicitByRef()) - { - GenTree* baseAddr = m_comp->gtNewLclvNode(inf.LclNum, dsc->TypeGet()); - store = StoreAtOffset(baseAddr, i * TARGET_POINTER_SIZE, value, TYP_REF); - // Implicit byref args are never on heap - store->gtFlags |= GTF_IND_TGT_NOT_HEAP; - } - else - { - store = m_comp->gtNewStoreLclFldNode(inf.LclNum, TYP_REF, i * TARGET_POINTER_SIZE, value); - } - - LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); - - gcRefIndex++; - } + store = m_comp->gtNewStoreLclVarNode(inf.LclNum, value); } + + LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); } } @@ -1952,7 +1716,6 @@ void AsyncTransformation::RestoreFromGCPointersOnResumption(unsigned // // Parameters: // block - The block containing the async call -// resumeObjectArrLclNum - Local that has the continuation object's GC pointers array // layout - Layout information for the continuation object // resumeBB - Basic block to append IR to // @@ -1962,7 +1725,6 @@ void AsyncTransformation::RestoreFromGCPointersOnResumption(unsigned // rethrow. // BasicBlock* AsyncTransformation::RethrowExceptionOnResumption(BasicBlock* block, - unsigned resumeObjectArrLclNum, const ContinuationLayout& layout, BasicBlock* resumeBB) { @@ -1996,9 +1758,9 @@ BasicBlock* AsyncTransformation::RethrowExceptionOnResumption(BasicBlock* // Check if we have an exception. unsigned exceptionLclNum = GetExceptionVar(); - GenTree* objectArr = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF); - unsigned exceptionOffset = OFFSETOF__CORINFO_Array__data + layout.ExceptionGCDataIndex * TARGET_POINTER_SIZE; - GenTree* exceptionInd = LoadFromOffset(objectArr, exceptionOffset, TYP_REF); + GenTree* continuation = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF); + unsigned exceptionOffset = OFFSETOF__CORINFO_Continuation__data + layout.ExceptionOffset; + GenTree* exceptionInd = LoadFromOffset(continuation, exceptionOffset, TYP_REF); GenTree* storeException = m_comp->gtNewStoreLclVarNode(exceptionLclNum, exceptionInd); LIR::AsRange(resumeBB).InsertAtEnd(LIR::SeqTree(m_comp, storeException)); @@ -2032,46 +1794,16 @@ BasicBlock* AsyncTransformation::RethrowExceptionOnResumption(BasicBlock* // call - The async call // callDefInfo - Information about the async call's definition // block - The block containing the async call -// resumeByteArrLclNum - Local that has the continuation object's data array -// resumeObjectArrLclNum - Local that has the continuation object's GC pointers array // layout - Layout information for the continuation object // storeResultBB - Basic block to append IR to // void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* call, const CallDefinitionInfo& callDefInfo, - unsigned resumeByteArrLclNum, - unsigned resumeObjectArrLclNum, const ContinuationLayout& layout, BasicBlock* storeResultBB) { - GenTree* resultBase; - unsigned resultOffset; - GenTreeFlags resultIndirFlags = GTF_IND_NONFAULTING; - if (layout.ReturnInGCData) - { - assert(resumeObjectArrLclNum != BAD_VAR_NUM); - resultBase = m_comp->gtNewLclvNode(resumeObjectArrLclNum, TYP_REF); - - if (call->gtReturnType == TYP_STRUCT) - { - // Boxed struct. - resultBase = LoadFromOffset(resultBase, OFFSETOF__CORINFO_Array__data, TYP_REF); - resultOffset = TARGET_POINTER_SIZE; // Offset of data inside box - } - else - { - assert(call->gtReturnType == TYP_REF); - resultOffset = OFFSETOF__CORINFO_Array__data; - } - } - else - { - assert(resumeByteArrLclNum != BAD_VAR_NUM); - resultBase = m_comp->gtNewLclvNode(resumeByteArrLclNum, TYP_REF); - resultOffset = OFFSETOF__CORINFO_Array__data + layout.ReturnValDataOffset; - if (layout.ReturnValDataOffset != 0) - resultIndirFlags = GTF_IND_UNALIGNED; - } + GenTree* resultBase = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF); + unsigned resultOffset = OFFSETOF__CORINFO_Continuation__data + layout.ReturnValOffset; assert(callDefInfo.DefinitionNode != nullptr); LclVarDsc* resultLcl = m_comp->lvaGetDesc(callDefInfo.DefinitionNode); @@ -2083,7 +1815,8 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* { GenTree* resultOffsetNode = m_comp->gtNewIconNode((ssize_t)resultOffset, TYP_I_IMPL); GenTree* resultAddr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, resultBase, resultOffsetNode); - GenTree* resultData = m_comp->gtNewLoadValueNode(layout.ReturnStructLayout, resultAddr, resultIndirFlags); + GenTree* resultData = + m_comp->gtNewLoadValueNode(layout.ReturnStructLayout, resultAddr, GTF_IND_NONFAULTING); GenTree* storeResult; if ((callDefInfo.DefinitionNode->GetLclOffs() == 0) && ClassLayout::AreCompatible(resultLcl->GetLayout(), layout.ReturnStructLayout)) @@ -2122,7 +1855,7 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* LclVarDsc* fieldDsc = m_comp->lvaGetDesc(fieldLclNum); unsigned fldOffset = resultOffset + fieldDsc->lvFldOffset; - GenTree* value = LoadFromOffset(resultBase, fldOffset, fieldDsc->TypeGet(), resultIndirFlags); + GenTree* value = LoadFromOffset(resultBase, fldOffset, fieldDsc->TypeGet(), GTF_IND_NONFAULTING); GenTree* store = m_comp->gtNewStoreLclVarNode(fieldLclNum, value); LIR::AsRange(storeResultBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); @@ -2135,7 +1868,7 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* } else { - GenTree* value = LoadFromOffset(resultBase, resultOffset, call->gtReturnType, resultIndirFlags); + GenTree* value = LoadFromOffset(resultBase, resultOffset, call->gtReturnType, GTF_IND_NONFAULTING); GenTree* storeResult; if (callDefInfo.DefinitionNode->OperIs(GT_STORE_LCL_VAR)) @@ -2202,49 +1935,6 @@ GenTreeStoreInd* AsyncTransformation::StoreAtOffset(GenTree* base, unsigned offs return store; } -//------------------------------------------------------------------------ -// AsyncTransformation::GetDataArrayVar: -// Create a new local to hold the data array of the continuation object. This -// local can be validly used for the entire suspension point; the returned -// local may be used by multiple suspension points. -// -// Returns: -// Local number. -// -unsigned AsyncTransformation::GetDataArrayVar() -{ - // Create separate locals unless we have many locals in the method for live - // range splitting purposes. This helps LSRA to avoid create additional - // callee saves that harm the prolog/epilog. - if ((m_dataArrayVar == BAD_VAR_NUM) || !m_comp->lvaHaveManyLocals()) - { - m_dataArrayVar = m_comp->lvaGrabTemp(false DEBUGARG("byte[] for continuation")); - m_comp->lvaGetDesc(m_dataArrayVar)->lvType = TYP_REF; - } - - return m_dataArrayVar; -} - -//------------------------------------------------------------------------ -// AsyncTransformation::GetGCDataArrayVar: -// Create a new local to hold the GC pointers array of the continuation -// object. This local can be validly used for the entire suspension point; -// the returned local may be used by multiple suspension points. -// -// Returns: -// Local number. -// -unsigned AsyncTransformation::GetGCDataArrayVar() -{ - if ((m_gcDataArrayVar == BAD_VAR_NUM) || !m_comp->lvaHaveManyLocals()) - { - m_gcDataArrayVar = m_comp->lvaGrabTemp(false DEBUGARG("object[] for continuation")); - m_comp->lvaGetDesc(m_gcDataArrayVar)->lvType = TYP_REF; - } - - return m_gcDataArrayVar; -} - //------------------------------------------------------------------------ // AsyncTransformation::GetResultBaseVar: // Create a new local to hold the base address of the incoming result from @@ -2477,10 +2167,8 @@ void AsyncTransformation::CreateResumptionSwitch() // We need to dispatch to the OSR version if the IL offset is non-negative. continuationArg = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF); - unsigned offsetOfData = m_comp->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationDataFldHnd); - GenTree* dataArr = LoadFromOffset(continuationArg, offsetOfData, TYP_REF); - unsigned offsetOfIlOffset = OFFSETOF__CORINFO_Array__data; - GenTree* ilOffset = LoadFromOffset(dataArr, offsetOfIlOffset, TYP_INT); + unsigned offsetOfIlOffset = OFFSETOF__CORINFO_Continuation__data; + GenTree* ilOffset = LoadFromOffset(continuationArg, offsetOfIlOffset, TYP_INT); unsigned ilOffsetLclNum = m_comp->lvaGrabTemp(false DEBUGARG("IL offset for tier0 OSR method")); m_comp->lvaGetDesc(ilOffsetLclNum)->lvType = TYP_INT; GenTree* storeIlOffset = m_comp->gtNewStoreLclVarNode(ilOffsetLclNum, ilOffset); @@ -2530,10 +2218,8 @@ void AsyncTransformation::CreateResumptionSwitch() JITDUMP(" Created " FMT_BB " to check for Tier-0 continuations\n", checkILOffsetBB->bbNum); continuationArg = m_comp->gtNewLclvNode(m_comp->lvaAsyncContinuationArg, TYP_REF); - unsigned offsetOfData = m_comp->info.compCompHnd->getFieldOffset(m_asyncInfo->continuationDataFldHnd); - GenTree* dataArr = LoadFromOffset(continuationArg, offsetOfData, TYP_REF); - unsigned offsetOfIlOffset = OFFSETOF__CORINFO_Array__data; - GenTree* ilOffset = LoadFromOffset(dataArr, offsetOfIlOffset, TYP_INT); + unsigned offsetOfIlOffset = OFFSETOF__CORINFO_Continuation__data; + GenTree* ilOffset = LoadFromOffset(continuationArg, offsetOfIlOffset, TYP_INT); GenTree* zero = m_comp->gtNewIconNode(0); GenTree* ltZero = m_comp->gtNewOperNode(GT_LT, TYP_INT, ilOffset, zero); GenTree* jtrue = m_comp->gtNewOperNode(GT_JTRUE, TYP_VOID, ltZero); diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index e30aaf760e6395..c70bbc5199fef3 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -5,10 +5,8 @@ struct LiveLocalInfo { unsigned LclNum; unsigned Alignment; - unsigned DataOffset; - unsigned DataSize; - unsigned GCDataIndex; - unsigned GCDataCount; + unsigned Offset; + unsigned Size; explicit LiveLocalInfo(unsigned lclNum) : LclNum(lclNum) @@ -18,16 +16,16 @@ struct LiveLocalInfo struct ContinuationLayout { - unsigned DataSize = 0; - unsigned GCRefsCount = 0; - ClassLayout* ReturnStructLayout = nullptr; - unsigned ReturnSize = 0; - bool ReturnInGCData = false; - unsigned ReturnValDataOffset = UINT_MAX; - unsigned ExceptionGCDataIndex = UINT_MAX; - unsigned ExecContextGCDataIndex = UINT_MAX; - unsigned ContinuationContextGCDataIndex = UINT_MAX; + unsigned Size = 0; + ClassLayout* ReturnStructLayout = nullptr; + unsigned ReturnAlignment = 0; + unsigned ReturnSize = 0; + unsigned ReturnValOffset = UINT_MAX; + unsigned ExceptionOffset = UINT_MAX; + unsigned ExecContextOffset = UINT_MAX; + unsigned ContinuationContextOffset = UINT_MAX; const jitstd::vector& Locals; + CORINFO_CLASS_HANDLE ClassHnd = NO_CLASS_HANDLE; explicit ContinuationLayout(const jitstd::vector& locals) : Locals(locals) @@ -90,39 +88,27 @@ class AsyncTransformation BasicBlock* CreateSuspension( BasicBlock* block, GenTreeCall* call, unsigned stateNum, AsyncLiveness& life, const ContinuationLayout& layout); - GenTreeCall* CreateAllocContinuationCall(AsyncLiveness& life, - GenTree* prevContinuation, - unsigned gcRefsCount, - unsigned int dataSize); - void FillInGCPointersOnSuspension(GenTreeCall* call, const ContinuationLayout& layout, BasicBlock* suspendBB); - void FillInDataOnSuspension(const jitstd::vector& liveLocals, BasicBlock* suspendBB); - void CreateCheckAndSuspendAfterCall(BasicBlock* block, - GenTreeCall* call, - const CallDefinitionInfo& callDefInfo, - AsyncLiveness& life, - BasicBlock* suspendBB, - BasicBlock** remainder); - BasicBlock* CreateResumption(BasicBlock* block, - BasicBlock* remainder, - GenTreeCall* call, - const CallDefinitionInfo& callDefInfo, - unsigned stateNum, - const ContinuationLayout& layout); - void SetSuspendedIndicator(BasicBlock* block, BasicBlock* callBlock, GenTreeCall* call); - void RestoreFromDataOnResumption(unsigned resumeByteArrLclNum, - const jitstd::vector& liveLocals, - BasicBlock* resumeBB); - void RestoreFromGCPointersOnResumption(unsigned resumeObjectArrLclNum, - const ContinuationLayout& layout, - BasicBlock* resumeBB); - BasicBlock* RethrowExceptionOnResumption(BasicBlock* block, - unsigned resumeObjectArrLclNum, - const ContinuationLayout& layout, - BasicBlock* resumeBB); + GenTreeCall* CreateAllocContinuationCall(AsyncLiveness& life, + GenTree* prevContinuation, + CORINFO_CLASS_HANDLE contClassHnd); + void FillInDataOnSuspension(GenTreeCall* call, const ContinuationLayout& layout, BasicBlock* suspendBB); + void CreateCheckAndSuspendAfterCall(BasicBlock* block, + GenTreeCall* call, + const CallDefinitionInfo& callDefInfo, + AsyncLiveness& life, + BasicBlock* suspendBB, + BasicBlock** remainder); + BasicBlock* CreateResumption(BasicBlock* block, + BasicBlock* remainder, + GenTreeCall* call, + const CallDefinitionInfo& callDefInfo, + unsigned stateNum, + const ContinuationLayout& layout); + void SetSuspendedIndicator(BasicBlock* block, BasicBlock* callBlock, GenTreeCall* call); + void RestoreFromDataOnResumption(const ContinuationLayout& layout, BasicBlock* resumeBB); + BasicBlock* RethrowExceptionOnResumption(BasicBlock* block, const ContinuationLayout& layout, BasicBlock* resumeBB); void CopyReturnValueOnResumption(GenTreeCall* call, const CallDefinitionInfo& callDefInfo, - unsigned resumeByteArrLclNum, - unsigned resumeObjectArrLclNum, const ContinuationLayout& layout, BasicBlock* storeResultBB); @@ -132,8 +118,6 @@ class AsyncTransformation GenTreeFlags indirFlags = GTF_IND_NONFAULTING); GenTreeStoreInd* StoreAtOffset(GenTree* base, unsigned offset, GenTree* value, var_types storeType); - unsigned GetDataArrayVar(); - unsigned GetGCDataArrayVar(); unsigned GetResultBaseVar(); unsigned GetExceptionVar(); diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index c0bd2516d730bf..ebb475785662c2 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -3361,6 +3361,13 @@ private void getAsyncInfo(ref CORINFO_ASYNC_INFO pAsyncInfoOut) throw new NotImplementedException(); } + private CORINFO_CLASS_STRUCT_* getContinuationType(UIntPtr dataSize, ref bool objRefs, ref CORINFO_CONTINUATION_DATA_OFFSETS dataOffsets) + { + // Hint for the developer: Use CorInfoImpl.RyuJit.cs and CorInfoImpl.ReadyToRun.cs if the implementation + // is not shared for NativeAOT and R2R. + throw new NotImplementedException("getContinuationType"); + } + private mdToken getMethodDefFromMethod(CORINFO_METHOD_STRUCT_* hMethod) { MethodDesc method = HandleToObject(hMethod); diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs index 8d0c83046277dd..68709418ba995e 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs @@ -2603,10 +2603,25 @@ private static uint _getJitFlags(IntPtr thisHandle, IntPtr* ppException, CORJIT_ } } + [UnmanagedCallersOnly] + private static CORINFO_CLASS_STRUCT_* _getContinuationType(IntPtr thisHandle, IntPtr* ppException, UIntPtr dataSize, bool* objRefs, CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) + { + var _this = GetThis(thisHandle); + try + { + return _this.getContinuationType(dataSize, ref *objRefs, ref *dataOffsets); + } + catch (Exception ex) + { + *ppException = _this.AllocException(ex); + return default; + } + } + private static IntPtr GetUnmanagedCallbacks() { - void** callbacks = (void**)Marshal.AllocCoTaskMem(sizeof(IntPtr) * 176); + void** callbacks = (void**)Marshal.AllocCoTaskMem(sizeof(IntPtr) * 177); callbacks[0] = (delegate* unmanaged)&_isIntrinsic; callbacks[1] = (delegate* unmanaged)&_notifyMethodInfoUsage; @@ -2784,6 +2799,7 @@ private static IntPtr GetUnmanagedCallbacks() callbacks[173] = (delegate* unmanaged)&_getExpectedTargetArchitecture; callbacks[174] = (delegate* unmanaged)&_getJitFlags; callbacks[175] = (delegate* unmanaged)&_getSpecialCopyHelper; + callbacks[176] = (delegate* unmanaged)&_getContinuationType; return (IntPtr)callbacks; } diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs index 2ae78da1fde8f0..fbe8020cd28579 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs @@ -884,6 +884,12 @@ public unsafe struct CORINFO_ASYNC_INFO public CORINFO_FIELD_STRUCT_* continuationGCDataFldHnd; } + public struct CORINFO_CONTINUATION_DATA_OFFSETS + { + public int ReturnValueOffset; + public int ExceptionOffset; + } + // Flags passed from JIT to runtime. public enum CORINFO_GET_TAILCALL_HELPERS_FLAGS { diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt index 4c4945475399c1..60eb7c40f5c81e 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt @@ -84,6 +84,7 @@ CORINFO_RESOLVED_TOKEN*,ref CORINFO_RESOLVED_TOKEN CORINFO_RESOLVED_TOKEN_PTR,CORINFO_RESOLVED_TOKEN*,CORINFO_RESOLVED_TOKEN*,CORINFO_RESOLVED_TOKEN* CORINFO_EE_INFO*,ref CORINFO_EE_INFO CORINFO_ASYNC_INFO*,ref CORINFO_ASYNC_INFO +const CORINFO_CONTINUATION_DATA_OFFSETS&,ref CORINFO_CONTINUATION_DATA_OFFSETS CORINFO_TAILCALL_HELPERS*,ref CORINFO_TAILCALL_HELPERS CORINFO_SWIFT_LOWERING*,ref CORINFO_SWIFT_LOWERING CORINFO_FPSTRUCT_LOWERING*,ref CORINFO_FPSTRUCT_LOWERING @@ -321,6 +322,7 @@ FUNCTIONS CORINFO_METHOD_HANDLE GetDelegateCtor(CORINFO_METHOD_HANDLE methHnd, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE targetMethodHnd, DelegateCtorArgs * pCtorData); void MethodCompileComplete(CORINFO_METHOD_HANDLE methHnd); bool getTailCallHelpers(CORINFO_RESOLVED_TOKEN* callToken, CORINFO_SIG_INFO* sig, CORINFO_GET_TAILCALL_HELPERS_FLAGS flags, CORINFO_TAILCALL_HELPERS* pResult); + CORINFO_CLASS_HANDLE getContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets); CORINFO_METHOD_HANDLE getAsyncResumptionStub(); bool convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN * pResolvedToken, bool mustConvert); bool notifyInstructionSetUsage(CORINFO_InstructionSet instructionSet,bool supportEnabled); diff --git a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h index 4069f29308dadd..f0066d95177d70 100644 --- a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h +++ b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h @@ -187,6 +187,7 @@ struct JitInterfaceCallbacks uint32_t (* getExpectedTargetArchitecture)(void * thisHandle, CorInfoExceptionClass** ppException); uint32_t (* getJitFlags)(void * thisHandle, CorInfoExceptionClass** ppException, CORJIT_FLAGS* flags, uint32_t sizeInBytes); CORINFO_METHOD_HANDLE (* getSpecialCopyHelper)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE type); + CORINFO_CLASS_HANDLE (* getContinuationType)(void * thisHandle, CorInfoExceptionClass** ppException, size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets); }; @@ -1921,4 +1922,15 @@ class JitInterfaceWrapper : public ICorJitInfo if (pException != nullptr) throw pException; return temp; } + + virtual CORINFO_CLASS_HANDLE getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + CorInfoExceptionClass* pException = nullptr; + CORINFO_CLASS_HANDLE temp = _callbacks->getContinuationType(_thisHandle, &pException, dataSize, objRefs, dataOffsets); + if (pException != nullptr) throw pException; + return temp; +} }; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h index 01ec01427fa862..9c93cf976ad90b 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h @@ -200,8 +200,6 @@ struct Agnostic_CORINFO_ASYNC_INFO DWORDLONG continuationResumeFldHnd; DWORDLONG continuationStateFldHnd; DWORDLONG continuationFlagsFldHnd; - DWORDLONG continuationDataFldHnd; - DWORDLONG continuationGCDataFldHnd; DWORD continuationsNeedMethodHandle; DWORDLONG captureExecutionContextMethHnd; DWORDLONG restoreExecutionContextMethHnd; @@ -662,6 +660,15 @@ struct Agnostic_GetFpStructLowering DWORD numLoweredElements; }; +struct Agnostic_GetContinuationTypeIn +{ + DWORDLONG dataSize; + DWORD objRefs; + DWORD result; + DWORD exception; + DWORD continuationContext; +}; + struct Agnostic_ResolveVirtualMethodKey { DWORDLONG virtualMethod; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h b/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h index cc0878ee3d7d66..a52578a288bc15 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/lwmlist.h @@ -130,6 +130,7 @@ LWM(GetSwiftLowering, DWORDLONG, Agnostic_GetSwiftLowering) LWM(GetFpStructLowering, DWORDLONG, Agnostic_GetFpStructLowering) LWM(GetTailCallHelpers, Agnostic_GetTailCallHelpers, Agnostic_CORINFO_TAILCALL_HELPERS) LWM(GetAsyncResumptionStub, DWORD, DWORDLONG) +LWM(GetContinuationType, Agnostic_GetContinuationTypeIn, DWORDLONG) LWM(UpdateEntryPointForTailCall, Agnostic_CORINFO_CONST_LOOKUP, Agnostic_CORINFO_CONST_LOOKUP) LWM(GetSpecialCopyHelper, DWORDLONG, DWORDLONG) LWM(GetThreadTLSIndex, DWORD, DLD) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index c2c5e7e0d5eae2..28a9c0f04b82ee 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -4457,8 +4457,6 @@ void MethodContext::recGetAsyncInfo(const CORINFO_ASYNC_INFO* pAsyncInfo) value.continuationResumeFldHnd = CastHandle(pAsyncInfo->continuationResumeFldHnd); value.continuationStateFldHnd = CastHandle(pAsyncInfo->continuationStateFldHnd); value.continuationFlagsFldHnd = CastHandle(pAsyncInfo->continuationFlagsFldHnd); - value.continuationDataFldHnd = CastHandle(pAsyncInfo->continuationDataFldHnd); - value.continuationGCDataFldHnd = CastHandle(pAsyncInfo->continuationGCDataFldHnd); value.continuationsNeedMethodHandle = pAsyncInfo->continuationsNeedMethodHandle ? 1 : 0; value.captureExecutionContextMethHnd = CastHandle(pAsyncInfo->captureExecutionContextMethHnd); value.restoreExecutionContextMethHnd = CastHandle(pAsyncInfo->restoreExecutionContextMethHnd); @@ -4472,10 +4470,9 @@ void MethodContext::recGetAsyncInfo(const CORINFO_ASYNC_INFO* pAsyncInfo) void MethodContext::dmpGetAsyncInfo(DWORD key, const Agnostic_CORINFO_ASYNC_INFO& value) { printf("GetAsyncInfo key %u value contClsHnd-%016" PRIX64 " contNextFldHnd-%016" PRIX64 " contResumeFldHnd-%016" PRIX64 - " contStateFldHnd-%016" PRIX64 " contFlagsFldHnd-%016" PRIX64 " contDataFldHnd-%016" PRIX64 " contGCDataFldHnd-%016" PRIX64 " contsNeedMethodHandle-%d", + " contStateFldHnd-%016" PRIX64 " contFlagsFldHnd-%016" PRIX64 " contsNeedMethodHandle-%d", key, value.continuationClsHnd, value.continuationNextFldHnd, value.continuationResumeFldHnd, - value.continuationStateFldHnd, value.continuationFlagsFldHnd, value.continuationDataFldHnd, - value.continuationGCDataFldHnd, value.continuationsNeedMethodHandle); + value.continuationStateFldHnd, value.continuationFlagsFldHnd, value.continuationsNeedMethodHandle); } void MethodContext::repGetAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut) { @@ -4485,8 +4482,6 @@ void MethodContext::repGetAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut) pAsyncInfoOut->continuationResumeFldHnd = (CORINFO_FIELD_HANDLE)value.continuationResumeFldHnd; pAsyncInfoOut->continuationStateFldHnd = (CORINFO_FIELD_HANDLE)value.continuationStateFldHnd; pAsyncInfoOut->continuationFlagsFldHnd = (CORINFO_FIELD_HANDLE)value.continuationFlagsFldHnd; - pAsyncInfoOut->continuationDataFldHnd = (CORINFO_FIELD_HANDLE)value.continuationDataFldHnd; - pAsyncInfoOut->continuationGCDataFldHnd = (CORINFO_FIELD_HANDLE)value.continuationGCDataFldHnd; pAsyncInfoOut->continuationsNeedMethodHandle = value.continuationsNeedMethodHandle != 0; pAsyncInfoOut->captureExecutionContextMethHnd = (CORINFO_METHOD_HANDLE)value.captureExecutionContextMethHnd; pAsyncInfoOut->restoreExecutionContextMethHnd = (CORINFO_METHOD_HANDLE)value.restoreExecutionContextMethHnd; @@ -6946,6 +6941,56 @@ CORINFO_METHOD_HANDLE MethodContext::repGetAsyncResumptionStub() return (CORINFO_METHOD_HANDLE)hnd; } +void MethodContext::recGetContinuationType(size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, + CORINFO_CLASS_HANDLE result) +{ + if (GetContinuationType == nullptr) + { + GetContinuationType = new LightWeightMap(); + } + + Agnostic_GetContinuationTypeIn key; + ZeroMemory(&key, sizeof(key)); + key.dataSize = (DWORDLONG)dataSize; + key.objRefs = (objRefs != nullptr) ? (*objRefs ? 1 : 0) : 0; // TODO add buffer + key.result = (DWORD)dataOffsets.Result; + key.exception = (DWORD)dataOffsets.Exception; + key.continuationContext = (DWORD)dataOffsets.ContinuationContext; + + DWORDLONG value = CastHandle(result); + + GetContinuationType->Add(key, value); + DEBUG_REC(dmpGetContinuationType(key, value)); +} + +void MethodContext::dmpGetContinuationType(const Agnostic_GetContinuationTypeIn& key, DWORDLONG value) +{ + printf("GetContinuationType dataSize-%016" PRIX64 ", objRefs-%u, result-%d, exception-%d, handle-%016" PRIX64, + key.dataSize, key.objRefs, key.result, key.exception, value); +} + +CORINFO_CLASS_HANDLE MethodContext::repGetContinuationType(size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + Agnostic_GetContinuationTypeIn key; + ZeroMemory(&key, sizeof(key)); + key.dataSize = (DWORDLONG)dataSize; + key.objRefs = (objRefs != nullptr) ? (*objRefs ? 1 : 0) : 0; + key.result = (DWORD)dataOffsets.Result; + key.exception = (DWORD)dataOffsets.Exception; + key.continuationContext = (DWORD)dataOffsets.ContinuationContext; + + DWORDLONG value = LookupByKeyOrMiss(GetContinuationType, key, + ": dataSize %016" PRIX64 ", objRefs %u, result %d, exception %d", + key.dataSize, key.objRefs, key.result, key.exception); + DEBUG_REP(dmpGetContinuationType(key, value)); + + return (CORINFO_CLASS_HANDLE)value; +} + void MethodContext::recUpdateEntryPointForTailCall( const CORINFO_CONST_LOOKUP& origEntryPoint, const CORINFO_CONST_LOOKUP& newEntryPoint) diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index c86944897c4f4f..076b59629d428b 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -867,6 +867,10 @@ class MethodContext void dmpGetAsyncResumptionStub(DWORD key, DWORDLONG handle); CORINFO_METHOD_HANDLE repGetAsyncResumptionStub(); + void recGetContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, CORINFO_CLASS_HANDLE result); + void dmpGetContinuationType(const Agnostic_GetContinuationTypeIn& key, DWORDLONG value); + CORINFO_CLASS_HANDLE repGetContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets); + void recUpdateEntryPointForTailCall(const CORINFO_CONST_LOOKUP& origEntryPoint, const CORINFO_CONST_LOOKUP& newEntryPoint); void dmpUpdateEntryPointForTailCall(const Agnostic_CORINFO_CONST_LOOKUP& origEntryPoint, const Agnostic_CORINFO_CONST_LOOKUP& newEntryPoint); void repUpdateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint); @@ -1208,6 +1212,7 @@ enum mcPackets Packet_GetAsyncResumptionStub = 231, Packet_GetCookieForInterpreterCalliSig = 232, Packet_GetHelperFtn = 233, + Packet_GetContinuationType = 234, }; void SetDebugDumpVariables(); diff --git a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp index 0a6118d73ab862..45e56396d7936f 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp @@ -1782,6 +1782,14 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub() return stub; } +CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + mc->cr->AddCall("getContinuationType"); + CORINFO_CLASS_HANDLE result = original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); + mc->recGetContinuationType(dataSize, objRefs, dataOffsets, result); + return result; +} + void interceptor_ICJI::updateEntryPointForTailCall(CORINFO_CONST_LOOKUP* entryPoint) { mc->cr->AddCall("updateEntryPointForTailCall"); @@ -2032,4 +2040,4 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getSpecialCopyHelper(CORINFO_CLASS_HANDL CORINFO_METHOD_HANDLE temp = original_ICorJitInfo->getSpecialCopyHelper(type); mc->recGetSpecialCopyHelper(type, temp); return temp; -} +} \ No newline at end of file diff --git a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp index ac148df7c4e295..1d9be26e158f76 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp @@ -1441,3 +1441,12 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getSpecialCopyHelper( return original_ICorJitInfo->getSpecialCopyHelper(type); } +CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + mcs->AddCall("getContinuationType"); + return original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); +} + diff --git a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp index 1f47ea740c0c48..075ab6c5f75b4b 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp @@ -1265,3 +1265,11 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getSpecialCopyHelper( return original_ICorJitInfo->getSpecialCopyHelper(type); } +CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + return original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); +} + diff --git a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp index a1b207c3de797e..685d7f807c9c89 100644 --- a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp @@ -1508,6 +1508,13 @@ CORINFO_METHOD_HANDLE MyICJI::getAsyncResumptionStub() return jitInstance->mc->repGetAsyncResumptionStub();; } +CORINFO_CLASS_HANDLE MyICJI::getContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + jitInstance->mc->cr->AddCall("getContinuationType"); + CORINFO_CLASS_HANDLE result = jitInstance->mc->repGetContinuationType(dataSize, objRefs, dataOffsets); + return result; +} + bool MyICJI::convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN* pResolvedToken, bool fMustConvert) { jitInstance->mc->cr->AddCall("convertPInvokeCalliToCall"); @@ -1841,4 +1848,4 @@ CORINFO_METHOD_HANDLE MyICJI::getSpecialCopyHelper(CORINFO_CLASS_HANDLE type) jitInstance->mc->cr->AddCall("getSpecialCopyHelper"); CORINFO_METHOD_HANDLE result = jitInstance->mc->repGetSpecialCopyHelper(type); return result; -} +} \ No newline at end of file diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index d10522a9985469..3c273d71bf2eab 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -2314,10 +2314,11 @@ static void EnumerateRunsOfObjRefs(unsigned size, bool* objRefs, TFunc func) end++; func((start - objRefs) * TARGET_POINTER_SIZE, (end - start) * TARGET_POINTER_SIZE); + start = end; } } -MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objRefs, class AllocMemTracker* pamTracker) +MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, class AllocMemTracker* pamTracker) { CONTRACTL { STANDARD_VM_CHECK; @@ -2325,46 +2326,75 @@ MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objR } CONTRACTL_END; MethodTable* pParentClass = CoreLibBinder::GetClass(CLASS__CONTINUATION); - unsigned parentInstanceSize = pParentClass->GetNumInstanceFieldBytes(); DWORD numVirtuals = pParentClass->GetNumVirtuals(); size_t cbMT = sizeof(MethodTable); cbMT += MethodTable::GetNumVtableIndirections(numVirtuals) * sizeof(MethodTable::VTableIndir_t); - unsigned numSeries = 0; + unsigned numObjRefRuns = 0; EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) { - numSeries++; + numObjRefRuns++; }); - _ASSERTE(!pParentClass->ContainsGCPointers()); - size_t cbGC = numSeries == 0 ? 0 : CGCDesc::ComputeSize(numSeries); + unsigned numParentPointerSeries = 0; + if (pParentClass->ContainsGCPointers()) + numParentPointerSeries = static_cast(CGCDesc::GetCGCDescFromMT(pParentClass)->GetNumSeries()); + + unsigned numPointerSeries = numParentPointerSeries + numObjRefRuns; + + size_t cbGC = numPointerSeries == 0 ? 0 : CGCDesc::ComputeSize(numPointerSeries); LoaderAllocator* pAllocator = GetLoaderAllocator(); - BYTE* pMemory = (BYTE*)pamTracker->Track(pAllocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); - memset(pMemory, 0, cbGC + cbMT); + BYTE* pMemory = (BYTE*)pamTracker->Track(pAllocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(CORINFO_CONTINUATION_DATA_OFFSETS)) + S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); + + CORINFO_CONTINUATION_DATA_OFFSETS* allocatedDataOffsets = new (pMemory) CORINFO_CONTINUATION_DATA_OFFSETS(dataOffsets); + + unsigned startOfDataInInstance = AlignUp(pParentClass->GetNumInstanceFieldBytes(), TARGET_POINTER_SIZE); + unsigned startOfDataInObject = OBJECT_SIZE + startOfDataInInstance; + + // Offsets passed in are relative to the data chunk, fix that up now to be relative to the start of the object. + if (allocatedDataOffsets->Result != UINT_MAX) + allocatedDataOffsets->Result += startOfDataInObject; + if (allocatedDataOffsets->Exception != UINT_MAX) + allocatedDataOffsets->Exception += startOfDataInObject; + if (allocatedDataOffsets->ContinuationContext != UINT_MAX) + allocatedDataOffsets->ContinuationContext += startOfDataInObject; + if (allocatedDataOffsets->KeepAlive != UINT_MAX) + allocatedDataOffsets->KeepAlive += startOfDataInObject; - MethodTable* pMT = (MethodTable*)(pMemory + cbGC); + MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); + pMT->SetIsContinuation(allocatedDataOffsets); pMT->AllocateAuxiliaryData(pAllocator, this, pamTracker, MethodTableStaticsFlags::None); pMT->SetLoaderAllocator(pAllocator); pMT->SetModule(this); pMT->SetNumVirtuals(static_cast(numVirtuals)); pMT->SetParentMethodTable(pParentClass); pMT->SetClass(pParentClass->GetClass()); // TODO: needs its own? - pMT->SetBaseSize(OBJECT_BASESIZE + parentInstanceSize + dataSize); + pMT->SetBaseSize(OBJECT_BASESIZE + startOfDataInInstance + dataSize); - if (numSeries > 0) + if (numPointerSeries > 0) { pMT->SetContainsGCPointers(); - CGCDesc::Init(pMT, numSeries); + CGCDesc::Init(pMT, numPointerSeries); CGCDescSeries* pSeries = CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries(); + + CGCDescSeries* pParentSeries = CGCDesc::GetCGCDescFromMT(pParentClass)->GetLowestSeries(); + for (unsigned i = 0; i < numParentPointerSeries; i++) + { + pSeries->SetSeriesSize((pParentSeries->GetSeriesSize() + pParentClass->GetBaseSize()) - pMT->GetBaseSize()); + pSeries->SetSeriesOffset(pParentSeries->GetSeriesOffset()); + pSeries++; + pParentSeries++; + } + auto writeSeries = [&](size_t start, size_t length) { pSeries->SetSeriesSize(length - pMT->GetBaseSize()); - pSeries->SetSeriesOffset(OBJECT_SIZE + parentInstanceSize + start); + pSeries->SetSeriesOffset(startOfDataInObject + start); pSeries++; }; EnumerateRunsOfObjRefs(dataSize, objRefs, writeSeries); - _ASSERTE(pSeries == CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries() + numSeries); + _ASSERTE(pSeries == CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries() + numPointerSeries); } pMT->SetClassInited(); diff --git a/src/coreclr/vm/ceeload.h b/src/coreclr/vm/ceeload.h index 31c5aaa6e2b8b7..47c069f1636166 100644 --- a/src/coreclr/vm/ceeload.h +++ b/src/coreclr/vm/ceeload.h @@ -1163,7 +1163,7 @@ class Module : public ModuleBase // the class load, which avoids the need for a 'being loaded' list MethodTable* CreateArrayMethodTable(TypeHandle elemType, CorElementType kind, unsigned rank, class AllocMemTracker *pamTracker); - MethodTable* CreateContinuationMethodTable(unsigned dataSize, bool* objRefs, class AllocMemTracker* pamTracker); + MethodTable* CreateContinuationMethodTable(unsigned dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, class AllocMemTracker* pamTracker); // Module/Assembly traversal Assembly * GetAssemblyIfLoaded( diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index 6de72c8918fa0a..27856ea277f2bc 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -862,8 +862,6 @@ DEFINE_FIELD(CONTINUATION, NEXT, Next) DEFINE_FIELD(CONTINUATION, RESUME, Resume) DEFINE_FIELD(CONTINUATION, STATE, State) DEFINE_FIELD(CONTINUATION, FLAGS, Flags) -DEFINE_FIELD(CONTINUATION, DATA, Data) -DEFINE_FIELD(CONTINUATION, GCDATA, GCData) DEFINE_CLASS(RUNTIME_WRAPPED_EXCEPTION, CompilerServices, RuntimeWrappedException) DEFINE_METHOD(RUNTIME_WRAPPED_EXCEPTION, OBJ_CTOR, .ctor, IM_Obj_RetVoid) diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index b6186a7382abc3..9622ec1fdc08a6 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -10242,8 +10242,6 @@ void CEEInfo::getAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut) pAsyncInfoOut->continuationResumeFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__RESUME)); pAsyncInfoOut->continuationStateFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__STATE)); pAsyncInfoOut->continuationFlagsFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__FLAGS)); - pAsyncInfoOut->continuationDataFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__DATA)); - pAsyncInfoOut->continuationGCDataFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__GCDATA)); pAsyncInfoOut->continuationsNeedMethodHandle = m_pMethodBeingCompiled->GetLoaderAllocator()->CanUnload(); pAsyncInfoOut->captureExecutionContextMethHnd = CORINFO_METHOD_HANDLE(CoreLibBinder::GetMethod(METHOD__ASYNC_HELPERS__CAPTURE_EXECUTION_CONTEXT)); pAsyncInfoOut->restoreExecutionContextMethHnd = CORINFO_METHOD_HANDLE(CoreLibBinder::GetMethod(METHOD__ASYNC_HELPERS__RESTORE_EXECUTION_CONTEXT)); @@ -10254,6 +10252,31 @@ void CEEInfo::getAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut) EE_TO_JIT_TRANSITION(); } +CORINFO_CLASS_HANDLE CEEInfo::getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + CONTRACTL { + THROWS; + GC_TRIGGERS; + MODE_PREEMPTIVE; + } CONTRACTL_END; + + CORINFO_CLASS_HANDLE result = NULL; + + JIT_TO_EE_TRANSITION(); + + AllocMemTracker amTracker; + // TODO: table to share/deduplicate these + result = (CORINFO_CLASS_HANDLE)m_pMethodBeingCompiled->GetModule()->CreateContinuationMethodTable((unsigned)dataSize, objRefs, dataOffsets, &amTracker); + amTracker.SuppressRelease(); + + EE_TO_JIT_TRANSITION(); + + return result; +} + // Return details about EE internal data structures uint32_t CEEInfo::getThreadTLSIndex(void **ppIndirection) { @@ -12388,6 +12411,7 @@ CORINFO_CLASS_HANDLE CEECodeGenInfo::getStaticFieldCurrentClass(CORINFO_FIELD_HA // TODO: Check if the jit is allowed to embed this handle in jitted code. // Note for the initonly cases it probably won't embed. + result = (CORINFO_CLASS_HANDLE) pObjMT; } } @@ -14495,9 +14519,11 @@ static Signature BuildResumptionStubSignature(LoaderAllocator* alloc, AllocMemTr { SigBuilder sigBuilder; sigBuilder.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT); - sigBuilder.AppendData(1); // 1 argument + sigBuilder.AppendData(2); // 2 argument sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT); // return type sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT); // continuation + sigBuilder.AppendElementType(ELEMENT_TYPE_BYREF); // result location + sigBuilder.AppendElementType(ELEMENT_TYPE_U1); return AllocateSignature(alloc, sigBuilder, pamTracker); } @@ -14701,93 +14727,9 @@ CORINFO_METHOD_HANDLE CEEJitInfo::getAsyncResumptionStub() pCode->EmitLDLOC(newContinuationLoc); pCode->EmitBRTRUE(doneResult); - // Load 'next' of current continuation - pCode->EmitLDARG(0); - pCode->EmitLDFLD(FIELD__CONTINUATION__NEXT); - - // Result is placed in GCData[0] if it has GC references (potentially boxing it). - bool isOrContainsGCPointers = false; - if (CorTypeInfo::IsObjRef(resultTypeHnd.GetInternalCorElementType()) || (resultTypeHnd.IsValueType() && resultTypeHnd.AsMethodTable()->ContainsGCPointers())) - { - // Load 'gcdata' of next continuation - pCode->EmitLDFLD(FIELD__CONTINUATION__GCDATA); - - // Now we have the GC array. At the first index is the result. - pCode->EmitLDC(0); - - // NOTE: that we are not using regular boxing (in EmitBOX sense) and allocate our own box instances via a helper. - // There are two reasons: - // - resultTypeHnd may be a nullable type and have different layout in boxed/unboxed forms. - // We do not want to deal with that. - // - resultTypeHnd may contain __Canon fields. Regular boxing would not allow that, but this box is used for a very - // specific internal purpose where we only require that the GC layout of the box matches the data - // that we store in it, thus we want to allow __Canon. - if (resultTypeHnd.IsValueType()) - { - // make a box and dup the ref - MethodDesc* md = CoreLibBinder::GetMethod(METHOD__ASYNC_HELPERS__ALLOC_CONTINUATION_RESULT_BOX); - pCode->EmitLDC((DWORD_PTR)resultTypeHnd.AsMethodTable()); - pCode->EmitCALL(pCode->GetToken(md), 1, 1); - pCode->EmitDUP(); - // dst is the offset of the first field in the box - pCode->EmitLDFLDA(FIELD__RAW_DATA__DATA); - // load the result - pCode->EmitLDLOC(resultLoc); - // store into the box - pCode->EmitSTOBJ(pCode->GetToken(resultTypeHnd)); - } - else - { - // load the result - pCode->EmitLDLOC(resultLoc); - } - - // Store the result. - pCode->EmitSTELEM_REF(); - } - else - { - // Otherwise it goes into Data, either at offset 0 or 4 depending - // on CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA. - ILCodeLabel* hasOsrILOffset = pCode->NewCodeLabel(); - - unsigned nextContinuationLcl = pCode->NewLocal(LocalDesc(continuationTypeHnd)); - pCode->EmitSTLOC(nextContinuationLcl); - - // Load 'flags' of next continuation - pCode->EmitLDLOC(nextContinuationLcl); - pCode->EmitLDFLD(FIELD__CONTINUATION__FLAGS); - pCode->EmitLDC(CORINFO_CONTINUATION_OSR_IL_OFFSET_IN_DATA); - pCode->EmitAND(); - pCode->EmitBRTRUE(hasOsrILOffset); - - // Load 'data' of next continuation - pCode->EmitLDLOC(nextContinuationLcl); - pCode->EmitLDFLD(FIELD__CONTINUATION__DATA); - // Load address of array at index 0 - pCode->EmitLDC(0); - pCode->EmitLDELEMA(pCode->GetToken(CoreLibBinder::GetClass(CLASS__BYTE))); - - // Store at index 0. - pCode->EmitLDLOC(resultLoc); - pCode->EmitSTOBJ(pCode->GetToken(resultTypeHnd)); - - pCode->EmitBR(doneResult); - - pCode->EmitLabel(hasOsrILOffset); - - // Load 'data' of next continuation - pCode->EmitLDLOC(nextContinuationLcl); - pCode->EmitLDFLD(FIELD__CONTINUATION__DATA); - // Load address of array at index 4 - pCode->EmitLDC(4); - pCode->EmitLDELEMA(pCode->GetToken(CoreLibBinder::GetClass(CLASS__BYTE))); - - // Store at index 4. - pCode->EmitLDLOC(resultLoc); - pCode->EmitUNALIGNED(1); - pCode->EmitSTOBJ(pCode->GetToken(resultTypeHnd)); - } + pCode->EmitLDARG(1); // resultLoc + pCode->EmitLDLOC(resultLoc); + pCode->EmitSTOBJ(pCode->GetToken(resultTypeHnd)); pCode->EmitLabel(doneResult); } diff --git a/src/coreclr/vm/methodtable.h b/src/coreclr/vm/methodtable.h index e161f01e0855a2..24e8dc0d0b3d83 100644 --- a/src/coreclr/vm/methodtable.h +++ b/src/coreclr/vm/methodtable.h @@ -2788,6 +2788,19 @@ class MethodTable SetFlag(enum_flag_Category_Nullable); } + inline BOOL IsContinuation() + { + LIMITED_METHOD_DAC_CONTRACT; + return GetFlag(enum_flag_IsContinuationType); + } + + inline void SetIsContinuation(const CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) + { + LIMITED_METHOD_CONTRACT; + SetFlag(enum_flag_IsContinuationType); + m_pContinuationOffsets = dataOffsets; + } + // The following methods are only valid for the method tables for array types. CorElementType GetArrayElementType() { @@ -2821,6 +2834,13 @@ class MethodTable return offsetof(MethodTable, m_ElementTypeHnd); } + const CORINFO_CONTINUATION_DATA_OFFSETS* GetContinuationOffsets() + { + LIMITED_METHOD_DAC_CONTRACT; + _ASSERTE(IsContinuation()); + return m_pContinuationOffsets; + } + //------------------------------------------------------------------- // UNDERLYING METADATA // @@ -3786,7 +3806,7 @@ public : enum_flag_HasDispatchMapSlot = 0x0004, enum_flag_wflags2_unused_2 = 0x0008, - //unused = 0x0010, + enum_flag_IsContinuationType = 0x0010, enum_flag_IsIntrinsicType = 0x0020, enum_flag_HasCctor = 0x0040, enum_flag_HasVirtualStaticMethods = 0x0080, @@ -3930,13 +3950,14 @@ public : // m_pPerInstInfo and m_pInterfaceMap have to be at fixed offsets because of performance sensitive // JITed code and JIT helpers. The space used by m_pPerInstInfo is used to represent the array - // element type handle for array MethodTables. + // element type handle for array MethodTables and continuation layout offsets for continuations. public: union { PerInstInfo_t m_pPerInstInfo; TADDR m_ElementTypeHnd; + const CORINFO_CONTINUATION_DATA_OFFSETS* m_pContinuationOffsets; }; union { From a642f42e639c20069329d906b821e005d1b931d3 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Oct 2025 01:14:34 +0200 Subject: [PATCH 04/48] Update type --- src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs index fbe8020cd28579..760daff80ec3f8 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs @@ -886,8 +886,10 @@ public unsafe struct CORINFO_ASYNC_INFO public struct CORINFO_CONTINUATION_DATA_OFFSETS { - public int ReturnValueOffset; - public int ExceptionOffset; + public int Result; + public int Exception; + public int ContinuationContext; + public int KeepAlive; } // Flags passed from JIT to runtime. From 03a640d3f07230b6b26139b0a779ca9a11133c3d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Oct 2025 01:24:13 +0200 Subject: [PATCH 05/48] Fix bug --- src/coreclr/vm/ceeload.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index 3c273d71bf2eab..74ec093136190b 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -2353,15 +2353,16 @@ MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objR unsigned startOfDataInInstance = AlignUp(pParentClass->GetNumInstanceFieldBytes(), TARGET_POINTER_SIZE); unsigned startOfDataInObject = OBJECT_SIZE + startOfDataInInstance; - // Offsets passed in are relative to the data chunk, fix that up now to be relative to the start of the object. + // Offsets passed in are relative to the data chunk, fix that up now to be + // relative to the start of the instance data. if (allocatedDataOffsets->Result != UINT_MAX) - allocatedDataOffsets->Result += startOfDataInObject; + allocatedDataOffsets->Result += startOfDataInInstance; if (allocatedDataOffsets->Exception != UINT_MAX) - allocatedDataOffsets->Exception += startOfDataInObject; + allocatedDataOffsets->Exception += startOfDataInInstance; if (allocatedDataOffsets->ContinuationContext != UINT_MAX) - allocatedDataOffsets->ContinuationContext += startOfDataInObject; + allocatedDataOffsets->ContinuationContext += startOfDataInInstance; if (allocatedDataOffsets->KeepAlive != UINT_MAX) - allocatedDataOffsets->KeepAlive += startOfDataInObject; + allocatedDataOffsets->KeepAlive += startOfDataInInstance; MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); pMT->SetIsContinuation(allocatedDataOffsets); From a99ce96696473d927a3a2899e67a208be060cec5 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Oct 2025 01:42:17 +0200 Subject: [PATCH 06/48] Fix some bugs --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 17 ++++++----------- .../CompilerServices/RuntimeHelpers.CoreCLR.cs | 7 ++++++- src/coreclr/jit/async.cpp | 5 ++--- src/coreclr/vm/methodtable.cpp | 2 +- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index f35ee277ece488..3fc3af40315f15 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -95,22 +95,18 @@ public unsafe object GetContinuationContext() public void SetException(Exception ex) { - // Only the special continuation sub types are expected. - Debug.Assert(GetType() != typeof(Continuation)); - MethodTable* mt = RuntimeHelpers.GetMethodTable(this); - Debug.Assert(mt->ContinuationOffsets->Exception != uint.MaxValue); - + // Only the special continuation sub types are expected. + Debug.Assert(mt->IsContinuation); ref byte data = ref RuntimeHelpers.GetRawData(this); Unsafe.As(ref Unsafe.Add(ref data, mt->ContinuationOffsets->Exception)) = ex; } public ref byte GetResultStorageOrNull() { - // Only the special continuation sub types are expected. - Debug.Assert(GetType() != typeof(Continuation)); - MethodTable* mt = RuntimeHelpers.GetMethodTable(this); + // Only the special continuation sub types are expected. + Debug.Assert(mt->IsContinuation); if (mt->ContinuationOffsets->Result == uint.MaxValue) return ref Unsafe.NullRef(); @@ -120,10 +116,9 @@ public ref byte GetResultStorageOrNull() public void SetKeepAlive(object? obj) { - // Only the special continuation sub types are expected. - Debug.Assert(GetType() != typeof(Continuation)); - MethodTable* mt = RuntimeHelpers.GetMethodTable(this); + // Only the special continuation sub types are expected. + Debug.Assert(mt->IsContinuation); Debug.Assert(mt->ContinuationOffsets->KeepAlive != uint.MaxValue); ref byte data = ref RuntimeHelpers.GetRawData(this); diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 67c35f122ee600..ed72de0485148b 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -688,8 +688,9 @@ internal unsafe struct MethodTable [FieldOffset(4)] public uint BaseSize; + [FieldOffset(8)] + public uint Flags2; // See additional native members in methodtable.h, not needed here yet. - // 0x8: m_dwFlags2 (additional flags and token in upper 24 bits) // 0xC: m_wNumVirtuals /// @@ -801,6 +802,8 @@ internal unsafe struct MethodTable | 0x10000000 // enum_flag_IDynamicInterfaceCastable; | 0x00040000; // enum_flag_Category_ValueType + private const uint enum_flag2_IsContinuationType = 0x10; + private const int DebugClassNamePtr = // adjust for debug_m_szClassName #if DEBUG #if TARGET_64BIT @@ -900,6 +903,8 @@ public int MultiDimensionalArrayRank public bool IsArray => (Flags & enum_flag_Category_Array_Mask) == enum_flag_Category_Array; + public bool IsContinuation => (Flags2 & enum_flag2_IsContinuationType) == enum_flag2_IsContinuationType; + public bool HasInstantiation => (Flags & enum_flag_HasComponentSize) == 0 && (Flags & enum_flag_GenericsMask) != enum_flag_GenericsMask_NonGeneric; public bool IsGenericTypeDefinition => (Flags & (enum_flag_HasComponentSize | enum_flag_GenericsMask)) == enum_flag_GenericsMask_TypicalInst; diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 0013992925d439..b9b3ae8898db52 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -624,12 +624,11 @@ PhaseStatus AsyncTransformation::Run() // ahead of time. for (BasicBlock* block : m_comp->Blocks()) { - unsigned suspPointsInBlock = 0; for (GenTree* tree : LIR::AsRange(block)) { if (tree->IsCall() && tree->AsCall()->IsAsync() && !tree->AsCall()->IsTailCall()) { - JITDUMP(FMT_BB " contains await(s) that will turn into suspension points\n", block->bbNum); + JITDUMP(FMT_BB " contains await(s)\n", block->bbNum); worklist.Push(block); break; } @@ -1026,7 +1025,7 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* { JITDUMP(" Method %s; keeping IL offset that inspired OSR method at the beginning of non-GC data\n", m_comp->doesMethodHavePatchpoints() ? "has patchpoints" : "is an OSR method"); - allocLayout(4, 4); + allocLayout(sizeof(int), sizeof(int)); } if (layout.ReturnSize > 0) diff --git a/src/coreclr/vm/methodtable.cpp b/src/coreclr/vm/methodtable.cpp index 56f704997fa332..7b3373e3ce649a 100644 --- a/src/coreclr/vm/methodtable.cpp +++ b/src/coreclr/vm/methodtable.cpp @@ -6245,7 +6245,7 @@ BOOL MethodTable::SanityCheck() if (GetNumGenericArgs() != 0) return (pCanonMT->GetClass() == pClass); else - return (pCanonMT == this) || IsArray(); + return (pCanonMT == this) || IsArray() || IsContinuation(); // TODO -- allocate an EEClass instead? } From fa1820adca9f4533c053afdbd0c917a21f1953a1 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sun, 5 Oct 2025 01:43:01 +0200 Subject: [PATCH 07/48] Run thunk generator --- src/coreclr/inc/icorjitinfoimpl_generated.h | 10 +-- src/coreclr/inc/jiteeversionguid.h | 10 +-- src/coreclr/jit/ICorJitInfo_names_generated.h | 2 +- .../jit/ICorJitInfo_wrapper_generated.hpp | 22 +++--- .../JitInterface/CorInfoImpl_generated.cs | 74 +++++++++---------- .../aot/jitinterface/jitinterface_generated.h | 24 +++--- .../icorjitinfo_generated.cpp | 18 ++--- .../icorjitinfo_generated.cpp | 16 ++-- 8 files changed, 88 insertions(+), 88 deletions(-) diff --git a/src/coreclr/inc/icorjitinfoimpl_generated.h b/src/coreclr/inc/icorjitinfoimpl_generated.h index ad8b89da83b8bf..e9f715e8062a64 100644 --- a/src/coreclr/inc/icorjitinfoimpl_generated.h +++ b/src/coreclr/inc/icorjitinfoimpl_generated.h @@ -655,6 +655,11 @@ bool getTailCallHelpers( CORINFO_GET_TAILCALL_HELPERS_FLAGS flags, CORINFO_TAILCALL_HELPERS* pResult) override; +CORINFO_CLASS_HANDLE getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) override; + CORINFO_METHOD_HANDLE getAsyncResumptionStub() override; bool convertPInvokeCalliToCall( @@ -746,11 +751,6 @@ uint32_t getJitFlags( CORINFO_METHOD_HANDLE getSpecialCopyHelper( CORINFO_CLASS_HANDLE type) override; -CORINFO_CLASS_HANDLE getContinuationType( - size_t dataSize, - bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) override; - /**********************************************************************************/ // clang-format on /**********************************************************************************/ diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 678c0095670c49..17f96a2f272f1c 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* 9bcc23e4-25da-470d-954f-ae08907e3b09 */ - 0x9bcc23e4, - 0x25da, - 0x470d, - {0x95, 0x4f, 0xae, 0x08, 0x90, 0x7e, 0x3b, 0x09} +constexpr GUID JITEEVersionIdentifier = { /* 156f1a7f-c06a-40fc-a984-987c8b4c2ba7 */ + 0x156f1a7f, + 0xc06a, + 0x40fc, + {0xa9, 0x84, 0x98, 0x7c, 0x8b, 0x4c, 0x2b, 0xa7} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/jit/ICorJitInfo_names_generated.h b/src/coreclr/jit/ICorJitInfo_names_generated.h index d23e331501cb57..e58987625b7d9e 100644 --- a/src/coreclr/jit/ICorJitInfo_names_generated.h +++ b/src/coreclr/jit/ICorJitInfo_names_generated.h @@ -159,6 +159,7 @@ DEF_CLR_API(getFieldThreadLocalStoreID) DEF_CLR_API(GetDelegateCtor) DEF_CLR_API(MethodCompileComplete) DEF_CLR_API(getTailCallHelpers) +DEF_CLR_API(getContinuationType) DEF_CLR_API(getAsyncResumptionStub) DEF_CLR_API(convertPInvokeCalliToCall) DEF_CLR_API(notifyInstructionSetUsage) @@ -180,6 +181,5 @@ DEF_CLR_API(getRelocTypeHint) DEF_CLR_API(getExpectedTargetArchitecture) DEF_CLR_API(getJitFlags) DEF_CLR_API(getSpecialCopyHelper) -DEF_CLR_API(getContinuationType) #undef DEF_CLR_API diff --git a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp index 7fcd7644f88422..72a01f4415bbb6 100644 --- a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp +++ b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp @@ -1536,6 +1536,17 @@ bool WrapICorJitInfo::getTailCallHelpers( return temp; } +CORINFO_CLASS_HANDLE WrapICorJitInfo::getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + API_ENTER(getContinuationType); + CORINFO_CLASS_HANDLE temp = wrapHnd->getContinuationType(dataSize, objRefs, dataOffsets); + API_LEAVE(getContinuationType); + return temp; +} + CORINFO_METHOD_HANDLE WrapICorJitInfo::getAsyncResumptionStub() { API_ENTER(getAsyncResumptionStub); @@ -1744,17 +1755,6 @@ CORINFO_METHOD_HANDLE WrapICorJitInfo::getSpecialCopyHelper( return temp; } -CORINFO_CLASS_HANDLE WrapICorJitInfo::getContinuationType( - size_t dataSize, - bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) -{ - API_ENTER(getContinuationType); - CORINFO_CLASS_HANDLE temp = wrapHnd->getContinuationType(dataSize, objRefs, dataOffsets); - API_LEAVE(getContinuationType); - return temp; -} - /**********************************************************************************/ // clang-format on /**********************************************************************************/ diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs index 68709418ba995e..4d95121dbc808d 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs @@ -2297,6 +2297,21 @@ private static byte _getTailCallHelpers(IntPtr thisHandle, IntPtr* ppException, } } + [UnmanagedCallersOnly] + private static CORINFO_CLASS_STRUCT_* _getContinuationType(IntPtr thisHandle, IntPtr* ppException, UIntPtr dataSize, bool* objRefs, CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) + { + var _this = GetThis(thisHandle); + try + { + return _this.getContinuationType(dataSize, ref *objRefs, ref *dataOffsets); + } + catch (Exception ex) + { + *ppException = _this.AllocException(ex); + return default; + } + } + [UnmanagedCallersOnly] private static CORINFO_METHOD_STRUCT_* _getAsyncResumptionStub(IntPtr thisHandle, IntPtr* ppException) { @@ -2603,21 +2618,6 @@ private static uint _getJitFlags(IntPtr thisHandle, IntPtr* ppException, CORJIT_ } } - [UnmanagedCallersOnly] - private static CORINFO_CLASS_STRUCT_* _getContinuationType(IntPtr thisHandle, IntPtr* ppException, UIntPtr dataSize, bool* objRefs, CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) - { - var _this = GetThis(thisHandle); - try - { - return _this.getContinuationType(dataSize, ref *objRefs, ref *dataOffsets); - } - catch (Exception ex) - { - *ppException = _this.AllocException(ex); - return default; - } - } - private static IntPtr GetUnmanagedCallbacks() { @@ -2778,28 +2778,28 @@ private static IntPtr GetUnmanagedCallbacks() callbacks[152] = (delegate* unmanaged)&_GetDelegateCtor; callbacks[153] = (delegate* unmanaged)&_MethodCompileComplete; callbacks[154] = (delegate* unmanaged)&_getTailCallHelpers; - callbacks[155] = (delegate* unmanaged)&_getAsyncResumptionStub; - callbacks[156] = (delegate* unmanaged)&_convertPInvokeCalliToCall; - callbacks[157] = (delegate* unmanaged)&_notifyInstructionSetUsage; - callbacks[158] = (delegate* unmanaged)&_updateEntryPointForTailCall; - callbacks[159] = (delegate* unmanaged)&_allocMem; - callbacks[160] = (delegate* unmanaged)&_reserveUnwindInfo; - callbacks[161] = (delegate* unmanaged)&_allocUnwindInfo; - callbacks[162] = (delegate* unmanaged)&_allocGCInfo; - callbacks[163] = (delegate* unmanaged)&_setEHcount; - callbacks[164] = (delegate* unmanaged)&_setEHinfo; - callbacks[165] = (delegate* unmanaged)&_logMsg; - callbacks[166] = (delegate* unmanaged)&_doAssert; - callbacks[167] = (delegate* unmanaged)&_reportFatalError; - callbacks[168] = (delegate* unmanaged)&_getPgoInstrumentationResults; - callbacks[169] = (delegate* unmanaged)&_allocPgoInstrumentationBySchema; - callbacks[170] = (delegate* unmanaged)&_recordCallSite; - callbacks[171] = (delegate* unmanaged)&_recordRelocation; - callbacks[172] = (delegate* unmanaged)&_getRelocTypeHint; - callbacks[173] = (delegate* unmanaged)&_getExpectedTargetArchitecture; - callbacks[174] = (delegate* unmanaged)&_getJitFlags; - callbacks[175] = (delegate* unmanaged)&_getSpecialCopyHelper; - callbacks[176] = (delegate* unmanaged)&_getContinuationType; + callbacks[155] = (delegate* unmanaged)&_getContinuationType; + callbacks[156] = (delegate* unmanaged)&_getAsyncResumptionStub; + callbacks[157] = (delegate* unmanaged)&_convertPInvokeCalliToCall; + callbacks[158] = (delegate* unmanaged)&_notifyInstructionSetUsage; + callbacks[159] = (delegate* unmanaged)&_updateEntryPointForTailCall; + callbacks[160] = (delegate* unmanaged)&_allocMem; + callbacks[161] = (delegate* unmanaged)&_reserveUnwindInfo; + callbacks[162] = (delegate* unmanaged)&_allocUnwindInfo; + callbacks[163] = (delegate* unmanaged)&_allocGCInfo; + callbacks[164] = (delegate* unmanaged)&_setEHcount; + callbacks[165] = (delegate* unmanaged)&_setEHinfo; + callbacks[166] = (delegate* unmanaged)&_logMsg; + callbacks[167] = (delegate* unmanaged)&_doAssert; + callbacks[168] = (delegate* unmanaged)&_reportFatalError; + callbacks[169] = (delegate* unmanaged)&_getPgoInstrumentationResults; + callbacks[170] = (delegate* unmanaged)&_allocPgoInstrumentationBySchema; + callbacks[171] = (delegate* unmanaged)&_recordCallSite; + callbacks[172] = (delegate* unmanaged)&_recordRelocation; + callbacks[173] = (delegate* unmanaged)&_getRelocTypeHint; + callbacks[174] = (delegate* unmanaged)&_getExpectedTargetArchitecture; + callbacks[175] = (delegate* unmanaged)&_getJitFlags; + callbacks[176] = (delegate* unmanaged)&_getSpecialCopyHelper; return (IntPtr)callbacks; } diff --git a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h index f0066d95177d70..e5b2c4f8c0dab0 100644 --- a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h +++ b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h @@ -166,6 +166,7 @@ struct JitInterfaceCallbacks CORINFO_METHOD_HANDLE (* GetDelegateCtor)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE methHnd, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE targetMethodHnd, DelegateCtorArgs* pCtorData); void (* MethodCompileComplete)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE methHnd); bool (* getTailCallHelpers)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* callToken, CORINFO_SIG_INFO* sig, CORINFO_GET_TAILCALL_HELPERS_FLAGS flags, CORINFO_TAILCALL_HELPERS* pResult); + CORINFO_CLASS_HANDLE (* getContinuationType)(void * thisHandle, CorInfoExceptionClass** ppException, size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets); CORINFO_METHOD_HANDLE (* getAsyncResumptionStub)(void * thisHandle, CorInfoExceptionClass** ppException); bool (* convertPInvokeCalliToCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool mustConvert); bool (* notifyInstructionSetUsage)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_InstructionSet instructionSet, bool supportEnabled); @@ -187,7 +188,6 @@ struct JitInterfaceCallbacks uint32_t (* getExpectedTargetArchitecture)(void * thisHandle, CorInfoExceptionClass** ppException); uint32_t (* getJitFlags)(void * thisHandle, CorInfoExceptionClass** ppException, CORJIT_FLAGS* flags, uint32_t sizeInBytes); CORINFO_METHOD_HANDLE (* getSpecialCopyHelper)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_CLASS_HANDLE type); - CORINFO_CLASS_HANDLE (* getContinuationType)(void * thisHandle, CorInfoExceptionClass** ppException, size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets); }; @@ -1715,6 +1715,17 @@ class JitInterfaceWrapper : public ICorJitInfo return temp; } + virtual CORINFO_CLASS_HANDLE getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + CorInfoExceptionClass* pException = nullptr; + CORINFO_CLASS_HANDLE temp = _callbacks->getContinuationType(_thisHandle, &pException, dataSize, objRefs, dataOffsets); + if (pException != nullptr) throw pException; + return temp; +} + virtual CORINFO_METHOD_HANDLE getAsyncResumptionStub() { CorInfoExceptionClass* pException = nullptr; @@ -1922,15 +1933,4 @@ class JitInterfaceWrapper : public ICorJitInfo if (pException != nullptr) throw pException; return temp; } - - virtual CORINFO_CLASS_HANDLE getContinuationType( - size_t dataSize, - bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) -{ - CorInfoExceptionClass* pException = nullptr; - CORINFO_CLASS_HANDLE temp = _callbacks->getContinuationType(_thisHandle, &pException, dataSize, objRefs, dataOffsets); - if (pException != nullptr) throw pException; - return temp; -} }; diff --git a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp index 1d9be26e158f76..ce0af8a86f23da 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp @@ -1266,6 +1266,15 @@ bool interceptor_ICJI::getTailCallHelpers( return original_ICorJitInfo->getTailCallHelpers(callToken, sig, flags, pResult); } +CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + mcs->AddCall("getContinuationType"); + return original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); +} + CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub() { mcs->AddCall("getAsyncResumptionStub"); @@ -1441,12 +1450,3 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getSpecialCopyHelper( return original_ICorJitInfo->getSpecialCopyHelper(type); } -CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType( - size_t dataSize, - bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) -{ - mcs->AddCall("getContinuationType"); - return original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); -} - diff --git a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp index 075ab6c5f75b4b..ce8857e2fb1401 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp @@ -1111,6 +1111,14 @@ bool interceptor_ICJI::getTailCallHelpers( return original_ICorJitInfo->getTailCallHelpers(callToken, sig, flags, pResult); } +CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType( + size_t dataSize, + bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +{ + return original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); +} + CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub() { return original_ICorJitInfo->getAsyncResumptionStub(); @@ -1265,11 +1273,3 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getSpecialCopyHelper( return original_ICorJitInfo->getSpecialCopyHelper(type); } -CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType( - size_t dataSize, - bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) -{ - return original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); -} - From e4a75f1780eb41ec34e679b38578f9129a1dc6ea Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 6 Oct 2025 15:59:38 +0200 Subject: [PATCH 08/48] Create a manager in the LoaderAllocator to hold map and create types --- src/coreclr/vm/CMakeLists.txt | 2 + src/coreclr/vm/asynccontinuations.cpp | 134 ++++++++++++++++++++++++++ src/coreclr/vm/asynccontinuations.h | 24 +++++ src/coreclr/vm/ceeload.cpp | 107 -------------------- src/coreclr/vm/ceeload.h | 2 - src/coreclr/vm/jitinterface.cpp | 5 +- src/coreclr/vm/loaderallocator.cpp | 33 +++++++ src/coreclr/vm/loaderallocator.hpp | 5 + 8 files changed, 201 insertions(+), 111 deletions(-) create mode 100644 src/coreclr/vm/asynccontinuations.cpp create mode 100644 src/coreclr/vm/asynccontinuations.h diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index f078c15b2a1b3c..93d191d335c911 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -52,6 +52,7 @@ set(VM_SOURCES_DAC_AND_WKS_COMMON array.cpp assembly.cpp assemblybinder.cpp + asynccontinuations.cpp binder.cpp castcache.cpp callcounting.cpp @@ -149,6 +150,7 @@ set(VM_HEADERS_DAC_AND_WKS_COMMON array.h assembly.hpp assemblybinder.h + asynccontinuations.h binder.h castcache.h callcounting.h diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp new file mode 100644 index 00000000000000..d2028a91d55d95 --- /dev/null +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -0,0 +1,134 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// =========================================================================== +// File: asynccontinuations.cpp +// + +// =========================================================================== +// This file contains the manager for creating new dynamic async continuation types. +// =========================================================================== +// + +#include "common.h" +#include "asynccontinuations.h" + +#ifndef DACCESS_COMPILE + +AsyncContinuationsManager::AsyncContinuationsManager(LoaderAllocator* allocator) + : m_allocator(allocator) +{ + LIMITED_METHOD_CONTRACT; +} + +template +static void EnumerateRunsOfObjRefs(unsigned size, const bool* objRefs, TFunc func) +{ + const bool* objRefsEnd = objRefs + (size + (TARGET_POINTER_SIZE - 1)) / TARGET_POINTER_SIZE; + const bool* start = objRefs; + while (start < objRefsEnd) + { + while (start < objRefsEnd && !*start) + start++; + + if (start >= objRefsEnd) + return; + + const bool* end = start; + while (end < objRefsEnd && *end) + end++; + + func((start - objRefs) * TARGET_POINTER_SIZE, (end - start) * TARGET_POINTER_SIZE); + start = end; + } +} + +MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker) +{ + STANDARD_VM_CONTRACT; + + MethodTable* pParentClass = CoreLibBinder::GetClass(CLASS__CONTINUATION); + DWORD numVirtuals = pParentClass->GetNumVirtuals(); + + size_t cbMT = sizeof(MethodTable); + cbMT += MethodTable::GetNumVtableIndirections(numVirtuals) * sizeof(MethodTable::VTableIndir_t); + + unsigned numObjRefRuns = 0; + EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) + { + numObjRefRuns++; + }); + + unsigned numParentPointerSeries = 0; + if (pParentClass->ContainsGCPointers()) + numParentPointerSeries = static_cast(CGCDesc::GetCGCDescFromMT(pParentClass)->GetNumSeries()); + + unsigned numPointerSeries = numParentPointerSeries + numObjRefRuns; + + size_t cbGC = numPointerSeries == 0 ? 0 : CGCDesc::ComputeSize(numPointerSeries); + + BYTE* pMemory = (BYTE*)pamTracker->Track(m_allocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(CORINFO_CONTINUATION_DATA_OFFSETS)) + S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); + + CORINFO_CONTINUATION_DATA_OFFSETS* allocatedDataOffsets = new (pMemory) CORINFO_CONTINUATION_DATA_OFFSETS(dataOffsets); + + unsigned startOfDataInInstance = AlignUp(pParentClass->GetNumInstanceFieldBytes(), TARGET_POINTER_SIZE); + unsigned startOfDataInObject = OBJECT_SIZE + startOfDataInInstance; + + // Offsets passed in are relative to the data chunk, fix that up now to be + // relative to the start of the instance data. + if (allocatedDataOffsets->Result != UINT_MAX) + allocatedDataOffsets->Result += startOfDataInInstance; + if (allocatedDataOffsets->Exception != UINT_MAX) + allocatedDataOffsets->Exception += startOfDataInInstance; + if (allocatedDataOffsets->ContinuationContext != UINT_MAX) + allocatedDataOffsets->ContinuationContext += startOfDataInInstance; + if (allocatedDataOffsets->KeepAlive != UINT_MAX) + allocatedDataOffsets->KeepAlive += startOfDataInInstance; + + MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); + pMT->SetIsContinuation(allocatedDataOffsets); + pMT->AllocateAuxiliaryData(m_allocator, asyncMethod->GetLoaderModule(), pamTracker, MethodTableStaticsFlags::None); + pMT->SetLoaderAllocator(m_allocator); + pMT->SetModule(asyncMethod->GetModule()); + pMT->SetNumVirtuals(static_cast(numVirtuals)); + pMT->SetParentMethodTable(pParentClass); + pMT->SetClass(pParentClass->GetClass()); // TODO: needs its own? + pMT->SetBaseSize(OBJECT_BASESIZE + startOfDataInInstance + dataSize); + + if (numPointerSeries > 0) + { + pMT->SetContainsGCPointers(); + CGCDesc::Init(pMT, numPointerSeries); + CGCDescSeries* pSeries = CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries(); + + CGCDescSeries* pParentSeries = CGCDesc::GetCGCDescFromMT(pParentClass)->GetLowestSeries(); + for (unsigned i = 0; i < numParentPointerSeries; i++) + { + pSeries->SetSeriesSize((pParentSeries->GetSeriesSize() + pParentClass->GetBaseSize()) - pMT->GetBaseSize()); + pSeries->SetSeriesOffset(pParentSeries->GetSeriesOffset()); + pSeries++; + pParentSeries++; + } + + auto writeSeries = [&](size_t start, size_t length) { + pSeries->SetSeriesSize(length - pMT->GetBaseSize()); + pSeries->SetSeriesOffset(startOfDataInObject + start); + pSeries++; + }; + EnumerateRunsOfObjRefs(dataSize, objRefs, writeSeries); + _ASSERTE(pSeries == CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries() + numPointerSeries); + } + + pMT->SetClassInited(); + + return pMT; +} + +MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker) +{ + STANDARD_VM_CONTRACT; + + // TODO: table to share/deduplicate these. + return CreateNewContinuationMethodTable(dataSize, objRefs, dataOffsets, asyncMethod, pamTracker); +} + +#endif diff --git a/src/coreclr/vm/asynccontinuations.h b/src/coreclr/vm/asynccontinuations.h new file mode 100644 index 00000000000000..0abcda416ad650 --- /dev/null +++ b/src/coreclr/vm/asynccontinuations.h @@ -0,0 +1,24 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// =========================================================================== +// File: asynccontinuations.h +// +// =========================================================================== + +#ifndef ASYNCCONTINUATIONS_H +#define ASYNCCONTINUATIONS_H + +class AsyncContinuationsManager +{ + LoaderAllocator* m_allocator; + + MethodTable* CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker); + +public: + AsyncContinuationsManager(LoaderAllocator* allocator); + MethodTable* LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker); +}; + +typedef DPTR(AsyncContinuationsManager) PTR_AsyncContinuationsManager; + +#endif diff --git a/src/coreclr/vm/ceeload.cpp b/src/coreclr/vm/ceeload.cpp index 74ec093136190b..bb0f2037816852 100644 --- a/src/coreclr/vm/ceeload.cpp +++ b/src/coreclr/vm/ceeload.cpp @@ -2296,113 +2296,6 @@ BYTE *Module::GetProfilerBase() } } -template -static void EnumerateRunsOfObjRefs(unsigned size, bool* objRefs, TFunc func) -{ - bool* objRefsEnd = objRefs + (size + (TARGET_POINTER_SIZE - 1)) / TARGET_POINTER_SIZE; - bool* start = objRefs; - while (start < objRefsEnd) - { - while (start < objRefsEnd && !*start) - start++; - - if (start >= objRefsEnd) - return; - - bool* end = start; - while (end < objRefsEnd && *end) - end++; - - func((start - objRefs) * TARGET_POINTER_SIZE, (end - start) * TARGET_POINTER_SIZE); - start = end; - } -} - -MethodTable* Module::CreateContinuationMethodTable(unsigned dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, class AllocMemTracker* pamTracker) -{ - CONTRACTL { - STANDARD_VM_CHECK; - PRECONDITION(dataSize > 0); - } CONTRACTL_END; - - MethodTable* pParentClass = CoreLibBinder::GetClass(CLASS__CONTINUATION); - DWORD numVirtuals = pParentClass->GetNumVirtuals(); - - size_t cbMT = sizeof(MethodTable); - cbMT += MethodTable::GetNumVtableIndirections(numVirtuals) * sizeof(MethodTable::VTableIndir_t); - - unsigned numObjRefRuns = 0; - EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) - { - numObjRefRuns++; - }); - - unsigned numParentPointerSeries = 0; - if (pParentClass->ContainsGCPointers()) - numParentPointerSeries = static_cast(CGCDesc::GetCGCDescFromMT(pParentClass)->GetNumSeries()); - - unsigned numPointerSeries = numParentPointerSeries + numObjRefRuns; - - size_t cbGC = numPointerSeries == 0 ? 0 : CGCDesc::ComputeSize(numPointerSeries); - - LoaderAllocator* pAllocator = GetLoaderAllocator(); - BYTE* pMemory = (BYTE*)pamTracker->Track(pAllocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(CORINFO_CONTINUATION_DATA_OFFSETS)) + S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); - - CORINFO_CONTINUATION_DATA_OFFSETS* allocatedDataOffsets = new (pMemory) CORINFO_CONTINUATION_DATA_OFFSETS(dataOffsets); - - unsigned startOfDataInInstance = AlignUp(pParentClass->GetNumInstanceFieldBytes(), TARGET_POINTER_SIZE); - unsigned startOfDataInObject = OBJECT_SIZE + startOfDataInInstance; - - // Offsets passed in are relative to the data chunk, fix that up now to be - // relative to the start of the instance data. - if (allocatedDataOffsets->Result != UINT_MAX) - allocatedDataOffsets->Result += startOfDataInInstance; - if (allocatedDataOffsets->Exception != UINT_MAX) - allocatedDataOffsets->Exception += startOfDataInInstance; - if (allocatedDataOffsets->ContinuationContext != UINT_MAX) - allocatedDataOffsets->ContinuationContext += startOfDataInInstance; - if (allocatedDataOffsets->KeepAlive != UINT_MAX) - allocatedDataOffsets->KeepAlive += startOfDataInInstance; - - MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); - pMT->SetIsContinuation(allocatedDataOffsets); - pMT->AllocateAuxiliaryData(pAllocator, this, pamTracker, MethodTableStaticsFlags::None); - pMT->SetLoaderAllocator(pAllocator); - pMT->SetModule(this); - pMT->SetNumVirtuals(static_cast(numVirtuals)); - pMT->SetParentMethodTable(pParentClass); - pMT->SetClass(pParentClass->GetClass()); // TODO: needs its own? - pMT->SetBaseSize(OBJECT_BASESIZE + startOfDataInInstance + dataSize); - - if (numPointerSeries > 0) - { - pMT->SetContainsGCPointers(); - CGCDesc::Init(pMT, numPointerSeries); - CGCDescSeries* pSeries = CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries(); - - CGCDescSeries* pParentSeries = CGCDesc::GetCGCDescFromMT(pParentClass)->GetLowestSeries(); - for (unsigned i = 0; i < numParentPointerSeries; i++) - { - pSeries->SetSeriesSize((pParentSeries->GetSeriesSize() + pParentClass->GetBaseSize()) - pMT->GetBaseSize()); - pSeries->SetSeriesOffset(pParentSeries->GetSeriesOffset()); - pSeries++; - pParentSeries++; - } - - auto writeSeries = [&](size_t start, size_t length) { - pSeries->SetSeriesSize(length - pMT->GetBaseSize()); - pSeries->SetSeriesOffset(startOfDataInObject + start); - pSeries++; - }; - EnumerateRunsOfObjRefs(dataSize, objRefs, writeSeries); - _ASSERTE(pSeries == CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries() + numPointerSeries); - } - - pMT->SetClassInited(); - - return pMT; -} - #endif //!DACCESS_COMPILE Assembly * diff --git a/src/coreclr/vm/ceeload.h b/src/coreclr/vm/ceeload.h index 47c069f1636166..c4d02b6e02dbb6 100644 --- a/src/coreclr/vm/ceeload.h +++ b/src/coreclr/vm/ceeload.h @@ -1163,8 +1163,6 @@ class Module : public ModuleBase // the class load, which avoids the need for a 'being loaded' list MethodTable* CreateArrayMethodTable(TypeHandle elemType, CorElementType kind, unsigned rank, class AllocMemTracker *pamTracker); - MethodTable* CreateContinuationMethodTable(unsigned dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, class AllocMemTracker* pamTracker); - // Module/Assembly traversal Assembly * GetAssemblyIfLoaded( mdAssemblyRef kAssemblyRef, diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 9622ec1fdc08a6..a500db11903fb9 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -10267,9 +10267,10 @@ CORINFO_CLASS_HANDLE CEEInfo::getContinuationType( JIT_TO_EE_TRANSITION(); + LoaderAllocator* allocator = m_pMethodBeingCompiled->GetLoaderAllocator(); + AsyncContinuationsManager* asyncConts = allocator->GetAsyncContinuationsManager(); AllocMemTracker amTracker; - // TODO: table to share/deduplicate these - result = (CORINFO_CLASS_HANDLE)m_pMethodBeingCompiled->GetModule()->CreateContinuationMethodTable((unsigned)dataSize, objRefs, dataOffsets, &amTracker); + result = (CORINFO_CLASS_HANDLE)asyncConts->LookupOrCreateContinuationMethodTable((unsigned)dataSize, objRefs, dataOffsets, m_pMethodBeingCompiled, &amTracker); amTracker.SuppressRelease(); EE_TO_JIT_TRANSITION(); diff --git a/src/coreclr/vm/loaderallocator.cpp b/src/coreclr/vm/loaderallocator.cpp index f33bd9de06e62e..62c47f2b756044 100644 --- a/src/coreclr/vm/loaderallocator.cpp +++ b/src/coreclr/vm/loaderallocator.cpp @@ -63,6 +63,8 @@ LoaderAllocator::LoaderAllocator(bool collectible) : m_onStackReplacementManager = NULL; #endif + m_asyncContinuationsManager = NULL; + m_fGCPressure = false; m_fTerminated = false; m_fUnloaded = false; @@ -1410,6 +1412,12 @@ void LoaderAllocator::Terminate() } #endif + if (m_asyncContinuationsManager != NULL) + { + delete m_asyncContinuationsManager; + m_asyncContinuationsManager = NULL; + } + // In collectible types we merge the low frequency and high frequency heaps // So don't destroy them twice. if ((m_pLowFrequencyHeap != NULL) && (m_pLowFrequencyHeap != m_pHighFrequencyHeap)) @@ -2294,6 +2302,31 @@ PTR_OnStackReplacementManager LoaderAllocator::GetOnStackReplacementManager() #endif // FEATURE_ON_STACK_REPLACEMENT #ifndef DACCESS_COMPILE +PTR_AsyncContinuationsManager LoaderAllocator::GetAsyncContinuationsManager() +{ + CONTRACTL + { + THROWS; + GC_TRIGGERS; + MODE_ANY; + INJECT_FAULT(COMPlusThrowOM();); + } + CONTRACTL_END; + + if (m_asyncContinuationsManager == NULL) + { + AsyncContinuationsManager* newManager = new AsyncContinuationsManager(this); + + if (InterlockedCompareExchangeT(&m_asyncContinuationsManager, newManager, NULL) != NULL) + { + // some thread swooped in and set the field + delete newManager; + } + } + _ASSERTE(m_asyncContinuationsManager != NULL); + return m_asyncContinuationsManager; +} + void LoaderAllocator::AllocateBytesForStaticVariables(DynamicStaticsInfo* pStaticsInfo, uint32_t cbMem, bool isClassInitedByUpdatingStaticPointer) { CONTRACTL diff --git a/src/coreclr/vm/loaderallocator.hpp b/src/coreclr/vm/loaderallocator.hpp index eaf327588f9a99..52e54d98d42a88 100644 --- a/src/coreclr/vm/loaderallocator.hpp +++ b/src/coreclr/vm/loaderallocator.hpp @@ -19,6 +19,7 @@ class FuncPtrStubs; #include "qcall.h" #include "ilstubcache.h" +#include "asynccontinuations.h" #include "callcounting.h" #include "methoddescbackpatchinfo.h" #include "crossloaderallocatorhash.h" @@ -484,6 +485,8 @@ class LoaderAllocator PTR_OnStackReplacementManager m_onStackReplacementManager; #endif + PTR_AsyncContinuationsManager m_asyncContinuationsManager; + #ifndef DACCESS_COMPILE public: @@ -894,6 +897,8 @@ class LoaderAllocator PTR_OnStackReplacementManager GetOnStackReplacementManager(); #endif // FEATURE_ON_STACK_REPLACEMENT + PTR_AsyncContinuationsManager GetAsyncContinuationsManager(); + #ifndef DACCESS_COMPILE public: virtual void RegisterDependentHandleToNativeObjectForCleanup(LADependentHandleToNativeObject *dependentHandle) {}; From 82995d586a8121cf33cd4a59b5a843dde5e15b07 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 6 Oct 2025 16:38:40 +0200 Subject: [PATCH 09/48] Use SPC for continuation's module --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 14 ++++++++------ .../CompilerServices/RuntimeHelpers.CoreCLR.cs | 7 ++----- src/coreclr/vm/asynccontinuations.cpp | 7 ++++--- src/coreclr/vm/methodtable.h | 11 +++-------- src/coreclr/vm/methodtable.inl | 7 +++++++ 5 files changed, 24 insertions(+), 22 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 3fc3af40315f15..95908e26ec10a0 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -83,10 +83,10 @@ internal unsafe class Continuation public unsafe object GetContinuationContext() { - // Only the special continuation sub types are expected. - Debug.Assert(GetType() != typeof(Continuation)); - MethodTable* mt = RuntimeHelpers.GetMethodTable(this); + + // Only the special continuation sub types have continuation offsets. + Debug.Assert(mt->IsContinuation); Debug.Assert(mt->ContinuationOffsets->ContinuationContext != uint.MaxValue); ref byte data = ref RuntimeHelpers.GetRawData(this); @@ -96,7 +96,8 @@ public unsafe object GetContinuationContext() public void SetException(Exception ex) { MethodTable* mt = RuntimeHelpers.GetMethodTable(this); - // Only the special continuation sub types are expected. + + // Only the special continuation sub types have continuation offsets. Debug.Assert(mt->IsContinuation); ref byte data = ref RuntimeHelpers.GetRawData(this); Unsafe.As(ref Unsafe.Add(ref data, mt->ContinuationOffsets->Exception)) = ex; @@ -105,8 +106,9 @@ public void SetException(Exception ex) public ref byte GetResultStorageOrNull() { MethodTable* mt = RuntimeHelpers.GetMethodTable(this); - // Only the special continuation sub types are expected. + // Only the special continuation sub types have continuation offsets. Debug.Assert(mt->IsContinuation); + if (mt->ContinuationOffsets->Result == uint.MaxValue) return ref Unsafe.NullRef(); @@ -117,7 +119,7 @@ public ref byte GetResultStorageOrNull() public void SetKeepAlive(object? obj) { MethodTable* mt = RuntimeHelpers.GetMethodTable(this); - // Only the special continuation sub types are expected. + // Only the special continuation sub types have continuation offsets. Debug.Assert(mt->IsContinuation); Debug.Assert(mt->ContinuationOffsets->KeepAlive != uint.MaxValue); diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index ed72de0485148b..fc1a578a60c3d7 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -688,9 +688,8 @@ internal unsafe struct MethodTable [FieldOffset(4)] public uint BaseSize; - [FieldOffset(8)] - public uint Flags2; // See additional native members in methodtable.h, not needed here yet. + // 0x8: m_dwFlags2 (additional flags and token in upper 24 bits) // 0xC: m_wNumVirtuals /// @@ -802,8 +801,6 @@ internal unsafe struct MethodTable | 0x10000000 // enum_flag_IDynamicInterfaceCastable; | 0x00040000; // enum_flag_Category_ValueType - private const uint enum_flag2_IsContinuationType = 0x10; - private const int DebugClassNamePtr = // adjust for debug_m_szClassName #if DEBUG #if TARGET_64BIT @@ -903,7 +900,7 @@ public int MultiDimensionalArrayRank public bool IsArray => (Flags & enum_flag_Category_Array_Mask) == enum_flag_Category_Array; - public bool IsContinuation => (Flags2 & enum_flag2_IsContinuationType) == enum_flag2_IsContinuationType; + public bool IsContinuation => ParentMethodTable == (MethodTable*)typeof(Continuation).TypeHandle.Value; public bool HasInstantiation => (Flags & enum_flag_HasComponentSize) == 0 && (Flags & enum_flag_GenericsMask) != enum_flag_GenericsMask_NonGeneric; diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index d2028a91d55d95..69fbb64b3df7dd 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -47,6 +47,7 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne STANDARD_VM_CONTRACT; MethodTable* pParentClass = CoreLibBinder::GetClass(CLASS__CONTINUATION); + DWORD numVirtuals = pParentClass->GetNumVirtuals(); size_t cbMT = sizeof(MethodTable); @@ -85,12 +86,12 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne allocatedDataOffsets->KeepAlive += startOfDataInInstance; MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); - pMT->SetIsContinuation(allocatedDataOffsets); + pMT->SetParentMethodTable(pParentClass); + pMT->SetContinuationDataOffsets(allocatedDataOffsets); pMT->AllocateAuxiliaryData(m_allocator, asyncMethod->GetLoaderModule(), pamTracker, MethodTableStaticsFlags::None); pMT->SetLoaderAllocator(m_allocator); - pMT->SetModule(asyncMethod->GetModule()); + pMT->SetModule(pParentClass->GetModule()); pMT->SetNumVirtuals(static_cast(numVirtuals)); - pMT->SetParentMethodTable(pParentClass); pMT->SetClass(pParentClass->GetClass()); // TODO: needs its own? pMT->SetBaseSize(OBJECT_BASESIZE + startOfDataInInstance + dataSize); diff --git a/src/coreclr/vm/methodtable.h b/src/coreclr/vm/methodtable.h index 24e8dc0d0b3d83..198f1f14d64284 100644 --- a/src/coreclr/vm/methodtable.h +++ b/src/coreclr/vm/methodtable.h @@ -2788,16 +2788,11 @@ class MethodTable SetFlag(enum_flag_Category_Nullable); } - inline BOOL IsContinuation() - { - LIMITED_METHOD_DAC_CONTRACT; - return GetFlag(enum_flag_IsContinuationType); - } + inline BOOL IsContinuation(); - inline void SetIsContinuation(const CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) + inline void SetContinuationDataOffsets(const CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) { LIMITED_METHOD_CONTRACT; - SetFlag(enum_flag_IsContinuationType); m_pContinuationOffsets = dataOffsets; } @@ -3806,7 +3801,7 @@ public : enum_flag_HasDispatchMapSlot = 0x0004, enum_flag_wflags2_unused_2 = 0x0008, - enum_flag_IsContinuationType = 0x0010, + enum_flag_wflags2_unused_3 = 0x0010, enum_flag_IsIntrinsicType = 0x0020, enum_flag_HasCctor = 0x0040, enum_flag_HasVirtualStaticMethods = 0x0080, diff --git a/src/coreclr/vm/methodtable.inl b/src/coreclr/vm/methodtable.inl index 6bdf3aa40a0bc2..098de59cb6aced 100644 --- a/src/coreclr/vm/methodtable.inl +++ b/src/coreclr/vm/methodtable.inl @@ -299,6 +299,13 @@ inline BOOL MethodTable::IsValueType() return GetFlag(enum_flag_Category_ValueType_Mask) == enum_flag_Category_ValueType; } +inline BOOL MethodTable::IsContinuation() +{ + LIMITED_METHOD_DAC_CONTRACT; + + return m_pParentMethodTable == CoreLibBinder::GetClassIfExist(CLASS__CONTINUATION); +} + //========================================================================================== inline DWORD MethodTable::GetRank() { From 38bdf11de345b2414dcea39a17111b47e9f4cb0a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 6 Oct 2025 16:39:04 +0200 Subject: [PATCH 10/48] Remove unnecessary continuation allocation Now that we store directly into the ThunkTask's result we do not need to allocate a tail continuation at all. --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 60 ++++++++----------- src/coreclr/vm/asyncthunks.cpp | 7 +-- .../Runtime/CompilerServices/AsyncHelpers.cs | 16 ++--- 3 files changed, 35 insertions(+), 48 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 95908e26ec10a0..39d76584849c56 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -140,7 +140,7 @@ public static partial class AsyncHelpers // Methods like FinalizeTaskReturningThunk will unlink the state and wrap into a Task. private struct RuntimeAsyncAwaitState { - public Continuation? SentinelContinuation; + public Continuation? HeadContinuation; public INotifyCompletion? Notifier; } @@ -316,15 +316,14 @@ public static unsafe void MoveNext(T task) where T : Task where TOps : { ExecutionAndSyncBlockStore contexts = default; contexts.Push(); - Continuation continuation = TOps.GetContinuationState(task); + Continuation? continuation = TOps.GetContinuationState(task); while (true) { + Debug.Assert(continuation != null); try { - Debug.Assert(continuation.Next != null); - - ref byte resultLoc = ref continuation.Next.Resume != null ? ref continuation.Next.GetResultStorageOrNull() : ref TOps.GetResultStorage(task); + ref byte resultLoc = ref continuation.Next != null ? ref continuation.Next.GetResultStorageOrNull() : ref TOps.GetResultStorage(task); Continuation? newContinuation = continuation.Resume(continuation, ref resultLoc); if (newContinuation != null) @@ -339,8 +338,9 @@ public static unsafe void MoveNext(T task) where T : Task where TOps : } catch (Exception ex) { - Continuation nextContinuation = UnwindToPossibleHandler(continuation); - if (nextContinuation.Resume == null) + Debug.Assert(continuation != null); + Continuation? nextContinuation = UnwindToPossibleHandler(continuation); + if (nextContinuation == null) { // Tail of AsyncTaskMethodBuilderT.SetException bool successfullySet = ex is OperationCanceledException oce ? @@ -362,7 +362,7 @@ public static unsafe void MoveNext(T task) where T : Task where TOps : continuation = nextContinuation; } - if (continuation.Resume == null) + if (continuation == null) { bool successfullySet = TOps.SetCompleted(task); @@ -384,14 +384,18 @@ public static unsafe void MoveNext(T task) where T : Task where TOps : } } - private static Continuation UnwindToPossibleHandler(Continuation continuation) + private static Continuation? UnwindToPossibleHandler(Continuation continuation) { while (true) { - Debug.Assert(continuation.Next != null); - continuation = continuation.Next; - if ((continuation.Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION) != 0) - return continuation; + Continuation? nextContinuation = continuation.Next; + if (nextContinuation == null) + return null; + + if ((nextContinuation.Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION) != 0) + return nextContinuation; + + continuation = nextContinuation; } } @@ -433,9 +437,9 @@ private static Continuation UnlinkHeadContinuation(out INotifyCompletion? notifi notifier = state.Notifier; state.Notifier = null; - Continuation sentinelContinuation = state.SentinelContinuation!; - Continuation head = sentinelContinuation.Next!; - sentinelContinuation.Next = null; + Continuation headContinuation = state.HeadContinuation!; + Continuation head = headContinuation.Next!; + headContinuation.Next = null; return head; } @@ -507,42 +511,30 @@ private static bool QueueContinuationFollowUpActionIfNecessary(T task, #pragma warning disable CA1859 // When a Task-returning thunk gets a continuation result // it calls here to make a Task that awaits on the current async state. - private static Task FinalizeTaskReturningThunk(Continuation continuation) + private static Task FinalizeTaskReturningThunk() { - Continuation finalContinuation = new Continuation - { - Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION, - }; - continuation.Next = finalContinuation; - ThunkTask result = new(); result.HandleSuspended(); return result; } - private static Task FinalizeTaskReturningThunk(Continuation continuation) + private static Task FinalizeTaskReturningThunk() { - Continuation finalContinuation = new Continuation - { - Flags = CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION, - }; - continuation.Next = finalContinuation; - ThunkTask result = new(); result.HandleSuspended(); return result; } - private static ValueTask FinalizeValueTaskReturningThunk(Continuation continuation) + private static ValueTask FinalizeValueTaskReturningThunk() { // We only come to these methods in the expensive case (already // suspended), so ValueTask optimization here is not relevant. - return new ValueTask(FinalizeTaskReturningThunk(continuation)); + return new ValueTask(FinalizeTaskReturningThunk()); } - private static ValueTask FinalizeValueTaskReturningThunk(Continuation continuation) + private static ValueTask FinalizeValueTaskReturningThunk() { - return new ValueTask(FinalizeTaskReturningThunk(continuation)); + return new ValueTask(FinalizeTaskReturningThunk()); } [MethodImpl(MethodImplOptions.AggressiveInlining)] diff --git a/src/coreclr/vm/asyncthunks.cpp b/src/coreclr/vm/asyncthunks.cpp index bf2a385cd6d8ec..de6aac7509e72c 100644 --- a/src/coreclr/vm/asyncthunks.cpp +++ b/src/coreclr/vm/asyncthunks.cpp @@ -64,8 +64,6 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncOtherVariant, MetaSig& ILCodeStream* pCode = pSL->NewCodeStream(ILStubLinker::kDispatch); - unsigned continuationLocal = pCode->NewLocal(LocalDesc(CoreLibBinder::GetClass(CLASS__CONTINUATION))); - TypeHandle thTaskRet = thunkMsig.GetRetTypeHandleThrowing(); bool isValueTask = thTaskRet.GetMethodTable()->IsValueType(); @@ -167,8 +165,6 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncOtherVariant, MetaSig& if (logicalResultLocal != UINT_MAX) pCode->EmitSTLOC(logicalResultLocal); pCode->EmitCALL(METHOD__STUBHELPERS__ASYNC_CALL_CONTINUATION, 0, 1); - pCode->EmitSTLOC(continuationLocal); - pCode->EmitLDLOC(continuationLocal); pCode->EmitBRFALSE(finishedLabel); pCode->EmitLEAVE(suspendedLabel); @@ -255,8 +251,7 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncOtherVariant, MetaSig& md = CoreLibBinder::GetMethod(METHOD__ASYNC_HELPERS__FINALIZE_TASK_RETURNING_THUNK); finalizeTaskReturningThunkToken = pCode->GetToken(md); } - pCode->EmitLDLOC(continuationLocal); - pCode->EmitCALL(finalizeTaskReturningThunkToken, 1, 1); + pCode->EmitCALL(finalizeTaskReturningThunkToken, 0, 1); pCode->EmitSTLOC(returnTaskLocal); pCode->EmitLEAVE(returnTaskLabel); diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs index 7b2a94ce56b175..5531cb47467b59 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs @@ -24,12 +24,12 @@ public static partial class AsyncHelpers public static void AwaitAwaiter(TAwaiter awaiter) where TAwaiter : INotifyCompletion { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - Continuation? sentinelContinuation = state.SentinelContinuation; - if (sentinelContinuation == null) - state.SentinelContinuation = sentinelContinuation = new Continuation(); + Continuation? headContinuations = state.HeadContinuation; + if (headContinuations == null) + state.HeadContinuation = headContinuations = new Continuation(); state.Notifier = awaiter; - AsyncSuspend(sentinelContinuation); + AsyncSuspend(headContinuations); } // Must be NoInlining because we use AsyncSuspend to manufacture an explicit suspension point. @@ -40,12 +40,12 @@ public static void AwaitAwaiter(TAwaiter awaiter) where TAwaiter : INo public static void UnsafeAwaitAwaiter(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - Continuation? sentinelContinuation = state.SentinelContinuation; - if (sentinelContinuation == null) - state.SentinelContinuation = sentinelContinuation = new Continuation(); + Continuation? headContinuation = state.HeadContinuation; + if (headContinuation == null) + state.HeadContinuation = headContinuation = new Continuation(); state.Notifier = awaiter; - AsyncSuspend(sentinelContinuation); + AsyncSuspend(headContinuation); } [Intrinsic] From 97de35e37938f1e632faaf7250f48a1ad73e2dc3 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Mon, 6 Oct 2025 17:32:41 +0200 Subject: [PATCH 11/48] Fix some bugs --- src/coreclr/vm/asynccontinuations.cpp | 2 +- src/coreclr/vm/corelib.h | 8 ++++---- src/coreclr/vm/metasig.h | 6 ++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 69fbb64b3df7dd..db8b8c24a3485e 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -86,9 +86,9 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne allocatedDataOffsets->KeepAlive += startOfDataInInstance; MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); + pMT->AllocateAuxiliaryData(m_allocator, asyncMethod->GetLoaderModule(), pamTracker, MethodTableStaticsFlags::None); pMT->SetParentMethodTable(pParentClass); pMT->SetContinuationDataOffsets(allocatedDataOffsets); - pMT->AllocateAuxiliaryData(m_allocator, asyncMethod->GetLoaderModule(), pamTracker, MethodTableStaticsFlags::None); pMT->SetLoaderAllocator(m_allocator); pMT->SetModule(pParentClass->GetModule()); pMT->SetNumVirtuals(static_cast(numVirtuals)); diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index 27856ea277f2bc..91f010a9a85ece 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -731,10 +731,10 @@ DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION, AllocContinuation, DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION_METHOD, AllocContinuationMethod, NoSig) DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION_CLASS, AllocContinuationClass, NoSig) DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION_RESULT_BOX, AllocContinuationResultBox, SM_VoidPtr_RetObj) -DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK, FinalizeTaskReturningThunk, SM_Continuation_RetTask) -DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK_1, FinalizeTaskReturningThunk, GM_Continuation_RetTaskOfT) -DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK, FinalizeValueTaskReturningThunk, SM_Continuation_RetValueTask) -DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK_1, FinalizeValueTaskReturningThunk, GM_Continuation_RetValueTaskOfT) +DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK, FinalizeTaskReturningThunk, SM_RetTask) +DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK_1, FinalizeTaskReturningThunk, GM_RetTaskOfT) +DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK, FinalizeValueTaskReturningThunk, SM_RetValueTask) +DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_VALUETASK_RETURNING_THUNK_1, FinalizeValueTaskReturningThunk, GM_RetValueTaskOfT) DEFINE_METHOD(ASYNC_HELPERS, UNSAFE_AWAIT_AWAITER_1, UnsafeAwaitAwaiter, GM_T_RetVoid) DEFINE_METHOD(ASYNC_HELPERS, CAPTURE_EXECUTION_CONTEXT, CaptureExecutionContext, NoSig) DEFINE_METHOD(ASYNC_HELPERS, RESTORE_EXECUTION_CONTEXT, RestoreExecutionContext, NoSig) diff --git a/src/coreclr/vm/metasig.h b/src/coreclr/vm/metasig.h index ddb9b5ea403e63..efe0cc7595e120 100644 --- a/src/coreclr/vm/metasig.h +++ b/src/coreclr/vm/metasig.h @@ -625,10 +625,8 @@ DEFINE_METASIG(SM(PtrByte_RetVoid, P(b), v)) DEFINE_METASIG_T(SM(RetContinuation, , C(CONTINUATION))) DEFINE_METASIG(GM(T_RetVoid, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, M(0), v)) -DEFINE_METASIG_T(SM(Continuation_RetTask, C(CONTINUATION), C(TASK))) -DEFINE_METASIG_T(GM(Continuation_RetTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, C(CONTINUATION), GI(C(TASK_1), 1, M(0)))) -DEFINE_METASIG_T(SM(Continuation_RetValueTask, C(CONTINUATION), g(VALUETASK))) -DEFINE_METASIG_T(GM(Continuation_RetValueTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, C(CONTINUATION), GI(g(VALUETASK_1), 1, M(0)))) +DEFINE_METASIG_T(GM(RetTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, , GI(C(TASK_1), 1, M(0)))) +DEFINE_METASIG_T(GM(RetValueTaskOfT, IMAGE_CEE_CS_CALLCONV_DEFAULT, 1, , GI(g(VALUETASK_1), 1, M(0)))) // Undefine macros in case we include the file again in the compilation unit From fb296e3505c59ebf1a837e82f7772a46ac3293f7 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Oct 2025 16:40:11 +0200 Subject: [PATCH 12/48] Fix type GC info, add debug name, remove comments --- src/coreclr/jit/async.cpp | 29 ++++++++++++++++++- src/coreclr/vm/asynccontinuations.cpp | 41 +++++++++++++++++++-------- src/coreclr/vm/methodtable.cpp | 2 +- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index b9b3ae8898db52..945f80c69c2d75 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1078,13 +1078,15 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* #endif // Now create continuation type. First create bitmap of object refs. - bool* objRefs = new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE]; + bool* objRefs = new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE] {}; + unsigned numObjRefs = 0; for (LiveLocalInfo& inf : liveLocals) { LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); if (dsc->TypeIs(TYP_REF)) { objRefs[inf.Offset / TARGET_POINTER_SIZE] = true; + numObjRefs++; } else if (dsc->TypeIs(TYP_STRUCT)) { @@ -1095,11 +1097,36 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* if (layout->GetGCPtrType(slot) == TYP_REF) { objRefs[inf.Offset / TARGET_POINTER_SIZE + slot] = true; + numObjRefs++; } } } } +#ifdef DEBUG + if (m_comp->verbose) + { + printf("Getting continuation layout size = %u, numGCRefs = %u (Continuation_%s_%u_%u)\n", layout.Size, numObjRefs, m_comp->info.compMethodName, layout.Size, numObjRefs); + bool* start = objRefs; + bool* endOfObjRefs = objRefs + layout.Size / TARGET_POINTER_SIZE; + while (start < endOfObjRefs) + { + while (start < endOfObjRefs && !*start) + start++; + + if (start >= endOfObjRefs) + break; + + bool* end = start; + while (end < endOfObjRefs && *end) + end++; + + printf(" [%3u..%3u) obj refs\n", (start - objRefs) * TARGET_POINTER_SIZE, (end - objRefs) * TARGET_POINTER_SIZE); + start = end; + } + } +#endif + CORINFO_CONTINUATION_DATA_OFFSETS offsets; offsets.Result = layout.ReturnValOffset; offsets.Exception = layout.ExceptionOffset; diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index db8b8c24a3485e..9f282c6cdd4bc1 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -37,7 +37,7 @@ static void EnumerateRunsOfObjRefs(unsigned size, const bool* objRefs, TFunc fun while (end < objRefsEnd && *end) end++; - func((start - objRefs) * TARGET_POINTER_SIZE, (end - start) * TARGET_POINTER_SIZE); + func((start - objRefs) * TARGET_POINTER_SIZE, end - start); start = end; } } @@ -92,7 +92,7 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne pMT->SetLoaderAllocator(m_allocator); pMT->SetModule(pParentClass->GetModule()); pMT->SetNumVirtuals(static_cast(numVirtuals)); - pMT->SetClass(pParentClass->GetClass()); // TODO: needs its own? + pMT->SetClass(pParentClass->GetClass()); // EEClass of System.Runtime.CompilerServices.Continuation pMT->SetBaseSize(OBJECT_BASESIZE + startOfDataInInstance + dataSize); if (numPointerSeries > 0) @@ -101,24 +101,41 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne CGCDesc::Init(pMT, numPointerSeries); CGCDescSeries* pSeries = CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries(); + // Write GC runs. In memory they must be descending by offset, so we + // enumerate forwards but write them from the end. + unsigned curIndex = numPointerSeries; CGCDescSeries* pParentSeries = CGCDesc::GetCGCDescFromMT(pParentClass)->GetLowestSeries(); - for (unsigned i = 0; i < numParentPointerSeries; i++) + for (unsigned i = numParentPointerSeries; i--;) { - pSeries->SetSeriesSize((pParentSeries->GetSeriesSize() + pParentClass->GetBaseSize()) - pMT->GetBaseSize()); - pSeries->SetSeriesOffset(pParentSeries->GetSeriesOffset()); - pSeries++; - pParentSeries++; + curIndex--; + pSeries[curIndex].SetSeriesSize((pParentSeries[i].GetSeriesSize() + pParentClass->GetBaseSize()) - pMT->GetBaseSize()); + pSeries[curIndex].SetSeriesOffset(pParentSeries[i].GetSeriesOffset()); } - auto writeSeries = [&](size_t start, size_t length) { - pSeries->SetSeriesSize(length - pMT->GetBaseSize()); - pSeries->SetSeriesOffset(startOfDataInObject + start); - pSeries++; + auto writeSeries = [&](size_t start, size_t count) { + curIndex--; + pSeries[curIndex].SetSeriesSize((count * TARGET_POINTER_SIZE) - pMT->GetBaseSize()); + pSeries[curIndex].SetSeriesOffset(startOfDataInObject + start); }; EnumerateRunsOfObjRefs(dataSize, objRefs, writeSeries); - _ASSERTE(pSeries == CGCDesc::GetCGCDescFromMT(pMT)->GetLowestSeries() + numPointerSeries); + + _ASSERTE(curIndex == 0); } +#ifdef DEBUG + size_t numObjRefs = 0; + EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) { + numObjRefs += count; + }); + StackSString debugName; + debugName.Printf("Continuation_%s_%u_%zu", asyncMethod->m_pszDebugMethodName, dataSize, numObjRefs); + const char* debugNameUTF8 = debugName.GetUTF8(); + size_t len = strlen(debugNameUTF8) + 1; + char* name = (char*)pamTracker->Track(m_allocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(len))); + strcpy_s(name, len, debugNameUTF8); + pMT->SetDebugClassName(name); +#endif + pMT->SetClassInited(); return pMT; diff --git a/src/coreclr/vm/methodtable.cpp b/src/coreclr/vm/methodtable.cpp index 7b3373e3ce649a..8c3e61466776b2 100644 --- a/src/coreclr/vm/methodtable.cpp +++ b/src/coreclr/vm/methodtable.cpp @@ -6245,7 +6245,7 @@ BOOL MethodTable::SanityCheck() if (GetNumGenericArgs() != 0) return (pCanonMT->GetClass() == pClass); else - return (pCanonMT == this) || IsArray() || IsContinuation(); // TODO -- allocate an EEClass instead? + return (pCanonMT == this) || IsArray() || IsContinuation(); } From a3e223fae8d39b5981d545ef63bfdb7c5badf341 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Oct 2025 17:19:09 +0200 Subject: [PATCH 13/48] Refactor building of obj refs bitmap into helper class --- src/coreclr/jit/async.cpp | 86 ++++++++++++++++++++++++++++++--------- 1 file changed, 66 insertions(+), 20 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 945f80c69c2d75..3f1e388d8c6105 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -931,6 +931,58 @@ void AsyncTransformation::LiftLIREdges(BasicBlock* block, } } +class GCPointerBitMapBuilder +{ + bool* m_objRefs; + size_t m_size; + +public: + GCPointerBitMapBuilder(bool* objRefs, size_t size) + : m_objRefs(objRefs) + , m_size(size) + { + } + + INDEBUG(unsigned NumObjRefs = 0); + + void Set(unsigned offset) + { + assert((offset % TARGET_POINTER_SIZE) == 0); + assert(offset < m_size); + unsigned slot = offset / TARGET_POINTER_SIZE; + + // We do not expect to set the same offset multiple times + assert(!m_objRefs[slot]); + m_objRefs[slot] = true; + } + + void SetIfNotMax(unsigned offset) + { + if (offset != UINT_MAX) + { + Set(offset); + } + } + + void SetType(unsigned offset, var_types type, ClassLayout* layout) + { + if (type == TYP_REF) + { + Set(offset); + } + else if (type == TYP_STRUCT) + { + for (unsigned slot = 0; slot < layout->GetSlotCount(); slot++) + { + if (layout->GetGCPtrType(slot) == TYP_REF) + { + Set(offset + slot * TARGET_POINTER_SIZE); + } + } + } + } +}; + //------------------------------------------------------------------------ // AsyncTransformation::LayOutContinuation: // Create the layout of the GC pointer and data arrays in the continuation @@ -1079,34 +1131,28 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* // Now create continuation type. First create bitmap of object refs. bool* objRefs = new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE] {}; - unsigned numObjRefs = 0; + + GCPointerBitMapBuilder bitmapBuilder(objRefs, layout.Size); + bitmapBuilder.SetIfNotMax(layout.ExceptionOffset); + bitmapBuilder.SetIfNotMax(layout.ContinuationContextOffset); + bitmapBuilder.SetIfNotMax(layout.ExecContextOffset); + bitmapBuilder.SetIfNotMax(keepAliveOffset); + + if (layout.ReturnSize > 0) + { + bitmapBuilder.SetType(layout.ReturnValOffset, call->gtReturnType, layout.ReturnStructLayout); + } + for (LiveLocalInfo& inf : liveLocals) { LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); - if (dsc->TypeIs(TYP_REF)) - { - objRefs[inf.Offset / TARGET_POINTER_SIZE] = true; - numObjRefs++; - } - else if (dsc->TypeIs(TYP_STRUCT)) - { - ClassLayout* layout = dsc->GetLayout(); - - for (unsigned slot = 0; slot < layout->GetSlotCount(); slot++) - { - if (layout->GetGCPtrType(slot) == TYP_REF) - { - objRefs[inf.Offset / TARGET_POINTER_SIZE + slot] = true; - numObjRefs++; - } - } - } + bitmapBuilder.SetType(inf.Offset, dsc->TypeGet(), dsc->TypeIs(TYP_STRUCT) ? dsc->GetLayout() : nullptr); } #ifdef DEBUG if (m_comp->verbose) { - printf("Getting continuation layout size = %u, numGCRefs = %u (Continuation_%s_%u_%u)\n", layout.Size, numObjRefs, m_comp->info.compMethodName, layout.Size, numObjRefs); + printf("Getting continuation layout size = %u, numGCRefs = %u (Continuation_%s_%u_%u)\n", layout.Size, bitmapBuilder.NumObjRefs, m_comp->info.compMethodName, layout.Size, bitmapBuilder.NumObjRefs); bool* start = objRefs; bool* endOfObjRefs = objRefs + layout.Size / TARGET_POINTER_SIZE; while (start < endOfObjRefs) From 71b47b05ecbd94059ebeac78dc0c6ba0730b4ea7 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Oct 2025 17:23:41 +0200 Subject: [PATCH 14/48] Properly allocate space for KeepAlive object --- src/coreclr/jit/async.cpp | 59 +++++++++++++++++++++++++++++++-------- src/coreclr/jit/async.h | 3 ++ 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 3f1e388d8c6105..f3601a2f6bfe51 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -802,7 +802,7 @@ void AsyncTransformation::Transform( CreateLiveSetForSuspension(block, call, defs, life, liveLocals); - ContinuationLayout layout = LayOutContinuation(block, call, liveLocals); + ContinuationLayout layout = LayOutContinuation(block, call, ContinuationNeedsKeepAlive(life), liveLocals); ClearSuspendedIndicator(block, call); @@ -931,9 +931,36 @@ void AsyncTransformation::LiftLIREdges(BasicBlock* block, } } +//------------------------------------------------------------------------ +// AsyncTransformation::ContinuationNeedsKeepAlive: +// Check whether we need to allocate a "KeepAlive" field in the continuation. +// +// Parameters: +// life - Live locals +// +// Returns: +// True if we need to keep a LoaderAllocator for generic context or +// collectible method alive. +// +bool AsyncTransformation::ContinuationNeedsKeepAlive(AsyncLiveness& life) +{ + if (m_asyncInfo->continuationsNeedMethodHandle) + { + return true; + } + + const unsigned GENERICS_CTXT_FROM = CORINFO_GENERICS_CTXT_FROM_METHODDESC | CORINFO_GENERICS_CTXT_FROM_METHODTABLE; + if (((m_comp->info.compMethodInfo->options & GENERICS_CTXT_FROM) != 0) && life.IsLive(m_comp->info.compTypeCtxtArg)) + { + return true; + } + + return false; +} + class GCPointerBitMapBuilder { - bool* m_objRefs; + bool* m_objRefs; size_t m_size; public: @@ -989,16 +1016,18 @@ class GCPointerBitMapBuilder // object. // // Parameters: -// block - The block containing the async call -// call - The async call -// liveLocals - [in, out] Information about each live local. Size/alignment -// information is read and offset/index information is written. +// block - The block containing the async call +// call - The async call +// needsKeepAlive - Whether the layout needs a "keep alive" field allocated +// liveLocals - [in, out] Information about each live local. Size/alignment +// information is read and offset/index information is written. // // Returns: // Layout information. // ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* block, GenTreeCall* call, + bool needsKeepAlive, jitstd::vector& liveLocals) { ContinuationLayout layout(liveLocals); @@ -1111,6 +1140,12 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* layout.ExecContextOffset); } + unsigned keepAliveOffset = UINT_MAX; + if (needsKeepAlive) + { + keepAliveOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); + } + for (LiveLocalInfo& inf : liveLocals) { inf.Offset = allocLayout(inf.Alignment, inf.Size); @@ -1130,7 +1165,7 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* #endif // Now create continuation type. First create bitmap of object refs. - bool* objRefs = new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE] {}; + bool* objRefs = new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE]{}; GCPointerBitMapBuilder bitmapBuilder(objRefs, layout.Size); bitmapBuilder.SetIfNotMax(layout.ExceptionOffset); @@ -1152,8 +1187,9 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* #ifdef DEBUG if (m_comp->verbose) { - printf("Getting continuation layout size = %u, numGCRefs = %u (Continuation_%s_%u_%u)\n", layout.Size, bitmapBuilder.NumObjRefs, m_comp->info.compMethodName, layout.Size, bitmapBuilder.NumObjRefs); - bool* start = objRefs; + printf("Getting continuation layout size = %u, numGCRefs = %u (Continuation_%s_%u_%u)\n", layout.Size, + bitmapBuilder.NumObjRefs, m_comp->info.compMethodName, layout.Size, bitmapBuilder.NumObjRefs); + bool* start = objRefs; bool* endOfObjRefs = objRefs + layout.Size / TARGET_POINTER_SIZE; while (start < endOfObjRefs) { @@ -1167,7 +1203,8 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* while (end < endOfObjRefs && *end) end++; - printf(" [%3u..%3u) obj refs\n", (start - objRefs) * TARGET_POINTER_SIZE, (end - objRefs) * TARGET_POINTER_SIZE); + printf(" [%3u..%3u) obj refs\n", (start - objRefs) * TARGET_POINTER_SIZE, + (end - objRefs) * TARGET_POINTER_SIZE); start = end; } } @@ -1177,7 +1214,7 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* offsets.Result = layout.ReturnValOffset; offsets.Exception = layout.ExceptionOffset; offsets.ContinuationContext = layout.ContinuationContextOffset; - offsets.KeepAlive = UINT_MAX; + offsets.KeepAlive = keepAliveOffset; layout.ClassHnd = m_comp->info.compCompHnd->getContinuationType(layout.Size, objRefs, offsets); return layout; diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index c70bbc5199fef3..ce146d27700bc7 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -78,8 +78,11 @@ class AsyncTransformation const jitstd::vector& defs, jitstd::vector& liveLocals); + bool ContinuationNeedsKeepAlive(class AsyncLiveness& life); + ContinuationLayout LayOutContinuation(BasicBlock* block, GenTreeCall* call, + bool needsKeepAlive, jitstd::vector& liveLocals); void ClearSuspendedIndicator(BasicBlock* block, GenTreeCall* call); From badee7db393196f541234c3e6bcfbe84f282fdbe Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Oct 2025 18:04:37 +0200 Subject: [PATCH 15/48] Fix bitmap building for implicit byrefs --- src/coreclr/jit/async.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index f3601a2f6bfe51..2d831f547bc680 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1181,7 +1181,19 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* for (LiveLocalInfo& inf : liveLocals) { LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); - bitmapBuilder.SetType(inf.Offset, dsc->TypeGet(), dsc->TypeIs(TYP_STRUCT) ? dsc->GetLayout() : nullptr); + var_types storedType; + ClassLayout* layout; + if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) + { + storedType = TYP_STRUCT; + layout = dsc->GetLayout(); + } + else + { + storedType = dsc->TypeGet(); + layout = NULL; + } + bitmapBuilder.SetType(inf.Offset, storedType, layout); } #ifdef DEBUG From 56c403ab10e4f8970533cb365d8e4ddfc2b2f772 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 7 Oct 2025 18:48:34 +0200 Subject: [PATCH 16/48] Run jit-format, remove copilot comment --- src/coreclr/jit/async.cpp | 8 ++++---- src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs | 2 -- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 2d831f547bc680..550a1a26994eae 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1180,18 +1180,18 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* for (LiveLocalInfo& inf : liveLocals) { - LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); - var_types storedType; + LclVarDsc* dsc = m_comp->lvaGetDesc(inf.LclNum); + var_types storedType; ClassLayout* layout; if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) { storedType = TYP_STRUCT; - layout = dsc->GetLayout(); + layout = dsc->GetLayout(); } else { storedType = dsc->TypeGet(); - layout = NULL; + layout = NULL; } bitmapBuilder.SetType(inf.Offset, storedType, layout); } diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index ebb475785662c2..37a8771adb2a8b 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -3363,8 +3363,6 @@ private void getAsyncInfo(ref CORINFO_ASYNC_INFO pAsyncInfoOut) private CORINFO_CLASS_STRUCT_* getContinuationType(UIntPtr dataSize, ref bool objRefs, ref CORINFO_CONTINUATION_DATA_OFFSETS dataOffsets) { - // Hint for the developer: Use CorInfoImpl.RyuJit.cs and CorInfoImpl.ReadyToRun.cs if the implementation - // is not shared for NativeAOT and R2R. throw new NotImplementedException("getContinuationType"); } From ca7ef699eb3024ab0d4a248c7a6e5e2e8ac548df Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Oct 2025 14:37:52 +0200 Subject: [PATCH 17/48] Move Continuation MT into a static --- src/coreclr/inc/dacvars.h | 2 ++ src/coreclr/vm/asynccontinuations.cpp | 5 +++++ src/coreclr/vm/methodtable.inl | 3 ++- src/coreclr/vm/vars.cpp | 6 ++++++ src/coreclr/vm/vars.hpp | 6 ++++++ 5 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/coreclr/inc/dacvars.h b/src/coreclr/inc/dacvars.h index 151335e6daa60b..0be37f0098163a 100644 --- a/src/coreclr/inc/dacvars.h +++ b/src/coreclr/inc/dacvars.h @@ -175,6 +175,8 @@ DEFINE_DACVAR(UNKNOWN_POINTER_TYPE, dac__g_TypedReferenceMT, ::g_TypedReferenceM DEFINE_DACVAR(UNKNOWN_POINTER_TYPE, dac__g_pWeakReferenceClass, ::g_pWeakReferenceClass) DEFINE_DACVAR(UNKNOWN_POINTER_TYPE, dac__g_pWeakReferenceOfTClass, ::g_pWeakReferenceOfTClass) +DEFINE_DACVAR_VOLATILE(UNKNOWN_POINTER_TYPE, dac__g_pContinuationClassIfSubTypeCreated, ::g_pContinuationClassIfSubTypeCreated) + #ifdef FEATURE_COMINTEROP DEFINE_DACVAR(UNKNOWN_POINTER_TYPE, dac__g_pBaseCOMObject, ::g_pBaseCOMObject) #endif diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 9f282c6cdd4bc1..790c23f9f0185e 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -48,6 +48,11 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne MethodTable* pParentClass = CoreLibBinder::GetClass(CLASS__CONTINUATION); + if (g_pContinuationClassIfSubTypeCreated.Load() == NULL) + { + g_pContinuationClassIfSubTypeCreated.Store(pParentClass); + } + DWORD numVirtuals = pParentClass->GetNumVirtuals(); size_t cbMT = sizeof(MethodTable); diff --git a/src/coreclr/vm/methodtable.inl b/src/coreclr/vm/methodtable.inl index 098de59cb6aced..d24f788bbad5bc 100644 --- a/src/coreclr/vm/methodtable.inl +++ b/src/coreclr/vm/methodtable.inl @@ -303,7 +303,8 @@ inline BOOL MethodTable::IsContinuation() { LIMITED_METHOD_DAC_CONTRACT; - return m_pParentMethodTable == CoreLibBinder::GetClassIfExist(CLASS__CONTINUATION); + PTR_MethodTable contClass = g_pContinuationClassIfSubTypeCreated; + return contClass != NULL && m_pParentMethodTable == contClass; } //========================================================================================== diff --git a/src/coreclr/vm/vars.cpp b/src/coreclr/vm/vars.cpp index e5d1004ad97997..c5546edcb2c487 100644 --- a/src/coreclr/vm/vars.cpp +++ b/src/coreclr/vm/vars.cpp @@ -67,6 +67,12 @@ GPTR_IMPL(MethodTable, g_TypedReferenceMT); GPTR_IMPL(MethodTable, g_pWeakReferenceClass); GPTR_IMPL(MethodTable, g_pWeakReferenceOfTClass); +#ifdef DACCESS_COMPILE +GPTR_IMPL(MethodTable, g_pContinuationClassIfSubTypeCreated); +#else +GVAL_IMPL(Volatile, g_pContinuationClassIfSubTypeCreated); +#endif + #ifdef FEATURE_COMINTEROP GPTR_IMPL(MethodTable, g_pBaseCOMObject); #endif diff --git a/src/coreclr/vm/vars.hpp b/src/coreclr/vm/vars.hpp index 4ef7f2fbfcd0a6..08988b342e784b 100644 --- a/src/coreclr/vm/vars.hpp +++ b/src/coreclr/vm/vars.hpp @@ -345,6 +345,12 @@ GPTR_DECL(MethodTable, g_TypedReferenceMT); GPTR_DECL(MethodTable, g_pWeakReferenceClass); GPTR_DECL(MethodTable, g_pWeakReferenceOfTClass); +#ifdef DACCESS_COMPILE +GPTR_DECL(MethodTable, g_pContinuationClassIfSubTypeCreated); +#else +GVAL_DECL(Volatile, g_pContinuationClassIfSubTypeCreated); +#endif + #ifdef FEATURE_COMINTEROP GPTR_DECL(MethodTable, g_pBaseCOMObject); #endif From 9d6d275c3e6c0945a09f9746038f6984304bcfa3 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Oct 2025 14:46:23 +0200 Subject: [PATCH 18/48] Remove added newline --- src/coreclr/vm/jitinterface.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index a500db11903fb9..611223aa15ec63 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -12412,7 +12412,6 @@ CORINFO_CLASS_HANDLE CEECodeGenInfo::getStaticFieldCurrentClass(CORINFO_FIELD_HA // TODO: Check if the jit is allowed to embed this handle in jitted code. // Note for the initonly cases it probably won't embed. - result = (CORINFO_CLASS_HANDLE) pObjMT; } } From 04cd9f4117ce4cc19c1b4ed13fbe175d9ab8173d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Oct 2025 14:48:06 +0200 Subject: [PATCH 19/48] Undo another change --- src/coreclr/vm/methodtable.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/methodtable.h b/src/coreclr/vm/methodtable.h index 198f1f14d64284..7a086535b9b3a9 100644 --- a/src/coreclr/vm/methodtable.h +++ b/src/coreclr/vm/methodtable.h @@ -3801,7 +3801,7 @@ public : enum_flag_HasDispatchMapSlot = 0x0004, enum_flag_wflags2_unused_2 = 0x0008, - enum_flag_wflags2_unused_3 = 0x0010, + //unused = 0x0010, enum_flag_IsIntrinsicType = 0x0020, enum_flag_HasCctor = 0x0040, enum_flag_HasVirtualStaticMethods = 0x0080, From 2c30029d6584f34052fa3a288cf7617981fbe22d Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Oct 2025 14:50:44 +0200 Subject: [PATCH 20/48] Undo name change --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 8 ++++---- .../Runtime/CompilerServices/AsyncHelpers.cs | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 8d74483f5a4276..f66666738b547e 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -140,7 +140,7 @@ public static partial class AsyncHelpers // Methods like FinalizeTaskReturningThunk will unlink the state and wrap into a Task. private struct RuntimeAsyncAwaitState { - public Continuation? HeadContinuation; + public Continuation? SentinelContinuation; public INotifyCompletion? Notifier; } @@ -437,9 +437,9 @@ private static Continuation UnlinkHeadContinuation(out INotifyCompletion? notifi notifier = state.Notifier; state.Notifier = null; - Continuation headContinuation = state.HeadContinuation!; - Continuation head = headContinuation.Next!; - headContinuation.Next = null; + Continuation sentinelContinuation = state.SentinelContinuation!; + Continuation head = sentinelContinuation.Next!; + sentinelContinuation.Next = null; return head; } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs index 5531cb47467b59..7b2a94ce56b175 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.cs @@ -24,12 +24,12 @@ public static partial class AsyncHelpers public static void AwaitAwaiter(TAwaiter awaiter) where TAwaiter : INotifyCompletion { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - Continuation? headContinuations = state.HeadContinuation; - if (headContinuations == null) - state.HeadContinuation = headContinuations = new Continuation(); + Continuation? sentinelContinuation = state.SentinelContinuation; + if (sentinelContinuation == null) + state.SentinelContinuation = sentinelContinuation = new Continuation(); state.Notifier = awaiter; - AsyncSuspend(headContinuations); + AsyncSuspend(sentinelContinuation); } // Must be NoInlining because we use AsyncSuspend to manufacture an explicit suspension point. @@ -40,12 +40,12 @@ public static void AwaitAwaiter(TAwaiter awaiter) where TAwaiter : INo public static void UnsafeAwaitAwaiter(TAwaiter awaiter) where TAwaiter : ICriticalNotifyCompletion { ref RuntimeAsyncAwaitState state = ref t_runtimeAsyncAwaitState; - Continuation? headContinuation = state.HeadContinuation; - if (headContinuation == null) - state.HeadContinuation = headContinuation = new Continuation(); + Continuation? sentinelContinuation = state.SentinelContinuation; + if (sentinelContinuation == null) + state.SentinelContinuation = sentinelContinuation = new Continuation(); state.Notifier = awaiter; - AsyncSuspend(headContinuation); + AsyncSuspend(sentinelContinuation); } [Intrinsic] From 21ab2fc0302e5c729f0eb38c38ba0eca06ba8445 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Oct 2025 16:52:29 +0200 Subject: [PATCH 21/48] Add a cache for created layout types --- src/coreclr/vm/asynccontinuations.cpp | 202 +++++++++++++++++++++++++- src/coreclr/vm/asynccontinuations.h | 39 ++++- src/coreclr/vm/jitinterface.cpp | 11 +- 3 files changed, 241 insertions(+), 11 deletions(-) diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 790c23f9f0185e..22e0df339227f4 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -18,10 +18,14 @@ AsyncContinuationsManager::AsyncContinuationsManager(LoaderAllocator* allocator) : m_allocator(allocator) { LIMITED_METHOD_CONTRACT; + + m_layoutsLock.Init(CrstLeafLock); + LockOwner lock = {&m_layoutsLock, IsOwnerOfCrst}; + m_layouts.Init(16, &lock, m_allocator->GetLowFrequencyHeap()); } template -static void EnumerateRunsOfObjRefs(unsigned size, const bool* objRefs, TFunc func) +static bool EnumerateRunsOfObjRefs(unsigned size, const bool* objRefs, TFunc func) { const bool* objRefsEnd = objRefs + (size + (TARGET_POINTER_SIZE - 1)) / TARGET_POINTER_SIZE; const bool* start = objRefs; @@ -31,15 +35,19 @@ static void EnumerateRunsOfObjRefs(unsigned size, const bool* objRefs, TFunc fun start++; if (start >= objRefsEnd) - return; + return true; const bool* end = start; while (end < objRefsEnd && *end) end++; - func((start - objRefs) * TARGET_POINTER_SIZE, end - start); + if (!func((start - objRefs) * TARGET_POINTER_SIZE, end - start)) + return false; + start = end; } + + return true; } MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker) @@ -62,6 +70,7 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) { numObjRefRuns++; + return true; }); unsigned numParentPointerSeries = 0; @@ -121,6 +130,7 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne curIndex--; pSeries[curIndex].SetSeriesSize((count * TARGET_POINTER_SIZE) - pMT->GetBaseSize()); pSeries[curIndex].SetSeriesOffset(startOfDataInObject + start); + return true; }; EnumerateRunsOfObjRefs(dataSize, objRefs, writeSeries); @@ -131,6 +141,7 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne size_t numObjRefs = 0; EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) { numObjRefs += count; + return true; }); StackSString debugName; debugName.Printf("Continuation_%s_%u_%zu", asyncMethod->m_pszDebugMethodName, dataSize, numObjRefs); @@ -146,12 +157,191 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne return pMT; } -MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker) +MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod) { STANDARD_VM_CONTRACT; - // TODO: table to share/deduplicate these. - return CreateNewContinuationMethodTable(dataSize, objRefs, dataOffsets, asyncMethod, pamTracker); + ContinuationLayoutKeyData keyData(dataSize, objRefs, dataOffsets); + { + CrstHolder lock(&m_layoutsLock); + ContinuationLayoutKey key(&keyData); + MethodTable* lookupResult; + if (m_layouts.GetValue(key, (HashDatum*)&lookupResult)) + { + return lookupResult; + } + } + + AllocMemTracker amTracker; + MethodTable* result = CreateNewContinuationMethodTable(dataSize, objRefs, dataOffsets, asyncMethod, &amTracker); + { + CrstHolder lock(&m_layoutsLock); + ContinuationLayoutKey key(&keyData); + MethodTable* lookupResult; + if (m_layouts.GetValue(key, (HashDatum*)&lookupResult)) + { + result = lookupResult; + } + else + { + m_layouts.InsertValue(ContinuationLayoutKey(result), result); + amTracker.SuppressRelease(); + } + } + + return result; +} + +ContinuationLayoutKey::ContinuationLayoutKey(MethodTable* pMT) + : Data(reinterpret_cast(pMT) | 1) +{ +} + +ContinuationLayoutKey::ContinuationLayoutKey(ContinuationLayoutKeyData* pData) + : Data(reinterpret_cast(pData)) +{ +} + +EEHashEntry_t* ContinuationLayoutKeyHashTableHelper::AllocateEntry(ContinuationLayoutKey key, BOOL bDeepCopy, AllocationHeap heap) +{ + CONTRACTL + { + WRAPPER(THROWS); + WRAPPER(GC_NOTRIGGER); + INJECT_FAULT(return FALSE;); + } + CONTRACTL_END + + EEHashEntry_t* pEntry = (EEHashEntry_t*)new (nothrow) BYTE[SIZEOF_EEHASH_ENTRY + sizeof(ContinuationLayoutKey)]; + if (!pEntry) + return NULL; + memcpy(pEntry->Key, &key, sizeof(ContinuationLayoutKey)); + return pEntry; +} + +void ContinuationLayoutKeyHashTableHelper::DeleteEntry(EEHashEntry_t *pEntry, AllocationHeap heap) +{ + LIMITED_METHOD_CONTRACT; + + delete [] (BYTE*) pEntry; +} + +static CGCDescSeries* NextSeriesInData(CGCDescSeries* curSeries, CGCDescSeries* lowestSeries) +{ + while (curSeries >= lowestSeries && curSeries->GetSeriesOffset() < OFFSETOF__CORINFO_Continuation__data) + { + curSeries--; + } + + return curSeries; +} + +BOOL ContinuationLayoutKeyHashTableHelper::CompareKeys(EEHashEntry_t *pEntry, ContinuationLayoutKey key) +{ + LIMITED_METHOD_CONTRACT; + ContinuationLayoutKey storedKey; + memcpy(&storedKey, pEntry->Key, sizeof(ContinuationLayoutKey)); + + _ASSERTE((storedKey.Data & 1) != 0); // Always stores a MethodTable* as key + _ASSERTE((key.Data & 1) == 0); // Always uses ContinuationLayoutKeyData* as lookup key + + MethodTable* left = reinterpret_cast(storedKey.Data ^ 1); + ContinuationLayoutKeyData* right = reinterpret_cast(key.Data); + if (left->GetBaseSize() != (OBJHEADER_SIZE + OFFSETOF__CORINFO_Continuation__data + right->DataSize)) + return FALSE; + + const CORINFO_CONTINUATION_DATA_OFFSETS& leftOffsets = *left->GetContinuationOffsets(); + const CORINFO_CONTINUATION_DATA_OFFSETS& rightOffsets = right->DataOffsets; + if (leftOffsets.Result != rightOffsets.Result || + leftOffsets.Exception != rightOffsets.Exception || + leftOffsets.ContinuationContext != rightOffsets.ContinuationContext || + leftOffsets.KeepAlive != rightOffsets.KeepAlive) + { + return FALSE; + } + + // Now compare GC pointer series. + CGCDesc* gc = CGCDesc::GetCGCDescFromMT(left); + CGCDescSeries* curSeries = gc->GetHighestSeries(); + CGCDescSeries* lowestSeries = gc->GetLowestSeries(); + + // Skip ahead to first series inside the data region. + curSeries = NextSeriesInData(curSeries, lowestSeries); + + // Now verify that the series in the data match the bitmap. + auto compare = [=, &curSeries](size_t offset, size_t count) + { + if (curSeries < lowestSeries) + return false; + + if (OFFSETOF__CORINFO_Continuation__data + offset != curSeries->GetSeriesOffset()) + return false; + + if (count * TARGET_POINTER_SIZE != curSeries->GetSeriesSize() + left->GetBaseSize()) + return false; + + curSeries--; + return true; + }; + + if (!EnumerateRunsOfObjRefs(right->DataSize, right->ObjRefs, compare)) + { + return false; + } + + // Finally verify that we ran out of GC pointers simultaneously. + return curSeries + 1 == lowestSeries; +} + +DWORD ContinuationLayoutKeyHashTableHelper::Hash(ContinuationLayoutKey key) +{ + DWORD dwHash = 5381; + + const CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets; + if ((key.Data & 1) != 0) + { + MethodTable* pMT = reinterpret_cast(key.Data ^ 1); + dataOffsets = pMT->GetContinuationOffsets(); + + CGCDesc* gc = CGCDesc::GetCGCDescFromMT(pMT); + CGCDescSeries* curSeries = gc->GetHighestSeries(); + CGCDescSeries* lowestSeries = gc->GetLowestSeries(); + + // Skip ahead to first series inside the data region. + curSeries = NextSeriesInData(curSeries, lowestSeries); + + dwHash = ((dwHash << 5) + dwHash) ^ (pMT->GetBaseSize() - (OBJHEADER_SIZE + OFFSETOF__CORINFO_Continuation__data)); + while (curSeries >= lowestSeries) + { + dwHash = ((dwHash << 5) + dwHash) ^ (unsigned)(curSeries->GetSeriesOffset() - OFFSETOF__CORINFO_Continuation__data); + dwHash = ((dwHash << 5) + dwHash) ^ (unsigned)((curSeries->GetSeriesSize() + pMT->GetBaseSize()) / TARGET_POINTER_SIZE); + curSeries--; + } + } + else + { + ContinuationLayoutKeyData* keyData = reinterpret_cast(key.Data); + dataOffsets = &keyData->DataOffsets; + + dwHash = ((dwHash << 5) + dwHash) ^ keyData->DataSize; + EnumerateRunsOfObjRefs(keyData->DataSize, keyData->ObjRefs, [&dwHash](size_t offset, size_t count){ + dwHash = ((dwHash << 5) + dwHash) ^ (unsigned)offset; + dwHash = ((dwHash << 5) + dwHash) ^ (unsigned)count; + return true; + }); + } + + dwHash = ((dwHash << 5) + dwHash) ^ dataOffsets->Result; + dwHash = ((dwHash << 5) + dwHash) ^ dataOffsets->Exception; + dwHash = ((dwHash << 5) + dwHash) ^ dataOffsets->ContinuationContext; + dwHash = ((dwHash << 5) + dwHash) ^ dataOffsets->KeepAlive; + + return dwHash; +} + +void ContinuationLayoutKeyHashTableHelper::ReplaceKey(EEHashEntry_t *pEntry, ContinuationLayoutKey newKey) +{ + memcpy(pEntry->Key, &newKey, sizeof(ContinuationLayoutKey)); } #endif diff --git a/src/coreclr/vm/asynccontinuations.h b/src/coreclr/vm/asynccontinuations.h index 0abcda416ad650..70f87903035220 100644 --- a/src/coreclr/vm/asynccontinuations.h +++ b/src/coreclr/vm/asynccontinuations.h @@ -8,15 +8,52 @@ #ifndef ASYNCCONTINUATIONS_H #define ASYNCCONTINUATIONS_H +struct ContinuationLayoutKeyData +{ + unsigned DataSize; + const bool* ObjRefs; + const CORINFO_CONTINUATION_DATA_OFFSETS& DataOffsets; + + ContinuationLayoutKeyData(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) + : DataSize(dataSize) + , ObjRefs(objRefs) + , DataOffsets(dataOffsets) + { + } +}; + +struct ContinuationLayoutKey +{ + uintptr_t Data; + ContinuationLayoutKey() : Data(0) + { + } + ContinuationLayoutKey(MethodTable* pMT); + ContinuationLayoutKey(ContinuationLayoutKeyData* pData); +}; + +struct ContinuationLayoutKeyHashTableHelper +{ + static EEHashEntry_t * AllocateEntry(ContinuationLayoutKey key, BOOL bDeepCopy, AllocationHeap heap); + static void DeleteEntry(EEHashEntry_t *pEntry, AllocationHeap heap); + static BOOL CompareKeys(EEHashEntry_t *pEntry, ContinuationLayoutKey key); + static DWORD Hash(ContinuationLayoutKey key); + static void ReplaceKey(EEHashEntry_t *pEntry, ContinuationLayoutKey newKey); +}; + +typedef EEHashTable ContinuationLayoutHashTable; + class AsyncContinuationsManager { LoaderAllocator* m_allocator; + CrstExplicitInit m_layoutsLock; + ContinuationLayoutHashTable m_layouts; MethodTable* CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker); public: AsyncContinuationsManager(LoaderAllocator* allocator); - MethodTable* LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker); + MethodTable* LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod); }; typedef DPTR(AsyncContinuationsManager) PTR_AsyncContinuationsManager; diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 611223aa15ec63..0c1916936ba2bf 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -10269,9 +10269,12 @@ CORINFO_CLASS_HANDLE CEEInfo::getContinuationType( LoaderAllocator* allocator = m_pMethodBeingCompiled->GetLoaderAllocator(); AsyncContinuationsManager* asyncConts = allocator->GetAsyncContinuationsManager(); - AllocMemTracker amTracker; - result = (CORINFO_CLASS_HANDLE)asyncConts->LookupOrCreateContinuationMethodTable((unsigned)dataSize, objRefs, dataOffsets, m_pMethodBeingCompiled, &amTracker); - amTracker.SuppressRelease(); + result = (CORINFO_CLASS_HANDLE)asyncConts->LookupOrCreateContinuationMethodTable((unsigned)dataSize, objRefs, dataOffsets, m_pMethodBeingCompiled); + +#ifdef DEBUG + CORINFO_CLASS_HANDLE result2 = (CORINFO_CLASS_HANDLE)asyncConts->LookupOrCreateContinuationMethodTable((unsigned)dataSize, objRefs, dataOffsets, m_pMethodBeingCompiled); + _ASSERTE(result2 == result); +#endif EE_TO_JIT_TRANSITION(); @@ -14519,7 +14522,7 @@ static Signature BuildResumptionStubSignature(LoaderAllocator* alloc, AllocMemTr { SigBuilder sigBuilder; sigBuilder.AppendByte(IMAGE_CEE_CS_CALLCONV_DEFAULT); - sigBuilder.AppendData(2); // 2 argument + sigBuilder.AppendData(2); // 2 arguments sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT); // return type sigBuilder.AppendElementType(ELEMENT_TYPE_OBJECT); // continuation sigBuilder.AppendElementType(ELEMENT_TYPE_BYREF); // result location From a45427b06c9c96f68504b992a9c3f0926052c7b9 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Oct 2025 17:34:49 +0200 Subject: [PATCH 22/48] Fix a bug --- src/coreclr/vm/asynccontinuations.cpp | 30 +++++++++++++++------------ 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 22e0df339227f4..75eb07cc5307f8 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -88,17 +88,6 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne unsigned startOfDataInInstance = AlignUp(pParentClass->GetNumInstanceFieldBytes(), TARGET_POINTER_SIZE); unsigned startOfDataInObject = OBJECT_SIZE + startOfDataInInstance; - // Offsets passed in are relative to the data chunk, fix that up now to be - // relative to the start of the instance data. - if (allocatedDataOffsets->Result != UINT_MAX) - allocatedDataOffsets->Result += startOfDataInInstance; - if (allocatedDataOffsets->Exception != UINT_MAX) - allocatedDataOffsets->Exception += startOfDataInInstance; - if (allocatedDataOffsets->ContinuationContext != UINT_MAX) - allocatedDataOffsets->ContinuationContext += startOfDataInInstance; - if (allocatedDataOffsets->KeepAlive != UINT_MAX) - allocatedDataOffsets->KeepAlive += startOfDataInInstance; - MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); pMT->AllocateAuxiliaryData(m_allocator, asyncMethod->GetLoaderModule(), pamTracker, MethodTableStaticsFlags::None); pMT->SetParentMethodTable(pParentClass); @@ -161,7 +150,22 @@ MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(un { STANDARD_VM_CONTRACT; - ContinuationLayoutKeyData keyData(dataSize, objRefs, dataOffsets); + // The API we expose has all offsets relative to the data, but we prefer to + // have offsets relative to the start of instance data so that + // RuntimeHelpers.GetRawData(obj) + offset returns the right offset. Adjust + // it here. + CORINFO_CONTINUATION_DATA_OFFSETS adjustedDataOffsets = dataOffsets; + const uint32_t startOfDataInInstance = OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object; + if (dataOffsets.Result != UINT_MAX) + adjustedDataOffsets.Result += startOfDataInInstance; + if (dataOffsets.Exception != UINT_MAX) + adjustedDataOffsets.Exception += startOfDataInInstance; + if (dataOffsets.ContinuationContext != UINT_MAX) + adjustedDataOffsets.ContinuationContext += startOfDataInInstance; + if (dataOffsets.KeepAlive != UINT_MAX) + adjustedDataOffsets.KeepAlive += startOfDataInInstance; + + ContinuationLayoutKeyData keyData(dataSize, objRefs, adjustedDataOffsets); { CrstHolder lock(&m_layoutsLock); ContinuationLayoutKey key(&keyData); @@ -173,7 +177,7 @@ MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(un } AllocMemTracker amTracker; - MethodTable* result = CreateNewContinuationMethodTable(dataSize, objRefs, dataOffsets, asyncMethod, &amTracker); + MethodTable* result = CreateNewContinuationMethodTable(dataSize, objRefs, adjustedDataOffsets, asyncMethod, &amTracker); { CrstHolder lock(&m_layoutsLock); ContinuationLayoutKey key(&keyData); From 37fd4341bd6e4b1ab4582a565ced632d51ad4bbe Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Oct 2025 18:07:40 +0200 Subject: [PATCH 23/48] Fix another bug --- src/coreclr/jit/async.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 550a1a26994eae..8294eef4de4171 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1111,7 +1111,7 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* if (layout.ReturnSize > 0) { - layout.ReturnValOffset = allocLayout(1, layout.ReturnSize); + layout.ReturnValOffset = allocLayout(layout.ReturnAlignment, layout.ReturnSize); JITDUMP(" Will store return of type %s, size %u at offset %u\n", call->gtReturnType == TYP_STRUCT ? layout.ReturnStructLayout->GetClassName() From 15f205ad830a8789b5f2ad13072c4408f453fede Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 8 Oct 2025 18:16:25 +0200 Subject: [PATCH 24/48] Fix debug counting --- src/coreclr/jit/async.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 8294eef4de4171..b5ac904ffc043d 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -981,6 +981,7 @@ class GCPointerBitMapBuilder // We do not expect to set the same offset multiple times assert(!m_objRefs[slot]); m_objRefs[slot] = true; + INDEBUG(NumObjRefs++); } void SetIfNotMax(unsigned offset) From 8a6564e3a2b922212bef89227f9a5b0b000f938a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 10 Oct 2025 10:26:14 +0200 Subject: [PATCH 25/48] Fix MethodTable::ValidateWithPossibleAV for new types --- src/coreclr/vm/methodtable.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/methodtable.cpp b/src/coreclr/vm/methodtable.cpp index 8c3e61466776b2..96df170a0513c7 100644 --- a/src/coreclr/vm/methodtable.cpp +++ b/src/coreclr/vm/methodtable.cpp @@ -375,7 +375,7 @@ BOOL MethodTable::ValidateWithPossibleAV() } // generic instantiation check - return (HasInstantiation() || IsArray()) && (pEEClassFromMethodTable->GetClassWithPossibleAV() == pEEClass); + return (HasInstantiation() || IsArray() || IsContinuation()) && (pEEClassFromMethodTable->GetClassWithPossibleAV() == pEEClass); } From a92ca94b51e27ba59c92a9384022a306a119c2c0 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 10 Oct 2025 11:23:43 +0200 Subject: [PATCH 26/48] Add assert in `GetModule()`, make `GetType()` return `typeof(Continuation)` --- src/coreclr/vm/methodtable.cpp | 5 +++-- src/coreclr/vm/methodtable.h | 1 + src/coreclr/vm/typehandle.cpp | 13 +++++++++++++ src/coreclr/vm/typehandle.h | 3 +++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/coreclr/vm/methodtable.cpp b/src/coreclr/vm/methodtable.cpp index 96df170a0513c7..fa8114b1bf7e32 100644 --- a/src/coreclr/vm/methodtable.cpp +++ b/src/coreclr/vm/methodtable.cpp @@ -421,7 +421,7 @@ WORD MethodTable::GetNumMethods() PTR_MethodTable MethodTable::GetTypicalMethodTable() { LIMITED_METHOD_DAC_CONTRACT; - if (IsArray()) + if (IsArray() || IsContinuation()) return (PTR_MethodTable)this; PTR_MethodTable methodTableMaybe = GetModule()->LookupTypeDef(GetCl()).AsMethodTable(); @@ -1341,6 +1341,7 @@ BOOL MethodTable::CanCastToClass(MethodTable *pTargetMT, TypeHandlePairList *pVi PRECONDITION(CheckPointer(pTargetMT)); PRECONDITION(!pTargetMT->IsArray()); PRECONDITION(!pTargetMT->IsInterface()); + PRECONDITION(!pTargetMT->IsContinuation()); } CONTRACTL_END @@ -7491,7 +7492,7 @@ CHECK MethodTable::CheckInstanceActivated() { WRAPPER_NO_CONTRACT; - if (IsArray()) + if (IsArray() || IsContinuation()) CHECK_OK; diff --git a/src/coreclr/vm/methodtable.h b/src/coreclr/vm/methodtable.h index 7a086535b9b3a9..24e314ccc086df 100644 --- a/src/coreclr/vm/methodtable.h +++ b/src/coreclr/vm/methodtable.h @@ -939,6 +939,7 @@ class MethodTable PTR_Module GetModule() { LIMITED_METHOD_CONTRACT; + _ASSERTE(!IsContinuation()); return m_pModule; } diff --git a/src/coreclr/vm/typehandle.cpp b/src/coreclr/vm/typehandle.cpp index 05b80e1fd662fa..0071874b913b53 100644 --- a/src/coreclr/vm/typehandle.cpp +++ b/src/coreclr/vm/typehandle.cpp @@ -91,6 +91,13 @@ BOOL TypeHandle::IsString() const return !IsTypeDesc() && AsMethodTable()->IsString(); } +BOOL TypeHandle::IsContinuation() const +{ + LIMITED_METHOD_CONTRACT; + + return !IsTypeDesc() && AsMethodTable()->IsContinuation(); +} + BOOL TypeHandle::IsGenericVariable() const { LIMITED_METHOD_DAC_CONTRACT; @@ -318,6 +325,12 @@ bool TypeHandle::IsManagedClassObjectPinned() const void TypeHandle::AllocateManagedClassObject(RUNTIMETYPEHANDLE* pDest) { + if (IsContinuation()) + { + *pDest = OBJECTREFToObject(GetParent().GetManagedClassObject()); + return; + } + REFLECTCLASSBASEREF refClass = NULL; PTR_LoaderAllocator allocator = GetLoaderAllocator(); diff --git a/src/coreclr/vm/typehandle.h b/src/coreclr/vm/typehandle.h index c4fb31c7324be4..92af9c513a7641 100644 --- a/src/coreclr/vm/typehandle.h +++ b/src/coreclr/vm/typehandle.h @@ -440,6 +440,9 @@ class TypeHandle // String BOOL IsString() const; + // Continuation sub types + BOOL IsContinuation() const; + // True if this type *is* a formal generic type parameter or any component of it is a formal generic type parameter BOOL ContainsGenericVariables(BOOL methodOnly=FALSE) const; From 4d99c5b8dd7f9ba42ca48ebdc64786b31a3fd481 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 14 Oct 2025 12:15:27 +0200 Subject: [PATCH 27/48] Minor cleanups --- src/coreclr/vm/asynccontinuations.cpp | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 75eb07cc5307f8..246342fe307d45 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -168,9 +168,8 @@ MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(un ContinuationLayoutKeyData keyData(dataSize, objRefs, adjustedDataOffsets); { CrstHolder lock(&m_layoutsLock); - ContinuationLayoutKey key(&keyData); MethodTable* lookupResult; - if (m_layouts.GetValue(key, (HashDatum*)&lookupResult)) + if (m_layouts.GetValue(ContinuationLayoutKey(&keyData), (HashDatum*)&lookupResult)) { return lookupResult; } @@ -180,9 +179,8 @@ MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(un MethodTable* result = CreateNewContinuationMethodTable(dataSize, objRefs, adjustedDataOffsets, asyncMethod, &amTracker); { CrstHolder lock(&m_layoutsLock); - ContinuationLayoutKey key(&keyData); MethodTable* lookupResult; - if (m_layouts.GetValue(key, (HashDatum*)&lookupResult)) + if (m_layouts.GetValue(ContinuationLayoutKey(&keyData), (HashDatum*)&lookupResult)) { result = lookupResult; } @@ -217,7 +215,7 @@ EEHashEntry_t* ContinuationLayoutKeyHashTableHelper::AllocateEntry(ContinuationL CONTRACTL_END EEHashEntry_t* pEntry = (EEHashEntry_t*)new (nothrow) BYTE[SIZEOF_EEHASH_ENTRY + sizeof(ContinuationLayoutKey)]; - if (!pEntry) + if (pEntry == NULL) return NULL; memcpy(pEntry->Key, &key, sizeof(ContinuationLayoutKey)); return pEntry; From cd4fbd537d9b33b9844dfae9c8ad065fdf608cdb Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 14 Oct 2025 12:15:53 +0200 Subject: [PATCH 28/48] Throw `NotSupportedException` for Object.GetType() called on continuations --- src/coreclr/vm/typehandle.cpp | 10 +++++++++- .../System.Private.CoreLib/src/Resources/Strings.resx | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/coreclr/vm/typehandle.cpp b/src/coreclr/vm/typehandle.cpp index 0071874b913b53..3726307d1d9d45 100644 --- a/src/coreclr/vm/typehandle.cpp +++ b/src/coreclr/vm/typehandle.cpp @@ -325,9 +325,17 @@ bool TypeHandle::IsManagedClassObjectPinned() const void TypeHandle::AllocateManagedClassObject(RUNTIMETYPEHANDLE* pDest) { + CONTRACTL + { + THROWS; + GC_TRIGGERS; + MODE_COOPERATIVE; + } + CONTRACTL_END + if (IsContinuation()) { - *pDest = OBJECTREFToObject(GetParent().GetManagedClassObject()); + COMPlusThrow(kNotSupportedException, W("NotSupported_Continuation")); return; } diff --git a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx index 34977b2e22b910..7404f6124e5b16 100644 --- a/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx +++ b/src/libraries/System.Private.CoreLib/src/Resources/Strings.resx @@ -4058,6 +4058,9 @@ C++/CLI activation has been disabled via a feature switch. See https://aka.ms/dotnet-illink/nativehost for more information. + + Object.GetType() is not supported for continuations + Queue empty. From 3bde22a19a59b51a8b23b3bc5e288881aa0abb28 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 14 Oct 2025 12:18:14 +0200 Subject: [PATCH 29/48] Update src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs Co-authored-by: Jan Kotas --- .../System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index fc1a578a60c3d7..dd542085724a53 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -900,7 +900,7 @@ public int MultiDimensionalArrayRank public bool IsArray => (Flags & enum_flag_Category_Array_Mask) == enum_flag_Category_Array; - public bool IsContinuation => ParentMethodTable == (MethodTable*)typeof(Continuation).TypeHandle.Value; + public bool IsContinuation => ParentMethodTable == TypeHandle.TypeHandleOf().AsMethodTable(); public bool HasInstantiation => (Flags & enum_flag_HasComponentSize) == 0 && (Flags & enum_flag_GenericsMask) != enum_flag_GenericsMask_NonGeneric; From 11033f63a12c7c274586d9bc40b9cfdf547fb302 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 14 Oct 2025 14:01:41 +0200 Subject: [PATCH 30/48] Handle large alignment requirements --- src/coreclr/jit/async.cpp | 33 +++++++++++++++++++++------------ src/coreclr/jit/async.h | 16 +++++++++++++++- 2 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index b5ac904ffc043d..971f49fbf7d1dd 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1112,7 +1112,7 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* if (layout.ReturnSize > 0) { - layout.ReturnValOffset = allocLayout(layout.ReturnAlignment, layout.ReturnSize); + layout.ReturnValOffset = allocLayout(layout.ReturnHeapAlignment(), layout.ReturnSize); JITDUMP(" Will store return of type %s, size %u at offset %u\n", call->gtReturnType == TYP_STRUCT ? layout.ReturnStructLayout->GetClassName() @@ -1149,7 +1149,7 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* for (LiveLocalInfo& inf : liveLocals) { - inf.Offset = allocLayout(inf.Alignment, inf.Size); + inf.Offset = allocLayout(inf.HeapAlignment(), inf.Size); } layout.Size = roundUp(layout.Size, TARGET_POINTER_SIZE); @@ -1560,16 +1560,19 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, value = m_comp->gtNewLclVarNode(inf.LclNum); } + GenTreeFlags indirFlags = + GTF_IND_NONFAULTING | (inf.HeapAlignment() < inf.Alignment ? GTF_IND_UNALIGNED : GTF_EMPTY); + GenTree* store; if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) { GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); GenTree* addr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, newContinuation, cns); - store = m_comp->gtNewStoreValueNode(dsc->GetLayout(), addr, value, GTF_IND_NONFAULTING); + store = m_comp->gtNewStoreValueNode(dsc->GetLayout(), addr, value, indirFlags); } else { - store = StoreAtOffset(newContinuation, offset, value, dsc->TypeGet()); + store = StoreAtOffset(newContinuation, offset, value, dsc->TypeGet(), indirFlags); } LIR::AsRange(suspendBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); @@ -1805,14 +1808,16 @@ void AsyncTransformation::RestoreFromDataOnResumption(const ContinuationLayout& GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); GenTree* addr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, continuation, cns); + GenTreeFlags indirFlags = + GTF_IND_NONFAULTING | (inf.HeapAlignment() < inf.Alignment ? GTF_IND_UNALIGNED : GTF_EMPTY); GenTree* value; if (dsc->TypeIs(TYP_STRUCT) || dsc->IsImplicitByRef()) { - value = m_comp->gtNewLoadValueNode(dsc->GetLayout(), addr, GTF_IND_NONFAULTING); + value = m_comp->gtNewLoadValueNode(dsc->GetLayout(), addr, indirFlags); } else { - value = m_comp->gtNewIndir(dsc->TypeGet(), addr, GTF_IND_NONFAULTING); + value = m_comp->gtNewIndir(dsc->TypeGet(), addr, indirFlags); } GenTree* store; @@ -1930,6 +1935,9 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* assert(callDefInfo.DefinitionNode != nullptr); LclVarDsc* resultLcl = m_comp->lvaGetDesc(callDefInfo.DefinitionNode); + GenTreeFlags indirFlags = + GTF_IND_NONFAULTING | (layout.ReturnHeapAlignment() < layout.ReturnAlignment ? GTF_IND_UNALIGNED : GTF_EMPTY); + // TODO-TP: We can use liveness to avoid generating a lot of this IR. if (call->gtReturnType == TYP_STRUCT) { @@ -1937,8 +1945,7 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* { GenTree* resultOffsetNode = m_comp->gtNewIconNode((ssize_t)resultOffset, TYP_I_IMPL); GenTree* resultAddr = m_comp->gtNewOperNode(GT_ADD, TYP_BYREF, resultBase, resultOffsetNode); - GenTree* resultData = - m_comp->gtNewLoadValueNode(layout.ReturnStructLayout, resultAddr, GTF_IND_NONFAULTING); + GenTree* resultData = m_comp->gtNewLoadValueNode(layout.ReturnStructLayout, resultAddr, indirFlags); GenTree* storeResult; if ((callDefInfo.DefinitionNode->GetLclOffs() == 0) && ClassLayout::AreCompatible(resultLcl->GetLayout(), layout.ReturnStructLayout)) @@ -1977,7 +1984,7 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* LclVarDsc* fieldDsc = m_comp->lvaGetDesc(fieldLclNum); unsigned fldOffset = resultOffset + fieldDsc->lvFldOffset; - GenTree* value = LoadFromOffset(resultBase, fldOffset, fieldDsc->TypeGet(), GTF_IND_NONFAULTING); + GenTree* value = LoadFromOffset(resultBase, fldOffset, fieldDsc->TypeGet(), indirFlags); GenTree* store = m_comp->gtNewStoreLclVarNode(fieldLclNum, value); LIR::AsRange(storeResultBB).InsertAtEnd(LIR::SeqTree(m_comp, store)); @@ -1990,7 +1997,7 @@ void AsyncTransformation::CopyReturnValueOnResumption(GenTreeCall* } else { - GenTree* value = LoadFromOffset(resultBase, resultOffset, call->gtReturnType, GTF_IND_NONFAULTING); + GenTree* value = LoadFromOffset(resultBase, resultOffset, call->gtReturnType, indirFlags); GenTree* storeResult; if (callDefInfo.DefinitionNode->OperIs(GT_STORE_LCL_VAR)) @@ -2043,17 +2050,19 @@ GenTreeIndir* AsyncTransformation::LoadFromOffset(GenTree* base, // offset - Offset to add on top of the base address // value - Value to store // storeType - Type of store +// indirFlags - Indirection flags // // Returns: // IR node of the store. // -GenTreeStoreInd* AsyncTransformation::StoreAtOffset(GenTree* base, unsigned offset, GenTree* value, var_types storeType) +GenTreeStoreInd* AsyncTransformation::StoreAtOffset( + GenTree* base, unsigned offset, GenTree* value, var_types storeType, GenTreeFlags indirFlags) { assert(base->TypeIs(TYP_REF, TYP_BYREF, TYP_I_IMPL)); GenTree* cns = m_comp->gtNewIconNode((ssize_t)offset, TYP_I_IMPL); var_types addrType = base->TypeIs(TYP_I_IMPL) ? TYP_I_IMPL : TYP_BYREF; GenTree* addr = m_comp->gtNewOperNode(GT_ADD, addrType, base, cns); - GenTreeStoreInd* store = m_comp->gtNewStoreIndNode(storeType, addr, value, GTF_IND_NONFAULTING); + GenTreeStoreInd* store = m_comp->gtNewStoreIndNode(storeType, addr, value, indirFlags); return store; } diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index ce146d27700bc7..79df5435608891 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -12,6 +12,11 @@ struct LiveLocalInfo : LclNum(lclNum) { } + + unsigned HeapAlignment() const + { + return std::min(Alignment, (unsigned)TARGET_POINTER_SIZE); + } }; struct ContinuationLayout @@ -31,6 +36,11 @@ struct ContinuationLayout : Locals(locals) { } + + unsigned ReturnHeapAlignment() const + { + return std::min(ReturnAlignment, (unsigned)TARGET_POINTER_SIZE); + } }; struct CallDefinitionInfo @@ -119,7 +129,11 @@ class AsyncTransformation unsigned offset, var_types type, GenTreeFlags indirFlags = GTF_IND_NONFAULTING); - GenTreeStoreInd* StoreAtOffset(GenTree* base, unsigned offset, GenTree* value, var_types storeType); + GenTreeStoreInd* StoreAtOffset(GenTree* base, + unsigned offset, + GenTree* value, + var_types storeType, + GenTreeFlags indirFlags = GTF_IND_NONFAULTING); unsigned GetResultBaseVar(); unsigned GetExceptionVar(); From 83bcde7bb5b16c23cd4b78cf446789ee786a0540 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 14 Oct 2025 14:11:06 +0200 Subject: [PATCH 31/48] Fix after merge --- src/coreclr/inc/jiteeversionguid.h | 10 ++-- .../JitInterface/CorInfoImpl_generated.cs | 54 ++----------------- 2 files changed, 9 insertions(+), 55 deletions(-) diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index 17f96a2f272f1c..d44407a0f5a818 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* 156f1a7f-c06a-40fc-a984-987c8b4c2ba7 */ - 0x156f1a7f, - 0xc06a, - 0x40fc, - {0xa9, 0x84, 0x98, 0x7c, 0x8b, 0x4c, 0x2b, 0xa7} +constexpr GUID JITEEVersionIdentifier = { /* 2309660f-0396-4aab-8a77-fe66133a1ba0 */ + 0x2309660f, + 0x0396, + 0x4aab, + {0x8a, 0x77, 0xfe, 0x66, 0x13, 0x3a, 0x1b, 0xa0} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs index b7a76218f5733f..88fba3f9a6299b 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs @@ -2298,7 +2298,7 @@ private static byte _getTailCallHelpers(IntPtr thisHandle, IntPtr* ppException, } [UnmanagedCallersOnly] - private static CORINFO_CLASS_STRUCT_* _getContinuationType(IntPtr thisHandle, IntPtr* ppException, UIntPtr dataSize, bool* objRefs, CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) + private static CORINFO_CLASS_STRUCT_* _getContinuationType(IntPtr thisHandle, IntPtr* ppException, nuint dataSize, bool* objRefs, CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) { var _this = GetThis(thisHandle); try @@ -2621,7 +2621,7 @@ private static uint _getJitFlags(IntPtr thisHandle, IntPtr* ppException, CORJIT_ private static IntPtr GetUnmanagedCallbacks() { - void** callbacks = (void**)NativeMemory.Alloc((nuint)(sizeof(void*) * 176)); + void** callbacks = (void**)NativeMemory.Alloc((nuint)(sizeof(void*) * 177)); callbacks[0] = (delegate* unmanaged)&_isIntrinsic; callbacks[1] = (delegate* unmanaged)&_notifyMethodInfoUsage; @@ -2778,8 +2778,7 @@ private static IntPtr GetUnmanagedCallbacks() callbacks[152] = (delegate* unmanaged)&_GetDelegateCtor; callbacks[153] = (delegate* unmanaged)&_MethodCompileComplete; callbacks[154] = (delegate* unmanaged)&_getTailCallHelpers; -<<<<<<< HEAD - callbacks[155] = (delegate* unmanaged)&_getContinuationType; + callbacks[155] = (delegate* unmanaged)&_getContinuationType; callbacks[156] = (delegate* unmanaged)&_getAsyncResumptionStub; callbacks[157] = (delegate* unmanaged)&_convertPInvokeCalliToCall; callbacks[158] = (delegate* unmanaged)&_notifyInstructionSetUsage; @@ -2787,7 +2786,7 @@ private static IntPtr GetUnmanagedCallbacks() callbacks[160] = (delegate* unmanaged)&_allocMem; callbacks[161] = (delegate* unmanaged)&_reserveUnwindInfo; callbacks[162] = (delegate* unmanaged)&_allocUnwindInfo; - callbacks[163] = (delegate* unmanaged)&_allocGCInfo; + callbacks[163] = (delegate* unmanaged)&_allocGCInfo; callbacks[164] = (delegate* unmanaged)&_setEHcount; callbacks[165] = (delegate* unmanaged)&_setEHinfo; callbacks[166] = (delegate* unmanaged)&_logMsg; @@ -2801,51 +2800,6 @@ private static IntPtr GetUnmanagedCallbacks() callbacks[174] = (delegate* unmanaged)&_getExpectedTargetArchitecture; callbacks[175] = (delegate* unmanaged)&_getJitFlags; callbacks[176] = (delegate* unmanaged)&_getSpecialCopyHelper; -||||||| 337fea7e187 - callbacks[155] = (delegate* unmanaged)&_getAsyncResumptionStub; - callbacks[156] = (delegate* unmanaged)&_convertPInvokeCalliToCall; - callbacks[157] = (delegate* unmanaged)&_notifyInstructionSetUsage; - callbacks[158] = (delegate* unmanaged)&_updateEntryPointForTailCall; - callbacks[159] = (delegate* unmanaged)&_allocMem; - callbacks[160] = (delegate* unmanaged)&_reserveUnwindInfo; - callbacks[161] = (delegate* unmanaged)&_allocUnwindInfo; - callbacks[162] = (delegate* unmanaged)&_allocGCInfo; - callbacks[163] = (delegate* unmanaged)&_setEHcount; - callbacks[164] = (delegate* unmanaged)&_setEHinfo; - callbacks[165] = (delegate* unmanaged)&_logMsg; - callbacks[166] = (delegate* unmanaged)&_doAssert; - callbacks[167] = (delegate* unmanaged)&_reportFatalError; - callbacks[168] = (delegate* unmanaged)&_getPgoInstrumentationResults; - callbacks[169] = (delegate* unmanaged)&_allocPgoInstrumentationBySchema; - callbacks[170] = (delegate* unmanaged)&_recordCallSite; - callbacks[171] = (delegate* unmanaged)&_recordRelocation; - callbacks[172] = (delegate* unmanaged)&_getRelocTypeHint; - callbacks[173] = (delegate* unmanaged)&_getExpectedTargetArchitecture; - callbacks[174] = (delegate* unmanaged)&_getJitFlags; - callbacks[175] = (delegate* unmanaged)&_getSpecialCopyHelper; -======= - callbacks[155] = (delegate* unmanaged)&_getAsyncResumptionStub; - callbacks[156] = (delegate* unmanaged)&_convertPInvokeCalliToCall; - callbacks[157] = (delegate* unmanaged)&_notifyInstructionSetUsage; - callbacks[158] = (delegate* unmanaged)&_updateEntryPointForTailCall; - callbacks[159] = (delegate* unmanaged)&_allocMem; - callbacks[160] = (delegate* unmanaged)&_reserveUnwindInfo; - callbacks[161] = (delegate* unmanaged)&_allocUnwindInfo; - callbacks[162] = (delegate* unmanaged)&_allocGCInfo; - callbacks[163] = (delegate* unmanaged)&_setEHcount; - callbacks[164] = (delegate* unmanaged)&_setEHinfo; - callbacks[165] = (delegate* unmanaged)&_logMsg; - callbacks[166] = (delegate* unmanaged)&_doAssert; - callbacks[167] = (delegate* unmanaged)&_reportFatalError; - callbacks[168] = (delegate* unmanaged)&_getPgoInstrumentationResults; - callbacks[169] = (delegate* unmanaged)&_allocPgoInstrumentationBySchema; - callbacks[170] = (delegate* unmanaged)&_recordCallSite; - callbacks[171] = (delegate* unmanaged)&_recordRelocation; - callbacks[172] = (delegate* unmanaged)&_getRelocTypeHint; - callbacks[173] = (delegate* unmanaged)&_getExpectedTargetArchitecture; - callbacks[174] = (delegate* unmanaged)&_getJitFlags; - callbacks[175] = (delegate* unmanaged)&_getSpecialCopyHelper; ->>>>>>> 8aad2ab6331322b510786cee457deb992c19b9ad return (IntPtr)callbacks; } From b6ad42edd659a87d103bc11b5d8635fa7d168fa3 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 14 Oct 2025 16:50:28 +0200 Subject: [PATCH 32/48] Allocate singleton subclassed continuation with a corresponding `EEClass` --- src/coreclr/vm/asynccontinuations.cpp | 84 ++++++++++++++++++++++++--- src/coreclr/vm/asynccontinuations.h | 3 + 2 files changed, 79 insertions(+), 8 deletions(-) diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 246342fe307d45..5a99600ebebc52 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -24,6 +24,47 @@ AsyncContinuationsManager::AsyncContinuationsManager(LoaderAllocator* allocator) m_layouts.Init(16, &lock, m_allocator->GetLowFrequencyHeap()); } +static EEClass* volatile g_singletonContinuationEEClass; + +EEClass* AsyncContinuationsManager::GetOrCreateSingletonSubContinuationEEClass() +{ + if (g_singletonContinuationEEClass != NULL) + return g_singletonContinuationEEClass; + + return CreateSingletonSubContinuationEEClass(); +} + +EEClass* AsyncContinuationsManager::CreateSingletonSubContinuationEEClass() +{ + AllocMemTracker amTracker; + + Module* spc = SystemDomain::SystemModule(); + LoaderAllocator* allocator = SystemDomain::GetGlobalLoaderAllocator(); + + EEClass* pClass = EEClass::CreateMinimalClass(allocator->GetHighFrequencyHeap(), &amTracker); + + CORINFO_CONTINUATION_DATA_OFFSETS offsets; + offsets.Result = UINT_MAX; + offsets.Exception = UINT_MAX; + offsets.ContinuationContext = UINT_MAX; + offsets.KeepAlive = UINT_MAX; + MethodTable* pMT = CreateNewContinuationMethodTable(0, NULL, offsets, pClass, allocator, spc, &amTracker); + + pClass->SetMethodTable(pMT); + +#ifdef _DEBUG + pClass->SetDebugClassName("dynamicContinuation"); + pMT->SetDebugClassName("dynamicContinuation"); +#endif + + if (InterlockedCompareExchangeT(&g_singletonContinuationEEClass, pClass, NULL) == NULL) + { + amTracker.SuppressRelease(); + } + + return g_singletonContinuationEEClass; +} + template static bool EnumerateRunsOfObjRefs(unsigned size, const bool* objRefs, TFunc func) { @@ -50,7 +91,14 @@ static bool EnumerateRunsOfObjRefs(unsigned size, const bool* objRefs, TFunc fun return true; } -MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker) +MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable( + unsigned dataSize, + const bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, + EEClass* pClass, + LoaderAllocator* allocator, + Module* loaderModule, + AllocMemTracker* pamTracker) { STANDARD_VM_CONTRACT; @@ -81,7 +129,7 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne size_t cbGC = numPointerSeries == 0 ? 0 : CGCDesc::ComputeSize(numPointerSeries); - BYTE* pMemory = (BYTE*)pamTracker->Track(m_allocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(CORINFO_CONTINUATION_DATA_OFFSETS)) + S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); + BYTE* pMemory = (BYTE*)pamTracker->Track(allocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(CORINFO_CONTINUATION_DATA_OFFSETS)) + S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); CORINFO_CONTINUATION_DATA_OFFSETS* allocatedDataOffsets = new (pMemory) CORINFO_CONTINUATION_DATA_OFFSETS(dataOffsets); @@ -89,13 +137,13 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne unsigned startOfDataInObject = OBJECT_SIZE + startOfDataInInstance; MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); - pMT->AllocateAuxiliaryData(m_allocator, asyncMethod->GetLoaderModule(), pamTracker, MethodTableStaticsFlags::None); + pMT->AllocateAuxiliaryData(allocator, loaderModule, pamTracker, MethodTableStaticsFlags::None); pMT->SetParentMethodTable(pParentClass); pMT->SetContinuationDataOffsets(allocatedDataOffsets); - pMT->SetLoaderAllocator(m_allocator); + pMT->SetLoaderAllocator(allocator); pMT->SetModule(pParentClass->GetModule()); pMT->SetNumVirtuals(static_cast(numVirtuals)); - pMT->SetClass(pParentClass->GetClass()); // EEClass of System.Runtime.CompilerServices.Continuation + pMT->SetClass(pClass); pMT->SetBaseSize(OBJECT_BASESIZE + startOfDataInInstance + dataSize); if (numPointerSeries > 0) @@ -126,6 +174,28 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne _ASSERTE(curIndex == 0); } + pMT->SetClassInited(); + INDEBUG(pMT->GetAuxiliaryDataForWrite()->SetIsPublished()); + + return pMT; +} + +MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable( + unsigned dataSize, + const bool* objRefs, + const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, + MethodDesc* asyncMethod, + AllocMemTracker* pamTracker) +{ + MethodTable* pMT = CreateNewContinuationMethodTable( + dataSize, + objRefs, + dataOffsets, + GetOrCreateSingletonSubContinuationEEClass(), + m_allocator, + asyncMethod->GetLoaderModule(), + pamTracker); + #ifdef DEBUG size_t numObjRefs = 0; EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) { @@ -133,7 +203,7 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne return true; }); StackSString debugName; - debugName.Printf("Continuation_%s_%u_%zu", asyncMethod->m_pszDebugMethodName, dataSize, numObjRefs); + debugName.Printf("dynamicContinuation_%s_%u_%zu", asyncMethod->m_pszDebugMethodName, dataSize, numObjRefs); const char* debugNameUTF8 = debugName.GetUTF8(); size_t len = strlen(debugNameUTF8) + 1; char* name = (char*)pamTracker->Track(m_allocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(len))); @@ -141,8 +211,6 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable(unsigne pMT->SetDebugClassName(name); #endif - pMT->SetClassInited(); - return pMT; } diff --git a/src/coreclr/vm/asynccontinuations.h b/src/coreclr/vm/asynccontinuations.h index 70f87903035220..eaf0e761b5c145 100644 --- a/src/coreclr/vm/asynccontinuations.h +++ b/src/coreclr/vm/asynccontinuations.h @@ -51,6 +51,9 @@ class AsyncContinuationsManager MethodTable* CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker); + static MethodTable* CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, EEClass* eeClass, LoaderAllocator* allocator, Module* loaderModule, AllocMemTracker* pamTracker); + static PTR_EEClass GetOrCreateSingletonSubContinuationEEClass(); + static PTR_EEClass CreateSingletonSubContinuationEEClass(); public: AsyncContinuationsManager(LoaderAllocator* allocator); MethodTable* LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod); From 5e4fb407c7a2d378bbbd491877a9e8961b01878a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Tue, 14 Oct 2025 19:28:02 +0200 Subject: [PATCH 33/48] Enable runtime async by default for testing --- src/coreclr/inc/clrconfigvalues.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 542510f79b4a74..3c2a9c3dab3dd8 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -720,7 +720,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zbb, W("EnableRiscV64 #endif // Runtime-async -RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_RuntimeAsync, W("RuntimeAsync"), 0, "Enables runtime async method support") +RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_RuntimeAsync, W("RuntimeAsync"), 1, "Enables runtime async method support") /// /// Uncategorized From a21a7a8d8e03a802f03e0e80a4c3b3600d466c4f Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 15 Oct 2025 15:37:28 +0200 Subject: [PATCH 34/48] Fix bug --- src/coreclr/jit/async.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 971f49fbf7d1dd..9573042e159839 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1166,7 +1166,7 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* #endif // Now create continuation type. First create bitmap of object refs. - bool* objRefs = new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE]{}; + bool* objRefs = layout.Size < TARGET_POINTER_SIZE ? nullptr : new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE]{}; GCPointerBitMapBuilder bitmapBuilder(objRefs, layout.Size); bitmapBuilder.SetIfNotMax(layout.ExceptionOffset); From 84b205cd6527f5ddcc33bb81eb839dfc720db125 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 15 Oct 2025 15:37:59 +0200 Subject: [PATCH 35/48] Run jit-format --- src/coreclr/jit/async.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 9573042e159839..b0e7cec93e2fdf 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1166,7 +1166,8 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* #endif // Now create continuation type. First create bitmap of object refs. - bool* objRefs = layout.Size < TARGET_POINTER_SIZE ? nullptr : new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE]{}; + bool* objRefs = + layout.Size < TARGET_POINTER_SIZE ? nullptr : new (m_comp, CMK_Async) bool[layout.Size / TARGET_POINTER_SIZE]{}; GCPointerBitMapBuilder bitmapBuilder(objRefs, layout.Size); bitmapBuilder.SetIfNotMax(layout.ExceptionOffset); From 7b5f8a7a78730b2f72203a4393a354894f845514 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 15 Oct 2025 17:13:48 +0200 Subject: [PATCH 36/48] Give continuations names, fire profiler/ETW events --- src/coreclr/vm/asynccontinuations.cpp | 59 +++++++++++++++++++++------ src/coreclr/vm/asynccontinuations.h | 25 ++++++++++++ src/coreclr/vm/clsload.cpp | 2 +- src/coreclr/vm/clsload.hpp | 2 + src/coreclr/vm/jitinterface.cpp | 22 +++++++++- src/coreclr/vm/loaderallocator.cpp | 2 + src/coreclr/vm/typestring.cpp | 19 +++++++-- src/coreclr/vm/typestring.h | 1 + 8 files changed, 112 insertions(+), 20 deletions(-) diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 5a99600ebebc52..242c832d8ebfdb 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -24,6 +24,23 @@ AsyncContinuationsManager::AsyncContinuationsManager(LoaderAllocator* allocator) m_layouts.Init(16, &lock, m_allocator->GetLowFrequencyHeap()); } +void AsyncContinuationsManager::NotifyUnloadingClasses() +{ + if (!CORProfilerTrackClasses()) + { + return; + } + + EEHashTableIteration iter; + m_layouts.IterateStart(&iter); + while (m_layouts.IterateNext(&iter)) + { + MethodTable* pMT = (MethodTable*)m_layouts.IterateGetValue(&iter); + ClassLoader::NotifyUnload(pMT, true); + ClassLoader::NotifyUnload(pMT, false); + } +} + static EEClass* volatile g_singletonContinuationEEClass; EEClass* AsyncContinuationsManager::GetOrCreateSingletonSubContinuationEEClass() @@ -53,8 +70,8 @@ EEClass* AsyncContinuationsManager::CreateSingletonSubContinuationEEClass() pClass->SetMethodTable(pMT); #ifdef _DEBUG - pClass->SetDebugClassName("dynamicContinuation"); - pMT->SetDebugClassName("dynamicContinuation"); + pClass->SetDebugClassName("Continuation"); + pMT->SetDebugClassName("Continuation"); #endif if (InterlockedCompareExchangeT(&g_singletonContinuationEEClass, pClass, NULL) == NULL) @@ -197,13 +214,11 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable( pamTracker); #ifdef DEBUG - size_t numObjRefs = 0; - EnumerateRunsOfObjRefs(dataSize, objRefs, [&](size_t start, size_t count) { - numObjRefs += count; - return true; - }); StackSString debugName; - debugName.Printf("dynamicContinuation_%s_%u_%zu", asyncMethod->m_pszDebugMethodName, dataSize, numObjRefs); + PrintContinuationName( + pMT, + [&](LPCSTR str, LPCWSTR wstr) { debugName.AppendUTF8(str); }, + [&](unsigned num) { debugName.AppendPrintf("%u", num); }); const char* debugNameUTF8 = debugName.GetUTF8(); size_t len = strlen(debugNameUTF8) + 1; char* name = (char*)pamTracker->Track(m_allocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(len))); @@ -243,23 +258,41 @@ MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(un } } +#ifdef FEATURE_EVENT_TRACE + UINT32 typeLoad = ETW::TypeSystemLog::TypeLoadBegin(); +#endif + AllocMemTracker amTracker; - MethodTable* result = CreateNewContinuationMethodTable(dataSize, objRefs, adjustedDataOffsets, asyncMethod, &amTracker); + MethodTable* pNewMT = CreateNewContinuationMethodTable(dataSize, objRefs, adjustedDataOffsets, asyncMethod, &amTracker); + MethodTable* pReturnedMT = pNewMT; { CrstHolder lock(&m_layoutsLock); MethodTable* lookupResult; if (m_layouts.GetValue(ContinuationLayoutKey(&keyData), (HashDatum*)&lookupResult)) { - result = lookupResult; + pReturnedMT = lookupResult; } else { - m_layouts.InsertValue(ContinuationLayoutKey(result), result); - amTracker.SuppressRelease(); + m_layouts.InsertValue(ContinuationLayoutKey(pNewMT), pNewMT); + } + } + + if (pReturnedMT == pNewMT) + { + amTracker.SuppressRelease(); + + ClassLoader::NotifyLoad(TypeHandle(pNewMT)); + +#ifdef FEATURE_EVENT_TRACE + if (ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_DOTNET_Context, TypeLoadStop)) + { + ETW::TypeSystemLog::TypeLoadEnd(typeLoad, TypeHandle(pNewMT), CLASS_LOADED); } +#endif } - return result; + return pReturnedMT; } ContinuationLayoutKey::ContinuationLayoutKey(MethodTable* pMT) diff --git a/src/coreclr/vm/asynccontinuations.h b/src/coreclr/vm/asynccontinuations.h index eaf0e761b5c145..54883dbe4883ce 100644 --- a/src/coreclr/vm/asynccontinuations.h +++ b/src/coreclr/vm/asynccontinuations.h @@ -57,8 +57,33 @@ class AsyncContinuationsManager public: AsyncContinuationsManager(LoaderAllocator* allocator); MethodTable* LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod); + void NotifyUnloadingClasses(); + + template + static void PrintContinuationName(MethodTable* pMT, AppendString append, AppendNum appendNum); }; typedef DPTR(AsyncContinuationsManager) PTR_AsyncContinuationsManager; +template +void AsyncContinuationsManager::PrintContinuationName(MethodTable* pMT, AppendString append, AppendNum appendNum) +{ + append("Continuation_", W("Continuation_")); + appendNum(pMT->GetBaseSize() - (OBJHEADER_SIZE + OFFSETOF__CORINFO_Continuation__data)); + CGCDesc* desc = CGCDesc::GetCGCDescFromMT(pMT); + CGCDescSeries* lowestSeries = desc->GetLowestSeries(); + for (CGCDescSeries* curSeries = desc->GetHighestSeries(); curSeries >= lowestSeries; curSeries--) + { + if (curSeries->GetSeriesOffset() < OFFSETOF__CORINFO_Continuation__data) + { + continue; + } + + append("_", W("_")); + appendNum((unsigned)(curSeries->GetSeriesOffset() - OFFSETOF__CORINFO_Continuation__data)); + append("_", W("_")); + appendNum((unsigned)((curSeries->GetSeriesSize() + pMT->GetBaseSize()) / TARGET_POINTER_SIZE)); + } +} + #endif diff --git a/src/coreclr/vm/clsload.cpp b/src/coreclr/vm/clsload.cpp index 72ca527baa01c4..eedfec3d11848e 100644 --- a/src/coreclr/vm/clsload.cpp +++ b/src/coreclr/vm/clsload.cpp @@ -2879,7 +2879,7 @@ void ClassLoader::NotifyLoad(TypeHandle typeHnd) } #endif //PROFILING_SUPPORTED - if (pMT->IsTypicalTypeDefinition()) + if (pMT->IsTypicalTypeDefinition() && !IsNilToken(pMT->GetCl())) { LOG((LF_CLASSLOADER, LL_INFO100, "Successfully loaded class %s\n", pMT->GetDebugClassName())); diff --git a/src/coreclr/vm/clsload.hpp b/src/coreclr/vm/clsload.hpp index c50dd4a5fc7987..9d98e76c46200b 100644 --- a/src/coreclr/vm/clsload.hpp +++ b/src/coreclr/vm/clsload.hpp @@ -915,12 +915,14 @@ class ClassLoader // Publish the type in the loader's tables static TypeHandle PublishType(const TypeKey *pTypeKey, TypeHandle typeHnd); +public: // Notify profiler and debugger that a type load has completed // Also update perf counters static void NotifyLoad(TypeHandle typeHnd); // Notify profiler that a MethodTable is being unloaded static void NotifyUnload(MethodTable* pMT, bool unloadStarted); +private: // Phase CLASS_LOAD_EXACTPARENTS of class loading // Load exact parents and interfaces and dependent structures (generics dictionary, vtable fixes) static void LoadExactParents(MethodTable* pMT); diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 7a72356576c5ee..33131a01ad2be3 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -3417,7 +3417,6 @@ size_t CEEInfo::printClassName(CORINFO_CLASS_HANDLE cls, char* buffer, size_t bu size_t requiredBufferSize = 0; TypeHandle th(cls); - IMDInternalImport* pImport = th.GetMethodTable()->GetMDImport(); auto append = [buffer, bufferSize, &bytesWritten, &requiredBufferSize](const char* str) { @@ -3440,6 +3439,13 @@ size_t CEEInfo::printClassName(CORINFO_CLASS_HANDLE cls, char* buffer, size_t bu requiredBufferSize += strLen; }; + auto appendNum = [&](unsigned num) + { + char str[16]; + sprintf_s(str, "%u", num); + append(str); + }; + // Subset of TypeString that does just what we need while staying in UTF8 // and avoiding expensive copies. This function is called a lot in checked // builds. @@ -3451,10 +3457,22 @@ size_t CEEInfo::printClassName(CORINFO_CLASS_HANDLE cls, char* buffer, size_t bu mdTypeDef td = th.GetCl(); if (IsNilToken(td)) { - append("(dynamicClass)"); + if (th.IsContinuation()) + { + AsyncContinuationsManager::PrintContinuationName( + th.AsMethodTable(), + [&](LPCSTR str, LPCWSTR wstr) { append(str); }, + appendNum); + } + else + { + append("(dynamicClass)"); + } } else { + IMDInternalImport* pImport = th.GetMethodTable()->GetMDImport(); + DWORD attr; IfFailThrow(pImport->GetTypeDefProps(td, &attr, NULL)); diff --git a/src/coreclr/vm/loaderallocator.cpp b/src/coreclr/vm/loaderallocator.cpp index 62c47f2b756044..e8aceec349e07b 100644 --- a/src/coreclr/vm/loaderallocator.cpp +++ b/src/coreclr/vm/loaderallocator.cpp @@ -1414,6 +1414,8 @@ void LoaderAllocator::Terminate() if (m_asyncContinuationsManager != NULL) { + m_asyncContinuationsManager->NotifyUnloadingClasses(); + delete m_asyncContinuationsManager; m_asyncContinuationsManager = NULL; } diff --git a/src/coreclr/vm/typestring.cpp b/src/coreclr/vm/typestring.cpp index 4321492601e3f7..2c22666561a7ea 100644 --- a/src/coreclr/vm/typestring.cpp +++ b/src/coreclr/vm/typestring.cpp @@ -834,14 +834,25 @@ void TypeString::AppendType(TypeNameBuilder& tnb, TypeHandle ty, Instantiation t else { // Get the TypeDef token and attributes - IMDInternalImport *pImport = ty.GetMethodTable()->GetMDImport(); mdTypeDef td = ty.GetCl(); - if (IsNilToken(td)) { - // This type does not exist in metadata. Simply append "dynamicClass". - tnb.AddName(W("(dynamicClass)")); + if (IsNilToken(td)) + { + if (ty.IsContinuation()) + { + AsyncContinuationsManager::PrintContinuationName( + ty.AsMethodTable(), + [&](LPCSTR str, LPCWSTR wstr) { tnb.Append(wstr); }, + [&](unsigned num) { tnb.AppendNum(num); }); + } + else + { + // This type does not exist in metadata. Simply append "dynamicClass". + tnb.AddName(W("(dynamicClass)")); + } } else { + IMDInternalImport *pImport = ty.GetMethodTable()->GetMDImport(); #ifdef _DEBUG if (format & FormatDebug) { diff --git a/src/coreclr/vm/typestring.h b/src/coreclr/vm/typestring.h index c8706144c6ebe0..4bfe0b9692483e 100644 --- a/src/coreclr/vm/typestring.h +++ b/src/coreclr/vm/typestring.h @@ -102,6 +102,7 @@ class TypeNameBuilder void Append(WCHAR c) { WRAPPER_NO_CONTRACT; m_pStr->Append(c); } void Append(LPCUTF8 pStr) { WRAPPER_NO_CONTRACT; m_pStr->AppendUTF8(pStr); } void Append(UTF8 c) { WRAPPER_NO_CONTRACT; m_pStr->AppendUTF8(c); } + void AppendNum(unsigned num) { WRAPPER_NO_CONTRACT; m_pStr->AppendPrintf("%u", num); } SString* GetString() { WRAPPER_NO_CONTRACT; return m_pStr; } private: From df38cebb8be050b98746206a12b3ecdd6613c0e6 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 15 Oct 2025 17:37:02 +0200 Subject: [PATCH 37/48] Get Continuation class name from EE --- src/coreclr/jit/async.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index b0e7cec93e2fdf..f2e5e128347ca5 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1201,8 +1201,7 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* #ifdef DEBUG if (m_comp->verbose) { - printf("Getting continuation layout size = %u, numGCRefs = %u (Continuation_%s_%u_%u)\n", layout.Size, - bitmapBuilder.NumObjRefs, m_comp->info.compMethodName, layout.Size, bitmapBuilder.NumObjRefs); + printf("Getting continuation layout size = %u, numGCRefs = %u\n", layout.Size, bitmapBuilder.NumObjRefs); bool* start = objRefs; bool* endOfObjRefs = objRefs + layout.Size / TARGET_POINTER_SIZE; while (start < endOfObjRefs) @@ -1231,6 +1230,11 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* offsets.KeepAlive = keepAliveOffset; layout.ClassHnd = m_comp->info.compCompHnd->getContinuationType(layout.Size, objRefs, offsets); +#ifdef DEBUG + char buffer[256]; + JITDUMP(" Result = %s\n", m_comp->eeGetClassName(layout.ClassHnd, buffer, ArrLen(buffer))); +#endif + return layout; } From 91158a15c7b4d6f00d2511a854e43a43fe8e2f1b Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 15 Oct 2025 17:37:23 +0200 Subject: [PATCH 38/48] Fire duplicate TypeLoadEnd events instead of no matching pair --- src/coreclr/vm/asynccontinuations.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 242c832d8ebfdb..3557ac75cdb320 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -283,14 +283,14 @@ MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(un amTracker.SuppressRelease(); ClassLoader::NotifyLoad(TypeHandle(pNewMT)); + } #ifdef FEATURE_EVENT_TRACE - if (ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_DOTNET_Context, TypeLoadStop)) - { - ETW::TypeSystemLog::TypeLoadEnd(typeLoad, TypeHandle(pNewMT), CLASS_LOADED); - } -#endif + if (ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_DOTNET_Context, TypeLoadStop)) + { + ETW::TypeSystemLog::TypeLoadEnd(typeLoad, TypeHandle(pReturnedMT), CLASS_LOADED); } +#endif return pReturnedMT; } From 98917c0e8547f3e533a452752fa2dcc55e1aa988 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Wed, 15 Oct 2025 17:37:35 +0200 Subject: [PATCH 39/48] Fix non Windows build --- src/coreclr/vm/jitinterface.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 33131a01ad2be3..03507fc34cbe75 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -3442,7 +3442,7 @@ size_t CEEInfo::printClassName(CORINFO_CLASS_HANDLE cls, char* buffer, size_t bu auto appendNum = [&](unsigned num) { char str[16]; - sprintf_s(str, "%u", num); + sprintf_s(str, ARRAY_SIZE(str), "%u", num); append(str); }; From ec99d53bcc704b39b8252ed3787ff6cb6946dc55 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 16 Oct 2025 15:49:33 +0200 Subject: [PATCH 40/48] Fix OFFSETOF__CORINFO_Continuation__data for 32 bit --- src/coreclr/inc/corinfo.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index 8d9e3eddfc1939..e01c6a8eeb1560 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1958,7 +1958,7 @@ struct CORINFO_FPSTRUCT_LOWERING #define OFFSETOF__CORINFO_Span__reference 0 #define OFFSETOF__CORINFO_Span__length TARGET_POINTER_SIZE -#define OFFSETOF__CORINFO_Continuation__data (SIZEOF__CORINFO_Object + TARGET_POINTER_SIZE /* Next */ + TARGET_POINTER_SIZE /* Resume */ + TARGET_POINTER_SIZE /* Flags + maybe padding */) +#define OFFSETOF__CORINFO_Continuation__data (SIZEOF__CORINFO_Object + TARGET_POINTER_SIZE /* Next */ + TARGET_POINTER_SIZE /* Resume */ + 8 /* Flags + State */) /* data to optimize delegate construction */ From 9b3eea91b2ab517767d69117af182bc0f840864a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 16 Oct 2025 22:21:36 +0200 Subject: [PATCH 41/48] Remove CORINFO_CONTINUATION_DATA_OFFSETS; implement SuperPMI support --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 65 +++++++++---------- .../RuntimeHelpers.CoreCLR.cs | 3 - src/coreclr/inc/corinfo.h | 23 +++---- src/coreclr/inc/icorjitinfoimpl_generated.h | 2 +- src/coreclr/inc/jiteeversionguid.h | 10 +-- .../jit/ICorJitInfo_wrapper_generated.hpp | 4 +- src/coreclr/jit/async.cpp | 58 ++++++++++------- src/coreclr/jit/async.h | 6 +- .../tools/Common/JitInterface/CorInfoImpl.cs | 3 +- .../JitInterface/CorInfoImpl_generated.cs | 6 +- .../tools/Common/JitInterface/CorInfoTypes.cs | 8 --- .../ThunkGenerator/ThunkInput.txt | 3 +- .../aot/jitinterface/jitinterface_generated.h | 6 +- .../tools/superpmi/superpmi-shared/agnostic.h | 4 +- .../superpmi-shared/methodcontext.cpp | 34 +++++----- .../superpmi/superpmi-shared/methodcontext.h | 4 +- .../superpmi-shim-collector/icorjitinfo.cpp | 8 +-- .../icorjitinfo_generated.cpp | 4 +- .../icorjitinfo_generated.cpp | 4 +- .../tools/superpmi/superpmi/icorjitinfo.cpp | 6 +- src/coreclr/vm/asynccontinuations.cpp | 57 ++-------------- src/coreclr/vm/asynccontinuations.h | 10 ++- src/coreclr/vm/jitinterface.cpp | 7 +- src/coreclr/vm/methodtable.h | 16 +---- 24 files changed, 138 insertions(+), 213 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index e62cc0d6abd6f6..34da47a7ecfe1c 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -4,6 +4,7 @@ using System.Buffers.Binary; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; +using System.Numerics; using System.Reflection; using System.Runtime.InteropServices; using System.Runtime.Serialization; @@ -49,28 +50,24 @@ public void Pop() [Flags] internal enum CorInfoContinuationFlags { + CORINFO_CONTINUATION_HAS_OSR_ILOFFSET = 1, // If this bit is set the continuation resumes inside a try block and thus // if an exception is being propagated, needs to be resumed. The exception // should be placed at index 0 or 1 depending on whether the continuation // also expects a result. - CORINFO_CONTINUATION_NEEDS_EXCEPTION = 1, + CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2, + CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT = 4, + CORINFO_CONTINUATION_HAS_KEEPALIVE = 8, + CORINFO_CONTINUATION_HAS_RESULT = 16, // If this bit is set the continuation should continue on the thread // pool. - CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 2, + CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 32, // If this bit is set the continuation has a SynchronizationContext // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 4, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 64, // If this bit is set the continuation has a TaskScheduler // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 8, - } - - internal struct CORINFO_CONTINUATION_DATA_OFFSETS - { - public uint Result; - public uint Exception; - public uint ContinuationContext; - public uint KeepAlive; + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 128, } #pragma warning disable CA1852 // "Type can be sealed" -- no it cannot because the runtime constructs subtypes dynamically @@ -81,50 +78,46 @@ internal unsafe class Continuation public CorInfoContinuationFlags Flags; public int State; - public unsafe object GetContinuationContext() - { - MethodTable* mt = RuntimeHelpers.GetMethodTable(this); +#if TARGET_64BIT + private const int POINTER_SIZE = 8; +#else + private const int POINTER_SIZE = 4; +#endif - // Only the special continuation sub types have continuation offsets. - Debug.Assert(mt->IsContinuation); - Debug.Assert(mt->ContinuationOffsets->ContinuationContext != uint.MaxValue); + private const int CONTINUATION_OFFSET_TO_DATA = POINTER_SIZE /* Next */ + POINTER_SIZE /* Resume */ + 8 /* Flags + State */; + public unsafe object GetContinuationContext() + { + Debug.Assert((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT) != 0); + uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT - 1)); ref byte data = ref RuntimeHelpers.GetRawData(this); - return Unsafe.As(ref Unsafe.Add(ref data, mt->ContinuationOffsets->ContinuationContext)); + return Unsafe.As(ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE)); } public void SetException(Exception ex) { - MethodTable* mt = RuntimeHelpers.GetMethodTable(this); - - // Only the special continuation sub types have continuation offsets. - Debug.Assert(mt->IsContinuation); + Debug.Assert((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION) != 0); + uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION - 1)); ref byte data = ref RuntimeHelpers.GetRawData(this); - Unsafe.As(ref Unsafe.Add(ref data, mt->ContinuationOffsets->Exception)) = ex; + Unsafe.As(ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE)) = ex; } public ref byte GetResultStorageOrNull() { - MethodTable* mt = RuntimeHelpers.GetMethodTable(this); - // Only the special continuation sub types have continuation offsets. - Debug.Assert(mt->IsContinuation); - - if (mt->ContinuationOffsets->Result == uint.MaxValue) + if ((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_RESULT) == 0) return ref Unsafe.NullRef(); + uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_RESULT - 1)); ref byte data = ref RuntimeHelpers.GetRawData(this); - return ref Unsafe.Add(ref data, mt->ContinuationOffsets->Result); + return ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE); } public void SetKeepAlive(object? obj) { - MethodTable* mt = RuntimeHelpers.GetMethodTable(this); - // Only the special continuation sub types have continuation offsets. - Debug.Assert(mt->IsContinuation); - Debug.Assert(mt->ContinuationOffsets->KeepAlive != uint.MaxValue); - + Debug.Assert((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_KEEPALIVE) != 0); + uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_KEEPALIVE - 1)); ref byte data = ref RuntimeHelpers.GetRawData(this); - Unsafe.As(ref Unsafe.Add(ref data, mt->ContinuationOffsets->KeepAlive)) = obj; + Unsafe.As(ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE)) = obj; } } diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index dd542085724a53..259475edd6d465 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -739,9 +739,6 @@ internal unsafe struct MethodTable [FieldOffset(ElementTypeOffset)] public MethodTable*** PerInstInfo; - [FieldOffset(ElementTypeOffset)] - public CORINFO_CONTINUATION_DATA_OFFSETS* ContinuationOffsets; - /// /// This interface map used to list out the set of interfaces. Only meaningful if InterfaceCount is non-zero. /// diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index e01c6a8eeb1560..9f98ab4695e4e8 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1700,20 +1700,24 @@ struct CORINFO_EE_INFO enum CorInfoContinuationFlags { + CORINFO_CONTINUATION_HAS_OSR_ILOFFSET = 1, // If this bit is set the continuation resumes inside a try block and thus // if an exception is being propagated, needs to be resumed. The exception // should be placed at index 0 or 1 depending on whether the continuation // also expects a result. - CORINFO_CONTINUATION_NEEDS_EXCEPTION = 1, + CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2, + CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT = 4, + CORINFO_CONTINUATION_HAS_KEEPALIVE = 8, + CORINFO_CONTINUATION_HAS_RESULT = 16, // If this bit is set the continuation should continue on the thread // pool. - CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 2, + CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 32, // If this bit is set the continuation has a SynchronizationContext // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 4, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 64, // If this bit is set the continuation has a TaskScheduler // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 8, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 128, }; struct CORINFO_ASYNC_INFO @@ -1740,15 +1744,6 @@ struct CORINFO_ASYNC_INFO CORINFO_METHOD_HANDLE restoreContextsMethHnd; }; -// Offsets for continuation data layout -struct CORINFO_CONTINUATION_DATA_OFFSETS -{ - uint32_t Result; - uint32_t Exception; - uint32_t ContinuationContext; - uint32_t KeepAlive; -}; - // Flags passed from JIT to runtime. enum CORINFO_GET_TAILCALL_HELPERS_FLAGS { @@ -3345,7 +3340,7 @@ class ICorDynamicInfo : public ICorStaticInfo virtual CORINFO_CLASS_HANDLE getContinuationType( size_t dataSize, bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets + size_t objRefsSize ) = 0; // Optionally, convert calli to regular method call. This is for PInvoke argument marshalling. diff --git a/src/coreclr/inc/icorjitinfoimpl_generated.h b/src/coreclr/inc/icorjitinfoimpl_generated.h index f3e07b95b30876..0dc4f5fa089732 100644 --- a/src/coreclr/inc/icorjitinfoimpl_generated.h +++ b/src/coreclr/inc/icorjitinfoimpl_generated.h @@ -658,7 +658,7 @@ bool getTailCallHelpers( CORINFO_CLASS_HANDLE getContinuationType( size_t dataSize, bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) override; + size_t objRefsSize) override; CORINFO_METHOD_HANDLE getAsyncResumptionStub() override; diff --git a/src/coreclr/inc/jiteeversionguid.h b/src/coreclr/inc/jiteeversionguid.h index d44407a0f5a818..2b4ee632d71684 100644 --- a/src/coreclr/inc/jiteeversionguid.h +++ b/src/coreclr/inc/jiteeversionguid.h @@ -37,11 +37,11 @@ #include -constexpr GUID JITEEVersionIdentifier = { /* 2309660f-0396-4aab-8a77-fe66133a1ba0 */ - 0x2309660f, - 0x0396, - 0x4aab, - {0x8a, 0x77, 0xfe, 0x66, 0x13, 0x3a, 0x1b, 0xa0} +constexpr GUID JITEEVersionIdentifier = { /* 68e93e9d-dd28-49ca-9ebc-a01a54532bb3 */ + 0x68e93e9d, + 0xdd28, + 0x49ca, + {0x9e, 0xbc, 0xa0, 0x1a, 0x54, 0x53, 0x2b, 0xb3} }; #endif // JIT_EE_VERSIONING_GUID_H diff --git a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp index 2d06899182d94d..b1014f476f739e 100644 --- a/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp +++ b/src/coreclr/jit/ICorJitInfo_wrapper_generated.hpp @@ -1539,10 +1539,10 @@ bool WrapICorJitInfo::getTailCallHelpers( CORINFO_CLASS_HANDLE WrapICorJitInfo::getContinuationType( size_t dataSize, bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) + size_t objRefsSize) { API_ENTER(getContinuationType); - CORINFO_CLASS_HANDLE temp = wrapHnd->getContinuationType(dataSize, objRefs, dataOffsets); + CORINFO_CLASS_HANDLE temp = wrapHnd->getContinuationType(dataSize, objRefs, objRefsSize); API_LEAVE(getContinuationType); return temp; } diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index f2e5e128347ca5..66a8b9db3788c0 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1107,17 +1107,8 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* { JITDUMP(" Method %s; keeping IL offset that inspired OSR method at the beginning of non-GC data\n", m_comp->doesMethodHavePatchpoints() ? "has patchpoints" : "is an OSR method"); - allocLayout(sizeof(int), sizeof(int)); - } - - if (layout.ReturnSize > 0) - { - layout.ReturnValOffset = allocLayout(layout.ReturnHeapAlignment(), layout.ReturnSize); - - JITDUMP(" Will store return of type %s, size %u at offset %u\n", - call->gtReturnType == TYP_STRUCT ? layout.ReturnStructLayout->GetClassName() - : varTypeName(call->gtReturnType), - layout.ReturnSize, layout.ReturnValOffset); + // Must be pointer sized for compatibility with Continuation methods that access fields + layout.OSRILOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); } if (block->hasTryIndex()) @@ -1134,6 +1125,22 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* layout.ContinuationContextOffset); } + if (needsKeepAlive) + { + layout.KeepAliveOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); + JITDUMP(" Continuation needs keep alive object; will be at offset %u\n", layout.KeepAliveOffset); + } + + if (layout.ReturnSize > 0) + { + layout.ReturnValOffset = allocLayout(layout.ReturnHeapAlignment(), layout.ReturnSize); + + JITDUMP(" Will store return of type %s, size %u at offset %u\n", + call->gtReturnType == TYP_STRUCT ? layout.ReturnStructLayout->GetClassName() + : varTypeName(call->gtReturnType), + layout.ReturnSize, layout.ReturnValOffset); + } + if (call->GetAsyncInfo().ExecutionContextHandling == ExecutionContextHandling::AsyncSaveAndRestore) { layout.ExecContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); @@ -1141,12 +1148,6 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* layout.ExecContextOffset); } - unsigned keepAliveOffset = UINT_MAX; - if (needsKeepAlive) - { - keepAliveOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); - } - for (LiveLocalInfo& inf : liveLocals) { inf.Offset = allocLayout(inf.HeapAlignment(), inf.Size); @@ -1172,8 +1173,8 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* GCPointerBitMapBuilder bitmapBuilder(objRefs, layout.Size); bitmapBuilder.SetIfNotMax(layout.ExceptionOffset); bitmapBuilder.SetIfNotMax(layout.ContinuationContextOffset); + bitmapBuilder.SetIfNotMax(layout.KeepAliveOffset); bitmapBuilder.SetIfNotMax(layout.ExecContextOffset); - bitmapBuilder.SetIfNotMax(keepAliveOffset); if (layout.ReturnSize > 0) { @@ -1223,12 +1224,8 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* } #endif - CORINFO_CONTINUATION_DATA_OFFSETS offsets; - offsets.Result = layout.ReturnValOffset; - offsets.Exception = layout.ExceptionOffset; - offsets.ContinuationContext = layout.ContinuationContextOffset; - offsets.KeepAlive = keepAliveOffset; - layout.ClassHnd = m_comp->info.compCompHnd->getContinuationType(layout.Size, objRefs, offsets); + layout.ClassHnd = + m_comp->info.compCompHnd->getContinuationType(layout.Size, objRefs, layout.Size / TARGET_POINTER_SIZE); #ifdef DEBUG char buffer[256]; @@ -1439,8 +1436,16 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Fill in 'flags' const AsyncCallInfo& callInfo = call->GetAsyncInfo(); unsigned continuationFlags = 0; - if (block->hasTryIndex()) + if (layout.OSRILOffset != UINT_MAX) + continuationFlags |= CORINFO_CONTINUATION_HAS_OSR_ILOFFSET; + if (layout.ExceptionOffset != UINT_MAX) continuationFlags |= CORINFO_CONTINUATION_NEEDS_EXCEPTION; + if (layout.ContinuationContextOffset != UINT_MAX) + continuationFlags |= CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT; + if (layout.KeepAliveOffset != UINT_MAX) + continuationFlags |= CORINFO_CONTINUATION_HAS_KEEPALIVE; + if (layout.ReturnValOffset != UINT_MAX) + continuationFlags |= CORINFO_CONTINUATION_HAS_RESULT; if (callInfo.ContinuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) continuationFlags |= CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL; @@ -1538,6 +1543,9 @@ void AsyncTransformation::FillInDataOnSuspension(GenTreeCall* call, else ilOffsetToStore = m_comp->gtNewIconNode((int)m_comp->info.compILEntry); + // OSR IL offset needs to be at offset 0 because OSR and tier0 methods + // need to agree on that. + assert(layout.OSRILOffset == 0); GenTree* newContinuation = m_comp->gtNewLclvNode(m_newContinuationVar, TYP_REF); unsigned offset = OFFSETOF__CORINFO_Continuation__data; GenTree* storePatchpointOffset = StoreAtOffset(newContinuation, offset, ilOffsetToStore, TYP_INT); diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 79df5435608891..20505461d53075 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -22,13 +22,15 @@ struct LiveLocalInfo struct ContinuationLayout { unsigned Size = 0; + unsigned OSRILOffset = UINT_MAX; + unsigned ExceptionOffset = UINT_MAX; + unsigned ContinuationContextOffset = UINT_MAX; + unsigned KeepAliveOffset = UINT_MAX; ClassLayout* ReturnStructLayout = nullptr; unsigned ReturnAlignment = 0; unsigned ReturnSize = 0; unsigned ReturnValOffset = UINT_MAX; - unsigned ExceptionOffset = UINT_MAX; unsigned ExecContextOffset = UINT_MAX; - unsigned ContinuationContextOffset = UINT_MAX; const jitstd::vector& Locals; CORINFO_CLASS_HANDLE ClassHnd = NO_CLASS_HANDLE; diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs index 7e8c5805887157..9f1900b1ba3496 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl.cs @@ -3361,8 +3361,9 @@ private void getAsyncInfo(ref CORINFO_ASYNC_INFO pAsyncInfoOut) throw new NotImplementedException(); } - private CORINFO_CLASS_STRUCT_* getContinuationType(UIntPtr dataSize, ref bool objRefs, ref CORINFO_CONTINUATION_DATA_OFFSETS dataOffsets) + private CORINFO_CLASS_STRUCT_* getContinuationType(nuint dataSize, ref bool objRefs, nuint objRefsSize) { + Debug.Assert(objRefsSize == (dataSize + (nuint)(PointerSize - 1)) / (nuint)PointerSize); throw new NotImplementedException("getContinuationType"); } diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs index 88fba3f9a6299b..b0a1781661517d 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoImpl_generated.cs @@ -2298,12 +2298,12 @@ private static byte _getTailCallHelpers(IntPtr thisHandle, IntPtr* ppException, } [UnmanagedCallersOnly] - private static CORINFO_CLASS_STRUCT_* _getContinuationType(IntPtr thisHandle, IntPtr* ppException, nuint dataSize, bool* objRefs, CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) + private static CORINFO_CLASS_STRUCT_* _getContinuationType(IntPtr thisHandle, IntPtr* ppException, nuint dataSize, bool* objRefs, nuint objRefsSize) { var _this = GetThis(thisHandle); try { - return _this.getContinuationType(dataSize, ref *objRefs, ref *dataOffsets); + return _this.getContinuationType(dataSize, ref *objRefs, objRefsSize); } catch (Exception ex) { @@ -2778,7 +2778,7 @@ private static IntPtr GetUnmanagedCallbacks() callbacks[152] = (delegate* unmanaged)&_GetDelegateCtor; callbacks[153] = (delegate* unmanaged)&_MethodCompileComplete; callbacks[154] = (delegate* unmanaged)&_getTailCallHelpers; - callbacks[155] = (delegate* unmanaged)&_getContinuationType; + callbacks[155] = (delegate* unmanaged)&_getContinuationType; callbacks[156] = (delegate* unmanaged)&_getAsyncResumptionStub; callbacks[157] = (delegate* unmanaged)&_convertPInvokeCalliToCall; callbacks[158] = (delegate* unmanaged)&_notifyInstructionSetUsage; diff --git a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs index 718e77ccc7c792..797ae1c7d3061e 100644 --- a/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs +++ b/src/coreclr/tools/Common/JitInterface/CorInfoTypes.cs @@ -884,14 +884,6 @@ public unsafe struct CORINFO_ASYNC_INFO public CORINFO_FIELD_STRUCT_* continuationGCDataFldHnd; } - public struct CORINFO_CONTINUATION_DATA_OFFSETS - { - public int Result; - public int Exception; - public int ContinuationContext; - public int KeepAlive; - } - // Flags passed from JIT to runtime. public enum CORINFO_GET_TAILCALL_HELPERS_FLAGS { diff --git a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt index 5cf814d1d7e39f..2a06ed81d01bd5 100644 --- a/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt +++ b/src/coreclr/tools/Common/JitInterface/ThunkGenerator/ThunkInput.txt @@ -84,7 +84,6 @@ CORINFO_RESOLVED_TOKEN*,ref CORINFO_RESOLVED_TOKEN CORINFO_RESOLVED_TOKEN_PTR,CORINFO_RESOLVED_TOKEN*,CORINFO_RESOLVED_TOKEN*,CORINFO_RESOLVED_TOKEN* CORINFO_EE_INFO*,ref CORINFO_EE_INFO CORINFO_ASYNC_INFO*,ref CORINFO_ASYNC_INFO -const CORINFO_CONTINUATION_DATA_OFFSETS&,ref CORINFO_CONTINUATION_DATA_OFFSETS CORINFO_TAILCALL_HELPERS*,ref CORINFO_TAILCALL_HELPERS CORINFO_SWIFT_LOWERING*,ref CORINFO_SWIFT_LOWERING CORINFO_FPSTRUCT_LOWERING*,ref CORINFO_FPSTRUCT_LOWERING @@ -322,7 +321,7 @@ FUNCTIONS CORINFO_METHOD_HANDLE GetDelegateCtor(CORINFO_METHOD_HANDLE methHnd, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE targetMethodHnd, DelegateCtorArgs * pCtorData); void MethodCompileComplete(CORINFO_METHOD_HANDLE methHnd); bool getTailCallHelpers(CORINFO_RESOLVED_TOKEN* callToken, CORINFO_SIG_INFO* sig, CORINFO_GET_TAILCALL_HELPERS_FLAGS flags, CORINFO_TAILCALL_HELPERS* pResult); - CORINFO_CLASS_HANDLE getContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets); + CORINFO_CLASS_HANDLE getContinuationType(size_t dataSize, bool* objRefs, size_t objRefsSize); CORINFO_METHOD_HANDLE getAsyncResumptionStub(); bool convertPInvokeCalliToCall(CORINFO_RESOLVED_TOKEN * pResolvedToken, bool mustConvert); bool notifyInstructionSetUsage(CORINFO_InstructionSet instructionSet,bool supportEnabled); diff --git a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h index 95115ba69d2443..cf8ea9deb5b2ff 100644 --- a/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h +++ b/src/coreclr/tools/aot/jitinterface/jitinterface_generated.h @@ -166,7 +166,7 @@ struct JitInterfaceCallbacks CORINFO_METHOD_HANDLE (* GetDelegateCtor)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE methHnd, CORINFO_CLASS_HANDLE clsHnd, CORINFO_METHOD_HANDLE targetMethodHnd, DelegateCtorArgs* pCtorData); void (* MethodCompileComplete)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_METHOD_HANDLE methHnd); bool (* getTailCallHelpers)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* callToken, CORINFO_SIG_INFO* sig, CORINFO_GET_TAILCALL_HELPERS_FLAGS flags, CORINFO_TAILCALL_HELPERS* pResult); - CORINFO_CLASS_HANDLE (* getContinuationType)(void * thisHandle, CorInfoExceptionClass** ppException, size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets); + CORINFO_CLASS_HANDLE (* getContinuationType)(void * thisHandle, CorInfoExceptionClass** ppException, size_t dataSize, bool* objRefs, size_t objRefsSize); CORINFO_METHOD_HANDLE (* getAsyncResumptionStub)(void * thisHandle, CorInfoExceptionClass** ppException); bool (* convertPInvokeCalliToCall)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_RESOLVED_TOKEN* pResolvedToken, bool mustConvert); bool (* notifyInstructionSetUsage)(void * thisHandle, CorInfoExceptionClass** ppException, CORINFO_InstructionSet instructionSet, bool supportEnabled); @@ -1718,10 +1718,10 @@ class JitInterfaceWrapper : public ICorJitInfo virtual CORINFO_CLASS_HANDLE getContinuationType( size_t dataSize, bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) + size_t objRefsSize) { CorInfoExceptionClass* pException = nullptr; - CORINFO_CLASS_HANDLE temp = _callbacks->getContinuationType(_thisHandle, &pException, dataSize, objRefs, dataOffsets); + CORINFO_CLASS_HANDLE temp = _callbacks->getContinuationType(_thisHandle, &pException, dataSize, objRefs, objRefsSize); if (pException != nullptr) throw pException; return temp; } diff --git a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h index 9c93cf976ad90b..633eb333371807 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h @@ -664,9 +664,7 @@ struct Agnostic_GetContinuationTypeIn { DWORDLONG dataSize; DWORD objRefs; - DWORD result; - DWORD exception; - DWORD continuationContext; + DWORD objRefsSize; }; struct Agnostic_ResolveVirtualMethodKey diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 0234dd0a86faf6..2dbce14e317eb9 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -6943,7 +6943,7 @@ CORINFO_METHOD_HANDLE MethodContext::repGetAsyncResumptionStub() void MethodContext::recGetContinuationType(size_t dataSize, bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, + size_t objRefsSize, CORINFO_CLASS_HANDLE result) { if (GetContinuationType == nullptr) @@ -6954,10 +6954,8 @@ void MethodContext::recGetContinuationType(size_t dataSize, Agnostic_GetContinuationTypeIn key; ZeroMemory(&key, sizeof(key)); key.dataSize = (DWORDLONG)dataSize; - key.objRefs = (objRefs != nullptr) ? (*objRefs ? 1 : 0) : 0; // TODO add buffer - key.result = (DWORD)dataOffsets.Result; - key.exception = (DWORD)dataOffsets.Exception; - key.continuationContext = (DWORD)dataOffsets.ContinuationContext; + key.objRefs = GetContinuationType->AddBuffer((unsigned char*)objRefs, (unsigned)objRefsSize); + key.objRefsSize = (DWORD)objRefsSize; DWORDLONG value = CastHandle(result); @@ -6967,25 +6965,27 @@ void MethodContext::recGetContinuationType(size_t dataSize, void MethodContext::dmpGetContinuationType(const Agnostic_GetContinuationTypeIn& key, DWORDLONG value) { - printf("GetContinuationType dataSize-%016" PRIX64 ", objRefs-%u, result-%d, exception-%d, handle-%016" PRIX64, - key.dataSize, key.objRefs, key.result, key.exception, value); + printf("GetContinuationType dataSize-%016" PRIX64 ", objRefs-%u, handle-%016" PRIX64, + key.dataSize, key.objRefs, value); } CORINFO_CLASS_HANDLE MethodContext::repGetContinuationType(size_t dataSize, - bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) + bool* objRefs, + size_t objRefsSize) { + AssertMapExistsNoMessage(GetContinuationType); + + int objRefsBuffer = GetContinuationType->Contains((unsigned char*)objRefs, (unsigned)objRefsSize); + if (objRefsBuffer == -1) + LogException(EXCEPTIONCODE_MC, "SuperPMI assertion failed (missing obj refs buffer)", ""); + Agnostic_GetContinuationTypeIn key; ZeroMemory(&key, sizeof(key)); key.dataSize = (DWORDLONG)dataSize; - key.objRefs = (objRefs != nullptr) ? (*objRefs ? 1 : 0) : 0; - key.result = (DWORD)dataOffsets.Result; - key.exception = (DWORD)dataOffsets.Exception; - key.continuationContext = (DWORD)dataOffsets.ContinuationContext; - - DWORDLONG value = LookupByKeyOrMiss(GetContinuationType, key, - ": dataSize %016" PRIX64 ", objRefs %u, result %d, exception %d", - key.dataSize, key.objRefs, key.result, key.exception); + key.objRefs = objRefsBuffer; + key.objRefsSize = (DWORD)objRefsSize; + + DWORDLONG value = LookupByKeyOrMiss(GetContinuationType, key, ": dataSize %016" PRIX64 ", objRefs %u", key.dataSize, key.objRefs); DEBUG_REP(dmpGetContinuationType(key, value)); return (CORINFO_CLASS_HANDLE)value; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h index ebbe2334ab20d6..c5d7f82923841e 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.h @@ -867,9 +867,9 @@ class MethodContext void dmpGetAsyncResumptionStub(DWORD key, DWORDLONG handle); CORINFO_METHOD_HANDLE repGetAsyncResumptionStub(); - void recGetContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, CORINFO_CLASS_HANDLE result); + void recGetContinuationType(size_t dataSize, bool* objRefs, size_t objRefsSize, CORINFO_CLASS_HANDLE result); void dmpGetContinuationType(const Agnostic_GetContinuationTypeIn& key, DWORDLONG value); - CORINFO_CLASS_HANDLE repGetContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets); + CORINFO_CLASS_HANDLE repGetContinuationType(size_t dataSize, bool* objRefs, size_t objRefsSize); void recUpdateEntryPointForTailCall(const CORINFO_CONST_LOOKUP& origEntryPoint, const CORINFO_CONST_LOOKUP& newEntryPoint); void dmpUpdateEntryPointForTailCall(const Agnostic_CORINFO_CONST_LOOKUP& origEntryPoint, const Agnostic_CORINFO_CONST_LOOKUP& newEntryPoint); diff --git a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp index 8a30587ddc1ba1..e8250d6a45c483 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-collector/icorjitinfo.cpp @@ -1782,11 +1782,11 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub() return stub; } -CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType(size_t dataSize, bool* objRefs, size_t objRefsSize) { mc->cr->AddCall("getContinuationType"); - CORINFO_CLASS_HANDLE result = original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); - mc->recGetContinuationType(dataSize, objRefs, dataOffsets, result); + CORINFO_CLASS_HANDLE result = original_ICorJitInfo->getContinuationType(dataSize, objRefs, objRefsSize); + mc->recGetContinuationType(dataSize, objRefs, objRefsSize, result); return result; } @@ -2040,4 +2040,4 @@ CORINFO_METHOD_HANDLE interceptor_ICJI::getSpecialCopyHelper(CORINFO_CLASS_HANDL CORINFO_METHOD_HANDLE temp = original_ICorJitInfo->getSpecialCopyHelper(type); mc->recGetSpecialCopyHelper(type, temp); return temp; -} \ No newline at end of file +} diff --git a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp index eb54dfa0420440..e78124fc9642fa 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-counter/icorjitinfo_generated.cpp @@ -1269,10 +1269,10 @@ bool interceptor_ICJI::getTailCallHelpers( CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType( size_t dataSize, bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) + size_t objRefsSize) { mcs->AddCall("getContinuationType"); - return original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); + return original_ICorJitInfo->getContinuationType(dataSize, objRefs, objRefsSize); } CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub() diff --git a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp index 005b6aaaf093be..7937d140a0d275 100644 --- a/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shim-simple/icorjitinfo_generated.cpp @@ -1114,9 +1114,9 @@ bool interceptor_ICJI::getTailCallHelpers( CORINFO_CLASS_HANDLE interceptor_ICJI::getContinuationType( size_t dataSize, bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) + size_t objRefsSize) { - return original_ICorJitInfo->getContinuationType(dataSize, objRefs, dataOffsets); + return original_ICorJitInfo->getContinuationType(dataSize, objRefs, objRefsSize); } CORINFO_METHOD_HANDLE interceptor_ICJI::getAsyncResumptionStub() diff --git a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp index 78781fc1be7c78..b022efbc7c600b 100644 --- a/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp +++ b/src/coreclr/tools/superpmi/superpmi/icorjitinfo.cpp @@ -1508,10 +1508,10 @@ CORINFO_METHOD_HANDLE MyICJI::getAsyncResumptionStub() return jitInstance->mc->repGetAsyncResumptionStub();; } -CORINFO_CLASS_HANDLE MyICJI::getContinuationType(size_t dataSize, bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) +CORINFO_CLASS_HANDLE MyICJI::getContinuationType(size_t dataSize, bool* objRefs, size_t objRefsSize) { jitInstance->mc->cr->AddCall("getContinuationType"); - CORINFO_CLASS_HANDLE result = jitInstance->mc->repGetContinuationType(dataSize, objRefs, dataOffsets); + CORINFO_CLASS_HANDLE result = jitInstance->mc->repGetContinuationType(dataSize, objRefs, objRefsSize); return result; } @@ -1848,4 +1848,4 @@ CORINFO_METHOD_HANDLE MyICJI::getSpecialCopyHelper(CORINFO_CLASS_HANDLE type) jitInstance->mc->cr->AddCall("getSpecialCopyHelper"); CORINFO_METHOD_HANDLE result = jitInstance->mc->repGetSpecialCopyHelper(type); return result; -} \ No newline at end of file +} diff --git a/src/coreclr/vm/asynccontinuations.cpp b/src/coreclr/vm/asynccontinuations.cpp index 3557ac75cdb320..c9f954015d76c6 100644 --- a/src/coreclr/vm/asynccontinuations.cpp +++ b/src/coreclr/vm/asynccontinuations.cpp @@ -60,12 +60,7 @@ EEClass* AsyncContinuationsManager::CreateSingletonSubContinuationEEClass() EEClass* pClass = EEClass::CreateMinimalClass(allocator->GetHighFrequencyHeap(), &amTracker); - CORINFO_CONTINUATION_DATA_OFFSETS offsets; - offsets.Result = UINT_MAX; - offsets.Exception = UINT_MAX; - offsets.ContinuationContext = UINT_MAX; - offsets.KeepAlive = UINT_MAX; - MethodTable* pMT = CreateNewContinuationMethodTable(0, NULL, offsets, pClass, allocator, spc, &amTracker); + MethodTable* pMT = CreateNewContinuationMethodTable(0, NULL, pClass, allocator, spc, &amTracker); pClass->SetMethodTable(pMT); @@ -111,7 +106,6 @@ static bool EnumerateRunsOfObjRefs(unsigned size, const bool* objRefs, TFunc fun MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable( unsigned dataSize, const bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, EEClass* pClass, LoaderAllocator* allocator, Module* loaderModule, @@ -146,17 +140,14 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable( size_t cbGC = numPointerSeries == 0 ? 0 : CGCDesc::ComputeSize(numPointerSeries); - BYTE* pMemory = (BYTE*)pamTracker->Track(allocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(sizeof(CORINFO_CONTINUATION_DATA_OFFSETS)) + S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); - - CORINFO_CONTINUATION_DATA_OFFSETS* allocatedDataOffsets = new (pMemory) CORINFO_CONTINUATION_DATA_OFFSETS(dataOffsets); + BYTE* pMemory = (BYTE*)pamTracker->Track(allocator->GetHighFrequencyHeap()->AllocMem(S_SIZE_T(cbGC) + S_SIZE_T(cbMT))); unsigned startOfDataInInstance = AlignUp(pParentClass->GetNumInstanceFieldBytes(), TARGET_POINTER_SIZE); unsigned startOfDataInObject = OBJECT_SIZE + startOfDataInInstance; - MethodTable* pMT = (MethodTable*)(pMemory + sizeof(CORINFO_CONTINUATION_DATA_OFFSETS) + cbGC); + MethodTable* pMT = (MethodTable*)(pMemory + cbGC); pMT->AllocateAuxiliaryData(allocator, loaderModule, pamTracker, MethodTableStaticsFlags::None); pMT->SetParentMethodTable(pParentClass); - pMT->SetContinuationDataOffsets(allocatedDataOffsets); pMT->SetLoaderAllocator(allocator); pMT->SetModule(pParentClass->GetModule()); pMT->SetNumVirtuals(static_cast(numVirtuals)); @@ -200,14 +191,12 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable( MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable( unsigned dataSize, const bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker) { MethodTable* pMT = CreateNewContinuationMethodTable( dataSize, objRefs, - dataOffsets, GetOrCreateSingletonSubContinuationEEClass(), m_allocator, asyncMethod->GetLoaderModule(), @@ -229,26 +218,11 @@ MethodTable* AsyncContinuationsManager::CreateNewContinuationMethodTable( return pMT; } -MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod) +MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, MethodDesc* asyncMethod) { STANDARD_VM_CONTRACT; - // The API we expose has all offsets relative to the data, but we prefer to - // have offsets relative to the start of instance data so that - // RuntimeHelpers.GetRawData(obj) + offset returns the right offset. Adjust - // it here. - CORINFO_CONTINUATION_DATA_OFFSETS adjustedDataOffsets = dataOffsets; - const uint32_t startOfDataInInstance = OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object; - if (dataOffsets.Result != UINT_MAX) - adjustedDataOffsets.Result += startOfDataInInstance; - if (dataOffsets.Exception != UINT_MAX) - adjustedDataOffsets.Exception += startOfDataInInstance; - if (dataOffsets.ContinuationContext != UINT_MAX) - adjustedDataOffsets.ContinuationContext += startOfDataInInstance; - if (dataOffsets.KeepAlive != UINT_MAX) - adjustedDataOffsets.KeepAlive += startOfDataInInstance; - - ContinuationLayoutKeyData keyData(dataSize, objRefs, adjustedDataOffsets); + ContinuationLayoutKeyData keyData(dataSize, objRefs); { CrstHolder lock(&m_layoutsLock); MethodTable* lookupResult; @@ -263,7 +237,7 @@ MethodTable* AsyncContinuationsManager::LookupOrCreateContinuationMethodTable(un #endif AllocMemTracker amTracker; - MethodTable* pNewMT = CreateNewContinuationMethodTable(dataSize, objRefs, adjustedDataOffsets, asyncMethod, &amTracker); + MethodTable* pNewMT = CreateNewContinuationMethodTable(dataSize, objRefs, asyncMethod, &amTracker); MethodTable* pReturnedMT = pNewMT; { CrstHolder lock(&m_layoutsLock); @@ -353,16 +327,6 @@ BOOL ContinuationLayoutKeyHashTableHelper::CompareKeys(EEHashEntry_t *pEntry, Co if (left->GetBaseSize() != (OBJHEADER_SIZE + OFFSETOF__CORINFO_Continuation__data + right->DataSize)) return FALSE; - const CORINFO_CONTINUATION_DATA_OFFSETS& leftOffsets = *left->GetContinuationOffsets(); - const CORINFO_CONTINUATION_DATA_OFFSETS& rightOffsets = right->DataOffsets; - if (leftOffsets.Result != rightOffsets.Result || - leftOffsets.Exception != rightOffsets.Exception || - leftOffsets.ContinuationContext != rightOffsets.ContinuationContext || - leftOffsets.KeepAlive != rightOffsets.KeepAlive) - { - return FALSE; - } - // Now compare GC pointer series. CGCDesc* gc = CGCDesc::GetCGCDescFromMT(left); CGCDescSeries* curSeries = gc->GetHighestSeries(); @@ -400,12 +364,9 @@ DWORD ContinuationLayoutKeyHashTableHelper::Hash(ContinuationLayoutKey key) { DWORD dwHash = 5381; - const CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets; if ((key.Data & 1) != 0) { MethodTable* pMT = reinterpret_cast(key.Data ^ 1); - dataOffsets = pMT->GetContinuationOffsets(); - CGCDesc* gc = CGCDesc::GetCGCDescFromMT(pMT); CGCDescSeries* curSeries = gc->GetHighestSeries(); CGCDescSeries* lowestSeries = gc->GetLowestSeries(); @@ -424,7 +385,6 @@ DWORD ContinuationLayoutKeyHashTableHelper::Hash(ContinuationLayoutKey key) else { ContinuationLayoutKeyData* keyData = reinterpret_cast(key.Data); - dataOffsets = &keyData->DataOffsets; dwHash = ((dwHash << 5) + dwHash) ^ keyData->DataSize; EnumerateRunsOfObjRefs(keyData->DataSize, keyData->ObjRefs, [&dwHash](size_t offset, size_t count){ @@ -434,11 +394,6 @@ DWORD ContinuationLayoutKeyHashTableHelper::Hash(ContinuationLayoutKey key) }); } - dwHash = ((dwHash << 5) + dwHash) ^ dataOffsets->Result; - dwHash = ((dwHash << 5) + dwHash) ^ dataOffsets->Exception; - dwHash = ((dwHash << 5) + dwHash) ^ dataOffsets->ContinuationContext; - dwHash = ((dwHash << 5) + dwHash) ^ dataOffsets->KeepAlive; - return dwHash; } diff --git a/src/coreclr/vm/asynccontinuations.h b/src/coreclr/vm/asynccontinuations.h index 54883dbe4883ce..ad577fb4fa00b3 100644 --- a/src/coreclr/vm/asynccontinuations.h +++ b/src/coreclr/vm/asynccontinuations.h @@ -12,12 +12,10 @@ struct ContinuationLayoutKeyData { unsigned DataSize; const bool* ObjRefs; - const CORINFO_CONTINUATION_DATA_OFFSETS& DataOffsets; - ContinuationLayoutKeyData(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) + ContinuationLayoutKeyData(unsigned dataSize, const bool* objRefs) : DataSize(dataSize) , ObjRefs(objRefs) - , DataOffsets(dataOffsets) { } }; @@ -49,14 +47,14 @@ class AsyncContinuationsManager CrstExplicitInit m_layoutsLock; ContinuationLayoutHashTable m_layouts; - MethodTable* CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod, AllocMemTracker* pamTracker); + MethodTable* CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, MethodDesc* asyncMethod, AllocMemTracker* pamTracker); - static MethodTable* CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, EEClass* eeClass, LoaderAllocator* allocator, Module* loaderModule, AllocMemTracker* pamTracker); + static MethodTable* CreateNewContinuationMethodTable(unsigned dataSize, const bool* objRefs, EEClass* eeClass, LoaderAllocator* allocator, Module* loaderModule, AllocMemTracker* pamTracker); static PTR_EEClass GetOrCreateSingletonSubContinuationEEClass(); static PTR_EEClass CreateSingletonSubContinuationEEClass(); public: AsyncContinuationsManager(LoaderAllocator* allocator); - MethodTable* LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets, MethodDesc* asyncMethod); + MethodTable* LookupOrCreateContinuationMethodTable(unsigned dataSize, const bool* objRefs, MethodDesc* asyncMethod); void NotifyUnloadingClasses(); template diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index 03507fc34cbe75..cefb438ccbf7d2 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -10273,7 +10273,7 @@ void CEEInfo::getAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut) CORINFO_CLASS_HANDLE CEEInfo::getContinuationType( size_t dataSize, bool* objRefs, - const CORINFO_CONTINUATION_DATA_OFFSETS& dataOffsets) + size_t objRefsSize) { CONTRACTL { THROWS; @@ -10285,12 +10285,13 @@ CORINFO_CLASS_HANDLE CEEInfo::getContinuationType( JIT_TO_EE_TRANSITION(); + _ASSERTE(objRefsSize == (dataSize + (TARGET_POINTER_SIZE - 1)) / TARGET_POINTER_SIZE); LoaderAllocator* allocator = m_pMethodBeingCompiled->GetLoaderAllocator(); AsyncContinuationsManager* asyncConts = allocator->GetAsyncContinuationsManager(); - result = (CORINFO_CLASS_HANDLE)asyncConts->LookupOrCreateContinuationMethodTable((unsigned)dataSize, objRefs, dataOffsets, m_pMethodBeingCompiled); + result = (CORINFO_CLASS_HANDLE)asyncConts->LookupOrCreateContinuationMethodTable((unsigned)dataSize, objRefs, m_pMethodBeingCompiled); #ifdef DEBUG - CORINFO_CLASS_HANDLE result2 = (CORINFO_CLASS_HANDLE)asyncConts->LookupOrCreateContinuationMethodTable((unsigned)dataSize, objRefs, dataOffsets, m_pMethodBeingCompiled); + CORINFO_CLASS_HANDLE result2 = (CORINFO_CLASS_HANDLE)asyncConts->LookupOrCreateContinuationMethodTable((unsigned)dataSize, objRefs, m_pMethodBeingCompiled); _ASSERTE(result2 == result); #endif diff --git a/src/coreclr/vm/methodtable.h b/src/coreclr/vm/methodtable.h index 24e314ccc086df..4a77cb157dae66 100644 --- a/src/coreclr/vm/methodtable.h +++ b/src/coreclr/vm/methodtable.h @@ -2791,12 +2791,6 @@ class MethodTable inline BOOL IsContinuation(); - inline void SetContinuationDataOffsets(const CORINFO_CONTINUATION_DATA_OFFSETS* dataOffsets) - { - LIMITED_METHOD_CONTRACT; - m_pContinuationOffsets = dataOffsets; - } - // The following methods are only valid for the method tables for array types. CorElementType GetArrayElementType() { @@ -2830,13 +2824,6 @@ class MethodTable return offsetof(MethodTable, m_ElementTypeHnd); } - const CORINFO_CONTINUATION_DATA_OFFSETS* GetContinuationOffsets() - { - LIMITED_METHOD_DAC_CONTRACT; - _ASSERTE(IsContinuation()); - return m_pContinuationOffsets; - } - //------------------------------------------------------------------- // UNDERLYING METADATA // @@ -3946,14 +3933,13 @@ public : // m_pPerInstInfo and m_pInterfaceMap have to be at fixed offsets because of performance sensitive // JITed code and JIT helpers. The space used by m_pPerInstInfo is used to represent the array - // element type handle for array MethodTables and continuation layout offsets for continuations. + // element type handle for array MethodTables. public: union { PerInstInfo_t m_pPerInstInfo; TADDR m_ElementTypeHnd; - const CORINFO_CONTINUATION_DATA_OFFSETS* m_pContinuationOffsets; }; union { From 21698eafc006d8117cb744b26d66af5b4b2a2fa9 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 16 Oct 2025 22:28:28 +0200 Subject: [PATCH 42/48] Sort all object refs to be first --- src/coreclr/jit/async.cpp | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 66a8b9db3788c0..58565dd640ccf0 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1067,15 +1067,24 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* } } - jitstd::sort(liveLocals.begin(), liveLocals.end(), [](const LiveLocalInfo& lhs, const LiveLocalInfo& rhs) { - if (lhs.Alignment == rhs.Alignment) + jitstd::sort(liveLocals.begin(), liveLocals.end(), [=](const LiveLocalInfo& lhs, const LiveLocalInfo& rhs) { + bool lhsIsRef = m_comp->lvaGetDesc(lhs.LclNum)->TypeIs(TYP_REF); + bool rhsIsRef = m_comp->lvaGetDesc(rhs.LclNum)->TypeIs(TYP_REF); + + // Keep object refs first to improve sharability of continuation types. + if (lhsIsRef != rhsIsRef) + { + return lhsIsRef; + } + + // Otherwise prefer higher alignment first. + if (lhs.HeapAlignment() != rhs.HeapAlignment()) { - // Prefer lowest local num first for same alignment. - return lhs.LclNum < rhs.LclNum; + return lhs.HeapAlignment() > rhs.HeapAlignment(); } - // Otherwise prefer highest alignment first. - return lhs.Alignment > rhs.Alignment; + // Prefer lowest local num first for tiebreaker. + return lhs.LclNum < rhs.LclNum; }); if (call->gtReturnType == TYP_STRUCT) From 69f8eb60c41905fa43329dfb26dccd469653bb3a Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 16 Oct 2025 22:52:40 +0200 Subject: [PATCH 43/48] Pass keep alive offset as an argument to complex allocation helpers --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 25 ++++-------- src/coreclr/inc/corinfo.h | 9 ++--- src/coreclr/jit/async.cpp | 38 ++++++++++--------- src/coreclr/jit/async.h | 6 +-- 4 files changed, 36 insertions(+), 42 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 34da47a7ecfe1c..04bc993e9f3660 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -57,17 +57,16 @@ internal enum CorInfoContinuationFlags // also expects a result. CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2, CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT = 4, - CORINFO_CONTINUATION_HAS_KEEPALIVE = 8, - CORINFO_CONTINUATION_HAS_RESULT = 16, + CORINFO_CONTINUATION_HAS_RESULT = 8, // If this bit is set the continuation should continue on the thread // pool. - CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 32, + CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 16, // If this bit is set the continuation has a SynchronizationContext // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 64, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 32, // If this bit is set the continuation has a TaskScheduler // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 128, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 64, } #pragma warning disable CA1852 // "Type can be sealed" -- no it cannot because the runtime constructs subtypes dynamically @@ -111,14 +110,6 @@ public ref byte GetResultStorageOrNull() ref byte data = ref RuntimeHelpers.GetRawData(this); return ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE); } - - public void SetKeepAlive(object? obj) - { - Debug.Assert((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_KEEPALIVE) != 0); - uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_KEEPALIVE - 1)); - ref byte data = ref RuntimeHelpers.GetRawData(this); - Unsafe.As(ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE)) = obj; - } } public static partial class AsyncHelpers @@ -149,16 +140,16 @@ private static unsafe Continuation AllocContinuation(Continuation prevContinuati return newContinuation; } - private static unsafe Continuation AllocContinuationMethod(Continuation prevContinuation, MethodTable* contMT, MethodDesc* method) + private static unsafe Continuation AllocContinuationMethod(Continuation prevContinuation, MethodTable* contMT, int keepAliveOffset, MethodDesc* method) { LoaderAllocator loaderAllocator = RuntimeMethodHandle.GetLoaderAllocator(new RuntimeMethodHandleInternal((IntPtr)method)); Continuation newContinuation = (Continuation)RuntimeTypeHandle.InternalAllocNoChecks(contMT); - newContinuation.SetKeepAlive(loaderAllocator); + Unsafe.As(ref Unsafe.Add(ref RuntimeHelpers.GetRawData(newContinuation), keepAliveOffset)) = loaderAllocator; prevContinuation.Next = newContinuation; return newContinuation; } - private static unsafe Continuation AllocContinuationClass(Continuation prevContinuation, MethodTable* contMT, MethodTable* methodTable) + private static unsafe Continuation AllocContinuationClass(Continuation prevContinuation, MethodTable* contMT, int keepAliveOffset, MethodTable* methodTable) { IntPtr loaderAllocatorHandle = methodTable->GetLoaderAllocatorHandle(); @@ -166,7 +157,7 @@ private static unsafe Continuation AllocContinuationClass(Continuation prevConti prevContinuation.Next = newContinuation; if (loaderAllocatorHandle != IntPtr.Zero) { - newContinuation.SetKeepAlive(GCHandle.FromIntPtr(loaderAllocatorHandle).Target); + Unsafe.As(ref Unsafe.Add(ref RuntimeHelpers.GetRawData(newContinuation), keepAliveOffset)) = GCHandle.FromIntPtr(loaderAllocatorHandle).Target; } return newContinuation; } diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index 9f98ab4695e4e8..eb720abed430ae 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1707,17 +1707,16 @@ enum CorInfoContinuationFlags // also expects a result. CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2, CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT = 4, - CORINFO_CONTINUATION_HAS_KEEPALIVE = 8, - CORINFO_CONTINUATION_HAS_RESULT = 16, + CORINFO_CONTINUATION_HAS_RESULT = 8, // If this bit is set the continuation should continue on the thread // pool. - CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 32, + CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 16, // If this bit is set the continuation has a SynchronizationContext // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 64, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 32, // If this bit is set the continuation has a TaskScheduler // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 128, + CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 64, }; struct CORINFO_ASYNC_INFO diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 58565dd640ccf0..b30a465f3f21dc 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1134,12 +1134,6 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* layout.ContinuationContextOffset); } - if (needsKeepAlive) - { - layout.KeepAliveOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); - JITDUMP(" Continuation needs keep alive object; will be at offset %u\n", layout.KeepAliveOffset); - } - if (layout.ReturnSize > 0) { layout.ReturnValOffset = allocLayout(layout.ReturnHeapAlignment(), layout.ReturnSize); @@ -1150,6 +1144,12 @@ ContinuationLayout AsyncTransformation::LayOutContinuation(BasicBlock* layout.ReturnSize, layout.ReturnValOffset); } + if (needsKeepAlive) + { + layout.KeepAliveOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); + JITDUMP(" Continuation needs keep alive object; will be at offset %u\n", layout.KeepAliveOffset); + } + if (call->GetAsyncInfo().ExecutionContextHandling == ExecutionContextHandling::AsyncSaveAndRestore) { layout.ExecContextOffset = allocLayout(TARGET_POINTER_SIZE, TARGET_POINTER_SIZE); @@ -1418,7 +1418,7 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Allocate continuation GenTree* returnedContinuation = m_comp->gtNewLclvNode(m_returnedContinuationVar, TYP_REF); - GenTreeCall* allocContinuation = CreateAllocContinuationCall(life, returnedContinuation, layout.ClassHnd); + GenTreeCall* allocContinuation = CreateAllocContinuationCall(life, returnedContinuation, layout); m_comp->compCurBB = suspendBB; m_comp->fgMorphTree(allocContinuation); @@ -1451,8 +1451,6 @@ BasicBlock* AsyncTransformation::CreateSuspension( continuationFlags |= CORINFO_CONTINUATION_NEEDS_EXCEPTION; if (layout.ContinuationContextOffset != UINT_MAX) continuationFlags |= CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT; - if (layout.KeepAliveOffset != UINT_MAX) - continuationFlags |= CORINFO_CONTINUATION_HAS_KEEPALIVE; if (layout.ReturnValOffset != UINT_MAX) continuationFlags |= CORINFO_CONTINUATION_HAS_RESULT; if (callInfo.ContinuationContextHandling == ContinuationContextHandling::ContinueOnThreadPool) @@ -1486,17 +1484,16 @@ BasicBlock* AsyncTransformation::CreateSuspension( // Parameters: // life - Liveness information about live locals // prevContinuation - IR node that has the value of the previous continuation object -// gcRefsCount - Number of GC refs to allocate in the continuation object -// dataSize - Number of bytes to allocate in the continuation object +// layout - Layout information // // Returns: // IR node representing the allocation. // -GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& life, - GenTree* prevContinuation, - CORINFO_CLASS_HANDLE contClassHnd) +GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& life, + GenTree* prevContinuation, + const ContinuationLayout& layout) { - GenTree* contClassHndNode = m_comp->gtNewIconEmbClsHndNode(contClassHnd); + GenTree* contClassHndNode = m_comp->gtNewIconEmbClsHndNode(layout.ClassHnd); // If VM requests that we report the method handle, or if we have a shared generic context method handle // that is live here, then we need to call a different helper to keep the loader alive. GenTree* methodHandleArg = nullptr; @@ -1518,14 +1515,21 @@ GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& if (methodHandleArg != nullptr) { + assert(layout.KeepAliveOffset != UINT_MAX); + // Offset passed to function is relative to instance data. + int keepAliveOffset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + layout.KeepAliveOffset; + GenTree* keepAliveOffsetNode = m_comp->gtNewIconNode(keepAliveOffset); return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION_METHOD, TYP_REF, prevContinuation, - contClassHndNode, methodHandleArg); + contClassHndNode, keepAliveOffsetNode, methodHandleArg); } if (classHandleArg != nullptr) { + assert(layout.KeepAliveOffset != UINT_MAX); + int keepAliveOffset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + layout.KeepAliveOffset; + GenTree* keepAliveOffsetNode = m_comp->gtNewIconNode(keepAliveOffset); return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION_CLASS, TYP_REF, prevContinuation, - contClassHndNode, classHandleArg); + contClassHndNode, keepAliveOffsetNode, classHandleArg); } return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION, TYP_REF, prevContinuation, contClassHndNode); diff --git a/src/coreclr/jit/async.h b/src/coreclr/jit/async.h index 20505461d53075..6546ba02b7afde 100644 --- a/src/coreclr/jit/async.h +++ b/src/coreclr/jit/async.h @@ -103,9 +103,9 @@ class AsyncTransformation BasicBlock* CreateSuspension( BasicBlock* block, GenTreeCall* call, unsigned stateNum, AsyncLiveness& life, const ContinuationLayout& layout); - GenTreeCall* CreateAllocContinuationCall(AsyncLiveness& life, - GenTree* prevContinuation, - CORINFO_CLASS_HANDLE contClassHnd); + GenTreeCall* CreateAllocContinuationCall(AsyncLiveness& life, + GenTree* prevContinuation, + const ContinuationLayout& layout); void FillInDataOnSuspension(GenTreeCall* call, const ContinuationLayout& layout, BasicBlock* suspendBB); void CreateCheckAndSuspendAfterCall(BasicBlock* block, GenTreeCall* call, From f5bb49927a2e7206107178291fa7b2415fc2ce42 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Thu, 16 Oct 2025 23:23:01 +0200 Subject: [PATCH 44/48] Remove managed MethodTable.IsContinuation --- .../System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs index 259475edd6d465..9fa1f1f004d9b5 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/RuntimeHelpers.CoreCLR.cs @@ -897,8 +897,6 @@ public int MultiDimensionalArrayRank public bool IsArray => (Flags & enum_flag_Category_Array_Mask) == enum_flag_Category_Array; - public bool IsContinuation => ParentMethodTable == TypeHandle.TypeHandleOf().AsMethodTable(); - public bool HasInstantiation => (Flags & enum_flag_HasComponentSize) == 0 && (Flags & enum_flag_GenericsMask) != enum_flag_GenericsMask_NonGeneric; public bool IsGenericTypeDefinition => (Flags & (enum_flag_HasComponentSize | enum_flag_GenericsMask)) == enum_flag_GenericsMask_TypicalInst; From ee717bd18ac18fa93733a6d64ccb7d4e0d1eaba1 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Fri, 17 Oct 2025 09:31:01 +0200 Subject: [PATCH 45/48] Address feedback --- .../CompilerServices/AsyncHelpers.CoreCLR.cs | 91 ++++++++++--------- src/coreclr/inc/clrconfigvalues.h | 2 +- src/coreclr/inc/corinfo.h | 27 ++++-- src/coreclr/jit/async.cpp | 2 +- src/tests/async/Directory.Build.targets | 13 +++ 5 files changed, 83 insertions(+), 52 deletions(-) create mode 100644 src/tests/async/Directory.Build.targets diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 04bc993e9f3660..2a3540b10aebba 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -48,25 +48,34 @@ public void Pop() } [Flags] - internal enum CorInfoContinuationFlags + // Keep in sync with CORINFO_CONTINUATION_FLAGS + internal enum ContinuationFlags { - CORINFO_CONTINUATION_HAS_OSR_ILOFFSET = 1, - // If this bit is set the continuation resumes inside a try block and thus - // if an exception is being propagated, needs to be resumed. The exception - // should be placed at index 0 or 1 depending on whether the continuation - // also expects a result. - CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2, - CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT = 4, - CORINFO_CONTINUATION_HAS_RESULT = 8, + // Note: the following 'Has' members determine the members present at + // the beginning of the continuation's data chunk. Each field is + // pointer sized when present, apart from the result that has variable + // size. + + // Whether or not the continuation starts with an OSR IL offset. + HasOsrILOffset = 1, + // If this bit is set the continuation resumes inside a try block and + // thus if an exception is being propagated, needs to be resumed. + HasException = 2, + // If this bit is set the continuation has space for a continuation + // context. + HasContinuationContext = 4, + // If this bit is set the continuation has space to store a result + // returned by the callee. + HasResult = 8, // If this bit is set the continuation should continue on the thread // pool. - CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 16, - // If this bit is set the continuation has a SynchronizationContext - // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 32, - // If this bit is set the continuation has a TaskScheduler - // that we should continue on. - CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 64, + ContinueOnThreadPool = 16, + // If this bit is set the continuation context is a + // SynchronizationContext that we should continue on. + ContinueOnCapturedSynchronizationContext = 32, + // If this bit is set the continuation context is a TaskScheduler that + // we should continue on. + ContinueOnCapturedTaskScheduler = 64, } #pragma warning disable CA1852 // "Type can be sealed" -- no it cannot because the runtime constructs subtypes dynamically @@ -74,41 +83,41 @@ internal unsafe class Continuation { public Continuation? Next; public delegate* Resume; - public CorInfoContinuationFlags Flags; + public ContinuationFlags Flags; public int State; #if TARGET_64BIT - private const int POINTER_SIZE = 8; + private const int PointerSize = 8; #else - private const int POINTER_SIZE = 4; + private const int PointerSize = 4; #endif - private const int CONTINUATION_OFFSET_TO_DATA = POINTER_SIZE /* Next */ + POINTER_SIZE /* Resume */ + 8 /* Flags + State */; + private const int DataOffset = PointerSize /* Next */ + PointerSize /* Resume */ + 8 /* Flags + State */; public unsafe object GetContinuationContext() { - Debug.Assert((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT) != 0); - uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT - 1)); + Debug.Assert((Flags & ContinuationFlags.HasContinuationContext) != 0); + uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)ContinuationFlags.HasContinuationContext - 1)); ref byte data = ref RuntimeHelpers.GetRawData(this); - return Unsafe.As(ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE)); + return Unsafe.As(ref Unsafe.Add(ref data, DataOffset + contIndex * PointerSize)); } public void SetException(Exception ex) { - Debug.Assert((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION) != 0); - uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION - 1)); + Debug.Assert((Flags & ContinuationFlags.HasException) != 0); + uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)ContinuationFlags.HasException - 1)); ref byte data = ref RuntimeHelpers.GetRawData(this); - Unsafe.As(ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE)) = ex; + Unsafe.As(ref Unsafe.Add(ref data, DataOffset + contIndex * PointerSize)) = ex; } public ref byte GetResultStorageOrNull() { - if ((Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_RESULT) == 0) + if ((Flags & ContinuationFlags.HasResult) == 0) return ref Unsafe.NullRef(); - uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)CorInfoContinuationFlags.CORINFO_CONTINUATION_HAS_RESULT - 1)); + uint contIndex = (uint)BitOperations.PopCount((uint)Flags & ((uint)ContinuationFlags.HasResult - 1)); ref byte data = ref RuntimeHelpers.GetRawData(this); - return ref Unsafe.Add(ref data, CONTINUATION_OFFSET_TO_DATA + contIndex * POINTER_SIZE); + return ref Unsafe.Add(ref data, DataOffset + contIndex * PointerSize); } } @@ -412,7 +421,7 @@ public static unsafe void DispatchContinuations(T task) where T : Task, if (nextContinuation == null) return null; - if ((nextContinuation.Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_NEEDS_EXCEPTION) != 0) + if ((nextContinuation.Flags & ContinuationFlags.HasException) != 0) return nextContinuation; continuation = nextContinuation; @@ -436,10 +445,10 @@ public static void HandleSuspended(T task) where T : Task, ITaskComplet // Head continuation should be the result of async call to AwaitAwaiter or UnsafeAwaitAwaiter. // These never have special continuation handling. - const CorInfoContinuationFlags continueFlags = - CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT | - CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL | - CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER; + const ContinuationFlags continueFlags = + ContinuationFlags.ContinueOnCapturedSynchronizationContext | + ContinuationFlags.ContinueOnThreadPool | + ContinuationFlags.ContinueOnCapturedTaskScheduler; Debug.Assert((headContinuation.Flags & continueFlags) == 0); TOps.SetContinuationState(task, headContinuation); @@ -474,7 +483,7 @@ public static void HandleSuspended(T task) where T : Task, ITaskComplet private static bool QueueContinuationFollowUpActionIfNecessary(T task, Continuation continuation) where T : Task where TOps : IRuntimeAsyncTaskOps { - if ((continuation.Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL) != 0) + if ((continuation.Flags & ContinuationFlags.ContinueOnThreadPool) != 0) { SynchronizationContext? ctx = Thread.CurrentThreadAssumedInitialized._synchronizationContext; if (ctx == null || ctx.GetType() == typeof(SynchronizationContext)) @@ -492,7 +501,7 @@ private static bool QueueContinuationFollowUpActionIfNecessary(T task, return true; } - if ((continuation.Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT) != 0) + if ((continuation.Flags & ContinuationFlags.ContinueOnCapturedSynchronizationContext) != 0) { object continuationContext = continuation.GetContinuationContext(); Debug.Assert(continuationContext is SynchronizationContext { }); @@ -518,7 +527,7 @@ private static bool QueueContinuationFollowUpActionIfNecessary(T task, return true; } - if ((continuation.Flags & CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER) != 0) + if ((continuation.Flags & ContinuationFlags.ContinueOnCapturedTaskScheduler) != 0) { object continuationContext = continuation.GetContinuationContext(); Debug.Assert(continuationContext is TaskScheduler { }); @@ -642,11 +651,11 @@ private static void RestoreContexts(bool suspended, ExecutionContext? previousEx } } - private static void CaptureContinuationContext(SynchronizationContext syncCtx, ref object context, ref CorInfoContinuationFlags flags) + private static void CaptureContinuationContext(SynchronizationContext syncCtx, ref object context, ref ContinuationFlags flags) { if (syncCtx != null && syncCtx.GetType() != typeof(SynchronizationContext)) { - flags |= CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT; + flags |= ContinuationFlags.ContinueOnCapturedSynchronizationContext; context = syncCtx; return; } @@ -654,12 +663,12 @@ private static void CaptureContinuationContext(SynchronizationContext syncCtx, r TaskScheduler? sched = TaskScheduler.InternalCurrent; if (sched != null && sched != TaskScheduler.Default) { - flags |= CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER; + flags |= ContinuationFlags.ContinueOnCapturedTaskScheduler; context = sched; return; } - flags |= CorInfoContinuationFlags.CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL; + flags |= ContinuationFlags.ContinueOnThreadPool; } internal static T CompletedTaskResult(Task task) diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 3c2a9c3dab3dd8..542510f79b4a74 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -720,7 +720,7 @@ RETAIL_CONFIG_DWORD_INFO(EXTERNAL_EnableRiscV64Zbb, W("EnableRiscV64 #endif // Runtime-async -RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_RuntimeAsync, W("RuntimeAsync"), 1, "Enables runtime async method support") +RETAIL_CONFIG_DWORD_INFO(UNSUPPORTED_RuntimeAsync, W("RuntimeAsync"), 0, "Enables runtime async method support") /// /// Uncategorized diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index eb720abed430ae..30a94de13ab3dc 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1698,24 +1698,33 @@ struct CORINFO_EE_INFO CORINFO_OS osType; }; +// Keep in sync with ContinuationFlags enum in BCL sources enum CorInfoContinuationFlags { + // Note: the following 'Has' members determine the members present at + // the beginning of the continuation's data chunk. Each field is + // pointer sized when present, apart from the result that has variable + // size. + + // Whether or not the continuation starts with an OSR IL offset. CORINFO_CONTINUATION_HAS_OSR_ILOFFSET = 1, - // If this bit is set the continuation resumes inside a try block and thus - // if an exception is being propagated, needs to be resumed. The exception - // should be placed at index 0 or 1 depending on whether the continuation - // also expects a result. - CORINFO_CONTINUATION_NEEDS_EXCEPTION = 2, + // If this bit is set the continuation resumes inside a try block and + // thus if an exception is being propagated, needs to be resumed. + CORINFO_CONTINUATION_HAS_EXCEPTION = 2, + // If this bit is set the continuation has space for a continuation + // context. CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT = 4, + // If this bit is set the continuation has space to store a result + // returned by the callee. CORINFO_CONTINUATION_HAS_RESULT = 8, // If this bit is set the continuation should continue on the thread // pool. CORINFO_CONTINUATION_CONTINUE_ON_THREAD_POOL = 16, - // If this bit is set the continuation has a SynchronizationContext - // that we should continue on. + // If this bit is set the continuation context is a + // SynchronizationContext that we should continue on. CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_SYNCHRONIZATION_CONTEXT = 32, - // If this bit is set the continuation has a TaskScheduler - // that we should continue on. + // If this bit is set the continuation context is a TaskScheduler that + // we should continue on. CORINFO_CONTINUATION_CONTINUE_ON_CAPTURED_TASK_SCHEDULER = 64, }; diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index b30a465f3f21dc..5c5f8099d4df32 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -1448,7 +1448,7 @@ BasicBlock* AsyncTransformation::CreateSuspension( if (layout.OSRILOffset != UINT_MAX) continuationFlags |= CORINFO_CONTINUATION_HAS_OSR_ILOFFSET; if (layout.ExceptionOffset != UINT_MAX) - continuationFlags |= CORINFO_CONTINUATION_NEEDS_EXCEPTION; + continuationFlags |= CORINFO_CONTINUATION_HAS_EXCEPTION; if (layout.ContinuationContextOffset != UINT_MAX) continuationFlags |= CORINFO_CONTINUATION_HAS_CONTINUATION_CONTEXT; if (layout.ReturnValOffset != UINT_MAX) diff --git a/src/tests/async/Directory.Build.targets b/src/tests/async/Directory.Build.targets new file mode 100644 index 00000000000000..5a4d413a9e1f62 --- /dev/null +++ b/src/tests/async/Directory.Build.targets @@ -0,0 +1,13 @@ + + + + true + + + + + true + + + + From 1520a7fce771a08e457ff6762840c86ebc0b33c8 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 18 Oct 2025 14:05:30 +0200 Subject: [PATCH 46/48] Remove AllocContinuationResultBox --- .../Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs | 12 ------------ src/coreclr/vm/corelib.h | 1 - 2 files changed, 13 deletions(-) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs index 2a3540b10aebba..291fd8802d3016 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Runtime/CompilerServices/AsyncHelpers.CoreCLR.cs @@ -171,18 +171,6 @@ private static unsafe Continuation AllocContinuationClass(Continuation prevConti return newContinuation; } - // Used to box the return value before storing into caller's continuation - // if the value is an object-containing struct. - // We are allocating a box directly instead of relying on regular boxing because we want - // to store structs without changing layout, including nullables. - private static unsafe object AllocContinuationResultBox(void* ptr) - { - MethodTable* pMT = (MethodTable*)ptr; - Debug.Assert(pMT->IsValueType); - // We need no type/cctor checks since we will be storing an instance that already exists. - return RuntimeTypeHandle.InternalAllocNoChecks((MethodTable*)pMT); - } - [BypassReadyToRun] [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.Async)] [RequiresPreviewFeatures] diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index bb18c3da45cbf0..7227608a0dba17 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -718,7 +718,6 @@ DEFINE_CLASS(ASYNC_HELPERS, CompilerServices, AsyncHelpers) DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION, AllocContinuation, NoSig) DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION_METHOD, AllocContinuationMethod, NoSig) DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION_CLASS, AllocContinuationClass, NoSig) -DEFINE_METHOD(ASYNC_HELPERS, ALLOC_CONTINUATION_RESULT_BOX, AllocContinuationResultBox, SM_VoidPtr_RetObj) DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK, FinalizeTaskReturningThunk, SM_RetTask) DEFINE_METHOD(ASYNC_HELPERS, FINALIZE_TASK_RETURNING_THUNK_1, FinalizeTaskReturningThunk, GM_RetTaskOfT) From c8d6e57cade6ecbd53c8a72e2a97f5044142a90e Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 18 Oct 2025 14:09:08 +0200 Subject: [PATCH 47/48] Remove CORINFO_ASYNC_INFO::continuationsNeedMethodHandle --- src/coreclr/inc/corinfo.h | 3 --- src/coreclr/jit/async.cpp | 27 +++---------------- .../tools/superpmi/superpmi-shared/agnostic.h | 1 - .../superpmi-shared/methodcontext.cpp | 6 ++--- src/coreclr/vm/jitinterface.cpp | 1 - 5 files changed, 6 insertions(+), 32 deletions(-) diff --git a/src/coreclr/inc/corinfo.h b/src/coreclr/inc/corinfo.h index 30a94de13ab3dc..bdedfa685b37e8 100644 --- a/src/coreclr/inc/corinfo.h +++ b/src/coreclr/inc/corinfo.h @@ -1740,9 +1740,6 @@ struct CORINFO_ASYNC_INFO CORINFO_FIELD_HANDLE continuationStateFldHnd; // 'Flags' field CORINFO_FIELD_HANDLE continuationFlagsFldHnd; - // Whether or not the continuation needs to be allocated through the - // helper that also takes a method handle - bool continuationsNeedMethodHandle; // Method handle for AsyncHelpers.CaptureExecutionContext CORINFO_METHOD_HANDLE captureExecutionContextMethHnd; // Method handle for AsyncHelpers.RestoreExecutionContext diff --git a/src/coreclr/jit/async.cpp b/src/coreclr/jit/async.cpp index 5c5f8099d4df32..4d43458c89e569 100644 --- a/src/coreclr/jit/async.cpp +++ b/src/coreclr/jit/async.cpp @@ -944,11 +944,6 @@ void AsyncTransformation::LiftLIREdges(BasicBlock* block, // bool AsyncTransformation::ContinuationNeedsKeepAlive(AsyncLiveness& life) { - if (m_asyncInfo->continuationsNeedMethodHandle) - { - return true; - } - const unsigned GENERICS_CTXT_FROM = CORINFO_GENERICS_CTXT_FROM_METHODDESC | CORINFO_GENERICS_CTXT_FROM_METHODTABLE; if (((m_comp->info.compMethodInfo->options & GENERICS_CTXT_FROM) != 0) && life.IsLive(m_comp->info.compTypeCtxtArg)) { @@ -1496,36 +1491,22 @@ GenTreeCall* AsyncTransformation::CreateAllocContinuationCall(AsyncLiveness& GenTree* contClassHndNode = m_comp->gtNewIconEmbClsHndNode(layout.ClassHnd); // If VM requests that we report the method handle, or if we have a shared generic context method handle // that is live here, then we need to call a different helper to keep the loader alive. - GenTree* methodHandleArg = nullptr; - GenTree* classHandleArg = nullptr; if (((m_comp->info.compMethodInfo->options & CORINFO_GENERICS_CTXT_FROM_METHODDESC) != 0) && life.IsLive(m_comp->info.compTypeCtxtArg)) - { - methodHandleArg = m_comp->gtNewLclvNode(m_comp->info.compTypeCtxtArg, TYP_I_IMPL); - } - else if (((m_comp->info.compMethodInfo->options & CORINFO_GENERICS_CTXT_FROM_METHODTABLE) != 0) && - life.IsLive(m_comp->info.compTypeCtxtArg)) - { - classHandleArg = m_comp->gtNewLclvNode(m_comp->info.compTypeCtxtArg, TYP_I_IMPL); - } - else if (m_asyncInfo->continuationsNeedMethodHandle) - { - methodHandleArg = m_comp->gtNewIconEmbMethHndNode(m_comp->info.compMethodHnd); - } - - if (methodHandleArg != nullptr) { assert(layout.KeepAliveOffset != UINT_MAX); + GenTree* methodHandleArg = m_comp->gtNewLclvNode(m_comp->info.compTypeCtxtArg, TYP_I_IMPL); // Offset passed to function is relative to instance data. int keepAliveOffset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + layout.KeepAliveOffset; GenTree* keepAliveOffsetNode = m_comp->gtNewIconNode(keepAliveOffset); return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION_METHOD, TYP_REF, prevContinuation, contClassHndNode, keepAliveOffsetNode, methodHandleArg); } - - if (classHandleArg != nullptr) + else if (((m_comp->info.compMethodInfo->options & CORINFO_GENERICS_CTXT_FROM_METHODTABLE) != 0) && + life.IsLive(m_comp->info.compTypeCtxtArg)) { assert(layout.KeepAliveOffset != UINT_MAX); + GenTree* classHandleArg = m_comp->gtNewLclvNode(m_comp->info.compTypeCtxtArg, TYP_I_IMPL); int keepAliveOffset = (OFFSETOF__CORINFO_Continuation__data - SIZEOF__CORINFO_Object) + layout.KeepAliveOffset; GenTree* keepAliveOffsetNode = m_comp->gtNewIconNode(keepAliveOffset); return m_comp->gtNewHelperCallNode(CORINFO_HELP_ALLOC_CONTINUATION_CLASS, TYP_REF, prevContinuation, diff --git a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h index 633eb333371807..d45676ab5a8784 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h +++ b/src/coreclr/tools/superpmi/superpmi-shared/agnostic.h @@ -200,7 +200,6 @@ struct Agnostic_CORINFO_ASYNC_INFO DWORDLONG continuationResumeFldHnd; DWORDLONG continuationStateFldHnd; DWORDLONG continuationFlagsFldHnd; - DWORD continuationsNeedMethodHandle; DWORDLONG captureExecutionContextMethHnd; DWORDLONG restoreExecutionContextMethHnd; DWORDLONG captureContinuationContextMethHnd; diff --git a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp index 2dbce14e317eb9..7dad7bac1fc57b 100644 --- a/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp +++ b/src/coreclr/tools/superpmi/superpmi-shared/methodcontext.cpp @@ -4457,7 +4457,6 @@ void MethodContext::recGetAsyncInfo(const CORINFO_ASYNC_INFO* pAsyncInfo) value.continuationResumeFldHnd = CastHandle(pAsyncInfo->continuationResumeFldHnd); value.continuationStateFldHnd = CastHandle(pAsyncInfo->continuationStateFldHnd); value.continuationFlagsFldHnd = CastHandle(pAsyncInfo->continuationFlagsFldHnd); - value.continuationsNeedMethodHandle = pAsyncInfo->continuationsNeedMethodHandle ? 1 : 0; value.captureExecutionContextMethHnd = CastHandle(pAsyncInfo->captureExecutionContextMethHnd); value.restoreExecutionContextMethHnd = CastHandle(pAsyncInfo->restoreExecutionContextMethHnd); value.captureContinuationContextMethHnd = CastHandle(pAsyncInfo->captureContinuationContextMethHnd); @@ -4470,9 +4469,9 @@ void MethodContext::recGetAsyncInfo(const CORINFO_ASYNC_INFO* pAsyncInfo) void MethodContext::dmpGetAsyncInfo(DWORD key, const Agnostic_CORINFO_ASYNC_INFO& value) { printf("GetAsyncInfo key %u value contClsHnd-%016" PRIX64 " contNextFldHnd-%016" PRIX64 " contResumeFldHnd-%016" PRIX64 - " contStateFldHnd-%016" PRIX64 " contFlagsFldHnd-%016" PRIX64 " contsNeedMethodHandle-%d", + " contStateFldHnd-%016" PRIX64 " contFlagsFldHnd-%016" PRIX64, key, value.continuationClsHnd, value.continuationNextFldHnd, value.continuationResumeFldHnd, - value.continuationStateFldHnd, value.continuationFlagsFldHnd, value.continuationsNeedMethodHandle); + value.continuationStateFldHnd, value.continuationFlagsFldHnd); } void MethodContext::repGetAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut) { @@ -4482,7 +4481,6 @@ void MethodContext::repGetAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut) pAsyncInfoOut->continuationResumeFldHnd = (CORINFO_FIELD_HANDLE)value.continuationResumeFldHnd; pAsyncInfoOut->continuationStateFldHnd = (CORINFO_FIELD_HANDLE)value.continuationStateFldHnd; pAsyncInfoOut->continuationFlagsFldHnd = (CORINFO_FIELD_HANDLE)value.continuationFlagsFldHnd; - pAsyncInfoOut->continuationsNeedMethodHandle = value.continuationsNeedMethodHandle != 0; pAsyncInfoOut->captureExecutionContextMethHnd = (CORINFO_METHOD_HANDLE)value.captureExecutionContextMethHnd; pAsyncInfoOut->restoreExecutionContextMethHnd = (CORINFO_METHOD_HANDLE)value.restoreExecutionContextMethHnd; pAsyncInfoOut->captureContinuationContextMethHnd = (CORINFO_METHOD_HANDLE)value.captureContinuationContextMethHnd; diff --git a/src/coreclr/vm/jitinterface.cpp b/src/coreclr/vm/jitinterface.cpp index cefb438ccbf7d2..8bb8e1967ed874 100644 --- a/src/coreclr/vm/jitinterface.cpp +++ b/src/coreclr/vm/jitinterface.cpp @@ -10260,7 +10260,6 @@ void CEEInfo::getAsyncInfo(CORINFO_ASYNC_INFO* pAsyncInfoOut) pAsyncInfoOut->continuationResumeFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__RESUME)); pAsyncInfoOut->continuationStateFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__STATE)); pAsyncInfoOut->continuationFlagsFldHnd = CORINFO_FIELD_HANDLE(CoreLibBinder::GetField(FIELD__CONTINUATION__FLAGS)); - pAsyncInfoOut->continuationsNeedMethodHandle = m_pMethodBeingCompiled->GetLoaderAllocator()->CanUnload(); pAsyncInfoOut->captureExecutionContextMethHnd = CORINFO_METHOD_HANDLE(CoreLibBinder::GetMethod(METHOD__ASYNC_HELPERS__CAPTURE_EXECUTION_CONTEXT)); pAsyncInfoOut->restoreExecutionContextMethHnd = CORINFO_METHOD_HANDLE(CoreLibBinder::GetMethod(METHOD__ASYNC_HELPERS__RESTORE_EXECUTION_CONTEXT)); pAsyncInfoOut->captureContinuationContextMethHnd = CORINFO_METHOD_HANDLE(CoreLibBinder::GetMethod(METHOD__ASYNC_HELPERS__CAPTURE_CONTINUATION_CONTEXT)); From f80f4e33fa00e5660a1125ca581c0b6ff2054d68 Mon Sep 17 00:00:00 2001 From: Jakob Botsch Nielsen Date: Sat, 18 Oct 2025 14:22:58 +0200 Subject: [PATCH 48/48] Update pseudocode --- src/coreclr/vm/asyncthunks.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/coreclr/vm/asyncthunks.cpp b/src/coreclr/vm/asyncthunks.cpp index 3d788a43f5c4cb..fcb0c06d059936 100644 --- a/src/coreclr/vm/asyncthunks.cpp +++ b/src/coreclr/vm/asyncthunks.cpp @@ -69,16 +69,14 @@ void MethodDesc::EmitTaskReturningThunk(MethodDesc* pAsyncOtherVariant, MetaSig& // store.Push(); // try // { - // Continuation cont; // try // { // T result = Inner(args); // // call an intrisic to see if the call above produced a continuation - // cont = StubHelpers.AsyncCallContinuation(); - // if (cont == null) + // if (StubHelpers.AsyncCallContinuation() == null) // return Task.FromResult(result); // - // return FinalizeTaskReturningThunk(cont); + // return FinalizeTaskReturningThunk(); // } // catch (Exception ex) // {