Reduce allocations (#6132)

- Remove per request allocations by implementing IThreadPoolWorkItem on the IISHttpContext.
- Removed per operation allocations by using UnsafeQueueUserWorkItem in AsyncIOOperation.
- This should also reduce overhead by removing non-essential ExecutionContext propagation logic

Author: davidfowl

src/Servers/IIS/src/Microsoft.AspNetCore.Server.IIS/Core/IISHttpContext.cs

@@ -25,7 +25,7 @@
 
 namespace Microsoft.AspNetCore.Server.IIS.Core
 {
-    internal abstract partial class IISHttpContext : NativeRequestContext, IDisposable
+    internal abstract partial class IISHttpContext : NativeRequestContext, IThreadPoolWorkItem, IDisposable
     {
         private const int MinAllocBufferSize = 2048;
         private const int PauseWriterThreshold = 65536;
@@ -531,5 +531,40 @@ private WindowsPrincipal GetWindowsPrincipal()
             }
             return null;
         }
+
+        // Invoked by the thread pool
+        public void Execute()
+        {
+            _ = HandleRequest();
+        }
+
+        private async Task HandleRequest()
+        {
+            bool successfulRequest = false;
+            try
+            {
+                successfulRequest = await ProcessRequestAsync();
+            }
+            catch (Exception ex)
+            {
+               _logger.LogError(0, ex, $"Unexpected exception in {nameof(IISHttpContext)}.{nameof(HandleRequest)}.");
+            }
+            finally
+            {
+                // Post completion after completing the request to resume the state machine
+                PostCompletion(ConvertRequestCompletionResults(successfulRequest));
+
+                Server.DecrementRequests();
+
+                // Dispose the context
+                Dispose();
+            }
+        }
+
+        private static NativeMethods.REQUEST_NOTIFICATION_STATUS ConvertRequestCompletionResults(bool success)
+        {
+            return success ? NativeMethods.REQUEST_NOTIFICATION_STATUS.RQ_NOTIFICATION_CONTINUE
+                           : NativeMethods.REQUEST_NOTIFICATION_STATUS.RQ_NOTIFICATION_FINISH_REQUEST;
+        }
     }
 }

src/Servers/IIS/src/Microsoft.AspNetCore.Server.IIS/Core/IISHttpServer.cs

@@ -152,18 +152,34 @@ public void Dispose()
             _nativeApplication.Dispose();
         }
 
+        private void IncrementRequests()
+        {
+            Interlocked.Increment(ref _outstandingRequests);
+        }
+
+        internal void DecrementRequests()
+        {
+            if (Interlocked.Decrement(ref _outstandingRequests) == 0 && Stopping)
+            {
+                // All requests have been drained.
+                _nativeApplication.StopCallsIntoManaged();
+                _shutdownSignal.TrySetResult(null);
+            }
+        }
+
         private static NativeMethods.REQUEST_NOTIFICATION_STATUS HandleRequest(IntPtr pInProcessHandler, IntPtr pvRequestContext)
         {
             IISHttpServer server = null;
             try
             {
                 // Unwrap the server so we can create an http context and process the request
                 server = (IISHttpServer)GCHandle.FromIntPtr(pvRequestContext).Target;
-                Interlocked.Increment(ref server._outstandingRequests);
+                server.IncrementRequests();
 
                 var context = server._iisContextFactory.CreateHttpContext(pInProcessHandler);
 
-                ThreadPool.QueueUserWorkItem(state => _ = HandleRequest((IISHttpContext)state), context);
+                ThreadPool.UnsafeQueueUserWorkItem(context, preferLocal: false);
+
                 return NativeMethods.REQUEST_NOTIFICATION_STATUS.RQ_NOTIFICATION_PENDING;
             }
             catch (Exception ex)
@@ -174,23 +190,6 @@ private static NativeMethods.REQUEST_NOTIFICATION_STATUS HandleRequest(IntPtr pI
             }
         }
 
-        private static async Task HandleRequest(IISHttpContext context)
-        {
-            bool successfulRequest = false;
-            try
-            {
-                successfulRequest = await context.ProcessRequestAsync();
-            }
-            catch (Exception ex)
-            {
-                context.Server._logger.LogError(0, ex, $"Unexpected exception in {nameof(IISHttpServer)}.{nameof(HandleRequest)}.");
-            }
-            finally
-            {
-                CompleteRequest(context, successfulRequest);
-            }
-        }
-
         private static bool HandleShutdown(IntPtr pvRequestContext)
         {
             IISHttpServer server = null;
@@ -238,28 +237,6 @@ private static NativeMethods.REQUEST_NOTIFICATION_STATUS OnAsyncCompletion(IntPt
             }
         }
 
-        private static void CompleteRequest(IISHttpContext context, bool result)
-        {
-            // Post completion after completing the request to resume the state machine
-            context.PostCompletion(ConvertRequestCompletionResults(result));
-
-            if (Interlocked.Decrement(ref context.Server._outstandingRequests) == 0 && context.Server.Stopping)
-            {
-                // All requests have been drained.
-                context.Server._nativeApplication.StopCallsIntoManaged();
-                context.Server._shutdownSignal.TrySetResult(null);
-            }
-
-            // Dispose the context
-            context.Dispose();
-        }
-
-        private static NativeMethods.REQUEST_NOTIFICATION_STATUS ConvertRequestCompletionResults(bool success)
-        {
-            return success ? NativeMethods.REQUEST_NOTIFICATION_STATUS.RQ_NOTIFICATION_CONTINUE
-                                                     : NativeMethods.REQUEST_NOTIFICATION_STATUS.RQ_NOTIFICATION_FINISH_REQUEST;
-        }
-
         private class IISContextFactory<T> : IISContextFactory
         {
             private readonly IHttpApplication<T> _application;

src/Servers/IIS/src/Microsoft.AspNetCore.Server.IIS/Core/IO/AsyncIOOperation.cs

@@ -151,10 +151,7 @@ public void Invoke()
             {
                 if (Continuation != null)
                 {
-                    // TODO: use generic overload when code moved to be netcoreapp only
-                    var continuation = Continuation;
-                    var state = State;
-                    ThreadPool.QueueUserWorkItem(_ => continuation(state));
+                    ThreadPool.UnsafeQueueUserWorkItem(Continuation, State, preferLocal: false);
                 }
             }
         }