RpcException handling.

release_notes.md

@@ -18,3 +18,4 @@
 - Update Python Worker Version to [4.38.0](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.38.0)
 - Only start the Diagnostic Events flush logs timer when events are present, preventing unnecessary flush attempts (#11100).
 - Switched memory usage reporting to use CGroup metrics by default for Linux consumption (#11114)
+- Avoid logging RpcException details (#11116)

src/WebJobs.Script.WebHost/Diagnostics/SystemLogger.cs

@@ -7,29 +7,35 @@
 using Microsoft.Azure.WebJobs.Host.Executors.Internal;
 using Microsoft.Azure.WebJobs.Host.Indexers;
 using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 
 namespace Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics
 {
-    public class SystemLogger : ILogger
+    public class SystemLogger : ILogger, IDisposable
     {
         private readonly string _categoryName;
         private readonly string _functionName;
         private readonly string _hostInstanceId;
         private readonly bool _isUserFunction;
+        private readonly string _rpcExceptionName = "Microsoft.Azure.WebJobs.Script.Workers.Rpc.RpcException";
         private readonly LogLevel _logLevel;
         private readonly IEnvironment _environment;
         private readonly IEventGenerator _eventGenerator;
         private readonly IDebugStateProvider _debugStateProvider;
         private readonly IScriptEventManager _eventManager;
         private readonly IExternalScopeProvider _scopeProvider;
+        private readonly IDisposable _hostingConfigOptionsOnChangeListener;
+        private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
+        private readonly IDisposable _appServiceOptionsOnChangeListener;
         private AppServiceOptions _appServiceOptions;
+        private bool _logRpcExceptionDetails = false;
 
         public SystemLogger(string hostInstanceId, string categoryName, IEventGenerator eventGenerator, IEnvironment environment, IDebugStateProvider debugStateProvider,
-           IScriptEventManager eventManager, IExternalScopeProvider scopeProvider, IOptionsMonitor<AppServiceOptions> appServiceOptionsMonitor)
+           IScriptEventManager eventManager, IExternalScopeProvider scopeProvider, IOptionsMonitor<AppServiceOptions> appServiceOptionsMonitor, IOptionsMonitor<FunctionsHostingConfigOptions> hostingConfigOptions)
         {
             _environment = environment;
             _eventGenerator = eventGenerator;
@@ -42,8 +48,18 @@ public SystemLogger(string hostInstanceId, string categoryName, IEventGenerator
             _eventManager = eventManager;
             _scopeProvider = scopeProvider;
 
-            appServiceOptionsMonitor.OnChange(newOptions => _appServiceOptions = newOptions);
+            _appServiceOptionsOnChangeListener = appServiceOptionsMonitor.OnChange(newOptions => _appServiceOptions = newOptions);
             _appServiceOptions = appServiceOptionsMonitor.CurrentValue;
+
+            _hostingConfigOptions = hostingConfigOptions ?? throw new ArgumentNullException(nameof(hostingConfigOptions));
+            _logRpcExceptionDetails = hostingConfigOptions.CurrentValue.LogRpcExceptionDetails;
+            _hostingConfigOptionsOnChangeListener = _hostingConfigOptions.OnChange(newOptions =>
+            {
+                if (newOptions.LogRpcExceptionDetails != _logRpcExceptionDetails)
+                {
+                    _logRpcExceptionDetails = newOptions.LogRpcExceptionDetails;
+                }
+            });
         }
 
         public IDisposable BeginScope<TState>(TState state) => _scopeProvider.Push(state);
@@ -57,7 +73,7 @@ public bool IsEnabled(LogLevel logLevel)
 
         public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception exception, Func<TState, Exception, string> formatter)
         {
-            if (!IsEnabled(logLevel) || _isUserFunction || FunctionInvoker.CurrentScope == FunctionInvocationScope.User)
+            if (_isUserFunction || !IsEnabled(logLevel) || FunctionInvoker.CurrentScope == FunctionInvocationScope.User)
             {
                 return;
             }
@@ -173,12 +189,29 @@ public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Except
                     functionName = string.IsNullOrEmpty(fex.MethodName) ? string.Empty : fex.MethodName.Replace("Host.Functions.", string.Empty);
                 }
 
-                (innerExceptionType, innerExceptionMessage, details) = exception.GetExceptionDetails();
-                formattedMessage = Sanitizer.Sanitize(formattedMessage);
-                innerExceptionMessage = innerExceptionMessage ?? string.Empty;
+                var exceptionDetails = exception.GetExceptionDetails();
+
+                if (_logRpcExceptionDetails || !exceptionDetails.ExceptionType.Equals(_rpcExceptionName, StringComparison.Ordinal))
+                {
+                    // If _logExceptionDetails is true or the exception isn't an RPC exception, full details are logged.
+                    details = exceptionDetails.ExceptionDetails;
+                    innerExceptionType = exceptionDetails.ExceptionType;
+                    innerExceptionMessage = exceptionDetails.ExceptionMessage;
+                }
+                else
+                {
+                    details = "An exception occurred during invocation, but its details are redacted. Customers with AppInsights or OTel enabled can access full exception details.";
+                    innerExceptionType = exceptionDetails.ExceptionType;
+                }
             }
 
             _eventGenerator.LogFunctionTraceEvent(logLevel, subscriptionId, appName, functionName, eventName, source, details, formattedMessage, innerExceptionType, innerExceptionMessage, invocationId, _hostInstanceId, activityId, runtimeSiteName, slotName, DateTime.UtcNow);
         }
+
+        public void Dispose()
+        {
+            _appServiceOptionsOnChangeListener?.Dispose();
+            _hostingConfigOptionsOnChangeListener?.Dispose();
+        }
     }
 }

src/WebJobs.Script.WebHost/Diagnostics/SystemLoggerProvider.cs

@@ -3,6 +3,7 @@
 
 using System;
 using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Azure.WebJobs.Script.Workers;
@@ -20,21 +21,23 @@ public class SystemLoggerProvider : ILoggerProvider, ISupportExternalScope
         private readonly IDebugStateProvider _debugStateProvider;
         private readonly IScriptEventManager _eventManager;
         private readonly IOptionsMonitor<AppServiceOptions> _appServiceOptions;
+        private readonly IOptionsMonitor<FunctionsHostingConfigOptions> _hostingConfigOptions;
         private IExternalScopeProvider _scopeProvider;
 
-        public SystemLoggerProvider(IOptions<ScriptJobHostOptions> scriptOptions, IEventGenerator eventGenerator, IEnvironment environment, IDebugStateProvider debugStateProvider, IScriptEventManager eventManager, IOptionsMonitor<AppServiceOptions> appServiceOptions)
-            : this(scriptOptions.Value.InstanceId, eventGenerator, environment, debugStateProvider, eventManager, appServiceOptions)
+        public SystemLoggerProvider(IOptions<ScriptJobHostOptions> scriptOptions, IEventGenerator eventGenerator, IEnvironment environment, IDebugStateProvider debugStateProvider, IScriptEventManager eventManager, IOptionsMonitor<AppServiceOptions> appServiceOptions, IOptionsMonitor<FunctionsHostingConfigOptions> functionsHostingConfigOptions)
+            : this(scriptOptions.Value.InstanceId, eventGenerator, environment, debugStateProvider, eventManager, appServiceOptions, functionsHostingConfigOptions)
         {
         }
 
-        protected SystemLoggerProvider(string hostInstanceId, IEventGenerator eventGenerator, IEnvironment environment, IDebugStateProvider debugStateProvider, IScriptEventManager eventManager, IOptionsMonitor<AppServiceOptions> appServiceOptions)
+        protected SystemLoggerProvider(string hostInstanceId, IEventGenerator eventGenerator, IEnvironment environment, IDebugStateProvider debugStateProvider, IScriptEventManager eventManager, IOptionsMonitor<AppServiceOptions> appServiceOptions, IOptionsMonitor<FunctionsHostingConfigOptions> functionsHostingConfigOptions)
         {
             _eventGenerator = eventGenerator;
             _environment = environment;
             _hostInstanceId = hostInstanceId;
             _debugStateProvider = debugStateProvider;
             _eventManager = eventManager;
             _appServiceOptions = appServiceOptions;
+            _hostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions));
         }
 
         public ILogger CreateLogger(string categoryName)
@@ -44,7 +47,7 @@ public ILogger CreateLogger(string categoryName)
                 // The SystemLogger is not used for user logs.
                 return NullLogger.Instance;
             }
-            return new SystemLogger(_hostInstanceId, categoryName, _eventGenerator, _environment, _debugStateProvider, _eventManager, _scopeProvider, _appServiceOptions);
+            return new SystemLogger(_hostInstanceId, categoryName, _eventGenerator, _environment, _debugStateProvider, _eventManager, _scopeProvider, _appServiceOptions, _hostingConfigOptions);
         }
 
         private bool IsUserLogCategory(string categoryName)

src/WebJobs.Script.WebHost/Diagnostics/WebHostSystemLoggerProvider.cs

@@ -1,6 +1,7 @@
 // Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.Eventing;
 using Microsoft.Extensions.Options;
@@ -13,8 +14,8 @@ namespace Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics
     /// </summary>
     public class WebHostSystemLoggerProvider : SystemLoggerProvider
     {
-        public WebHostSystemLoggerProvider(IEventGenerator eventGenerator, IEnvironment environment, IDebugStateProvider debugStateProvider, IScriptEventManager eventManager, IOptionsMonitor<AppServiceOptions> appServiceOptions)
-            : base(string.Empty, eventGenerator, environment, debugStateProvider, eventManager, appServiceOptions)
+        public WebHostSystemLoggerProvider(IEventGenerator eventGenerator, IEnvironment environment, IDebugStateProvider debugStateProvider, IScriptEventManager eventManager, IOptionsMonitor<AppServiceOptions> appServiceOptions, IOptionsMonitor<FunctionsHostingConfigOptions> hostingConfigOptions)
+            : base(string.Empty, eventGenerator, environment, debugStateProvider, eventManager, appServiceOptions, hostingConfigOptions)
         {
         }
     }

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -220,6 +220,22 @@ internal bool IsDotNetInProcDisabled
             }
         }
 
+        /// <summary>
+        /// Gets or sets a value indicating whether to log Rpc exception details returned by the worker.
+        /// </summary>
+        internal bool LogRpcExceptionDetails
+        {
+            get
+            {
+                return GetFeatureAsBooleanOrDefault(ScriptConstants.HostingConfigLogRpcExceptionDetails, false);
+            }
+
+            set
+            {
+                _features[ScriptConstants.HostingConfigLogRpcExceptionDetails] = value ? "1" : "0";
+            }
+        }
+
         /// <summary>
         /// Gets feature by name.
         /// </summary>

src/WebJobs.Script/ScriptConstants.cs

@@ -148,6 +148,7 @@ public static class ScriptConstants
         public const string HostingConfigSwtIssuerEnabled = "SwtIssuerEnabled";
         public const string HostingConfigInternalAuthApisAllowList = "InternalAuthApisAllowList";
         public const string HostingConfigDotNetInProcDisabled = "DotNetInProcDisabled";
+        public const string HostingConfigLogRpcExceptionDetails = "LogRpcExceptionDetails";
 
         public const string SiteAzureFunctionsUriFormat = "https://{0}.azurewebsites.net/azurefunctions";
         public const string ScmSiteUriFormat = "https://{0}.scm.azurewebsites.net";

test/WebJobs.Script.Tests/Eventing/SystemLoggerProviderTests.cs

@@ -3,6 +3,7 @@
 
 using System.IO;
 using Microsoft.Azure.WebJobs.Logging;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Configuration;
 using Microsoft.Azure.WebJobs.Script.WebHost.Diagnostics;
 using Microsoft.Extensions.Logging;
@@ -33,7 +34,9 @@ public SystemLoggerProviderTests()
             debugStateProvider.Setup(p => p.InDiagnosticMode).Returns(() => _inDiagnosticMode);
 
             var appServiceOptions = new TestOptionsMonitor<AppServiceOptions>(new AppServiceOptions());
-            _provider = new SystemLoggerProvider(_options, null, _environment, debugStateProvider.Object, null, appServiceOptions);
+            var testHostingConfigOptions = new TestOptionsMonitor<FunctionsHostingConfigOptions>(new FunctionsHostingConfigOptions());
+
+            _provider = new SystemLoggerProvider(_options, null, _environment, debugStateProvider.Object, null, appServiceOptions, testHostingConfigOptions);
         }
 
         [Fact]