Implement changes needed in the Host to decouple workers from the Host release

release_notes.md

@@ -17,3 +17,4 @@
 - Improvements to coldstart pipeline (#11102).
 - Update Python Worker Version to [4.38.0](https://github.com/Azure/azure-functions-python-worker/releases/tag/4.38.0)
 - Only start the Diagnostic Events flush logs timer when events are present, preventing unnecessary flush attempts (#11100).
+- Implement changes needed in the Host to decouple workers from the Host release (#11111)

src/WebJobs.Script/Config/FunctionsHostingConfigOptions.cs

@@ -123,6 +123,17 @@ internal string WorkerIndexingDisabledApps
             }
         }
 
+        /// <summary>
+        /// Gets a string delimited by '|' that contains the names of the language workers available through Probing paths outside of the Host.
+        /// </summary>
+        internal string WorkersAvailableForDynamicResolution
+        {
+            get
+            {
+                return GetFeature(RpcWorkerConstants.WorkersAvailableForDynamicResolution) ?? string.Empty;
+            }
+        }
+
         /// <summary>
         /// Gets a value indicating whether Linux Log Backoff is disabled in the hosting config.
         /// </summary>

src/WebJobs.Script/Environment/EnvironmentExtensions.cs

@@ -278,6 +278,16 @@ public static bool IsWindowsElasticPremium(this IEnvironment environment)
             return string.Equals(value, ScriptConstants.ElasticPremiumSku, StringComparison.OrdinalIgnoreCase);
         }
 
+        /// <summary>
+        /// Gets a value indicating whether the application is running in a Windows App Service environment.
+        /// </summary>
+        /// <param name="environment">The environment to verify.</param>
+        /// <returns><see cref="true"/> if running in a Windows App Service app; otherwise, false.</returns>
+        public static bool IsAnyWindows(this IEnvironment environment)
+        {
+            return environment.IsWindowsAzureManagedHosting() || environment.IsWindowsConsumption() || environment.IsWindowsElasticPremium();
+        }
+
         public static bool IsDynamicSku(this IEnvironment environment)
         {
             return environment.IsConsumptionSku() || environment.IsWindowsElasticPremium();

src/WebJobs.Script/Environment/EnvironmentSettingNames.cs

@@ -35,6 +35,7 @@ public static class EnvironmentSettingNames
         public const string AppInsightsAgent = "APPLICATIONINSIGHTS_ENABLE_AGENT";
         public const string FunctionsExtensionVersion = "FUNCTIONS_EXTENSION_VERSION";
         public const string FunctionWorkerRuntime = "FUNCTIONS_WORKER_RUNTIME";
+        public const string WorkerProbingPaths = "WORKER_PROBING_PATHS";
         public const string ContainerName = "CONTAINER_NAME";
         public const string WebsitePodName = "WEBSITE_POD_NAME";
         public const string LegionServiceHost = "LEGION_SERVICE_HOST";

src/WebJobs.Script/ScriptConstants.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT License. See License.txt in the project root for license information.
 
 using System;
+using System.Collections.Generic;
 using System.Collections.Immutable;
 using NuGet.Versioning;
 
@@ -133,6 +134,8 @@ public static class ScriptConstants
         public const string FeatureFlagDisableWebHostLogForwarding = "DisableWebHostLogForwarding";
         public const string FeatureFlagDisableMergedWebHostScriptHostConfiguration = "DisableMergedConfiguration";
         public const string FeatureFlagEnableWorkerIndexing = "EnableWorkerIndexing";
+        public const string FeatureFlagDisableDynamicWorkerResolution = "DisableDynamicWorkerResolution";
+        public const string FeatureFlagEnableDynamicWorkerResolution = "EnableDynamicWorkerResolution";
         public const string FeatureFlagEnableDebugTracing = "EnableDebugTracing";
         public const string FeatureFlagEnableProxies = "EnableProxies";
         public const string FeatureFlagStrictHISModeEnabled = "StrictHISModeEnabled";
@@ -250,6 +253,7 @@ public static class ScriptConstants
         public static readonly long DefaultMaxRequestBodySize = 104857600;
 
         public static readonly ImmutableArray<string> SystemLogCategoryPrefixes = ImmutableArray.Create("Microsoft.Azure.WebJobs.", "Function.", "Worker.", "Host.");
+        public static readonly HashSet<string> HostCapabilities = [];
 
         public static readonly string FunctionMetadataDirectTypeKey = "DirectType";
         public static readonly string LiveLogsSessionAIKey = "#AzFuncLiveLogsSessionId";

src/WebJobs.Script/Utility.cs

@@ -24,6 +24,7 @@
 using Microsoft.Azure.WebJobs.Script.Models;
 using Microsoft.Azure.WebJobs.Script.Workers.Rpc;
 using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
 using Newtonsoft.Json;
@@ -1065,6 +1066,35 @@ public static bool CanWorkerIndex(IEnumerable<RpcWorkerConfig> workerConfigs, IE
             return workerIndexingEnabled && workerIndexingAvailable;
         }
 
+        // Dynamic Worker Resolution can be enabled or disabled via feature flags or hosting config options. Feature flags take precedence over hosting config options.
+        // Users can enable or disable worker resolution via setting the appropriate feature flags.
+        // Worker resolution can also be enabled for specific workers at stamp level via the hosting config options.
+        public static bool IsDynamicWorkerResolutionEnabled(IEnvironment environment, HashSet<string> workersAvailableForResolutionViaHostingConfig)
+        {
+            bool isDynamicWorkerResolutionDisabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagDisableDynamicWorkerResolution, environment);
+
+            if (isDynamicWorkerResolutionDisabled)
+            {
+                return false;
+            }
+
+            bool isDynamicWorkerResolutionEnabled = FeatureFlags.IsEnabled(ScriptConstants.FeatureFlagEnableDynamicWorkerResolution, environment);
+
+            if (isDynamicWorkerResolutionEnabled)
+            {
+                return true;
+            }
+
+            string workerRuntime = environment.GetEnvironmentVariable(RpcWorkerConstants.FunctionWorkerRuntimeSettingName);
+
+            if (!environment.IsMultiLanguageRuntimeEnvironment() && !string.IsNullOrWhiteSpace(workerRuntime))
+            {
+                return workersAvailableForResolutionViaHostingConfig?.Contains(workerRuntime) ?? false;
+            }
+
+            return workersAvailableForResolutionViaHostingConfig?.Any() ?? false;
+        }
+
         public static void LogAutorestGeneratedJsonIfExists(string rootScriptPath, ILogger logger)
         {
             string autorestGeneratedJsonPath = Path.Combine(rootScriptPath, ScriptConstants.AutorestGeenratedMetadataFileName);
@@ -1127,6 +1157,11 @@ public static FunctionAppContentEditingState GetFunctionAppContentEditingState(I
             }
         }
 
+        public static string GetPlatformReleaseChannel(IEnvironment environment)
+        {
+            return environment.GetEnvironmentVariable(EnvironmentSettingNames.AntaresPlatformReleaseChannel) ?? ScriptConstants.LatestPlatformChannelNameUpper;
+        }
+
         public static bool TryReadAsBool(IDictionary<string, object> properties, string propertyKey, out bool result)
         {
             if (properties.TryGetValue(propertyKey, out object valueObject))

src/WebJobs.Script/Workers/Http/HttpWorkerDescription.cs

@@ -19,7 +19,7 @@ public class HttpWorkerDescription : WorkerDescription
 
         public override bool UseStdErrorStreamForErrorsOnly { get; set; } = true;
 
-        public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger)
+        public override void ApplyDefaultsAndValidate(string inputWorkerDirectory, ILogger logger, bool probingPathEnabled = false, HashSet<string> workersAvailableForResolutionViaHostingConfig = null)
         {
             if (inputWorkerDirectory == null)
             {

src/WebJobs.Script/Workers/ProcessManagement/WorkerDescription.cs

@@ -57,7 +57,7 @@ public abstract class WorkerDescription
         /// </summary>
         public bool? IsDisabled { get; set; }
 
-        public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger);
+        public abstract void ApplyDefaultsAndValidate(string workerDirectory, ILogger logger, bool probingPathEnabled = false, HashSet<string> workersAvailableForResolutionViaHostingConfig = null);
 
         internal void ThrowIfFileNotExists(string inputFile, string paramName)
         {

src/WebJobs.Script/Workers/Rpc/Configuration/LanguageWorkerOptionsSetup.cs

@@ -4,8 +4,14 @@
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Text.Json;
+using Microsoft.Azure.WebJobs.Script.Config;
 using Microsoft.Azure.WebJobs.Script.Diagnostics;
 using Microsoft.Azure.WebJobs.Script.Workers.Profiles;
+using Microsoft.Azure.WebJobs.Script.Workers.Rpc.Configuration;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -15,19 +21,22 @@ namespace Microsoft.Azure.WebJobs.Script.Workers.Rpc
 {
     internal class LanguageWorkerOptionsSetup : IConfigureOptions<LanguageWorkerOptions>
     {
+        private const string _windowsWorkerProbingPath = "C:\\home\\SiteExtensions\\workers"; // Harcoded site extensions path for Windows until Antares sets it as an Environment variable.
         private readonly IConfiguration _configuration;
         private readonly ILogger _logger;
         private readonly IEnvironment _environment;
         private readonly IMetricsLogger _metricsLogger;
         private readonly IWorkerProfileManager _workerProfileManager;
         private readonly IScriptHostManager _scriptHostManager;
+        private readonly IOptions<FunctionsHostingConfigOptions> _functionsHostingConfigOptions;
 
         public LanguageWorkerOptionsSetup(IConfiguration configuration,
                                           ILoggerFactory loggerFactory,
                                           IEnvironment environment,
                                           IMetricsLogger metricsLogger,
                                           IWorkerProfileManager workerProfileManager,
-                                          IScriptHostManager scriptHostManager)
+                                          IScriptHostManager scriptHostManager,
+                                          IOptions<FunctionsHostingConfigOptions> functionsHostingConfigOptions)
         {
             if (loggerFactory is null)
             {
@@ -39,6 +48,7 @@ public LanguageWorkerOptionsSetup(IConfiguration configuration,
             _environment = environment ?? throw new ArgumentNullException(nameof(environment));
             _metricsLogger = metricsLogger ?? throw new ArgumentNullException(nameof(metricsLogger));
             _workerProfileManager = workerProfileManager ?? throw new ArgumentNullException(nameof(workerProfileManager));
+            _functionsHostingConfigOptions = functionsHostingConfigOptions ?? throw new ArgumentNullException(nameof(functionsHostingConfigOptions));
 
             _logger = loggerFactory.CreateLogger("Host.LanguageWorkerConfig");
         }
@@ -72,9 +82,50 @@ public void Configure(LanguageWorkerOptions options)
                 }
             }
 
-            var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager);
+            HashSet<string> workers = GetWorkersAvailableForResolutionViaHostingConfig();
+            bool dynamicWorkerResolutionEnabled = Utility.IsDynamicWorkerResolutionEnabled(_environment, workers);
+            List<string> probingPaths = null;
+
+            if (dynamicWorkerResolutionEnabled)
+            {
+                probingPaths = GetWorkerProbingPaths();
+            }
+
+            var workerConfigurationResolver = new WorkerConfigurationResolver(configuration, _logger, _environment, _workerProfileManager, workers);
+            var configFactory = new RpcWorkerConfigFactory(configuration, _logger, SystemRuntimeInformation.Instance, _environment, _metricsLogger, _workerProfileManager, workerConfigurationResolver, dynamicWorkerResolutionEnabled, probingPaths, workers);
             options.WorkerConfigs = configFactory.GetConfigs();
         }
+
+        internal HashSet<string> GetWorkersAvailableForResolutionViaHostingConfig()
+        {
+            return _functionsHostingConfigOptions.Value
+                        ?.WorkersAvailableForDynamicResolution
+                        ?.ToLowerInvariant()
+                        .Split("|", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+                        .ToHashSet();
+        }
+
+        internal List<string> GetWorkerProbingPaths()
+        {
+            var probingPaths = new List<string>();
+
+            // If Environment variable is set, read probing paths from Environment
+            string probingPathsEnvValue = _environment.GetEnvironmentVariableOrDefault(EnvironmentSettingNames.WorkerProbingPaths, null);
+
+            if (!string.IsNullOrEmpty(probingPathsEnvValue))
+            {
+                probingPaths = probingPathsEnvValue.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).ToList();
+            }
+            else
+            {
+                if (_environment.IsAnyWindows())
+                {
+                    probingPaths.Add(_windowsWorkerProbingPath);
+                }
+            }
+
+            return probingPaths;
+        }
     }
 
     /// <summary>
@@ -100,4 +151,4 @@ public void PostConfigure(string name, LanguageWorkerOptions options)
             logger.LogInformation(message);
         }
     }
-}
+}