get policy content fixes (#654)

* get policy content fixes

* fix references in mock policy client

* rename policy cache

Co-authored-by: Dmitrii Korolev <dmkorolev@microsoft.com>

Author: DeagleGross

src/ArmTemplates/Commands/Executors/ExtractorExecutor.cs

@@ -717,9 +717,14 @@ async Task GenerateTemplates(
             var apiTemplateResources = apiTemplate.Resources.ToList();
             Template loggerTemplate = await this.loggerExtractor.GenerateLoggerTemplateAsync(this.extractorParameters, singleApiName, apiTemplateResources, apiLoggerId);
             List<TemplateResource> loggerResources = loggerTemplate.Resources.ToList();
-            Template namedValueTemplate = await this.propertyExtractor.GenerateNamedValuesTemplateAsync(singleApiName, apiTemplateResources, this.extractorParameters, this.backendExtractor, loggerResources);
+            Template namedValueTemplate = await this.propertyExtractor.GenerateNamedValuesTemplateAsync(singleApiName, apiTemplateResources, this.extractorParameters, this.backendExtractor, loggerResources, baseFilesGenerationDirectory);
             List<TemplateResource> namedValueResources = namedValueTemplate.Resources.ToList();
-            var backendResult = await this.backendExtractor.GenerateBackendsARMTemplateAsync(this.extractorParameters.SourceApimName, this.extractorParameters.ResourceGroup, singleApiName, apiTemplateResources, namedValueResources, this.extractorParameters);
+            var backendResult = await this.backendExtractor.GenerateBackendsARMTemplateAsync(
+                singleApiName, 
+                apiTemplate.TypedResources.GetAllPolicies(), 
+                namedValueResources, 
+                this.extractorParameters, 
+                baseFilesGenerationDirectory);
 
             Dictionary<string, string> loggerResourceIds = null;
             if (this.extractorParameters.ParameterizeLogResourceId)

src/ArmTemplates/Common/Extensions/ParameterNamingHelper.cs

@@ -7,6 +7,13 @@ static class ParameterNamingHelper
     {
         static readonly Regex ExcludeOtherFromLettersAndDigitsRegex = new Regex("[^a-zA-Z0-9]");
 
+        public static string GetSubstringBetweenTwoCharacters(char left, char right, string fullString)
+        {
+            var regex = new Regex($"(?<={left})(.*?)(?={right})");
+            var matchString = regex.Match(fullString);
+            return matchString.Captures.FirstOrDefault()?.Value ?? string.Empty;
+        }
+
         public static string GenerateValidParameterName(string apiName, string prefix)
         {
             if (string.IsNullOrEmpty(apiName))

src/ArmTemplates/Common/Templates/Apis/ApiTemplateResources.cs

@@ -38,6 +38,23 @@ public class ApiTemplateResources : TemplateResourcesBase, ITemplateResources
 
         public List<PolicyTemplateResource> ApiPolicies { get; set; } = new();
 
+        public List<PolicyTemplateResource> GetAllPolicies()
+        {
+            var policies = new List<PolicyTemplateResource>();
+
+            if (!this.ApiPolicies.IsNullOrEmpty())
+            {
+                policies.AddRange(this.ApiPolicies);
+            }
+
+            if (!this.ApiOperationsPolicies.IsNullOrEmpty())
+            {
+                policies.AddRange(this.ApiOperationsPolicies);
+            }
+
+            return policies;
+        }
+
         public void AddResourcesData(ApiTemplateResources otherResources)
         {
             if (otherResources is null)

src/ArmTemplates/Common/Templates/Policy/PolicyTemplateProperties.cs

@@ -10,6 +10,9 @@ namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates
 {
     public class PolicyTemplateProperties
     {
+        [JsonIgnore]
+        public string PolicyContentFileFullPath { get; set; }
+
         [JsonProperty("value")]
         public string PolicyContent { get; set; }
 

src/ArmTemplates/Common/Templates/Policy/PolicyTemplateUtils.cs

@@ -1,5 +1,6 @@
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.TemplateModels;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Policy;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
 using System;
 using System.Collections.Generic;
@@ -9,8 +10,21 @@ namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Entity
 {
     public interface IBackendExtractor
     {
-        Task<Tuple<Template, Dictionary<string, BackendApiParameters>>> GenerateBackendsARMTemplateAsync(string apimname, string resourceGroup, string singleApiName, List<TemplateResource> apiTemplateResources, List<TemplateResource> propertyResources, ExtractorParameters extractorParameters);
+        Task<Tuple<Template, Dictionary<string, BackendApiParameters>>> GenerateBackendsARMTemplateAsync(
+            string singleApiName, 
+            List<PolicyTemplateResource> policyTemplateResources,
+            List<TemplateResource> propertyResources, 
+            ExtractorParameters extractorParameters,
+            string baseFilesGenerationDirectory);
 
-        Task<bool> IsNamedValueUsedInBackends(string apimname, string resourceGroup, string singleApiName, List<TemplateResource> apiTemplateResources, ExtractorParameters extractorParameters, string propertyName, string propertyDisplayName);
+        Task<bool> IsNamedValueUsedInBackends(
+            string apimname, 
+            string resourceGroup, 
+            string singleApiName, 
+            List<TemplateResource> apiTemplateResources, 
+            ExtractorParameters extractorParameters, 
+            string propertyName, 
+            string propertyDisplayName,
+            string baseFilesGenerationDirectory);
     }
 }

src/ArmTemplates/Extractor/EntityExtractors/Abstractions/IBackendExtractor.cs

@@ -7,6 +7,8 @@ namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Entity
 {
     public interface IPolicyExtractor
     {
+        string GetCachedPolicyContent(PolicyTemplateResource policyTemplateResource, string baseFilesGenerationDirectory);
+
         Task<PolicyTemplateResource> GenerateProductPolicyTemplateAsync(
             ExtractorParameters extractorParameters,
             string productName,

src/ArmTemplates/Extractor/EntityExtractors/Abstractions/IPolicyExtractor.cs

@@ -7,6 +7,12 @@ namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Entity
 {
     public interface IPropertyExtractor
     {
-        Task<Template> GenerateNamedValuesTemplateAsync(string singleApiName, List<TemplateResource> apiTemplateResources, ExtractorParameters extractorParameters, IBackendExtractor backendExtractor, List<TemplateResource> loggerTemplateResources);
+        Task<Template> GenerateNamedValuesTemplateAsync(
+            string singleApiName, 
+            List<TemplateResource> apiTemplateResources, 
+            ExtractorParameters extractorParameters, 
+            IBackendExtractor backendExtractor, 
+            List<TemplateResource> loggerTemplateResources,
+            string baseFilesGenerationDirectory);
     }
 }

src/ArmTemplates/Extractor/EntityExtractors/Abstractions/IPropertyExtractor.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using Newtonsoft.Json.Linq;
 using System.Linq;
-using Newtonsoft.Json;
 using System;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.TemplateModels;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Policy;
@@ -18,10 +17,14 @@ namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Entity
     public class BackendExtractor : EntityExtractorBase, IBackendExtractor
     {
         readonly ITemplateBuilder templateBuilder;
+        readonly IPolicyExtractor policyExtractor;
 
-        public BackendExtractor(ITemplateBuilder templateBuilder)
+        public BackendExtractor(
+            ITemplateBuilder templateBuilder,
+            IPolicyExtractor policyExtractor)
         {
             this.templateBuilder = templateBuilder;
+            this.policyExtractor = policyExtractor;
         }
 
         public async Task<string> GetBackendsAsync(string apiManagementName, string resourceGroupName, int skipNumOfRecords)
@@ -47,16 +50,13 @@ public async Task<string> GetBackendDetailsAsync(string apiManagementName, strin
         /// <summary>
         /// Generate the ARM assets for the backend resources
         /// </summary>
-        /// <param name="apimname"></param>
-        /// <param name="resourceGroup"></param>
-        /// <param name="singleApiName"></param>
-        /// <param name="apiTemplateResources"></param>
-        /// <param name="propertyResources"></param>
-        /// <param name="policyXMLBaseUrl"></param>
-        /// <param name="policyXMLSasToken"></param>
-        /// <param name="extractBackendParameters"></param>
         /// <returns>a combination of a Template and the value for the BackendSettings parameter</returns>
-        public async Task<Tuple<Template, Dictionary<string, BackendApiParameters>>> GenerateBackendsARMTemplateAsync(string apimname, string resourceGroup, string singleApiName, List<TemplateResource> apiTemplateResources, List<TemplateResource> propertyResources, ExtractorParameters extractorParameters)
+        public async Task<Tuple<Template, Dictionary<string, BackendApiParameters>>> GenerateBackendsARMTemplateAsync(
+            string singleApiName, 
+            List<PolicyTemplateResource> policyTemplateResources, 
+            List<TemplateResource> propertyResources, 
+            ExtractorParameters extractorParameters,
+            string baseFilesGenerationDirectory)
         {
             Console.WriteLine("------------------------------------------");
             Console.WriteLine("Extracting backends from service");
@@ -74,7 +74,6 @@ public async Task<Tuple<Template, Dictionary<string, BackendApiParameters>>> Gen
             List<TemplateResource> templateResources = new List<TemplateResource>();
 
             // isolate api and operation policy resources in the case of a single api extraction, as they may reference backends
-            var policyResources = apiTemplateResources.Where(resource => resource.Type == ResourceTypeConstants.APIPolicy || resource.Type == ResourceTypeConstants.APIOperationPolicy || resource.Type == ResourceTypeConstants.ProductPolicy);
             var namedValueResources = propertyResources.Where(resource => resource.Type == ResourceTypeConstants.Property);
 
             // pull all backends for service
@@ -84,13 +83,20 @@ public async Task<Tuple<Template, Dictionary<string, BackendApiParameters>>> Gen
 
             do
             {
-                string backends = await this.GetBackendsAsync(apimname, resourceGroup, skipNumberOfBackends);
+                string backends = await this.GetBackendsAsync(
+                    extractorParameters.SourceApimName,
+                    extractorParameters.ResourceGroup, 
+                    skipNumberOfBackends);
+
                 oBackends = JObject.Parse(backends);
 
                 foreach (var item in oBackends["value"])
                 {
                     string backendName = ((JValue)item["name"]).Value.ToString();
-                    string backend = await this.GetBackendDetailsAsync(apimname, resourceGroup, backendName);
+                    string backend = await this.GetBackendDetailsAsync(
+                        extractorParameters.SourceApimName,
+                        extractorParameters.ResourceGroup,
+                        backendName);
 
                     // convert returned backend to template resource class
                     BackendTemplateResource backendTemplateResource = backend.Deserialize<BackendTemplateResource>();
@@ -107,9 +113,9 @@ public async Task<Tuple<Template, Dictionary<string, BackendApiParameters>>> Gen
                     else
                     {
                         // check extracted policies to see if the backend is referenced.
-                        foreach (PolicyTemplateResource policyTemplateResource in policyResources)
+                        foreach (var policyTemplateResource in policyTemplateResources)
                         {
-                            string policyContent = PolicyTemplateUtils.GetPolicyContent(extractorParameters, policyTemplateResource);
+                            var policyContent = this.policyExtractor.GetCachedPolicyContent(policyTemplateResource, baseFilesGenerationDirectory);
 
                             if (this.DoesPolicyReferenceBackend(policyContent, namedValueResources, backendName, backendTemplateResource))
                             {
@@ -193,7 +199,15 @@ public bool DoesBackendReferenceNamedValue(TemplateResource namedValueResource,
                 || namedValue == backendTemplateResource.Properties.title;
         }
 
-        public async Task<bool> IsNamedValueUsedInBackends(string apimname, string resourceGroup, string singleApiName, List<TemplateResource> apiTemplateResources, ExtractorParameters extractorParameters, string propertyName, string propertyDisplayName)
+        public async Task<bool> IsNamedValueUsedInBackends(
+            string apimname, 
+            string resourceGroup, 
+            string singleApiName, 
+            List<TemplateResource> apiTemplateResources, 
+            ExtractorParameters extractorParameters, 
+            string propertyName, 
+            string propertyDisplayName,
+            string baseFilesGenerationDirectory)
         {
             // isolate api and operation policy resources in the case of a single api extraction, as they may reference backends
             var policyResources = apiTemplateResources.Where(resource => resource.Type == ResourceTypeConstants.APIPolicy || resource.Type == ResourceTypeConstants.APIOperationPolicy || resource.Type == ResourceTypeConstants.ProductPolicy);
@@ -238,7 +252,7 @@ public async Task<bool> IsNamedValueUsedInBackends(string apimname, string resou
                             // this is why an empty named values must be passed to this method for validation
                             foreach (PolicyTemplateResource policyTemplateResource in policyResources)
                             {
-                                string policyContent = PolicyTemplateUtils.GetPolicyContent(extractorParameters, policyTemplateResource);
+                                var policyContent = this.policyExtractor.GetCachedPolicyContent(policyTemplateResource, baseFilesGenerationDirectory);
 
                                 if (this.DoesPolicyReferenceBackend(policyContent, emptyNamedValueResources, backendName, backendTemplateResource))
                                 {

src/ArmTemplates/Extractor/EntityExtractors/BackendExtractor.cs

@@ -28,6 +28,7 @@ namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Entity
     public class MasterTemplateExtractor : ApiExtractor, IMasterTemplateExtractor
     {
         readonly ITemplateBuilder templateBuilder;
+        readonly IPolicyExtractor policyExtractor;
 
         public MasterTemplateExtractor(
             ILogger<ApiExtractor> logger,
@@ -42,6 +43,7 @@ public MasterTemplateExtractor(
             : base(logger, templateBuilder, apisClient, diagnosticExtractor, apiSchemaExtractor, policyExtractor, productApisExtractor, tagExtractor, apiOperationExtractor)
         {
             this.templateBuilder = templateBuilder;
+            this.policyExtractor = policyExtractor;
         }
 
         public Template GenerateLinkedMasterTemplate(
@@ -554,7 +556,7 @@ public async Task<Template> CreateMasterTemplateParameterValues(List<string> api
             if (extractorParameters.ParameterizeNamedValue)
             {
                 Dictionary<string, string> namedValues = new Dictionary<string, string>();
-                PropertyExtractor pExc = new PropertyExtractor(this.templateBuilder);
+                PropertyExtractor pExc = new PropertyExtractor(this.templateBuilder, this.policyExtractor);
                 string[] properties = await pExc.GetPropertiesAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup);
 
                 foreach (var extractedProperty in properties)
@@ -586,7 +588,7 @@ public async Task<Template> CreateMasterTemplateParameterValues(List<string> api
             if (extractorParameters.ParamNamedValuesKeyVaultSecrets)
             {
                 Dictionary<string, string> keyVaultNamedValues = new Dictionary<string, string>();
-                PropertyExtractor pExc = new PropertyExtractor(this.templateBuilder);
+                PropertyExtractor pExc = new PropertyExtractor(this.templateBuilder, this.policyExtractor);
                 string[] properties = await pExc.GetPropertiesAsync(extractorParameters.SourceApimName, extractorParameters.ResourceGroup);
 
                 foreach (var extractedProperty in properties)