Add possibility to override group/product guid to system predefined (#709)

* Add possibility to override group guid to system predefined

* Implement Product/ProductAPI name override functionality

* Add product data processor tests

* Save original name for groups in data process

* Add ProductApiDataProcessor tests

* Cover case with single api extraction with overrided names of product

Co-authored-by: Farhad Alizada <falizada@microsoft.com>

Author: f-alizada

src/ArmTemplates/Commands/Configurations/ExtractorConsoleAppConfiguration.cs

@@ -92,5 +92,11 @@ public class ExtractorConsoleAppConfiguration
 
         [Option(longName: "extractGateways", HelpText = "Attempt to extract information about Self-Hosted gateways")]
         public string ExtractGateways { get; set; }
+
+        [Option(longName: "overrideGroupGuids", HelpText = "Override group GUID identification to system generated")]
+        public string OverrideGroupGuids { get; set; }
+
+        [Option(longName: "overrideProductGuids", HelpText = "Override product GUID identification to system generated")]
+        public string OverrideProductGuids { get; set; }
     }
 }

src/ArmTemplates/Common/API/Clients/Groups/GroupsClient.cs

@@ -9,6 +9,7 @@
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Constants;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Groups;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Utilities.DataProcessors.Absctraction;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.API.Clients.Groups
 {
@@ -17,14 +18,24 @@ public class GroupsClient : ApiClientBase, IGroupsClient
         const string GetAllGroupsLinkedToProductRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/products/{4}/groups?api-version={5}";
         const string GetAllRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/groups?api-version={4}";
 
+        readonly IGroupDataProcessor groupDataProcessor;
+
+        public GroupsClient(IGroupDataProcessor groupDataProcessor)
+        {
+            this.groupDataProcessor = groupDataProcessor;
+        }
+
         public async Task<List<GroupTemplateResource>> GetAllAsync(ExtractorParameters extractorParameters)
         {
             var (azToken, azSubId) = await this.Auth.GetAccessToken();
 
             var requestUrl = string.Format(GetAllRequest,
                 this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, GlobalConstants.ApiVersion);
 
-            return await this.GetPagedResponseAsync<GroupTemplateResource>(azToken, requestUrl);
+            var groupTemplates = await this.GetPagedResponseAsync<GroupTemplateResource>(azToken, requestUrl);
+            this.groupDataProcessor.ProcessData(groupTemplates, extractorParameters);
+
+            return groupTemplates;
         }
 
         public async Task<List<GroupTemplateResource>> GetAllLinkedToProductAsync(string productName, ExtractorParameters extractorParameters)
@@ -34,7 +45,10 @@ public async Task<List<GroupTemplateResource>> GetAllLinkedToProductAsync(string
             var requestUrl = string.Format(GetAllGroupsLinkedToProductRequest,
                 this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, productName, GlobalConstants.ApiVersion);
 
-            return await this.GetPagedResponseAsync<GroupTemplateResource>(azToken, requestUrl);
+            var groupTemplates = await this.GetPagedResponseAsync<GroupTemplateResource>(azToken, requestUrl);
+            this.groupDataProcessor.ProcessData(groupTemplates, extractorParameters);
+
+            return groupTemplates;
         }
     }
 }

src/ArmTemplates/Common/API/Clients/Product/ProductsClient.cs

@@ -10,6 +10,7 @@
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ProductApis;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Products;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Utilities.DataProcessors.Absctraction;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.API.Clients.Product
 {
@@ -18,14 +19,26 @@ public class ProductsClient : ApiClientBase, IProductsClient
         const string GetAllProductsLinkedToApiRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/apis/{4}/products?api-version={5}";
         const string GetAllProductsRequest = "{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/products?api-version={4}";
 
+        readonly IProductDataProcessor productDataProcessor;
+        readonly IProductApiDataProcessor productApiDataProcessor;
+
+        public ProductsClient(IProductDataProcessor productDataProcessor, IProductApiDataProcessor productApiDataProcessor)
+        {
+            this.productDataProcessor = productDataProcessor;
+            this.productApiDataProcessor = productApiDataProcessor;
+        }
+
         public async Task<List<ProductsTemplateResource>> GetAllAsync(ExtractorParameters extractorParameters)
         {
             var (azToken, azSubId) = await this.Auth.GetAccessToken();
 
             string requestUrl = string.Format(GetAllProductsRequest,
                 this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, GlobalConstants.ApiVersion);
 
-            return await this.GetPagedResponseAsync<ProductsTemplateResource>(azToken, requestUrl);
+            var productTemplates = await this.GetPagedResponseAsync<ProductsTemplateResource>(azToken, requestUrl);
+            this.productDataProcessor.ProcessData(productTemplates, extractorParameters);
+
+            return productTemplates;
         }
 
         public async Task<List<ProductApiTemplateResource>> GetAllLinkedToApiAsync(string apiName, ExtractorParameters extractorParameters)
@@ -35,7 +48,10 @@ public async Task<List<ProductApiTemplateResource>> GetAllLinkedToApiAsync(strin
             string requestUrl = string.Format(GetAllProductsLinkedToApiRequest,
                this.BaseUrl, azSubId, extractorParameters.ResourceGroup, extractorParameters.SourceApimName, apiName, GlobalConstants.ApiVersion);
 
-            return await this.GetPagedResponseAsync<ProductApiTemplateResource>(azToken, requestUrl);
+            var productApiTemplateResources = await this.GetPagedResponseAsync<ProductApiTemplateResource>(azToken, requestUrl);
+            this.productApiDataProcessor.ProcessData(productApiTemplateResources, extractorParameters);
+
+            return productApiTemplateResources;
         }
     }
 }

src/ArmTemplates/Common/Templates/Groups/GroupTemplateResource.cs

@@ -3,12 +3,19 @@
 //  Licensed under the MIT License.
 // --------------------------------------------------------------------------
 
+using Newtonsoft.Json;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Groups
 {
     public class GroupTemplateResource : TemplateResource
     {
+        [JsonIgnore]
+        public string OriginalName { get; set; }
+
+        [JsonIgnore]
+        public string NewName { get; set; }
+
         public GroupProperties Properties { get; set; }
     }
 }

src/ArmTemplates/Common/Templates/ProductApis/ProductApiTemplateResource.cs

@@ -3,12 +3,19 @@
 //  Licensed under the MIT License.
 // --------------------------------------------------------------------------
 
+using Newtonsoft.Json;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.ProductApis
 {
     public class ProductApiTemplateResource : TemplateResource
     {
+        [JsonIgnore]
+        public string OriginalName { get; set; }
+
+        [JsonIgnore]
+        public string NewName { get; set; }
+
         public ProductApiProperties Properties { get; set; }
     }
 }

src/ArmTemplates/Common/Templates/Products/ProductsTemplateResource.cs

@@ -3,12 +3,19 @@
 //  Licensed under the MIT License.
 // --------------------------------------------------------------------------
 
+using Newtonsoft.Json;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Products
 {
     public class ProductsTemplateResource : TemplateResource
     {
+        [JsonIgnore]
+        public string OriginalName { get; set; }
+
+        [JsonIgnore]
+        public string NewName { get; set; }
+
         public ProductsProperties Properties { get; set; }
     }
 }

src/ArmTemplates/Extractor/EntityExtractors/Abstractions/IPolicyExtractor.cs

@@ -5,6 +5,7 @@
 
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Abstractions;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Policy;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Products;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.Models;
 using System.Threading.Tasks;
 
@@ -16,7 +17,7 @@ public interface IPolicyExtractor
 
         Task<PolicyTemplateResource> GenerateProductPolicyTemplateAsync(
             ExtractorParameters extractorParameters,
-            string productName,
+            ProductsTemplateResource productTemplateResource,
             string[] productResourceId,
             string baseFilesGenerationDirectory);
 

src/ArmTemplates/Extractor/EntityExtractors/PolicyExtractor.cs

@@ -18,6 +18,7 @@
 using Microsoft.Extensions.Logging;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Builders.Abstractions;
 using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Extensions;
+using Microsoft.Azure.Management.ApiManagement.ArmTemplates.Common.Templates.Products;
 
 namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Extractor.EntityExtractors
 {
@@ -154,21 +155,21 @@ public async Task<PolicyTemplateResource> GenerateApiPolicyResourceAsync(
 
         public async Task<PolicyTemplateResource> GenerateProductPolicyTemplateAsync(
             ExtractorParameters extractorParameters,
-            string productName,
+            ProductsTemplateResource productTemplateResource,
             string[] productResourceId,
             string baseFilesGenerationDirectory)
         {
-            var productPolicy = await this.policyClient.GetPolicyLinkedToProductAsync(productName, extractorParameters);
+            var productPolicy = await this.policyClient.GetPolicyLinkedToProductAsync(productTemplateResource.OriginalName, extractorParameters);
 
             if (productPolicy is null)
             {
-                this.logger.LogWarning($"Policy not found for product '{productName}'");
+                this.logger.LogWarning($"Policy not found for product '{productTemplateResource.OriginalName}'");
                 return productPolicy;
             }
 
-            this.logger.LogDebug($"Policy linked to {productName} product found successfuly");
+            this.logger.LogDebug($"Policy linked to {productTemplateResource.OriginalName} product found successfuly");
 
-            productPolicy.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{productName}/policy')]";
+            productPolicy.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{productTemplateResource.NewName}/policy')]";
             productPolicy.ApiVersion = GlobalConstants.ApiVersion;
             productPolicy.Scale = null;
             productPolicy.DependsOn = productResourceId;
@@ -177,7 +178,7 @@ public async Task<PolicyTemplateResource> GenerateProductPolicyTemplateAsync(
             // telling that it is needed to provide a policy Xml file
             if (extractorParameters.PolicyXMLBaseUrl is not null)
             {
-                var policyFileName = string.Format(ProductPolicyFileNameFormat, productName);
+                var policyFileName = string.Format(ProductPolicyFileNameFormat, productTemplateResource.NewName);
                 await this.SavePolicyXmlAsync(productPolicy, baseFilesGenerationDirectory, policyFileName);
 
                 this.SetPolicyTemplateResourcePolicyContentWithArmPresetValues(extractorParameters, productPolicy, policyFileName);

src/ArmTemplates/Extractor/EntityExtractors/ProductApisExtractor.cs

@@ -131,19 +131,18 @@ async Task<List<ProductApiTemplateResource>> GenerateProductApiTemplateResources
                 string lastProductAPIName = null;
                 foreach (var productApi in productApis)
                 {
-                    var originalProductApiName = productApi.Name;
-                    this.logger.LogInformation("'{0}' Product association found", originalProductApiName);
+                    this.logger.LogInformation("'{0}' Product association found", productApi.OriginalName);
 
                     // convert returned api product associations to template resource class
                     productApi.Type = ResourceTypeConstants.ProductApi;
-                    productApi.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{originalProductApiName}/{apiName}')]";
+                    productApi.Name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{productApi.NewName}/{apiName}')]";
                     productApi.ApiVersion = GlobalConstants.ApiVersion;
                     productApi.Scale = null;
                     productApi.DependsOn = lastProductAPIName != null ? new string[] { lastProductAPIName } : dependsOn;
 
                     productApiResources.Add(productApi);
 
-                    lastProductAPIName = $"[resourceId('Microsoft.ApiManagement/service/products/apis', parameters('{ParameterNames.ApimServiceName}'), '{originalProductApiName}', '{apiName}')]";
+                    lastProductAPIName = $"[resourceId('Microsoft.ApiManagement/service/products/apis', parameters('{ParameterNames.ApimServiceName}'), '{productApi.NewName}', '{apiName}')]";
                 }
             }
             catch (Exception ex)