Added support for Service diagnostics (All API level) (#519)

* changed property resourcetype from Microsoft.ApiManagement/service/properties to Microsoft.ApiManagement/service/namedValues inline with latest API version

* fixed generation of ProductAPI template

* Added extractor for All API service/diagnostics

Co-authored-by: Matthew Fortunka <mafortun@microsoft.com>

Author: fortunkam

src/APIM_ARMTemplate/apimtemplate/Common/Constants/ResourceTypeConstants.cs

@@ -6,6 +6,7 @@ internal static class ResourceTypeConstants
         public const string API = "Microsoft.ApiManagement/service/apis";
         public const string APIVersionSet = "Microsoft.ApiManagement/service/apiVersionSets";
         public const string APIDiagnostic = "Microsoft.ApiManagement/service/apis/diagnostics";
+        public const string APIServiceDiagnostic = "Microsoft.ApiManagement/service/diagnostics";
         public const string APIOperation = "Microsoft.ApiManagement/service/apis/operations";
         public const string APIOperationPolicy = "Microsoft.ApiManagement/service/apis/operations/policies";
         public const string APIOperationTag = "Microsoft.ApiManagement/service/apis/operations/tags";
@@ -22,7 +23,7 @@ internal static class ResourceTypeConstants
         public const string ProductGroup = "Microsoft.ApiManagement/service/products/groups";
         public const string ProductTag = "Microsoft.ApiManagement/service/products/tags";
         public const string ProductPolicy = "Microsoft.ApiManagement/service/products/policies";
-        public const string Property = "Microsoft.ApiManagement/service/properties";
+        public const string Property = "Microsoft.ApiManagement/service/namedValues";
         public const string Subscription = "Microsoft.ApiManagement/service/subscriptions";
         public const string Tag = "Microsoft.ApiManagement/service/tags";
     }

src/APIM_ARMTemplate/apimtemplate/Common/TemplateModels/DiagnosticTemplateResource.cs

@@ -11,6 +11,10 @@ public class DiagnosticTemplateProperties
     {
         public string alwaysLog { get; set; }
         public string loggerId { get; set; }
+        public string httpCorrelationProtocol { get; set; }
+        public string verbosity { get; set; }
+        public Nullable<bool> logClientIp { get; set; }
+
         public DiagnosticTemplateSampling sampling { get; set; }
         public DiagnosticTemplateFrontendBackend frontend { get; set; }
         public DiagnosticTemplateFrontendBackend backend { get; set; }

src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/APIExtractor.cs

@@ -214,6 +214,16 @@ public async Task<string> GetAPISchemaDetailsAsync(string ApiManagementName, str
             return await CallApiManagementAsync(azToken, requestUrl);
         }
 
+        public async Task<string> GetServiceDiagnosticsAsync(string ApiManagementName, string ResourceGroupName)
+        {
+            (string azToken, string azSubId) = await auth.GetAccessToken();
+
+            string requestUrl = string.Format("{0}/subscriptions/{1}/resourceGroups/{2}/providers/Microsoft.ApiManagement/service/{3}/diagnostics?api-version={4}",
+                baseUrl, azSubId, ResourceGroupName, ApiManagementName, GlobalConstants.APIVersion);
+
+            return await CallApiManagementAsync(azToken, requestUrl);
+        }
+
         public async Task<List<TemplateResource>> GenerateSingleAPIResourceAsync(string apiName, Extractor exc)
         {
             List<TemplateResource> templateResources = new List<TemplateResource>();
@@ -388,6 +398,12 @@ public async Task<Template> GenerateAPIsARMTemplateAsync(string singleApiName, L
                 }
             }
 
+
+            //Add the All API Diagnostics settings
+
+            templateResources.AddRange(await GetServiceDiagnosticsTemplateResourcesAsync(exc));
+
+
             armTemplate.resources = templateResources.ToArray();
             return armTemplate;
         }
@@ -766,5 +782,50 @@ private async Task<IEnumerable<TemplateResource>> GetRelatedTemplateResourcesAsy
 
             return templateResources;
         }
+
+        /// <summary>
+        /// Gets the "All API" level diagnostic resources, these are common to all APIs.
+        /// </summary>
+        /// <param name="exc">The extractor.</param>
+        /// <returns>a list of DiagnosticTemplateResources</returns>
+        private async Task<IEnumerable<TemplateResource>> GetServiceDiagnosticsTemplateResourcesAsync(Extractor exc)
+        {
+            List<TemplateResource> templateResources = new List<TemplateResource>();
+            string apimname = exc.sourceApimName, resourceGroup = exc.resourceGroup;
+
+            string serviceDiagnostics = await GetServiceDiagnosticsAsync(apimname, resourceGroup);
+            JObject oServiceDiagnostics = JObject.Parse(serviceDiagnostics);
+
+            foreach (var serviceDiagnostic in oServiceDiagnostics["value"])
+            {
+                string serviceDiagnosticName = ((JValue)serviceDiagnostic["name"]).Value.ToString();
+                Console.WriteLine("'{0}' Diagnostic found", serviceDiagnosticName);
+
+                // convert returned diagnostic to template resource class
+                DiagnosticTemplateResource serviceDiagnosticResource = serviceDiagnostic.ToObject<DiagnosticTemplateResource>();
+                serviceDiagnosticResource.name = $"[concat(parameters('{ParameterNames.ApimServiceName}'), '/{serviceDiagnosticName}')]";
+                serviceDiagnosticResource.type = ResourceTypeConstants.APIServiceDiagnostic;
+                serviceDiagnosticResource.apiVersion = GlobalConstants.APIVersion;
+                serviceDiagnosticResource.scale = null;
+                serviceDiagnosticResource.dependsOn = new string[] { };
+
+                if (exc.paramApiLoggerId)
+                {
+
+                    serviceDiagnosticResource.properties.loggerId = $"[parameters('{ParameterNames.ApiLoggerId}').{ExtractorUtils.GenValidParamName(serviceDiagnosticName, ParameterPrefix.Diagnostic)}]";
+                }
+
+                if (!serviceDiagnosticName.Contains("applicationinsights"))
+                {
+                    // enableHttpCorrelationHeaders only works for application insights, causes errors otherwise
+                    //TODO: Check this settings still valid?
+                    serviceDiagnosticResource.properties.enableHttpCorrelationHeaders = null;
+                }
+
+                templateResources.Add(serviceDiagnosticResource);
+            }
+
+            return templateResources.ToArray();
+        }
     }
 }

src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/EntityExtractor.cs

@@ -169,5 +169,12 @@ public Template GenerateEmptyApiTemplateWithParameters(Extractor exc)
             }
             return armTemplate;
         }
+
+        public Template GenerateEmptyProductApiTemplateWithParameters(Extractor exc)
+        {
+            Template armTemplate = GenerateEmptyTemplate();
+            armTemplate.parameters = new Dictionary<string, TemplateParameterProperties> { { ParameterNames.ApimServiceName, new TemplateParameterProperties() { type = "string" } } };
+            return armTemplate;
+        }
     }
 }

src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/MasterTemplateExtractor.cs

@@ -124,7 +124,7 @@ public Template GenerateLinkedMasterTemplate(Template apiTemplate,
             if (productAPIsTemplate != null && productAPIsTemplate.resources.Count() != 0)
             {
                 string productAPIsUri = GenerateLinkedTemplateUri(exc.linkedTemplatesUrlQueryString, exc.linkedTemplatesSasToken, fileNames.productAPIs);
-                resources.Add(this.CreateLinkedMasterTemplateResourceWithPolicyToken("productAPIsTemplate", productAPIsUri, productAPIDependsOn.ToArray(), exc));
+                resources.Add(this.CreateLinkedMasterTemplateResourceForProductAPITemplate("productAPIsTemplate", productAPIsUri, productAPIDependsOn.ToArray(), exc));
             }
 
             // apiTags
@@ -184,6 +184,15 @@ public MasterTemplateResource CreateLinkedMasterTemplateResourceWithPolicyToken(
             }
         }
 
+        public MasterTemplateResource CreateLinkedMasterTemplateResourceForProductAPITemplate(string name, string uriLink, string[] dependsOn, Extractor exc)
+        {
+            MasterTemplateResource masterResourceTemplate = this.CreateLinkedMasterTemplateResource(name, uriLink, dependsOn);
+            //Product API does not need the PolicyXMLBaseUrl parameter so remove it
+            //(this needs to be addressed at a higher level because this parameter isn't needed for some resources
+            masterResourceTemplate.properties.parameters.Remove(ParameterNames.PolicyXMLBaseUrl);
+            return masterResourceTemplate;
+        }
+
         public MasterTemplateResource CreateLinkedMasterTemplateResourceForLoggerTemplate(string name, string uriLink, string[] dependsOn, Extractor exc)
         {
             MasterTemplateResource masterResourceTemplate = this.CreateLinkedMasterTemplateResource(name, uriLink, dependsOn);

src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/ProductAPIExtractor.cs

@@ -63,7 +63,7 @@ public async Task<List<TemplateResource>> GenerateSingleProductAPIResourceAsync(
         public async Task<Template> GenerateAPIProductsARMTemplateAsync(string singleApiName, List<string> multipleApiNames, Extractor exc)
         {
             // initialize arm template
-            Template armTemplate = GenerateEmptyApiTemplateWithParameters(exc);
+            Template armTemplate = GenerateEmptyProductApiTemplateWithParameters(exc);
             List<TemplateResource> templateResources = new List<TemplateResource>();
             // when extract single API
             if (singleApiName != null)