Merge pull request #239 from Azure/lucashh-privatestorage

Lucashh privatestorage

Author: lucashuet93

src/APIM_ARMTemplate/README.md

@@ -22,6 +22,7 @@ The utility requires one argument, --configFile, which points to a yaml file tha
 |-----------------------|-----------------------|-----------------------|--------------------------------------------------|
 | version               | string                | Yes                   | Configuration version.                            |
 | apimServiceName       | string                | Yes                   | Name of the APIM service to deploy resources into.    |
+| policy                | string                | No                    | Location of the global service policy XML file. Can be url or local file.           |
 | apiVersionSets         | Array<[APIVersionSetConfiguration](#APIVersionSetConfiguration)> | No               | List of API Version Set configurations.                        |
 | apis                   | Array<[APIConfiguration](#APIConfiguration)>      | Yes                   | List of API configurations.                                |
 | products                   | Array<[ProductConfiguration](#ProductConfiguration)>      | No                   | List of Product configurations.                                |
@@ -31,6 +32,7 @@ The utility requires one argument, --configFile, which points to a yaml file tha
 | outputLocation        | string                | Yes                   | Local folder the utility will write templates to. |
 | linked                | boolean               | No                    | Determines whether the utility should create a master template that links to all generated templates. |
 | linkedTemplatesBaseUrl| string                | No                    | Location that stores linked templates. Required if 'linked' is set to true. |
+| linkedTemplatesUrlQueryString| string                | No                    | Query string appended to linked templates uris that enables retrieval from private storage. |
 
 #### APIConfiguration
 
@@ -101,6 +103,7 @@ The following is a full config.yml file with each property listed:
 ```
 version: 0.0.1
 apimServiceName: myAPIMService
+policy: C:\Users\myUsername\Projects\azure-api-management-devops-example\src\APIM_ARMTemplate\apimtemplate\Creator\ExampleFiles\XMLPolicies\globalServicePolicy.xml
 apiVersionSets:
     - id: myAPIVersionSetID
       displayName: swaggerPetstoreVersionSetLinked
@@ -227,6 +230,7 @@ backends:
 outputLocation: C:\Users\myUsername\GeneratedTemplates
 linked: false
 linkedTemplatesBaseUrl : https://mystorageaccount.blob.core.windows.net/mycontainer
+linkedTemplatesUrlQueryString : ?sv=2018-03-28&ss=bfqt&srt=sco&sp=rwdlacup&se=2019-12-22T23:12:53Z&st=2019-09-09T14:12:53Z&spr=https&sig=uFTldJEYPH888QVzKb7q7eLq0Xt%2Bu35UTqpFGUYo6uc%3D
 ```
 
 <a name="creator2"></a>
@@ -273,6 +277,7 @@ az account set --subscription <subscription_id>
 | fileFolder            | Yes                   | Path to output folder                             |
 | apiName               | No                    | Name of API. If provided, Extractor executes single API extraction. Otherwise, Extractor executes full extraction.  Note:  This is the "Name" value as seen in the API settings, not "Display Name" and is case sensitive.     |
 | linkedTemplatesBaseUrl| No                    | Linked templates remote location. If provided, Extractor generates master template and requires linked templates pushed to remote location.                                   |
+| linkedTemplatesUrlQueryString | string | No   | Query string appended to linked templates uris that enables retrieval from private storage. |
 | policyXMLBaseUrl      | No                    | Policy XML files remote location. If provided, Extractor generates policies folder with xml files, and requires they be pushed to remote location.                              |
 
 To run the Extractor with all arguments (executing a single API extraction with linked templates and policy file generation), use the following command: 

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorTests/MasterTemplateCreatorTests.cs

@@ -11,20 +11,21 @@ public class MasterTemplateCreatorTests
         public void ShouldCreateCorrectNumberOfDeploymentResources()
         {
             // arrange
-            CreatorConfig creatorConfig = new CreatorConfig() { apimServiceName = "apimService" };
+            CreatorConfig creatorConfig = new CreatorConfig() { apimServiceName = "apimService", linked = true };
             MasterTemplateCreator masterTemplateCreator = new MasterTemplateCreator();
             Template apiVersionSetsTemplate = new Template();
+            Template globalServicePolicyTemplate = new Template();
             Template productsTemplate = new Template();
             Template loggersTemplate = new Template();
             List<LinkedMasterTemplateAPIInformation> apiInfoList = new List<LinkedMasterTemplateAPIInformation>() { new LinkedMasterTemplateAPIInformation() { name = "api", isSplit = true } };
             FileNameGenerator fileNameGenerator = new FileNameGenerator();
             FileNames creatorFileNames = fileNameGenerator.GenerateFileNames(creatorConfig.apimServiceName);
 
-            // should create 5 resources (apiVersionSet, product, logger, both api templates)
-            int count = 5;
+            // should create 6 resources (globalServicePolicy, apiVersionSet, product, logger, both api templates)
+            int count = 6;
 
             // act
-            Template masterTemplate = masterTemplateCreator.CreateLinkedMasterTemplate(apiVersionSetsTemplate, productsTemplate, loggersTemplate, null, null, apiInfoList, creatorFileNames, creatorConfig.apimServiceName, fileNameGenerator);
+            Template masterTemplate = masterTemplateCreator.CreateLinkedMasterTemplate(creatorConfig, globalServicePolicyTemplate, apiVersionSetsTemplate, productsTemplate, loggersTemplate, null, null, apiInfoList, creatorFileNames, creatorConfig.apimServiceName, fileNameGenerator);
 
             // assert
             Assert.Equal(count, masterTemplate.resources.Length);
@@ -55,13 +56,13 @@ public void ShouldCreateCorrectNumberOfParameterValuesWhenLinked()
         public void ShouldCreateCorrectNumberOfParametersWhenUnlinked()
         {
             // arrange
+            CreatorConfig creatorConfig = new CreatorConfig() { apimServiceName = "apimService", linked = false };
             MasterTemplateCreator masterTemplateCreator = new MasterTemplateCreator();
-            bool linked = false;
             // unlinked templates result in 1 value
             int count = 1;
 
             // act
-            Dictionary<string, TemplateParameterProperties> masterTemplateParameters = masterTemplateCreator.CreateMasterTemplateParameters(linked);
+            Dictionary<string, TemplateParameterProperties> masterTemplateParameters = masterTemplateCreator.CreateMasterTemplateParameters(creatorConfig);
 
             // assert
             Assert.Equal(count, masterTemplateParameters.Keys.Count);
@@ -84,5 +85,20 @@ public void ShouldCreateLinkedMasterTemplateResourceFromValues()
             Assert.Equal(uriLink, masterTemplateResource.properties.templateLink.uri);
             Assert.Equal(dependsOn, masterTemplateResource.dependsOn);
         }
+
+        [Fact]
+        public void ShouldCreateCorrectLinkedUri()
+        {
+            // arrange
+            MasterTemplateCreator masterTemplateCreator = new MasterTemplateCreator();
+            CreatorConfig creatorConfig = new CreatorConfig() { apimServiceName = "apimService", linked = true, linkedTemplatesBaseUrl = "http://someurl.com", linkedTemplatesUrlQueryString = "?param=1" };
+            string apiVersionSetFileName = "/versionSet1-apiVersionSets.template.json";
+
+            // act
+            string linkedResourceUri = masterTemplateCreator.GenerateLinkedTemplateUri(creatorConfig, apiVersionSetFileName);
+
+            // assert
+            Assert.Equal($"[concat(parameters('LinkedTemplatesBaseUrl'), '{apiVersionSetFileName}', parameters('LinkedTemplatesUrlQueryString'))]", linkedResourceUri);
+        }
     }
 }

src/APIM_ARMTemplate/apimtemplate.test/Creator/TemplateCreatorTests/PolicyTemplateCreatorTests.cs

@@ -7,6 +7,23 @@ namespace Microsoft.Azure.Management.ApiManagement.ArmTemplates.Test
 {
     public class PolicyTemplateCreatorTests
     {
+        [Fact]
+        public void ShouldCreateGlobalServicePolicyTemplateResourceFromCreatorConfigWithCorrectContent()
+        {
+            // arrange
+            PolicyTemplateCreator policyTemplateCreator = PolicyTemplateCreatorFactory.GeneratePolicyTemplateCreator();
+            CreatorConfig creatorConfig = new CreatorConfig() { policy = "http://someurl.com" };
+
+            // act
+            Template policyTemplate = policyTemplateCreator.CreateGlobalServicePolicyTemplate(creatorConfig);
+            PolicyTemplateResource policyTemplateResource = policyTemplate.resources[0] as PolicyTemplateResource;
+
+            // assert
+            Assert.Equal($"[concat(parameters('ApimServiceName'), '/policy')]", policyTemplateResource.name);
+            Assert.Equal("rawxml-link", policyTemplateResource.properties.format);
+            Assert.Equal(creatorConfig.policy, policyTemplateResource.properties.value);
+        }
+
         [Fact]
         public void ShouldCreateAPIPolicyTemplateResourceFromCreatorConfigWithCorrectContent()
         {

src/APIM_ARMTemplate/apimtemplate/Commands/Create.cs

@@ -50,6 +50,9 @@ public CreateCommand()
                     MasterTemplateCreator masterTemplateCreator = new MasterTemplateCreator();
 
                     // create templates from provided configuration
+                    Console.WriteLine("Creating global service policy template");
+                    Console.WriteLine("------------------------------------------");
+                    Template globalServicePolicyTemplate = creatorConfig.policy != null ? policyTemplateCreator.CreateGlobalServicePolicyTemplate(creatorConfig) : null;
                     Console.WriteLine("Creating API version set template");
                     Console.WriteLine("------------------------------------------");
                     Template apiVersionSetsTemplate = creatorConfig.apiVersionSets != null ? apiVersionSetTemplateCreator.CreateAPIVersionSetTemplate(creatorConfig) : null;
@@ -81,6 +84,7 @@ public CreateCommand()
                         {
                             name = api.name,
                             isSplit = apiTemplateCreator.isSplitAPI(api),
+                            dependsOnGlobalServicePolicies = creatorConfig.policy != null,
                             dependsOnVersionSets = api.apiVersionSetId != null,
                             dependsOnProducts = api.products != null,
                             dependsOnLoggers = await masterTemplateCreator.DetermineIfAPIDependsOnLoggerAsync(api, fileReader),
@@ -96,17 +100,21 @@ public CreateCommand()
                     if (creatorConfig.linked == true)
                     {
                         // create linked master template
-                        Template masterTemplate = masterTemplateCreator.CreateLinkedMasterTemplate(apiVersionSetsTemplate, productsTemplate, loggersTemplate, backendsTemplate, authorizationServersTemplate, apiInformation, fileNames, creatorConfig.apimServiceName, fileNameGenerator);
+                        Template masterTemplate = masterTemplateCreator.CreateLinkedMasterTemplate(creatorConfig, globalServicePolicyTemplate, apiVersionSetsTemplate, productsTemplate, loggersTemplate, backendsTemplate, authorizationServersTemplate, apiInformation, fileNames, creatorConfig.apimServiceName, fileNameGenerator);
                         fileWriter.WriteJSONToFile(masterTemplate, String.Concat(creatorConfig.outputLocation, fileNames.linkedMaster));
                     }
                     foreach (Template apiTemplate in apiTemplates)
                     {
                         APITemplateResource apiResource = apiTemplate.resources.FirstOrDefault(resource => resource.type == ResourceTypeConstants.API) as APITemplateResource;
-                        APIConfig providedAPIConfiguration = creatorConfig.apis.FirstOrDefault(api => apiResource.properties.displayName.Equals(api.name, StringComparison.Ordinal));
+                        APIConfig providedAPIConfiguration = creatorConfig.apis.FirstOrDefault(api => apiResource.name.Contains(api.name, StringComparison.Ordinal));
                         // if the api version is not null the api is split into multiple templates. If the template is split and the content value has been set, then the template is for a subsequent api
                         string apiFileName = fileNameGenerator.GenerateCreatorAPIFileName(providedAPIConfiguration.name, apiTemplateCreator.isSplitAPI(providedAPIConfiguration), apiResource.properties.value == null, creatorConfig.apimServiceName);
                         fileWriter.WriteJSONToFile(apiTemplate, String.Concat(creatorConfig.outputLocation, apiFileName));
                     }
+                    if (globalServicePolicyTemplate != null)
+                    {
+                        fileWriter.WriteJSONToFile(globalServicePolicyTemplate, String.Concat(creatorConfig.outputLocation, fileNames.globalServicePolicy));
+                    }
                     if (apiVersionSetsTemplate != null)
                     {
                         fileWriter.WriteJSONToFile(apiVersionSetsTemplate, String.Concat(creatorConfig.outputLocation, fileNames.apiVersionSets));

src/APIM_ARMTemplate/apimtemplate/Commands/Extract.cs

@@ -19,6 +19,7 @@ public ExtractCommand()
             var fileFolderName = this.Option("--fileFolder <filefolder>", "ARM Template files folder", CommandOptionType.SingleValue);
             var apiName = this.Option("--apiName <apiName>", "API name", CommandOptionType.SingleValue);
             var linkedTemplatesBaseUrlName = this.Option("--linkedTemplatesBaseUrl <linkedTemplatesBaseUrl>", "Creates a master template with links", CommandOptionType.SingleValue);
+            var linkedTemplatesUrlQueryString = this.Option("--linkedTemplatesUrlQueryString <linkedTemplatesUrlQueryString>", "Query string appended to linked templates uris that enables retrieval from private storage", CommandOptionType.SingleValue);
             var policyXMLBaseUrlName = this.Option("--policyXMLBaseUrl <policyXMLBaseUrl>", "Writes policies to local XML files that require deployment to remote folder", CommandOptionType.SingleValue);
 
             this.HelpOption();
@@ -33,17 +34,18 @@ public ExtractCommand()
                     if (!fileFolderName.HasValue()) throw new Exception("Missing parameter <filefolder>.");
 
                     // isolate cli parameters
-                    string resourceGroup = resourceGroupName.Values[0].ToString();
-                    string sourceApim = sourceApimName.Values[0].ToString();
-                    string destinationApim = destinationAPIManagementName.Values[0].ToString();
-                    string fileFolder = fileFolderName.Values[0].ToString();
+                    string resourceGroup = resourceGroupName.Value().ToString();
+                    string sourceApim = sourceApimName.Value().ToString();
+                    string destinationApim = destinationAPIManagementName.Value().ToString();
+                    string fileFolder = fileFolderName.Value().ToString();
                     string linkedBaseUrl = linkedTemplatesBaseUrlName.HasValue() ? linkedTemplatesBaseUrlName.Value().ToString() : null;
+                    string linkedUrlQueryString = linkedTemplatesUrlQueryString.HasValue() ? linkedTemplatesUrlQueryString.Value().ToString() : null;
                     string policyXMLBaseUrl = policyXMLBaseUrlName.HasValue() ? policyXMLBaseUrlName.Value().ToString() : null;
                     string singleApiName = null;
 
                     if (apiName.Values.Count > 0)
                     {
-                        singleApiName = apiName.Values[0].ToString();
+                        singleApiName = apiName.Value().ToString();
                     }
 
                     Console.WriteLine("API Management Template");
@@ -69,11 +71,13 @@ public ExtractCommand()
                     AuthorizationServerExtractor authorizationServerExtractor = new AuthorizationServerExtractor();
                     BackendExtractor backendExtractor = new BackendExtractor();
                     LoggerExtractor loggerExtractor = new LoggerExtractor();
+                    PolicyExtractor policyExtractor = new PolicyExtractor(fileWriter);
                     PropertyExtractor propertyExtractor = new PropertyExtractor();
                     ProductExtractor productExtractor = new ProductExtractor(fileWriter);
                     MasterTemplateExtractor masterTemplateExtractor = new MasterTemplateExtractor();
 
                     // extract templates from apim service
+                    Template globalServicePolicyTemplate = await policyExtractor.GenerateGlobalServicePolicyTemplateAsync(sourceApim, resourceGroup, policyXMLBaseUrl, fileFolder);
                     Template apiTemplate = await apiExtractor.GenerateAPIsARMTemplateAsync(sourceApim, resourceGroup, singleApiName, policyXMLBaseUrl, fileFolder);
                     List<TemplateResource> apiTemplateResources = apiTemplate.resources.ToList();
                     Template apiVersionSetTemplate = await apiVersionSetExtractor.GenerateAPIVersionSetsARMTemplateAsync(sourceApim, resourceGroup, singleApiName, apiTemplateResources, policyXMLBaseUrl);
@@ -85,7 +89,7 @@ public ExtractCommand()
                     Template backendTemplate = await backendExtractor.GenerateBackendsARMTemplateAsync(sourceApim, resourceGroup, singleApiName, apiTemplateResources, namedValueResources, policyXMLBaseUrl);
 
                     // create parameters file
-                    Template templateParameters = masterTemplateExtractor.CreateMasterTemplateParameterValues(destinationApim, linkedBaseUrl, policyXMLBaseUrl);
+                    Template templateParameters = masterTemplateExtractor.CreateMasterTemplateParameterValues(destinationApim, linkedBaseUrl, linkedUrlQueryString, policyXMLBaseUrl);
 
                     // write templates to output file location
                     string apiFileName = fileNameGenerator.GenerateExtractorAPIFileName(singleApiName, sourceApim);
@@ -96,11 +100,12 @@ public ExtractCommand()
                     fileWriter.WriteJSONToFile(loggerTemplate, String.Concat(@fileFolder, fileNames.loggers));
                     fileWriter.WriteJSONToFile(namedValueTemplate, String.Concat(@fileFolder, fileNames.namedValues));
                     fileWriter.WriteJSONToFile(productTemplate, String.Concat(@fileFolder, fileNames.products));
+                    fileWriter.WriteJSONToFile(globalServicePolicyTemplate, String.Concat(@fileFolder, fileNames.globalServicePolicy));
 
                     if (linkedBaseUrl != null)
                     {
                         // create a master template that links to all other templates
-                        Template masterTemplate = masterTemplateExtractor.GenerateLinkedMasterTemplate(apiTemplate, apiVersionSetTemplate, productTemplate, loggerTemplate, backendTemplate, authorizationServerTemplate, namedValueTemplate, fileNames, apiFileName, policyXMLBaseUrl);
+                        Template masterTemplate = masterTemplateExtractor.GenerateLinkedMasterTemplate(apiTemplate, globalServicePolicyTemplate, apiVersionSetTemplate, productTemplate, loggerTemplate, backendTemplate, authorizationServerTemplate, namedValueTemplate, fileNames, apiFileName, linkedUrlQueryString, policyXMLBaseUrl);
                         fileWriter.WriteJSONToFile(masterTemplate, String.Concat(@fileFolder, fileNames.linkedMaster));
                     }
 

src/APIM_ARMTemplate/apimtemplate/Common/Constants/ResourceTypeConstants.cs

@@ -13,6 +13,7 @@ internal static class ResourceTypeConstants
         public const string APISchema = "Microsoft.ApiManagement/service/apis/schemas";
         public const string AuthorizationServer = "Microsoft.ApiManagement/service/authorizationServers";
         public const string Backend = "Microsoft.ApiManagement/service/backends";
+        public const string GlobalServicePolicy = "Microsoft.ApiManagement/service/policies";
         public const string Logger = "Microsoft.ApiManagement/service/loggers";
         public const string ProductAPI = "Microsoft.ApiManagement/service/products/apis";
         public const string Product = "Microsoft.ApiManagement/service/products";