Addition of batch processing operations (#483)

* Added flag to configuration for breaking operations into chained groups

* rebaselined with master

* fixed nesting in new method

* removed unused chunked method

* tidied spacing

Co-authored-by: Kit Amos <kamos@goodstart.org.au>
Co-authored-by: RupengLiu <rliu1211@terpmail.umd.edu>

Author: 3 people

src/APIM_ARMTemplate/apimtemplate/Extractor/EntityExtractors/APIExtractor.cs

@@ -500,7 +500,7 @@ private async Task<IEnumerable<TemplateResource>> GetRelatedTemplateResourcesAsy
             List<TemplateResource> templateResources = new List<TemplateResource>();
             string apimname = exc.sourceApimName, resourceGroup = exc.resourceGroup, fileFolder = exc.fileFolder, policyXMLBaseUrl = exc.policyXMLBaseUrl, policyXMLSasToken = exc.policyXMLSasToken;
 
-            #region Schemas            
+            #region Schemas
             // add schema resources to api template
             List<TemplateResource> schemaResources = await GenerateSchemasARMTemplate(apimname, apiName, resourceGroup, fileFolder);
             templateResources.AddRange(schemaResources);
@@ -509,9 +509,30 @@ private async Task<IEnumerable<TemplateResource>> GetRelatedTemplateResourcesAsy
             #region Operations
             // pull api operations for service
             string[] operationNames = await GetAllOperationNames(apimname, resourceGroup, apiName);
+            int numBatches = 0;
+
+            // create empty array for the batch operation owners
+            List<string> batchOwners = new List<string>();   
+
+            // if a batch size is specified            
+            if(exc.operationBatchSize > 0) {
+                // store the number of batches required based on exc.operationBatchSize
+                numBatches = (int)Math.Ceiling((double)operationNames.Length/ (double)exc.operationBatchSize);
+                //Console.WriteLine ("Number of batches: {0}", numBatches);
+            }
+            
 
             foreach (string operationName in operationNames)
             {
+                int opIndex = Array.IndexOf(operationNames, operationName);
+
+                //add batch owners into array
+                // ensure each owner is linked to the one before
+                if(exc.operationBatchSize > 0 && opIndex < numBatches) {
+                    batchOwners.Add(operationName);
+                    //Console.WriteLine("Adding operation {0} to owner list", operationName);
+                }
+
                 string operationDetails = await GetAPIOperationDetailsAsync(apimname, resourceGroup, apiName, operationName);
 
                 Console.WriteLine("'{0}' Operation found", operationName);
@@ -540,6 +561,24 @@ private async Task<IEnumerable<TemplateResource>> GetRelatedTemplateResourcesAsy
                     }
                 }
 
+                // add to batch if flagged
+                string batchdependsOn;
+
+                if(exc.operationBatchSize > 0 && opIndex > 0) {
+                    if(opIndex >= 1 && opIndex <= numBatches - 1) {
+                        // chain the owners to each other
+                        batchdependsOn = $"[resourceId('Microsoft.ApiManagement/service/apis/operations', parameters('{ParameterNames.ApimServiceName}'), '{apiName}', '{batchOwners[opIndex - 1]}')]";
+                        //Console.WriteLine("Owner chaining: this request {0} to previous {1}", operationName, batchOwners[opIndex-1]);
+                    } else {
+                        // chain the operation to respective owner
+                        int ownerIndex = (int)Math.Floor((opIndex - numBatches)/((double)exc.operationBatchSize-1));
+                        batchdependsOn = $"[resourceId('Microsoft.ApiManagement/service/apis/operations', parameters('{ParameterNames.ApimServiceName}'), '{apiName}', '{batchOwners[ownerIndex]}')]";
+                        //Console.WriteLine("Operation {0} chained to owner {1}", operationName, batchOwners[ownerIndex]);
+                    }
+                    
+                    operationDependsOn.Add(batchdependsOn);
+                }
+
                 operationResource.dependsOn = operationDependsOn.ToArray();
                 templateResources.Add(operationResource);
 
@@ -638,8 +677,8 @@ private async Task<IEnumerable<TemplateResource>> GetRelatedTemplateResourcesAsy
             catch (Exception) { }
             #endregion
 
-            #region API Tags
-            // add tags associated with the api to template
+	    #region API Tags				
+            // add tags associated with the api to template 
             try
             {
                 // pull tags associated with the api
@@ -661,10 +700,11 @@ private async Task<IEnumerable<TemplateResource>> GetRelatedTemplateResourcesAsy
                 }
             }
             catch (Exception) { }
-            #endregion
+			#endregion
 
-            #region API Products
+								
             // add product api associations to template
+            #region API Products
             try
             {
                 // pull product api associations
@@ -708,8 +748,8 @@ private async Task<IEnumerable<TemplateResource>> GetRelatedTemplateResourcesAsy
                 diagnosticResource.scale = null;
                 diagnosticResource.dependsOn = new string[] { $"[resourceId('Microsoft.ApiManagement/service/apis', parameters('{ParameterNames.ApimServiceName}'), '{apiName}')]" };
 
-                if (exc.paramApiLoggerId)
-                {
+                if (exc.paramApiLoggerId) {
+				 
                     diagnosticResource.properties.loggerId = $"[parameters('{ParameterNames.ApiLoggerId}').{ExtractorUtils.GenValidParamName(apiName, ParameterPrefix.Api)}.{ExtractorUtils.GenValidParamName(diagnosticName, ParameterPrefix.Diagnostic)}]";
                 }
 
@@ -720,6 +760,7 @@ private async Task<IEnumerable<TemplateResource>> GetRelatedTemplateResourcesAsy
                 }
 
                 templateResources.Add(diagnosticResource);
+
             }
             #endregion
 

src/APIM_ARMTemplate/apimtemplate/Extractor/Models/ExtractorConfiguration.cs

@@ -49,6 +49,8 @@ public class ExtractorConfig
         [Description("Should not include named values template")]
         public string notIncludeNamedValue { get; set; }
 
+        [Description("Group the operations into batches of x?")]
+        public int operationBatchSize {get;set;}
         public void Validate()
         {
             if (string.IsNullOrEmpty(sourceApimName)) throw new ArgumentException("Missing parameter <sourceApimName>.");
@@ -109,6 +111,8 @@ public class Extractor
         public bool paramLogResourceId { get; private set; }
         public bool notIncludeNamedValue { get; private set; }
 
+        public int operationBatchSize { get; private set;} 
+
         public Extractor(ExtractorConfig exc, string dirName)
         {
             this.sourceApimName = exc.sourceApimName;
@@ -128,6 +132,7 @@ public Extractor(ExtractorConfig exc, string dirName)
             this.paramApiLoggerId = exc.paramApiLoggerId != null && exc.paramApiLoggerId.Equals("true");
             this.paramLogResourceId = exc.paramLogResourceId != null && exc.paramLogResourceId.Equals("true");
             this.notIncludeNamedValue = exc.notIncludeNamedValue != null && exc.notIncludeNamedValue.Equals("true");
+            this.operationBatchSize  = exc.operationBatchSize;
         }
 
         public Extractor(ExtractorConfig exc) : this(exc, exc.fileFolder)