Added AreaPath and IterationPath migration with tests

Common/AreaAndIterationPathTree.cs

@@ -12,6 +12,8 @@ public class AreaAndIterationPathTree
         static ILogger Logger { get; } = MigratorLogging.CreateLogger<AreaAndIterationPathTree>();
         public ISet<string> AreaPathList { get; } = new HashSet<string>();
         public ISet<string> IterationPathList { get; } = new HashSet<string>();
+        public IDictionary<string, WorkItemClassificationNode> IterationPathListLookup { get; } = new Dictionary<string, WorkItemClassificationNode>();
+        public IDictionary<string, WorkItemClassificationNode> AreaPathListLookup { get; } = new Dictionary<string, WorkItemClassificationNode>();
 
         public AreaAndIterationPathTree(IList<WorkItemClassificationNode> nodeList)
         {
@@ -74,7 +76,7 @@ private void CreateAreaPathList(WorkItemClassificationNode headnode)
                 return;
             }
             //path for the headnode is null
-            ProcessNode(null, headnode, this.AreaPathList);
+            ProcessNode(null, headnode, this.AreaPathList, this.AreaPathListLookup);
         }
 
         private void CreateIterationPathList(WorkItemClassificationNode headnode)
@@ -85,10 +87,10 @@ private void CreateIterationPathList(WorkItemClassificationNode headnode)
                 return;
             }
             //path for the headnode is null
-            ProcessNode(null, headnode, this.IterationPathList);
+            ProcessNode(null, headnode, this.IterationPathList, this.IterationPathListLookup);
         }
 
-        private void ProcessNode(string path, WorkItemClassificationNode node, ISet<string> pathList)
+        private void ProcessNode(string path, WorkItemClassificationNode node, ISet<string> pathList, IDictionary<string, WorkItemClassificationNode> pathListLookup)
         {
             if (node == null)
             {
@@ -106,12 +108,14 @@ private void ProcessNode(string path, WorkItemClassificationNode node, ISet<stri
             }
 
             pathList.Add(currentpath);
+            pathListLookup.Add(currentpath, node);
+
 
             if (node.Children != null)
             {
                 foreach (var child in node.Children)
                 {
-                    ProcessNode(currentpath, child, pathList);
+                    ProcessNode(currentpath, child, pathList, pathListLookup);
                 }
             }
         }

Common/Config/ConfigJson.cs

@@ -42,6 +42,14 @@ public class ConfigJson
         [DefaultValue(true)]
         public bool SkipExisting { get; set; }
 
+        [JsonProperty(PropertyName = "move-area-paths", DefaultValueHandling = DefaultValueHandling.Populate)]
+        [DefaultValue(true)]
+        public bool MoveAreaPaths { get; set; }
+
+        [JsonProperty(PropertyName = "move-iterations", DefaultValueHandling = DefaultValueHandling.Populate)]
+        [DefaultValue(true)]
+        public bool MoveIterations{ get; set; }
+
         [JsonProperty(PropertyName = "move-history", DefaultValueHandling = DefaultValueHandling.Populate)]
         [DefaultValue(false)]
         public bool MoveHistory { get; set; }

Common/Migration/Contexts/IMigrationContext.cs

@@ -36,5 +36,7 @@ public interface IMigrationContext : IContext
         IList<string> UnsupportedFields { get; }
 
         IList<string> FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; set; }
+        AreaAndIterationPathTree SourceAreaAndIterationTree { get; set; }
+        AreaAndIterationPathTree TargetAreaAndIterationTree { get; set; }
     }
 }

Common/Migration/Contexts/MigrationContext.cs

@@ -35,6 +35,8 @@ public class MigrationContext : BaseContext, IMigrationContext
         public IList<string> UnsupportedFields => unsupportedFields;
 
         public IList<string> FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; set; }
+        public AreaAndIterationPathTree SourceAreaAndIterationTree { get; set; }
+        public AreaAndIterationPathTree TargetAreaAndIterationTree { get; set; }
 
         // List of fields that we do not support in migration because they are related to the board or another reason.
         private readonly IList<string> unsupportedFields = new ReadOnlyCollection<string>(new[]{

Common/Migration/Migrator.cs

@@ -348,6 +348,36 @@ public static async Task ReadSourceWorkItems(IMigrationContext migrationContext,
             batchContext.SourceWorkItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, workItemIds, expand: expand);
         }
 
+        /// <summary>
+        /// Populates batchContext.WorkItems
+        /// </summary>
+        /// <param name="migrationContext"></param>
+        /// <param name="workItemIds"></param>
+        /// <param name="batchContext"></param>
+        /// <param name="expand"></param>
+        /// <returns></returns>
+        public static async Task ReadSourceNodes(IMigrationContext migrationContext, string projectId)
+        {
+            var nodes = await WorkItemTrackingHelpers.GetClassificationNodes(migrationContext.SourceClient.WorkItemTrackingHttpClient, projectId);
+            migrationContext.SourceAreaAndIterationTree = new AreaAndIterationPathTree(nodes);
+        }
+
+        /// <summary>
+        /// Populates batchContext.WorkItems
+        /// </summary>
+        /// <param name="migrationContext"></param>
+        /// <param name="workItemIds"></param>
+        /// <param name="batchContext"></param>
+        /// <param name="expand"></param>
+        /// <returns></returns>
+        public static async Task ReadTargetNodes(IMigrationContext migrationContext, string projectId)
+        {
+            var nodes = await WorkItemTrackingHelpers.GetClassificationNodes(migrationContext.TargetClient.WorkItemTrackingHttpClient, projectId);
+            migrationContext.TargetAreaAndIterationTree = new AreaAndIterationPathTree(nodes);
+            migrationContext.TargetAreaPaths = migrationContext.TargetAreaAndIterationTree.AreaPathList;
+            migrationContext.TargetIterationPaths = migrationContext.TargetAreaAndIterationTree.IterationPathList;
+        }
+
         public static int GetTargetId(int sourceId, IEnumerable<WorkItemMigrationState> workItemMigrationStates)
         {
             return workItemMigrationStates.First(a => a.SourceId == sourceId).TargetId.Value;

Common/Migration/Phase1/PreProcessors/ClassificationNodesPreProcessor.cs

@@ -0,0 +1,136 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models;
+using Common.Config;
+using Logging;
+using Microsoft.TeamFoundation.TestManagement.WebApi;
+
+namespace Common.Migration
+{
+
+    public class ClassificationNodesPreProcessor : IPhase1PreProcessor
+    {
+        static ILogger Logger { get; } = MigratorLogging.CreateLogger<ClassificationNodesPreProcessor>();
+        private IMigrationContext context;
+
+        public string Name => "Classification Nodes (Area Paths/Iterations)";
+
+        public bool IsEnabled(ConfigJson config)
+        {
+            return true;
+        }
+
+        public async Task Prepare(IMigrationContext context)
+        {
+            this.context = context;
+        }
+
+        public async Task Process(IBatchMigrationContext batchContext)
+        {
+            int modificationCount = 0;
+            if (context.Config.MoveAreaPaths || context.Config.MoveIterations)
+            {
+                await Task.WhenAll(
+                    Migrator.ReadSourceNodes(context, context.Config.SourceConnection.Project),
+                    Migrator.ReadTargetNodes(context, context.Config.TargetConnection.Project));
+            }
+
+            #region Process area paths ..
+            if (context.Config.MoveAreaPaths)
+            {
+                modificationCount = await ProcessAreaPaths(batchContext, modificationCount);
+            }
+            #endregion
+
+            #region Process iterations ..
+            if (context.Config.MoveIterations)
+            {
+                modificationCount = await ProcessIterationPaths(batchContext, modificationCount);
+            }
+            #endregion
+
+            if (modificationCount > 0)
+            {
+                await Migrator.ReadTargetNodes(context, context.Config.TargetConnection.Project);
+            }
+        }
+
+        public async Task<int> ProcessIterationPaths(IBatchMigrationContext batchContext, int modificationCount)
+        {
+            Logger.LogInformation(LogDestination.All, $"Identified {context.SourceAreaAndIterationTree.IterationPathList.Count} iterations in source project.");
+
+            foreach (var iterationPath in context.SourceAreaAndIterationTree.IterationPathList)
+            {
+                string iterationPathInTarget = iterationPath.Replace(context.Config.SourceConnection.Project, context.Config.TargetConnection.Project);
+
+                // If the iteration path is not found in the work items we're currently processing then just ignore it.
+                if (!batchContext.SourceWorkItems.Any(w => w.Fields.ContainsKey("System.IterationPath") && w.Fields["System.IterationPath"].ToString().ToLower().Equals(iterationPath.ToLower())))
+                    continue;
+
+                if (context.TargetAreaAndIterationTree.IterationPathListLookup.ContainsKey(iterationPathInTarget))
+                {
+                    Logger.LogInformation(LogDestination.All, $"[Exists] {iterationPathInTarget}.");
+                }
+                else
+                {
+                    var sourceIterationNode = context.SourceAreaAndIterationTree.IterationPathListLookup[iterationPath];
+                    modificationCount += 1;
+                    await CreateIterationPath(iterationPath, sourceIterationNode);
+
+                    Logger.LogSuccess(LogDestination.All, $"[Created] {iterationPathInTarget}.");
+                }
+            }
+
+            Logger.LogInformation(LogDestination.All, $"Iterations synchronized.");
+            return modificationCount;
+        }
+
+        public async virtual Task CreateIterationPath(string iterationPath, WorkItemClassificationNode sourceIterationNode)
+        {
+            await WorkItemTrackingHelpers.CreateIterationAsync(
+                context.TargetClient.WorkItemTrackingHttpClient,
+                context.Config.TargetConnection.Project,
+                iterationPath,
+                sourceIterationNode.Attributes == null ? null : (DateTime?)sourceIterationNode.Attributes?["startDate"],
+                sourceIterationNode.Attributes == null ? null : (DateTime?)sourceIterationNode.Attributes["finishDate"]);
+        }
+
+        public async Task<int> ProcessAreaPaths(IBatchMigrationContext batchContext, int modificationCount)
+        {
+            Logger.LogInformation(LogDestination.All, $"Identified {context.SourceAreaAndIterationTree.AreaPathListLookup.Count} area paths in source project.");
+
+            foreach (var ap in context.SourceAreaAndIterationTree.AreaPathList)
+            {
+                string areaPathInTarget = ap.Replace(context.Config.SourceConnection.Project, context.Config.TargetConnection.Project);
+
+                // If the area path is not found in the work items we're currently processing then just ignore it.
+                if (!batchContext.SourceWorkItems.Any(w => w.Fields.ContainsKey("System.AreaPath") && w.Fields["System.AreaPath"].ToString().ToLower().Equals(ap.ToLower())))
+                {
+                    continue;
+                }
+                else if (context.TargetAreaAndIterationTree.AreaPathList.Any(a => a.ToLower() == areaPathInTarget.ToLower()))
+                {
+                    Logger.LogInformation(LogDestination.All, $"[Exists] {areaPathInTarget}.");
+                }
+                else
+                {
+                    modificationCount += 1;
+                    await CreateAreaPath(areaPathInTarget); 
+                    Logger.LogSuccess(LogDestination.All, $"[Created] {areaPathInTarget}.");
+                }
+            }
+
+            Logger.LogInformation(LogDestination.All, $"Area paths synchronized.");
+            return modificationCount;
+        }
+
+        public async virtual Task CreateAreaPath(string areaPathInTarget)
+        {
+            await WorkItemTrackingHelpers.CreateAreaPathAsync(context.TargetClient.WorkItemTrackingHttpClient, context.Config.TargetConnection.Project, areaPathInTarget);
+        }
+    }
+}

Common/Migration/Phase1/Processors/BaseWorkItemsProcessor.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
@@ -25,6 +26,7 @@ public abstract class BaseWorkItemsProcessor : IPhase1Processor
 
         public abstract BaseWitBatchRequestGenerator GetWitBatchRequestGenerator(IMigrationContext context, IBatchMigrationContext batchContext);
 
+
         public async Task Process(IMigrationContext context)
         {
             var workItemsAndStateToMigrate = this.GetWorkItemsAndStateToMigrate(context);

Common/Validation/WorkItem/ValidateClassificationNodes.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Linq;
 using System.Threading.Tasks;
 using Logging;
 using Microsoft.Extensions.Logging;

Common/WorkItemTrackingHelpers.cs

@@ -182,6 +182,42 @@ public async static Task<AttachmentReference> CreateAttachmentChunkedAsync(WorkI
             return attachmentReference;
         }
 
+        public async static Task<WorkItemClassificationNode> CreateAreaPathAsync(WorkItemTrackingHttpClient client, string projectId, string areaPath)
+        {
+            return await RetryHelper.RetryAsync(async () =>
+            {
+                return await client.CreateOrUpdateClassificationNodeAsync(new WorkItemClassificationNode()
+                {
+                    Name = areaPath.Contains("\\") ? areaPath.Split("\\").Last() : areaPath,
+                    StructureType = TreeNodeStructureType.Area,
+                }, projectId, TreeStructureGroup.Areas, areaPath.Replace(projectId, "").Replace(areaPath.Contains("\\") ? areaPath.Split("\\").Last() : areaPath, ""));
+            }, 5);
+        }
+
+        public async static Task<WorkItemClassificationNode> CreateIterationAsync(WorkItemTrackingHttpClient client, string projectId, string iteration, DateTime? startDate, DateTime? endDate)
+        {
+            return await RetryHelper.RetryAsync(async () =>
+            {
+                var attrs = new Dictionary<string, object>();
+                if (startDate.HasValue)
+                {
+                    attrs.Add("startDate", startDate);
+                }
+                if (endDate.HasValue)
+                {
+                    attrs.Add("finishDate", endDate);
+                }
+
+                return await client.CreateOrUpdateClassificationNodeAsync(new WorkItemClassificationNode()
+                {
+                    Name = iteration.Contains("\\") ? iteration.Split("\\").Last() : iteration,
+                    StructureType = TreeNodeStructureType.Iteration,
+                    Attributes = attrs
+
+                }, projectId, TreeStructureGroup.Iterations);
+            }, 5);
+        }
+
         public async static Task<Stream> GetAttachmentAsync(WorkItemTrackingHttpClient client, Guid id)
         {
             return await RetryHelper.RetryAsync(async () =>

Logging/BulkLogger.cs

@@ -22,7 +22,7 @@ public BulkLogger()
             this.bulkLoggerCheckTimer = new Timer(BulkLoggerCheck, "Some state", TimeSpan.FromSeconds(LoggingConstants.CheckInterval), TimeSpan.FromSeconds(LoggingConstants.CheckInterval));
             this.stopwatch = Stopwatch.StartNew();
             this.filePath = GetFilePathBasedOnTime();
-            Console.WriteLine($"Detailed logging sent to file: {Directory.GetCurrentDirectory()}\\{filePath}");
+            Console.WriteLine($"[Info] [{DateTime.Now.ToString("HH:mm:ss.fff")}] Detailed logging sent to file: {Directory.GetCurrentDirectory()}\\{filePath}");
         }
 
         public void WriteToQueue(LogItem logItem)

Logging/LogItem.cs

@@ -1,5 +1,6 @@
 using Microsoft.Extensions.Logging;
 using System;
+using System.Linq;
 
 namespace Logging
 {
@@ -44,7 +45,7 @@ public string OutputFormat(bool includeExceptionMessage, bool includeLogLevelTim
                 // HH specifies 24-hour time format
                 string timeStamp = DateTimeStampString();
                 string logLevelName = LogLevelName();
-                return $"[{logLevelName}   @{timeStamp}] {this.Message}";
+                return string.Join(Environment.NewLine, Message.Split(Environment.NewLine).Select(line => $"[{logLevelName}] [{timeStamp}] {line}"));
             }
             else
             {
@@ -54,7 +55,7 @@ public string OutputFormat(bool includeExceptionMessage, bool includeLogLevelTim
 
         public virtual string DateTimeStampString()
         {
-            return this.DateTimeStamp.ToString("HH.mm.ss.fff");
+            return this.DateTimeStamp.ToString("HH:mm:ss.fff");
         }
 
         public virtual string LogLevelName()

README.md

@@ -13,6 +13,7 @@ WiMigrator is a command line tool designed with the following goals in mind:
   * Git commit links (link to the source git commit)
   * Work item history (last 200 revisions as an attachment)
   * Tagging of the source items that have been migrated
+  * Area paths and iterations
 
 # Getting Started
 ## Requirements