diff --git a/Common/AreaAndIterationPathTree.cs b/Common/AreaAndIterationPathTree.cs index 0f46f1d..b6260c7 100644 --- a/Common/AreaAndIterationPathTree.cs +++ b/Common/AreaAndIterationPathTree.cs @@ -12,6 +12,8 @@ public class AreaAndIterationPathTree static ILogger Logger { get; } = MigratorLogging.CreateLogger(); public ISet AreaPathList { get; } = new HashSet(); public ISet IterationPathList { get; } = new HashSet(); + public IDictionary IterationPathListLookup { get; } = new Dictionary(); + public IDictionary AreaPathListLookup { get; } = new Dictionary(); public AreaAndIterationPathTree(IList nodeList) { @@ -74,7 +76,7 @@ private void CreateAreaPathList(WorkItemClassificationNode headnode) return; } //path for the headnode is null - ProcessNode(null, headnode, this.AreaPathList); + ProcessNode(null, headnode, this.AreaPathList, this.AreaPathListLookup); } private void CreateIterationPathList(WorkItemClassificationNode headnode) @@ -85,10 +87,10 @@ private void CreateIterationPathList(WorkItemClassificationNode headnode) return; } //path for the headnode is null - ProcessNode(null, headnode, this.IterationPathList); + ProcessNode(null, headnode, this.IterationPathList, this.IterationPathListLookup); } - private void ProcessNode(string path, WorkItemClassificationNode node, ISet pathList) + private void ProcessNode(string path, WorkItemClassificationNode node, ISet pathList, IDictionary pathListLookup) { if (node == null) { @@ -106,12 +108,14 @@ private void ProcessNode(string path, WorkItemClassificationNode node, ISet UnsupportedFields { get; } IList FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; set; } + AreaAndIterationPathTree SourceAreaAndIterationTree { get; set; } + AreaAndIterationPathTree TargetAreaAndIterationTree { get; set; } } } diff --git a/Common/Migration/Contexts/MigrationContext.cs b/Common/Migration/Contexts/MigrationContext.cs index 3daa915..244f5c3 100644 --- a/Common/Migration/Contexts/MigrationContext.cs +++ b/Common/Migration/Contexts/MigrationContext.cs @@ -35,6 +35,8 @@ public class MigrationContext : BaseContext, IMigrationContext public IList UnsupportedFields => unsupportedFields; public IList FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; set; } + public AreaAndIterationPathTree SourceAreaAndIterationTree { get; set; } + public AreaAndIterationPathTree TargetAreaAndIterationTree { get; set; } // List of fields that we do not support in migration because they are related to the board or another reason. private readonly IList unsupportedFields = new ReadOnlyCollection(new[]{ diff --git a/Common/Migration/Migrator.cs b/Common/Migration/Migrator.cs index 41a1b44..76b67eb 100644 --- a/Common/Migration/Migrator.cs +++ b/Common/Migration/Migrator.cs @@ -348,6 +348,32 @@ public static async Task ReadSourceWorkItems(IMigrationContext migrationContext, batchContext.SourceWorkItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, workItemIds, expand: expand); } + /// + /// Populates migrationContext.SourceAreaAndIterationTree + /// + /// + /// + /// + public static async Task ReadSourceNodes(IMigrationContext migrationContext, string projectId) + { + var nodes = await WorkItemTrackingHelpers.GetClassificationNodes(migrationContext.SourceClient.WorkItemTrackingHttpClient, projectId); + migrationContext.SourceAreaAndIterationTree = new AreaAndIterationPathTree(nodes); + } + + /// + /// Populates migrationContext.TargetAreaAndIterationTree, migrationContext.TargetAreaPaths, and migrationContext.TargetIterationPaths + /// + /// + /// + /// + public static async Task ReadTargetNodes(IMigrationContext migrationContext, string projectId) + { + var nodes = await WorkItemTrackingHelpers.GetClassificationNodes(migrationContext.TargetClient.WorkItemTrackingHttpClient, projectId); + migrationContext.TargetAreaAndIterationTree = new AreaAndIterationPathTree(nodes); + migrationContext.TargetAreaPaths = migrationContext.TargetAreaAndIterationTree.AreaPathList; + migrationContext.TargetIterationPaths = migrationContext.TargetAreaAndIterationTree.IterationPathList; + } + public static int GetTargetId(int sourceId, IEnumerable workItemMigrationStates) { return workItemMigrationStates.First(a => a.SourceId == sourceId).TargetId.Value; diff --git a/Common/Migration/Phase1/PreProcessors/ClassificationNodesPreProcessor.cs b/Common/Migration/Phase1/PreProcessors/ClassificationNodesPreProcessor.cs new file mode 100644 index 0000000..18a919f --- /dev/null +++ b/Common/Migration/Phase1/PreProcessors/ClassificationNodesPreProcessor.cs @@ -0,0 +1,130 @@ +using System; +using System.Linq; +using System.Threading.Tasks; +using Common.Config; +using Logging; +using Microsoft.Extensions.Logging; +using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; + +namespace Common.Migration +{ + + public class ClassificationNodesPreProcessor : IPhase1PreProcessor + { + static ILogger Logger { get; } = MigratorLogging.CreateLogger(); + private IMigrationContext context; + + public string Name => "Classification Nodes (Area Paths/Iterations)"; + + public bool IsEnabled(ConfigJson config) + { + return config.MoveAreaPaths || config.MoveIterations; + } + + public async Task Prepare(IMigrationContext context) + { + this.context = context; + } + + public async Task Process(IBatchMigrationContext batchContext) + { + int modificationCount = 0; + await Task.WhenAll( + Migrator.ReadSourceNodes(context, context.Config.SourceConnection.Project), + Migrator.ReadTargetNodes(context, context.Config.TargetConnection.Project)); + + if (context.Config.MoveAreaPaths) + { + modificationCount += await ProcessAreaPaths(batchContext); + } + + if (context.Config.MoveIterations) + { + modificationCount += await ProcessIterationPaths(batchContext); + } + + if (modificationCount > 0) + { + await Migrator.ReadTargetNodes(context, context.Config.TargetConnection.Project); + } + } + + public async Task ProcessIterationPaths(IBatchMigrationContext batchContext) + { + int modificationCount = 0; + Logger.LogInformation(LogDestination.All, $"Identified {context.SourceAreaAndIterationTree.IterationPathList.Count} iterations in source project."); + + foreach (var iterationPath in context.SourceAreaAndIterationTree.IterationPathList) + { + string iterationPathInTarget = AreaAndIterationPathTree.ReplaceLeadingProjectName(iterationPath, + context.Config.SourceConnection.Project, context.Config.TargetConnection.Project); + + // If the iteration path is not found in the work items we're currently processing then just ignore it. + if (!batchContext.SourceWorkItems.Any(w => w.Fields.ContainsKey("System.IterationPath") && w.Fields["System.IterationPath"].ToString().ToLower().Equals(iterationPath.ToLower()))) + continue; + + if (context.TargetAreaAndIterationTree.IterationPathListLookup.ContainsKey(iterationPathInTarget)) + { + Logger.LogInformation(LogDestination.All, $"[Exists] {iterationPathInTarget}."); + } + else + { + var sourceIterationNode = context.SourceAreaAndIterationTree.IterationPathListLookup[iterationPath]; + modificationCount += 1; + await CreateIterationPath(iterationPath, sourceIterationNode); + + Logger.LogSuccess(LogDestination.All, $"[Created] {iterationPathInTarget}."); + } + } + + Logger.LogInformation(LogDestination.All, $"Iterations synchronized."); + return modificationCount; + } + + public async virtual Task CreateIterationPath(string iterationPath, WorkItemClassificationNode sourceIterationNode) + { + await WorkItemTrackingHelpers.CreateIterationAsync( + context.TargetClient.WorkItemTrackingHttpClient, + context.Config.TargetConnection.Project, + iterationPath, + sourceIterationNode.Attributes == null ? null : (DateTime?)sourceIterationNode.Attributes?["startDate"], + sourceIterationNode.Attributes == null ? null : (DateTime?)sourceIterationNode.Attributes["finishDate"]); + } + + public async Task ProcessAreaPaths(IBatchMigrationContext batchContext) + { + int modificationCount = 0; + Logger.LogInformation(LogDestination.All, $"Identified {context.SourceAreaAndIterationTree.AreaPathListLookup.Count} area paths in source project."); + + foreach (var areaPath in context.SourceAreaAndIterationTree.AreaPathList) + { + string areaPathInTarget = AreaAndIterationPathTree.ReplaceLeadingProjectName(areaPath, + context.Config.SourceConnection.Project, context.Config.TargetConnection.Project); + + // If the area path is not found in the work items we're currently processing then just ignore it. + if (!batchContext.SourceWorkItems.Any(w => w.Fields.ContainsKey("System.AreaPath") && w.Fields["System.AreaPath"].ToString().ToLower().Equals(areaPath.ToLower()))) + { + continue; + } + else if (context.TargetAreaAndIterationTree.AreaPathList.Any(a => a.ToLower() == areaPathInTarget.ToLower())) + { + Logger.LogInformation(LogDestination.All, $"[Exists] {areaPathInTarget}."); + } + else + { + modificationCount += 1; + await CreateAreaPath(areaPathInTarget); + Logger.LogSuccess(LogDestination.All, $"[Created] {areaPathInTarget}."); + } + } + + Logger.LogInformation(LogDestination.All, $"Area paths synchronized."); + return modificationCount; + } + + public async virtual Task CreateAreaPath(string areaPathInTarget) + { + await WorkItemTrackingHelpers.CreateAreaPathAsync(context.TargetClient.WorkItemTrackingHttpClient, context.Config.TargetConnection.Project, areaPathInTarget); + } + } +} diff --git a/Common/Migration/Phase1/Processors/BaseWorkItemsProcessor.cs b/Common/Migration/Phase1/Processors/BaseWorkItemsProcessor.cs index 8aa27fd..b30d11b 100644 --- a/Common/Migration/Phase1/Processors/BaseWorkItemsProcessor.cs +++ b/Common/Migration/Phase1/Processors/BaseWorkItemsProcessor.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; @@ -24,7 +25,7 @@ public abstract class BaseWorkItemsProcessor : IPhase1Processor public abstract int GetWorkItemsToProcessCount(IBatchMigrationContext batchContext); public abstract BaseWitBatchRequestGenerator GetWitBatchRequestGenerator(IMigrationContext context, IBatchMigrationContext batchContext); - + public async Task Process(IMigrationContext context) { var workItemsAndStateToMigrate = this.GetWorkItemsAndStateToMigrate(context); diff --git a/Common/WorkItemTrackingHelpers.cs b/Common/WorkItemTrackingHelpers.cs index 5514c1f..4a3e024 100644 --- a/Common/WorkItemTrackingHelpers.cs +++ b/Common/WorkItemTrackingHelpers.cs @@ -182,6 +182,42 @@ public async static Task CreateAttachmentChunkedAsync(WorkI return attachmentReference; } + public async static Task CreateAreaPathAsync(WorkItemTrackingHttpClient client, string projectId, string areaPath) + { + return await RetryHelper.RetryAsync(async () => + { + return await client.CreateOrUpdateClassificationNodeAsync(new WorkItemClassificationNode() + { + Name = areaPath.Contains("\\") ? areaPath.Split("\\").Last() : areaPath, + StructureType = TreeNodeStructureType.Area, + }, projectId, TreeStructureGroup.Areas, areaPath.Replace(projectId, "").Replace(areaPath.Contains("\\") ? areaPath.Split("\\").Last() : areaPath, "")); + }, 5); + } + + public async static Task CreateIterationAsync(WorkItemTrackingHttpClient client, string projectId, string iteration, DateTime? startDate, DateTime? endDate) + { + return await RetryHelper.RetryAsync(async () => + { + var attrs = new Dictionary(); + if (startDate.HasValue) + { + attrs.Add("startDate", startDate); + } + if (endDate.HasValue) + { + attrs.Add("finishDate", endDate); + } + + return await client.CreateOrUpdateClassificationNodeAsync(new WorkItemClassificationNode() + { + Name = iteration.Contains("\\") ? iteration.Split("\\").Last() : iteration, + StructureType = TreeNodeStructureType.Iteration, + Attributes = attrs + + }, projectId, TreeStructureGroup.Iterations); + }, 5); + } + public async static Task GetAttachmentAsync(WorkItemTrackingHttpClient client, Guid id) { return await RetryHelper.RetryAsync(async () => diff --git a/Logging/BulkLogger.cs b/Logging/BulkLogger.cs index e60f4b0..f2ae4a2 100644 --- a/Logging/BulkLogger.cs +++ b/Logging/BulkLogger.cs @@ -22,7 +22,7 @@ public BulkLogger() this.bulkLoggerCheckTimer = new Timer(BulkLoggerCheck, "Some state", TimeSpan.FromSeconds(LoggingConstants.CheckInterval), TimeSpan.FromSeconds(LoggingConstants.CheckInterval)); this.stopwatch = Stopwatch.StartNew(); this.filePath = GetFilePathBasedOnTime(); - Console.WriteLine($"Detailed logging sent to file: {Directory.GetCurrentDirectory()}\\{filePath}"); + Console.WriteLine($"[Info] [{DateTime.Now.ToString("HH:mm:ss.fff")}] Detailed logging sent to file: {Directory.GetCurrentDirectory()}\\{filePath}"); } public void WriteToQueue(LogItem logItem) diff --git a/Logging/LogItem.cs b/Logging/LogItem.cs index 8f28980..987d25a 100644 --- a/Logging/LogItem.cs +++ b/Logging/LogItem.cs @@ -1,5 +1,6 @@ using Microsoft.Extensions.Logging; using System; +using System.Linq; namespace Logging { @@ -44,7 +45,7 @@ public string OutputFormat(bool includeExceptionMessage, bool includeLogLevelTim // HH specifies 24-hour time format string timeStamp = DateTimeStampString(); string logLevelName = LogLevelName(); - return $"[{logLevelName} @{timeStamp}] {this.Message}"; + return string.Join(Environment.NewLine, Message.Split(Environment.NewLine).Select(line => $"[{logLevelName}] [{timeStamp}] {line}")); } else { @@ -54,7 +55,7 @@ public string OutputFormat(bool includeExceptionMessage, bool includeLogLevelTim public virtual string DateTimeStampString() { - return this.DateTimeStamp.ToString("HH.mm.ss.fff"); + return this.DateTimeStamp.ToString("HH:mm:ss.fff"); } public virtual string LogLevelName() diff --git a/README.md b/README.md index a6da891..b65f765 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ WiMigrator is a command line tool designed with the following goals in mind: * Git commit links (link to the source git commit) * Work item history (last 200 revisions as an attachment) * Tagging of the source items that have been migrated + * Area paths and iterations # Getting Started ## Requirements diff --git a/UnitTests/Logging/LogItemTests.cs b/UnitTests/Logging/LogItemTests.cs index 3b5cc59..a5f0f03 100644 --- a/UnitTests/Logging/LogItemTests.cs +++ b/UnitTests/Logging/LogItemTests.cs @@ -12,7 +12,7 @@ public class LogItemTests [TestMethod] public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndIncludeLogLevelTimeStampIsTrueAndExceptionIsNull() { - string expected = "[Error @00.00.00.000] message"; + string expected = "[Error] [00.00.00.000] message"; bool includeExceptionMessage = false; bool includeLogLevelTimeStamp = true; int logDestination = LogDestination.All; @@ -34,7 +34,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndInc [TestMethod] public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndIncludeLogLevelTimeStampIsTrueAndExceptionExistsWithNullMessage() { - string expected = "[Error @00.00.00.000] message"; + string expected = "[Error] [00.00.00.000] message"; bool includeExceptionMessage = false; bool includeLogLevelTimeStamp = true; int logDestination = LogDestination.All; @@ -57,7 +57,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndInc [TestMethod] public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndIncludeLogLevelTimeStampIsTrueAndExceptionExistsWithMessage() { - string expected = "[Error @00.00.00.000] message"; + string expected = "[Error] [00.00.00.000] message"; bool includeExceptionMessage = false; bool includeLogLevelTimeStamp = true; int logDestination = LogDestination.All; @@ -80,7 +80,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndInc [TestMethod] public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncludeLogLevelTimeStampIsTrueAndExceptionIsNull() { - string expected = "[Error @00.00.00.000] message"; + string expected = "[Error] [00.00.00.000] message"; bool includeExceptionMessage = true; bool includeLogLevelTimeStamp = true; int logDestination = LogDestination.All; @@ -102,7 +102,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncl [TestMethod] public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncludeLogLevelTimeStampIsTrueAndExceptionExistsWithNullMessage() { - string expected = "[Error @00.00.00.000] message. Exception of type 'System.Exception' was thrown."; + string expected = "[Error] [00.00.00.000] message. Exception of type 'System.Exception' was thrown."; bool includeExceptionMessage = true; bool includeLogLevelTimeStamp = true; int logDestination = LogDestination.All; @@ -125,7 +125,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncl [TestMethod] public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncludeLogLevelTimeStampIsTrueAndExceptionExistsWithMessage() { - string expected = "[Error @00.00.00.000] message. This is sample Exception Message."; + string expected = "[Error] [00.00.00.000] message. This is sample Exception Message."; bool includeExceptionMessage = true; bool includeLogLevelTimeStamp = true; int logDestination = LogDestination.All; diff --git a/UnitTests/Preprocess/ClassificationNodesPreProcessorTests.cs b/UnitTests/Preprocess/ClassificationNodesPreProcessorTests.cs new file mode 100644 index 0000000..abbcd29 --- /dev/null +++ b/UnitTests/Preprocess/ClassificationNodesPreProcessorTests.cs @@ -0,0 +1,196 @@ +using System; +using System.Collections.Generic; +using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Common; +using Common.Migration; +using System.Collections.Concurrent; +using Common.Config; +using Moq; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; + +namespace UnitTests.Preprocess +{ + [TestClass] + public class ClassificationNodesPreProcessorTests + { + [TestMethod] + public async Task ProcessAreaPaths_Test() + { + var processorMock = new Mock(); + + // populate source area paths + AreaAndIterationPathTree sourceTree = new AreaAndIterationPathTree(new List() + { + new WorkItemClassificationNode() + { + Name = "Test Src Project", + Children = new List() + { + new WorkItemClassificationNode() + { + Name = "Child Node 1", + }, + new WorkItemClassificationNode() + { + Name = "Child Node 2", + }, + new WorkItemClassificationNode() + { + Name = "Child Node 3", + }, + } + } + }); + + // Partial target tree + AreaAndIterationPathTree targetTree = new AreaAndIterationPathTree(new List() + { + new WorkItemClassificationNode() + { + Name = "Test Target Project", + Children = new List() + { + new WorkItemClassificationNode() + { + Name = "Child Node 1", + }, + } + } + }); + + var contextMock = new Mock(); + var batchContextMock = new Mock(); + contextMock.SetupGet(ctx => ctx.Config).Returns(new ConfigJson() + { + SourceConnection = new ConfigConnection() + { + Project = "Test Src Project" + }, + TargetConnection = new ConfigConnection() + { + Project = "Test Target Project" + } + }); + contextMock.SetupGet(ctx => ctx.SourceAreaAndIterationTree).Returns(sourceTree); + contextMock.SetupGet(ctx => ctx.TargetAreaAndIterationTree).Returns(targetTree); + batchContextMock.SetupGet(ctx => ctx.SourceWorkItems).Returns(new List() { + new WorkItem() + { + Fields = new Dictionary() + { + { "System.AreaPath", "Test Src Project\\Child Node 3"} + } + }, + new WorkItem() + { + Fields = new Dictionary() + { + { "System.AreaPath", "Test Src Project\\Child Node 1"} + } + }, + new WorkItem() + { + Fields = new Dictionary() + { + { "System.AreaPath", "Test Src Project"} + } + }, + }); + + await processorMock.Object.Prepare(contextMock.Object); + int modified = await processorMock.Object.ProcessAreaPaths(batchContextMock.Object); + + processorMock.Verify(x => x.CreateAreaPath(It.IsIn("Test Target Project\\Child Node 3")), Times.Once); + Assert.IsTrue(modified == 1); + + } + + [TestMethod] + public async Task ProcessIterationPaths_Test() + { + var processorMock = new Mock(); + var sprint2Node = new WorkItemClassificationNode() + { + Name = "Sprint 2", + }; + // populate source area paths + AreaAndIterationPathTree sourceTree = new AreaAndIterationPathTree(new List() + { + new WorkItemClassificationNode() + { + Name = "Test Src Project", + StructureType = TreeNodeStructureType.Iteration, + Children = new List() + { + new WorkItemClassificationNode() + { + Name = "Sprint 1", + StructureType = TreeNodeStructureType.Iteration + }, + sprint2Node, + } + }, + }); + + // Partial target tree + AreaAndIterationPathTree targetTree = new AreaAndIterationPathTree(new List() + { + new WorkItemClassificationNode() + { + Name = "Test Target Project", + StructureType = TreeNodeStructureType.Iteration, + Children = new List() + { + new WorkItemClassificationNode() + { + Name = "Sprint 1", + StructureType = TreeNodeStructureType.Iteration + }, + } + }, + }); + + var contextMock = new Mock(); + var batchContextMock = new Mock(); + contextMock.SetupGet(ctx => ctx.Config).Returns(new ConfigJson() + { + SourceConnection = new ConfigConnection() + { + Project = "Test Src Project" + }, + TargetConnection = new ConfigConnection() + { + Project = "Test Target Project" + } + }); + contextMock.SetupGet(ctx => ctx.SourceAreaAndIterationTree).Returns(sourceTree); + contextMock.SetupGet(ctx => ctx.TargetAreaAndIterationTree).Returns(targetTree); + batchContextMock.SetupGet(ctx => ctx.SourceWorkItems).Returns(new List() { + new WorkItem() + { + Fields = new Dictionary() + { + { "System.IterationPath", "Test Src Project\\Sprint 1"} + } + }, + new WorkItem() + { + Fields = new Dictionary() + { + { "System.IterationPath", "Test Src Project\\Sprint 2"} + } + } + }); + + await processorMock.Object.Prepare(contextMock.Object); + int modified = await processorMock.Object.ProcessIterationPaths(batchContextMock.Object); + + processorMock.Verify(x => x.CreateIterationPath(It.IsIn("Test Src Project\\Sprint 2"), It.IsAny()), Times.Once); + Assert.AreEqual(modified, 1); + + } + } + +} \ No newline at end of file diff --git a/WiMigrator/migration-configuration.md b/WiMigrator/migration-configuration.md index d4cd775..d5d651d 100644 --- a/WiMigrator/migration-configuration.md +++ b/WiMigrator/migration-configuration.md @@ -43,6 +43,8 @@ #### ```move-attachments``` migrate attachments +#### ```move-area-paths``` migrate area paths +#### ```move-iterations``` migrate iterations #### ```move-links``` preserve and migrate work item links from source to target if the linked work item is part of the current query or it has been previously migrated.