Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 8 additions & 4 deletions Common/AreaAndIterationPathTree.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ public class AreaAndIterationPathTree
static ILogger Logger { get; } = MigratorLogging.CreateLogger<AreaAndIterationPathTree>();
public ISet<string> AreaPathList { get; } = new HashSet<string>();
public ISet<string> IterationPathList { get; } = new HashSet<string>();
public IDictionary<string, WorkItemClassificationNode> IterationPathListLookup { get; } = new Dictionary<string, WorkItemClassificationNode>();
public IDictionary<string, WorkItemClassificationNode> AreaPathListLookup { get; } = new Dictionary<string, WorkItemClassificationNode>();

public AreaAndIterationPathTree(IList<WorkItemClassificationNode> nodeList)
{
Expand Down Expand Up @@ -74,7 +76,7 @@ private void CreateAreaPathList(WorkItemClassificationNode headnode)
return;
}
//path for the headnode is null
ProcessNode(null, headnode, this.AreaPathList);
ProcessNode(null, headnode, this.AreaPathList, this.AreaPathListLookup);
}

private void CreateIterationPathList(WorkItemClassificationNode headnode)
Expand All @@ -85,10 +87,10 @@ private void CreateIterationPathList(WorkItemClassificationNode headnode)
return;
}
//path for the headnode is null
ProcessNode(null, headnode, this.IterationPathList);
ProcessNode(null, headnode, this.IterationPathList, this.IterationPathListLookup);
}

private void ProcessNode(string path, WorkItemClassificationNode node, ISet<string> pathList)
private void ProcessNode(string path, WorkItemClassificationNode node, ISet<string> pathList, IDictionary<string, WorkItemClassificationNode> pathListLookup)
{
if (node == null)
{
Expand All @@ -106,12 +108,14 @@ private void ProcessNode(string path, WorkItemClassificationNode node, ISet<stri
}

pathList.Add(currentpath);
pathListLookup.Add(currentpath, node);


if (node.Children != null)
{
foreach (var child in node.Children)
{
ProcessNode(currentpath, child, pathList);
ProcessNode(currentpath, child, pathList, pathListLookup);
}
}
}
Expand Down
8 changes: 8 additions & 0 deletions Common/Config/ConfigJson.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,14 @@ public class ConfigJson
[DefaultValue(true)]
public bool SkipExisting { get; set; }

[JsonProperty(PropertyName = "move-area-paths", DefaultValueHandling = DefaultValueHandling.Populate)]
[DefaultValue(false)]
public bool MoveAreaPaths { get; set; }

[JsonProperty(PropertyName = "move-iterations", DefaultValueHandling = DefaultValueHandling.Populate)]
[DefaultValue(false)]
public bool MoveIterations{ get; set; }

[JsonProperty(PropertyName = "move-history", DefaultValueHandling = DefaultValueHandling.Populate)]
[DefaultValue(false)]
public bool MoveHistory { get; set; }
Expand Down
2 changes: 2 additions & 0 deletions Common/Migration/Contexts/IMigrationContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,7 @@ public interface IMigrationContext : IContext
IList<string> UnsupportedFields { get; }

IList<string> FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; set; }
AreaAndIterationPathTree SourceAreaAndIterationTree { get; set; }
AreaAndIterationPathTree TargetAreaAndIterationTree { get; set; }
}
}
2 changes: 2 additions & 0 deletions Common/Migration/Contexts/MigrationContext.cs
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ public class MigrationContext : BaseContext, IMigrationContext
public IList<string> UnsupportedFields => unsupportedFields;

public IList<string> FieldsThatRequireSourceProjectToBeReplacedWithTargetProject { get; set; }
public AreaAndIterationPathTree SourceAreaAndIterationTree { get; set; }
public AreaAndIterationPathTree TargetAreaAndIterationTree { get; set; }

// List of fields that we do not support in migration because they are related to the board or another reason.
private readonly IList<string> unsupportedFields = new ReadOnlyCollection<string>(new[]{
Expand Down
26 changes: 26 additions & 0 deletions Common/Migration/Migrator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -348,6 +348,32 @@ public static async Task ReadSourceWorkItems(IMigrationContext migrationContext,
batchContext.SourceWorkItems = await WorkItemTrackingHelpers.GetWorkItemsAsync(migrationContext.SourceClient.WorkItemTrackingHttpClient, workItemIds, expand: expand);
}

/// <summary>
/// Populates migrationContext.SourceAreaAndIterationTree
/// </summary>
/// <param name="migrationContext"></param>
/// <param name="projectId"></param>
/// <returns></returns>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mind updating the comment block?

public static async Task ReadSourceNodes(IMigrationContext migrationContext, string projectId)
{
var nodes = await WorkItemTrackingHelpers.GetClassificationNodes(migrationContext.SourceClient.WorkItemTrackingHttpClient, projectId);
migrationContext.SourceAreaAndIterationTree = new AreaAndIterationPathTree(nodes);
}

/// <summary>
/// Populates migrationContext.TargetAreaAndIterationTree, migrationContext.TargetAreaPaths, and migrationContext.TargetIterationPaths
/// </summary>
/// <param name="migrationContext"></param>
/// <param name="projectId"></param>
/// <returns></returns>
public static async Task ReadTargetNodes(IMigrationContext migrationContext, string projectId)
{
var nodes = await WorkItemTrackingHelpers.GetClassificationNodes(migrationContext.TargetClient.WorkItemTrackingHttpClient, projectId);
migrationContext.TargetAreaAndIterationTree = new AreaAndIterationPathTree(nodes);
migrationContext.TargetAreaPaths = migrationContext.TargetAreaAndIterationTree.AreaPathList;
migrationContext.TargetIterationPaths = migrationContext.TargetAreaAndIterationTree.IterationPathList;
}

public static int GetTargetId(int sourceId, IEnumerable<WorkItemMigrationState> workItemMigrationStates)
{
return workItemMigrationStates.First(a => a.SourceId == sourceId).TargetId.Value;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Common.Config;
using Logging;
using Microsoft.Extensions.Logging;
using Microsoft.TeamFoundation.WorkItemTracking.WebApi.Models;

namespace Common.Migration
{

public class ClassificationNodesPreProcessor : IPhase1PreProcessor
{
static ILogger Logger { get; } = MigratorLogging.CreateLogger<ClassificationNodesPreProcessor>();
private IMigrationContext context;

public string Name => "Classification Nodes (Area Paths/Iterations)";

public bool IsEnabled(ConfigJson config)
{
return config.MoveAreaPaths || config.MoveIterations;
}

public async Task Prepare(IMigrationContext context)
{
this.context = context;
}

public async Task Process(IBatchMigrationContext batchContext)
{
int modificationCount = 0;
await Task.WhenAll(
Migrator.ReadSourceNodes(context, context.Config.SourceConnection.Project),
Migrator.ReadTargetNodes(context, context.Config.TargetConnection.Project));

if (context.Config.MoveAreaPaths)
{
modificationCount += await ProcessAreaPaths(batchContext);
}

if (context.Config.MoveIterations)
{
modificationCount += await ProcessIterationPaths(batchContext);
}

if (modificationCount > 0)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

i'm not a fan of the pattern of passing an int and then assigning it in the return. Either make it an out or just have two variables.

{
await Migrator.ReadTargetNodes(context, context.Config.TargetConnection.Project);
}
}

public async Task<int> ProcessIterationPaths(IBatchMigrationContext batchContext)
{
int modificationCount = 0;
Logger.LogInformation(LogDestination.All, $"Identified {context.SourceAreaAndIterationTree.IterationPathList.Count} iterations in source project.");

foreach (var iterationPath in context.SourceAreaAndIterationTree.IterationPathList)
{
string iterationPathInTarget = AreaAndIterationPathTree.ReplaceLeadingProjectName(iterationPath,
context.Config.SourceConnection.Project, context.Config.TargetConnection.Project);

// If the iteration path is not found in the work items we're currently processing then just ignore it.
if (!batchContext.SourceWorkItems.Any(w => w.Fields.ContainsKey("System.IterationPath") && w.Fields["System.IterationPath"].ToString().ToLower().Equals(iterationPath.ToLower())))
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

instead of all the tolowers, why not just do a stringcomparison.ignorecase

continue;

if (context.TargetAreaAndIterationTree.IterationPathListLookup.ContainsKey(iterationPathInTarget))
{
Logger.LogInformation(LogDestination.All, $"[Exists] {iterationPathInTarget}.");
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mind including a little more detail, like "[Iteration path exists]

}
else
{
var sourceIterationNode = context.SourceAreaAndIterationTree.IterationPathListLookup[iterationPath];
modificationCount += 1;
await CreateIterationPath(iterationPath, sourceIterationNode);

Logger.LogSuccess(LogDestination.All, $"[Created] {iterationPathInTarget}.");
}
}

Logger.LogInformation(LogDestination.All, $"Iterations synchronized.");
return modificationCount;
}

public async virtual Task CreateIterationPath(string iterationPath, WorkItemClassificationNode sourceIterationNode)
{
await WorkItemTrackingHelpers.CreateIterationAsync(
context.TargetClient.WorkItemTrackingHttpClient,
context.Config.TargetConnection.Project,
iterationPath,
sourceIterationNode.Attributes == null ? null : (DateTime?)sourceIterationNode.Attributes?["startDate"],
sourceIterationNode.Attributes == null ? null : (DateTime?)sourceIterationNode.Attributes["finishDate"]);
}

public async Task<int> ProcessAreaPaths(IBatchMigrationContext batchContext)
{
int modificationCount = 0;
Logger.LogInformation(LogDestination.All, $"Identified {context.SourceAreaAndIterationTree.AreaPathListLookup.Count} area paths in source project.");

foreach (var areaPath in context.SourceAreaAndIterationTree.AreaPathList)
{
string areaPathInTarget = AreaAndIterationPathTree.ReplaceLeadingProjectName(areaPath,
context.Config.SourceConnection.Project, context.Config.TargetConnection.Project);

// If the area path is not found in the work items we're currently processing then just ignore it.
if (!batchContext.SourceWorkItems.Any(w => w.Fields.ContainsKey("System.AreaPath") && w.Fields["System.AreaPath"].ToString().ToLower().Equals(areaPath.ToLower())))
{
continue;
}
else if (context.TargetAreaAndIterationTree.AreaPathList.Any(a => a.ToLower() == areaPathInTarget.ToLower()))
{
Logger.LogInformation(LogDestination.All, $"[Exists] {areaPathInTarget}.");
}
else
{
modificationCount += 1;
await CreateAreaPath(areaPathInTarget);
Logger.LogSuccess(LogDestination.All, $"[Created] {areaPathInTarget}.");
}
}

Logger.LogInformation(LogDestination.All, $"Area paths synchronized.");
return modificationCount;
}

public async virtual Task CreateAreaPath(string areaPathInTarget)
{
await WorkItemTrackingHelpers.CreateAreaPathAsync(context.TargetClient.WorkItemTrackingHttpClient, context.Config.TargetConnection.Project, areaPathInTarget);
}
}
}
5 changes: 3 additions & 2 deletions Common/Migration/Phase1/Processors/BaseWorkItemsProcessor.cs
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Threading.Tasks;
Expand All @@ -24,7 +25,7 @@ public abstract class BaseWorkItemsProcessor : IPhase1Processor
public abstract int GetWorkItemsToProcessCount(IBatchMigrationContext batchContext);

public abstract BaseWitBatchRequestGenerator GetWitBatchRequestGenerator(IMigrationContext context, IBatchMigrationContext batchContext);

public async Task Process(IMigrationContext context)
{
var workItemsAndStateToMigrate = this.GetWorkItemsAndStateToMigrate(context);
Expand Down
36 changes: 36 additions & 0 deletions Common/WorkItemTrackingHelpers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,42 @@ public async static Task<AttachmentReference> CreateAttachmentChunkedAsync(WorkI
return attachmentReference;
}

public async static Task<WorkItemClassificationNode> CreateAreaPathAsync(WorkItemTrackingHttpClient client, string projectId, string areaPath)
{
return await RetryHelper.RetryAsync(async () =>
{
return await client.CreateOrUpdateClassificationNodeAsync(new WorkItemClassificationNode()
{
Name = areaPath.Contains("\\") ? areaPath.Split("\\").Last() : areaPath,
StructureType = TreeNodeStructureType.Area,
}, projectId, TreeStructureGroup.Areas, areaPath.Replace(projectId, "").Replace(areaPath.Contains("\\") ? areaPath.Split("\\").Last() : areaPath, ""));
}, 5);
}

public async static Task<WorkItemClassificationNode> CreateIterationAsync(WorkItemTrackingHttpClient client, string projectId, string iteration, DateTime? startDate, DateTime? endDate)
{
return await RetryHelper.RetryAsync(async () =>
{
var attrs = new Dictionary<string, object>();
if (startDate.HasValue)
{
attrs.Add("startDate", startDate);
}
if (endDate.HasValue)
{
attrs.Add("finishDate", endDate);
}

return await client.CreateOrUpdateClassificationNodeAsync(new WorkItemClassificationNode()
{
Name = iteration.Contains("\\") ? iteration.Split("\\").Last() : iteration,
StructureType = TreeNodeStructureType.Iteration,
Attributes = attrs

}, projectId, TreeStructureGroup.Iterations);
}, 5);
}

public async static Task<Stream> GetAttachmentAsync(WorkItemTrackingHttpClient client, Guid id)
{
return await RetryHelper.RetryAsync(async () =>
Expand Down
2 changes: 1 addition & 1 deletion Logging/BulkLogger.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public BulkLogger()
this.bulkLoggerCheckTimer = new Timer(BulkLoggerCheck, "Some state", TimeSpan.FromSeconds(LoggingConstants.CheckInterval), TimeSpan.FromSeconds(LoggingConstants.CheckInterval));
this.stopwatch = Stopwatch.StartNew();
this.filePath = GetFilePathBasedOnTime();
Console.WriteLine($"Detailed logging sent to file: {Directory.GetCurrentDirectory()}\\{filePath}");
Console.WriteLine($"[Info] [{DateTime.Now.ToString("HH:mm:ss.fff")}] Detailed logging sent to file: {Directory.GetCurrentDirectory()}\\{filePath}");
}

public void WriteToQueue(LogItem logItem)
Expand Down
5 changes: 3 additions & 2 deletions Logging/LogItem.cs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using Microsoft.Extensions.Logging;
using System;
using System.Linq;

namespace Logging
{
Expand Down Expand Up @@ -44,7 +45,7 @@ public string OutputFormat(bool includeExceptionMessage, bool includeLogLevelTim
// HH specifies 24-hour time format
string timeStamp = DateTimeStampString();
string logLevelName = LogLevelName();
return $"[{logLevelName} @{timeStamp}] {this.Message}";
return string.Join(Environment.NewLine, Message.Split(Environment.NewLine).Select(line => $"[{logLevelName}] [{timeStamp}] {line}"));
}
else
{
Expand All @@ -54,7 +55,7 @@ public string OutputFormat(bool includeExceptionMessage, bool includeLogLevelTim

public virtual string DateTimeStampString()
{
return this.DateTimeStamp.ToString("HH.mm.ss.fff");
return this.DateTimeStamp.ToString("HH:mm:ss.fff");
}

public virtual string LogLevelName()
Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ WiMigrator is a command line tool designed with the following goals in mind:
* Git commit links (link to the source git commit)
* Work item history (last 200 revisions as an attachment)
* Tagging of the source items that have been migrated
* Area paths and iterations

# Getting Started
## Requirements
Expand Down
12 changes: 6 additions & 6 deletions UnitTests/Logging/LogItemTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ public class LogItemTests
[TestMethod]
public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndIncludeLogLevelTimeStampIsTrueAndExceptionIsNull()
{
string expected = "[Error @00.00.00.000] message";
string expected = "[Error] [00.00.00.000] message";
bool includeExceptionMessage = false;
bool includeLogLevelTimeStamp = true;
int logDestination = LogDestination.All;
Expand All @@ -34,7 +34,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndInc
[TestMethod]
public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndIncludeLogLevelTimeStampIsTrueAndExceptionExistsWithNullMessage()
{
string expected = "[Error @00.00.00.000] message";
string expected = "[Error] [00.00.00.000] message";
bool includeExceptionMessage = false;
bool includeLogLevelTimeStamp = true;
int logDestination = LogDestination.All;
Expand All @@ -57,7 +57,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndInc
[TestMethod]
public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndIncludeLogLevelTimeStampIsTrueAndExceptionExistsWithMessage()
{
string expected = "[Error @00.00.00.000] message";
string expected = "[Error] [00.00.00.000] message";
bool includeExceptionMessage = false;
bool includeLogLevelTimeStamp = true;
int logDestination = LogDestination.All;
Expand All @@ -80,7 +80,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsFalseAndInc
[TestMethod]
public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncludeLogLevelTimeStampIsTrueAndExceptionIsNull()
{
string expected = "[Error @00.00.00.000] message";
string expected = "[Error] [00.00.00.000] message";
bool includeExceptionMessage = true;
bool includeLogLevelTimeStamp = true;
int logDestination = LogDestination.All;
Expand All @@ -102,7 +102,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncl
[TestMethod]
public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncludeLogLevelTimeStampIsTrueAndExceptionExistsWithNullMessage()
{
string expected = "[Error @00.00.00.000] message. Exception of type 'System.Exception' was thrown.";
string expected = "[Error] [00.00.00.000] message. Exception of type 'System.Exception' was thrown.";
bool includeExceptionMessage = true;
bool includeLogLevelTimeStamp = true;
int logDestination = LogDestination.All;
Expand All @@ -125,7 +125,7 @@ public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncl
[TestMethod]
public void OutputFormat_FormatIsCorrectWhenIncludeExceptionMessageIsTrueAndIncludeLogLevelTimeStampIsTrueAndExceptionExistsWithMessage()
{
string expected = "[Error @00.00.00.000] message. This is sample Exception Message.";
string expected = "[Error] [00.00.00.000] message. This is sample Exception Message.";
bool includeExceptionMessage = true;
bool includeLogLevelTimeStamp = true;
int logDestination = LogDestination.All;
Expand Down
Loading