diff --git a/docs/data/classes/reference.tools.exportworkitemmappingtool.yaml b/docs/data/classes/reference.tools.exportworkitemmappingtool.yaml new file mode 100644 index 000000000..0c253cf1c --- /dev/null +++ b/docs/data/classes/reference.tools.exportworkitemmappingtool.yaml @@ -0,0 +1,55 @@ +optionsClassName: ExportWorkItemMappingToolOptions +optionsClassFullName: MigrationTools.Tools.ExportWorkItemMappingToolOptions +configurationSamples: +- name: defaults + order: 2 + description: + code: There are no defaults! Check the sample for options! + sampleFor: MigrationTools.Tools.ExportWorkItemMappingToolOptions +- name: sample + order: 1 + description: + code: There is no sample, but you can check the classic below for a general feel. + sampleFor: MigrationTools.Tools.ExportWorkItemMappingToolOptions +- name: classic + order: 3 + description: + code: >- + { + "$type": "ExportWorkItemMappingToolOptions", + "Enabled": false, + "TargetFile": "", + "PreserveExisting": true + } + sampleFor: MigrationTools.Tools.ExportWorkItemMappingToolOptions +description: >- + Tool for exporting mappings of work item IDs from source to target. + Work item migration processor uses this tool to record work item ID mappings. + The mappings will be saved to file defined in options at the end of the migration. +className: ExportWorkItemMappingTool +typeName: Tools +options: +- parameterName: Enabled + type: Boolean + description: If set to `true` then the tool will run. Set to `false` and the processor will not run. + defaultValue: true + isRequired: false + dotNetType: System.Boolean, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e +- parameterName: PreserveExisting + type: Boolean + description: >- + Indicates whether existing mappings in the target file should be preserved when saving new mappings. + Default value is `true`. + defaultValue: true + isRequired: false + dotNetType: System.Boolean, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e +- parameterName: TargetFile + type: String + description: Path to file, where work item mapping will be saved. + defaultValue: String.Empty + isRequired: false + dotNetType: System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e +status: missing XML code comments +processingTarget: missing XML code comments +classFile: src/MigrationTools/Tools/ExportWorkItemMappingTool.cs +optionsClassFile: src/MigrationTools/Tools/ExportWorkItemMappingToolOptions.cs diff --git a/docs/data/classes/reference.tools.tfschangesetmappingtool.yaml b/docs/data/classes/reference.tools.tfschangesetmappingtool.yaml index d86e03019..3d382ee85 100644 --- a/docs/data/classes/reference.tools.tfschangesetmappingtool.yaml +++ b/docs/data/classes/reference.tools.tfschangesetmappingtool.yaml @@ -49,8 +49,8 @@ typeName: Tools options: - parameterName: ChangeSetMappingFile type: String - description: missing XML code comments - defaultValue: missing XML code comments + description: Path to changeset mapping file. + defaultValue: null isRequired: false dotNetType: System.String, System.Private.CoreLib, Version=8.0.0.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e - parameterName: Enabled diff --git a/docs/static/schema/configuration.schema.json b/docs/static/schema/configuration.schema.json index 3d4d08795..3e4f5d928 100644 --- a/docs/static/schema/configuration.schema.json +++ b/docs/static/schema/configuration.schema.json @@ -1731,6 +1731,28 @@ "CommonTools": { "type": "object", "properties": { + "ExportWorkItemMappingTool": { + "title": "ExportWorkItemMappingTool", + "description": "Tool for exporting mappings of work item IDs from source to target.\r\n Work item migration processor uses this tool to record work item ID mappings.\r\n The mappings will be saved to file defined in options at the end of the migration.", + "type": "object", + "properties": { + "Enabled": { + "description": "If set to `true` then the tool will run. Set to `false` and the processor will not run.", + "type": "boolean", + "default": "true" + }, + "PreserveExisting": { + "description": "Indicates whether existing mappings in the target file should be preserved when saving new mappings.\r\n Default value is `true`.", + "type": "boolean", + "default": "true" + }, + "TargetFile": { + "description": "Path to file, where work item mapping will be saved.", + "type": "string", + "default": "String.Empty" + } + } + }, "FieldMappingTool": { "title": "FieldMappingTool", "description": "Tool for applying field mapping transformations to work items during migration, supporting various field mapping strategies like direct mapping, regex transformations, and value lookups.", @@ -2229,8 +2251,9 @@ "default": "true" }, "ChangeSetMappingFile": { - "description": "missing XML code comments", - "type": "string" + "description": "Path to changeset mapping file.", + "type": "string", + "default": "null" } } }, diff --git a/docs/static/schema/schema.tools.exportworkitemmappingtool.json b/docs/static/schema/schema.tools.exportworkitemmappingtool.json new file mode 100644 index 000000000..a50ab6036 --- /dev/null +++ b/docs/static/schema/schema.tools.exportworkitemmappingtool.json @@ -0,0 +1,24 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://devopsmigration.io/schema/schema.tools.exportworkitemmappingtool.json", + "title": "ExportWorkItemMappingTool", + "description": "Tool for exporting mappings of work item IDs from source to target.\r\n Work item migration processor uses this tool to record work item ID mappings.\r\n The mappings will be saved to file defined in options at the end of the migration.", + "type": "object", + "properties": { + "Enabled": { + "description": "If set to `true` then the tool will run. Set to `false` and the processor will not run.", + "type": "boolean", + "default": "true" + }, + "PreserveExisting": { + "description": "Indicates whether existing mappings in the target file should be preserved when saving new mappings.\r\n Default value is `true`.", + "type": "boolean", + "default": "true" + }, + "TargetFile": { + "description": "Path to file, where work item mapping will be saved.", + "type": "string", + "default": "String.Empty" + } + } +} \ No newline at end of file diff --git a/docs/static/schema/schema.tools.tfschangesetmappingtool.json b/docs/static/schema/schema.tools.tfschangesetmappingtool.json index f4dcd1abb..27cf9102d 100644 --- a/docs/static/schema/schema.tools.tfschangesetmappingtool.json +++ b/docs/static/schema/schema.tools.tfschangesetmappingtool.json @@ -6,8 +6,9 @@ "type": "object", "properties": { "ChangeSetMappingFile": { - "description": "missing XML code comments", - "type": "string" + "description": "Path to changeset mapping file.", + "type": "string", + "default": "null" }, "Enabled": { "description": "If set to `true` then the tool will run. Set to `false` and the processor will not run.", diff --git a/src/MigrationTools.Clients.AzureDevops.Rest.Tests/Processors/AzureDevOpsProcessorTests.cs b/src/MigrationTools.Clients.AzureDevops.Rest.Tests/Processors/AzureDevOpsProcessorTests.cs index 31a7ea289..5eb3f24a6 100644 --- a/src/MigrationTools.Clients.AzureDevops.Rest.Tests/Processors/AzureDevOpsProcessorTests.cs +++ b/src/MigrationTools.Clients.AzureDevops.Rest.Tests/Processors/AzureDevOpsProcessorTests.cs @@ -25,6 +25,7 @@ protected AzureDevOpsPipelineProcessor GetAzureDevOpsPipelineProcessor(AzureDevO services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); diff --git a/src/MigrationTools.Clients.TfsObjectModel.Tests/Processors/TfsProcessorTests.cs b/src/MigrationTools.Clients.TfsObjectModel.Tests/Processors/TfsProcessorTests.cs index 461d3d7ca..d1aa92155 100644 --- a/src/MigrationTools.Clients.TfsObjectModel.Tests/Processors/TfsProcessorTests.cs +++ b/src/MigrationTools.Clients.TfsObjectModel.Tests/Processors/TfsProcessorTests.cs @@ -38,6 +38,7 @@ protected TfsTeamSettingsProcessor GetTfsTeamSettingsProcessor(TfsTeamSettingsPr services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); @@ -107,6 +108,7 @@ protected TfsSharedQueryProcessor GetTfsSharedQueryProcessor(TfsSharedQueryProce services.AddSingleton(); services.AddSingleton(); services.AddSingleton(); + services.AddSingleton(); services.AddSingleton(); services.TryAddScoped(); diff --git a/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs b/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs index 3355c9ed9..b38d75374 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Processors/TfsWorkItemMigrationProcessor.cs @@ -233,6 +233,7 @@ protected override void InternalExecute() } finally { + CommonTools.ExportWorkItemMapping.SaveMappings(); if (Options.FixHtmlAttachmentLinks) { CommonTools.EmbededImages?.ProcessorExecutionEnd(null); @@ -601,6 +602,7 @@ private async Task ProcessWorkItemAsync(WorkItemData sourceWorkItem, ProgressTim } if (targetWorkItem != null) { + CommonTools.ExportWorkItemMapping.AddMapping(sourceWorkItem.Id, targetWorkItem.Id); targetWorkItem.ToWorkItem().Close(); } if (sourceWorkItem != null) diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsChangeSetMappingToolOptions.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsChangeSetMappingToolOptions.cs index aa2a15f2a..4bd9e2c07 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsChangeSetMappingToolOptions.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsChangeSetMappingToolOptions.cs @@ -1,13 +1,13 @@ -using System; -using MigrationTools.Enrichers; -using MigrationTools.Tools.Infrastructure; +using MigrationTools.Tools.Infrastructure; namespace MigrationTools.Tools { public class TfsChangeSetMappingToolOptions : ToolOptions { - + /// + /// Path to changeset mapping file. + /// + /// null public string ChangeSetMappingFile { get; set; } - } -} \ No newline at end of file +} diff --git a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsStaticTools.cs b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsStaticTools.cs index 17486e095..4ba014375 100644 --- a/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsStaticTools.cs +++ b/src/MigrationTools.Clients.TfsObjectModel/Tools/TfsStaticTools.cs @@ -22,6 +22,7 @@ public class TfsCommonTools : CommonTools /// Tool for git repository operations /// Tool for string field manipulation /// Tool for work item type mapping + /// Tool for exporting work item mapping. /// Tool for work item type validation. /// Tool for field mapping operations public TfsCommonTools( @@ -37,9 +38,10 @@ public TfsCommonTools( TfsGitRepositoryTool TfsGitRepositoryTool, IStringManipulatorTool StringManipulatorTool, IWorkItemTypeMappingTool workItemTypeMapping, + IExportWorkItemMappingTool exportWorkItemMapping, TfsWorkItemTypeValidatorTool workItemTypeValidatorTool, IFieldMappingTool fieldMappingTool - ) : base(StringManipulatorTool, workItemTypeMapping,fieldMappingTool) + ) : base(StringManipulatorTool, workItemTypeMapping, exportWorkItemMapping, fieldMappingTool) { UserMapping = userMappingEnricher; Attachment = attachmentEnricher; diff --git a/src/MigrationTools.Host/Commands/CommandBase.cs b/src/MigrationTools.Host/Commands/CommandBase.cs index 8dc0d476e..c7f52db8f 100644 --- a/src/MigrationTools.Host/Commands/CommandBase.cs +++ b/src/MigrationTools.Host/Commands/CommandBase.cs @@ -65,7 +65,6 @@ public sealed override async Task ExecuteAsync(CommandContext context, TSet { Log.Debug("Disabling Telemetry {CommandName}", this.GetType().Name); CommandActivity.AddTag("DisableTelemetry", settings.DisableTelemetry); - CommandActivity.Stop(); ActivitySourceProvider.DisableActivitySource(); } //Enable Debug Trace @@ -105,7 +104,7 @@ public sealed override async Task ExecuteAsync(CommandContext context, TSet internal virtual Task ExecuteInternalAsync(CommandContext context, TSettings settings) { - return Task.FromResult( 0); + return Task.FromResult(0); } public void RunStartupLogic(TSettings settings) diff --git a/src/MigrationTools.Shadows/Tools/MockExportWorkItemMappingTool.cs b/src/MigrationTools.Shadows/Tools/MockExportWorkItemMappingTool.cs new file mode 100644 index 000000000..9cdb7c962 --- /dev/null +++ b/src/MigrationTools.Shadows/Tools/MockExportWorkItemMappingTool.cs @@ -0,0 +1,15 @@ +using MigrationTools.Tools.Interfaces; + +namespace MigrationTools.Tools.Shadows +{ + public class MockExportWorkItemMappingTool : IExportWorkItemMappingTool + { + public void AddMapping(string sourceId, string targetId) + { + } + + public void SaveMappings() + { + } + } +} diff --git a/src/MigrationTools/Processors/Infrastructure/Processor.cs b/src/MigrationTools/Processors/Infrastructure/Processor.cs index 5ac015ae8..adcc9caeb 100644 --- a/src/MigrationTools/Processors/Infrastructure/Processor.cs +++ b/src/MigrationTools/Processors/Infrastructure/Processor.cs @@ -1,14 +1,11 @@ using System; using System.Collections.Generic; -using System.ComponentModel.Design; using System.Diagnostics; using System.Linq; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using MigrationTools._EngineV1.Configuration; using MigrationTools.Endpoints; -using MigrationTools.Endpoints.Infrastructure; using MigrationTools.Enrichers; using MigrationTools.Exceptions; using MigrationTools.Services; @@ -139,6 +136,7 @@ public void Execute() } finally { + ProcessorActivity.Stop(); Log.LogInformation("{ProcessorName} completed in {ProcessorDuration} ", Name, ProcessorActivity.Duration.ToString("c")); } @@ -172,4 +170,4 @@ protected static void AddParameter(string name, IDictionary stor if (!store.ContainsKey(name)) store.Add(name, value); } } -} \ No newline at end of file +} diff --git a/src/MigrationTools/ServiceCollectionExtensions.cs b/src/MigrationTools/ServiceCollectionExtensions.cs index fd5acee70..c2fea6bce 100644 --- a/src/MigrationTools/ServiceCollectionExtensions.cs +++ b/src/MigrationTools/ServiceCollectionExtensions.cs @@ -46,6 +46,7 @@ public static void AddMigrationToolServices(this IServiceCollection context, ICo context.AddSingleton().AddMigrationToolsOptions(configuration); context.AddSingleton().AddMigrationToolsOptions(configuration); + context.AddSingleton().AddMigrationToolsOptions(configuration); // context.AddSingleton().AddMigrationToolsOptions(configuration); diff --git a/src/MigrationTools/Tools/ExportWorkItemMappingTool.cs b/src/MigrationTools/Tools/ExportWorkItemMappingTool.cs new file mode 100644 index 000000000..f1a0fbc36 --- /dev/null +++ b/src/MigrationTools/Tools/ExportWorkItemMappingTool.cs @@ -0,0 +1,146 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using MigrationTools.Tools.Infrastructure; +using MigrationTools.Tools.Interfaces; + +namespace MigrationTools.Tools; + +/// +/// Tool for exporting mappings of work item IDs from source to target. +/// Work item migration processor uses this tool to record work item ID mappings. +/// The mappings will be saved to file defined in options at the end of the migration. +/// +public class ExportWorkItemMappingTool : Tool, IExportWorkItemMappingTool +{ + private static readonly JsonSerializerOptions _jsonOptions = new() + { + WriteIndented = true, + }; + + private readonly ConcurrentDictionary _mappings = new(StringComparer.OrdinalIgnoreCase); + + public ExportWorkItemMappingTool( + IOptions options, + IServiceProvider services, + ILogger logger, + ITelemetryLogger telemetry) + : base(options, services, logger, telemetry) + { + } + + /// + public void AddMapping(string sourceId, string targetId) + { + if (!Enabled) + { + return; + } + if (!_mappings.TryAdd(sourceId, targetId)) + { + if (_mappings.TryGetValue(sourceId, out string existingTargetId) + && !existingTargetId.Equals(targetId, StringComparison.OrdinalIgnoreCase)) + { + Log.LogError("Attempt to map source work item ID '{sourceId}' to target ID '{targetId}'." + + " This source work item ID is already mapped to different target ID '{existingTargetId}'." + + " Former mapping will be preserved.", + sourceId, targetId, existingTargetId); + } + return; + } + Log.LogDebug("Source work item ID '{sourceId}' is mapped to target work item ID '{targetId}'.", sourceId, targetId); + } + + /// + /// Save mappings to file defined in options. Mappings is saved as dictionary serialized to JSON. + /// + public void SaveMappings() + { + if (!Enabled) + { + return; + } + + if (string.IsNullOrEmpty(Options.TargetFile)) + { + Log.LogError("Cannot save work item mappings. Path to target file ('TargetFile' option) is not set."); + } + else + { + Log.LogInformation("Saving work item mappings to file '{targetFile}'.", Options.TargetFile); + Dictionary allMappings = new(_mappings); + if (Options.PreserveExisting) + { + Log.LogInformation($"'{nameof(Options.PreserveExisting)}' is set to 'true'." + + " Loading existing work item mappings from '{targetFile}'.", Options.TargetFile); + Dictionary existingMappings = LoadExistingMappings(); + allMappings = MergeWithExistingMappings(_mappings, existingMappings); + } + SaveMappingsCore(allMappings); + } + } + + private void SaveMappingsCore(Dictionary mappings) + { + try + { + string tempFile = Options.TargetFile + ".tmp"; + using (FileStream target = File.Create(tempFile)) + { + JsonSerializer.Serialize(target, mappings, _jsonOptions); + } + File.Copy(tempFile, Options.TargetFile, overwrite: true); + File.Delete(tempFile); + } + catch (Exception ex) + { + Log.LogError(ex, "Failed to save work item mappings to '{targetFile}'.", Options.TargetFile); + } + } + + private Dictionary LoadExistingMappings() + { + try + { + if (File.Exists(Options.TargetFile)) + { + using Stream source = File.OpenRead(Options.TargetFile); + Dictionary? existing = JsonSerializer.Deserialize>(source); + return existing is null ? [] : new Dictionary(existing, StringComparer.OrdinalIgnoreCase); + } + } + catch (Exception ex) + { + Log.LogError(ex, "Failed to load existing work item mappings from '{targetFile}'.", Options.TargetFile); + } + return []; + } + + private Dictionary MergeWithExistingMappings( + IDictionary mappings, + IDictionary existingMappings) + { + Dictionary result = new(mappings, StringComparer.OrdinalIgnoreCase); + + foreach (KeyValuePair existingMapping in existingMappings) + { + string sourceId = existingMapping.Key; + string existingTargetId = existingMapping.Value; + if (result.TryGetValue(sourceId, out string currentTargetId) + && !currentTargetId.Equals(existingTargetId, StringComparison.OrdinalIgnoreCase)) + { + Log.LogWarning("Current mapping for source work item ID '{sourceId}' is '{currentTargetId}'" + + " which is different from preserved target ID '{existingTargetId}'." + + " Preserved target ID will be discarded and current one will be used.", + sourceId, currentTargetId, existingTargetId); + continue; + } + result[sourceId] = existingTargetId; + } + return result; + } +} diff --git a/src/MigrationTools/Tools/ExportWorkItemMappingToolOptions.cs b/src/MigrationTools/Tools/ExportWorkItemMappingToolOptions.cs new file mode 100644 index 000000000..db216eb08 --- /dev/null +++ b/src/MigrationTools/Tools/ExportWorkItemMappingToolOptions.cs @@ -0,0 +1,22 @@ +using MigrationTools.Tools.Infrastructure; + +namespace MigrationTools.Tools; + +/// +/// Options for . +/// +public class ExportWorkItemMappingToolOptions : ToolOptions +{ + /// + /// Path to file, where work item mapping will be saved. + /// + /// String.Empty + public string TargetFile { get; set; } = string.Empty; + + /// + /// Indicates whether existing mappings in the target file should be preserved when saving new mappings. + /// Default value is . + /// + /// true + public bool PreserveExisting { get; set; } = true; +} diff --git a/src/MigrationTools/Tools/ExportWorkItemMappingToolOptionsValidator.cs b/src/MigrationTools/Tools/ExportWorkItemMappingToolOptionsValidator.cs new file mode 100644 index 000000000..fa474269c --- /dev/null +++ b/src/MigrationTools/Tools/ExportWorkItemMappingToolOptionsValidator.cs @@ -0,0 +1,17 @@ +using Microsoft.Extensions.Options; + +namespace MigrationTools.Tools +{ + internal class ExportWorkItemMappingToolOptionsValidator : IValidateOptions + { + public ValidateOptionsResult Validate(string name, ExportWorkItemMappingToolOptions options) + { + if (options.Enabled && string.IsNullOrWhiteSpace(options.TargetFile)) + { + const string msg = $"'{nameof(options.TargetFile)}' is not set, so work item mappings cannot be saved."; + return ValidateOptionsResult.Fail(msg); + } + return ValidateOptionsResult.Success; + } + } +} diff --git a/src/MigrationTools/Tools/Interfaces/IExportWorkItemMappingTool.cs b/src/MigrationTools/Tools/Interfaces/IExportWorkItemMappingTool.cs new file mode 100644 index 000000000..b5cd61fee --- /dev/null +++ b/src/MigrationTools/Tools/Interfaces/IExportWorkItemMappingTool.cs @@ -0,0 +1,19 @@ +namespace MigrationTools.Tools.Interfaces; + +/// +/// Tool for exporting mappings of work item IDs from source to target. +/// +public interface IExportWorkItemMappingTool +{ + /// + /// Add new work item mapping. + /// + /// Source work item ID. + /// Target work item ID. + void AddMapping(string sourceId, string targetId); + + /// + /// Save mappings. + /// + void SaveMappings(); +} diff --git a/src/MigrationTools/Tools/StaticTools.cs b/src/MigrationTools/Tools/StaticTools.cs index da9b61a5f..6f4dcfe54 100644 --- a/src/MigrationTools/Tools/StaticTools.cs +++ b/src/MigrationTools/Tools/StaticTools.cs @@ -17,6 +17,11 @@ public class CommonTools /// public IWorkItemTypeMappingTool WorkItemTypeMapping { get; private set; } + /// + /// Gets the work item mapping tool for exporting work item mappings. + /// + public IExportWorkItemMappingTool ExportWorkItemMapping { get; private set; } + /// /// Gets the field mapping tool for applying field transformations. /// @@ -27,14 +32,17 @@ public class CommonTools /// /// Tool for string field manipulation. /// Tool for work item type mapping. + /// Tool for exporting work item mapping. /// Tool for field mapping operations. public CommonTools( IStringManipulatorTool StringManipulatorTool, IWorkItemTypeMappingTool workItemTypeMapping, + IExportWorkItemMappingTool exportWorkItemMapping, IFieldMappingTool fieldMappingTool) { StringManipulator = StringManipulatorTool; WorkItemTypeMapping = workItemTypeMapping; + ExportWorkItemMapping = exportWorkItemMapping; FieldMappingTool = fieldMappingTool; } }