Merge pull request #31 from Techsola/directory_filter

Add ability to migrate only selected folders within the root folder

Author: jnm2

Readme.md

@@ -48,6 +48,12 @@ Options:
                                            first changeset under the given source path.
   --max-changeset <max-changeset>          The last changeset to migrate. Defaults to the most recent
                                            changeset under the given source path.
+  --directories <directories>              If this option is used, only the files within the specified
+                                           directories (relative to the root path) will be migrated. If
+                                           a file moves into this filter, the migrated result will
+                                           appear with no prior history. If a file moves out of this
+                                           filter, it will appear to be deleted. Wildcards are not
+                                           currently supported.
   --root-path-changes <root-path-changes>  Followed by one or more arguments with the format
                                            CS1234:$/New/Path. Changes the path that is mapped as the Git
                                            repository root to a new path during a specified changeset.

src/TfvcMigrator.Tests/EntryPointTests.cs

@@ -30,6 +30,7 @@ await Program.Main(new[]
                 "--out-dir", "somedir",
                 "--min-changeset", "42",
                 "--max-changeset", "43",
+                "--directories", "a/", "b/c",
                 "--root-path-changes", "CS1234:$/New/Path", "CS1235:$/Another/Path",
                 "--pat", "somepat",
             }));
@@ -40,11 +41,12 @@ await Program.Main(new[]
         arguments[3].ShouldBe("somedir");
         arguments[4].ShouldBe(42);
         arguments[5].ShouldBe(43);
-        arguments[6].ShouldBeOfType<ImmutableArray<RootPathChange>>().ShouldBe(new[]
+        arguments[6].ShouldBe(new[] { "a/", "b/c" });
+        arguments[7].ShouldBeOfType<ImmutableArray<RootPathChange>>().ShouldBe(new[]
         {
             new RootPathChange(1234, "$/New/Path"),
             new RootPathChange(1235, "$/Another/Path"),
         });
-        arguments[7].ShouldBe("somepat");
+        arguments[8].ShouldBe("somepat");
     }
 }

src/TfvcMigrator/PathUtils.cs

@@ -20,12 +20,12 @@ public static bool Contains(string parentPath, string otherPath)
             && otherPath[parentPath.Length] == '/' && otherPath.StartsWith(parentPath, StringComparison.OrdinalIgnoreCase);
     }
 
-    public static bool IsOrContains(string parentPath, string otherPath)
+    public static bool IsOrContains(ReadOnlySpan<char> parentPath, ReadOnlySpan<char> otherPath)
     {
-        if (parentPath.EndsWith('/'))
+        if (parentPath.EndsWith("/"))
             throw new ArgumentException("Path should not end with a trailing slash.", nameof(parentPath));
 
-        if (otherPath.EndsWith('/'))
+        if (otherPath.EndsWith("/"))
             throw new ArgumentException("Path should not end with a trailing slash.", nameof(otherPath));
 
         return otherPath.Length > parentPath.Length + 1

src/TfvcMigrator/Program.cs

@@ -2,6 +2,7 @@
 using System.CommandLine.NamingConventionBinder;
 using System.Globalization;
 using System.Text;
+using System.Text.RegularExpressions;
 using LibGit2Sharp;
 using Microsoft.TeamFoundation.SourceControl.WebApi;
 using Microsoft.VisualStudio.Services.Common;
@@ -26,6 +27,14 @@ public static Task<int> Main(string[] args)
             new Option<string?>("--out-dir") { Description = "The directory path at which to create a new Git repository. Defaults to the last segment in the root path under the current directory." },
             new Option<int?>("--min-changeset") { Description = "The changeset defining the initial commit. Defaults to the first changeset under the given source path." },
             new Option<int?>("--max-changeset") { Description = "The last changeset to migrate. Defaults to the most recent changeset under the given source path." },
+            new Option<ImmutableArray<string>>(
+                "--directories",
+                parseArgument: result => result.Tokens.Select(token => token.Value).ToImmutableArray())
+            {
+                Arity = ArgumentArity.OneOrMore,
+                AllowMultipleArgumentsPerToken = true,
+                Description = "If this option is used, only the files within the specified directories (relative to the root path) will be migrated. If a file moves into this filter, the migrated result will appear with no prior history. If a file moves out of this filter, it will appear to be deleted. Wildcards are not currently supported.",
+            },
             new Option<ImmutableArray<RootPathChange>>(
                 "--root-path-changes",
                 parseArgument: result => result.Tokens.Select(token => ParseRootPathChange(token.Value)).ToImmutableArray())
@@ -65,13 +74,22 @@ public static async Task<int> MigrateAsync(
         string? outDir = null,
         int? minChangeset = null,
         int? maxChangeset = null,
+        ImmutableArray<string> directories = default,
         ImmutableArray<RootPathChange> rootPathChanges = default,
         string? pat = null)
     {
         try
         {
+            if (directories.IsDefault) directories = ImmutableArray<string>.Empty;
             if (rootPathChanges.IsDefault) rootPathChanges = ImmutableArray<RootPathChange>.Empty;
 
+            directories = ImmutableArray.CreateRange(
+                directories,
+                directory => '/' + Regex.Replace(directory.Trim('/', '\\'), @"[/\\]+", "/"));
+
+            if (directories is [] || directories.Contains("/"))
+                directories = ImmutableArray.Create("");
+
             var outputDirectory = Path.GetFullPath(
                 new[] { outDir, PathUtils.GetLeaf(rootPath), projectCollectionUrl.Segments.LastOrDefault() }
                     .First(name => !string.IsNullOrEmpty(name))!);
@@ -90,28 +108,38 @@ pat is not null
 
             using var client = await connection.GetClientAsync<TfvcHttpClient>();
 
-            Console.WriteLine("Downloading changeset and label metadata...");
+            Console.WriteLine("Downloading changeset metadata...");
 
-            var (changesets, allLabels) = await (
+            var changesets = (await Task.WhenAll(directories.Select(directory =>
                 client.GetChangesetsAsync(
                     maxCommentLength: int.MaxValue,
                     top: int.MaxValue,
                     orderby: "ID asc",
                     searchCriteria: new TfvcChangesetSearchCriteria
                     {
                         FollowRenames = true,
-                        ItemPath = rootPath,
+                        ItemPath = rootPath + directory,
                         FromId = minChangeset ?? 0,
                         ToId = maxChangeset ?? 0,
-                    }),
+                    }))).ConfigureAwait(false))
+                .SelectMany(changesets => changesets)
+                .DistinctBy(changeset => changeset.ChangesetId)
+                .OrderBy(changeset => changeset.ChangesetId)
+                .ToList();
+
+            Console.WriteLine("Downloading labels...");
+
+            var allLabels = (await Task.WhenAll(directories.Select(directory =>
                 client.GetLabelsAsync(
-                    new TfvcLabelRequestData
-                    {
-                        MaxItemCount = int.MaxValue,
-                        LabelScope = rootPath,
-                    },
-                    top: int.MaxValue)
-            ).ConfigureAwait(false);
+                new TfvcLabelRequestData
+                {
+                    MaxItemCount = int.MaxValue,
+                    LabelScope = rootPath + directory,
+                },
+                top: int.MaxValue))).ConfigureAwait(false))
+                .SelectMany(labels => labels)
+                .DistinctBy(label => label.Id)
+                .ToList();
 
             var authorsLookup = await LoadAuthors(
                 authors,
@@ -135,14 +163,15 @@ pat is not null
             var commitsByChangeset = new Dictionary<int, List<(Commit Commit, BranchIdentity Branch, bool WasCreatedForChangeset)>>();
 
             await using var mappingStateAndItemsEnumerator =
-                EnumerateMappingStatesAsync(client, rootPathChanges, changesets, initialBranch)
+                EnumerateMappingStatesAsync(client, directories, rootPathChanges, changesets, initialBranch)
                     .SelectAwait(async state => (
                         MappingState: state,
                         // Make no attempt to reason about applying TFS item changes over time. Ask for the full set of files.
                         Items: await DownloadItemsAsync(
                             client,
-                            PathUtils.GetNonOverlappingPaths(
-                                state.BranchMappingsInDependentOperationOrder.Select(branchMapping => branchMapping.Mapping.RootDirectory)),
+                            from branchMapping in state.BranchMappingsInDependentOperationOrder
+                            from directory in directories
+                            select branchMapping.Mapping.RootDirectory + directory,
                             state.Changeset)))
                     .WithLookahead()
                     .GetAsyncEnumerator();
@@ -438,6 +467,7 @@ private static void ReportProgress(int changeset, int total, TimedProgress timed
 
     private static async IAsyncEnumerable<MappingState> EnumerateMappingStatesAsync(
         TfvcHttpClient client,
+        ImmutableArray<string> directories,
         ImmutableArray<RootPathChange> rootPathChanges,
         IReadOnlyList<TfvcChangesetRef> changesets,
         BranchIdentity initialBranch)
@@ -451,7 +481,7 @@ private static async IAsyncEnumerable<MappingState> EnumerateMappingStatesAsync(
 
         await using var changesetChangesEnumerator = changesets
             .Skip(1)
-            .SelectAwait(changeset => client.GetChangesetChangesAsync(changeset.ChangesetId, top: int.MaxValue - 1))
+            .SelectAwait(changeset => GetChangesAsync(client, changeset.ChangesetId, trunk.Path, directories))
             .WithLookahead()
             .GetAsyncEnumerator();
 
@@ -606,6 +636,36 @@ private static ImmutableDictionary<string, Identity> LoadAuthors(Stream authorsF
         return builder.ToImmutable();
     }
 
+    private static async Task<List<TfvcChange>> GetChangesAsync(TfvcHttpClient client, int changesetId, string rootPath, ImmutableArray<string> directories)
+    {
+        var changes = await client.GetChangesetChangesAsync(changesetId, top: int.MaxValue - 1);
+
+        changes.RemoveAll(change =>
+            !ItemPathPassesFilter(change.Item.Path, rootPath, directories)
+            && !ItemPathPassesFilter(change.SourceServerItem, rootPath, directories));
+
+        return changes;
+    }
+
+    private static bool ItemPathPassesFilter(string itemPath, string rootPath, ImmutableArray<string> directories)
+    {
+        if (!PathUtils.IsAbsolute(rootPath))
+            throw new ArgumentException("Root path must be absolute.", nameof(rootPath));
+
+        if (PathUtils.IsOrContains(rootPath, itemPath))
+        {
+            var remainingPath = itemPath.AsSpan(rootPath.Length);
+
+            foreach (var directory in directories)
+            {
+                if (PathUtils.IsOrContains(directory, remainingPath))
+                    return true;
+            }
+        }
+
+        return false;
+    }
+
     private static async Task<ImmutableArray<TfvcItem>> DownloadItemsAsync(TfvcHttpClient client, IEnumerable<string> scopePaths, int changeset)
     {
         var union = PathUtils.GetNonOverlappingPaths(scopePaths);