Merge pull request #71 from sandeepsnairms/main

sandeepsnairms · web-flow · commit b27a3f8c88b9 · 2025-04-15T21:40:04.000+05:30
Add Simulated Run
diff --git a/MongoMigrationWebApp/Components/MigrationDetails.razor b/MongoMigrationWebApp/Components/MigrationDetails.razor
@@ -43,6 +43,11 @@
                     <textarea id="nameSpaces" disabled="@(!newMode)" rows="4" placeholder="e.g. db1.col1,db1.col2,db2.col1,db2.col5" class="form-control" @bind="namespaces"></textarea>
                 </div>
 
+                <div class="mb-3">
+                    <label for="isSimulatedRun" class="form-label">Simulation Mode: No Target Writes</label>
+                    <input type="checkbox" id="isSimulatedRun" disabled="@(!newMode)" @bind="isSimulatedRun" />
+                </div>
+
                 @if (!string.IsNullOrEmpty(errorMessage))
                 {
                     <div class="alert alert-danger mt-2">@errorMessage</div>
@@ -93,7 +98,8 @@
     private string errorMessage = string.Empty;
     private bool useMongoDump = true; // Variable to track the selected value
     private string selectedOption = "Use MongoDump and MongoRestore"; // Default dropdown value
-
+    private bool isSimulatedRun = false; // Variable to track the dummy run option 
+    
     private void OnSelectionChanged(ChangeEventArgs e)
     {
         selectedOption = e.Value.ToString();
@@ -168,7 +174,8 @@
                 SourceEndpoint = tmpSrcEndpoint,
                 TargetEndpoint = tmpTgtEndpoint,
                 NameSpaces = namespaces,
-                UseMongoDump = useMongoDump
+                UseMongoDump = useMongoDump,
+                IsSimulatedRun = isSimulatedRun
             };
         await OnSubmit.InvokeAsync(job);
     }
diff --git a/MongoMigrationWebApp/MongoMigrationWebApp.csproj b/MongoMigrationWebApp/MongoMigrationWebApp.csproj
@@ -8,6 +8,16 @@
     <DockerDefaultTargetOS>Windows</DockerDefaultTargetOS>
   </PropertyGroup>
 
+  <ItemGroup>
+    <None Remove="readme.txt" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Content Include="readme.txt">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
   </ItemGroup>
diff --git a/MongoMigrationWebApp/Shared/MainLayout.razor b/MongoMigrationWebApp/Shared/MainLayout.razor
@@ -6,7 +6,7 @@
     <main>
         <div class="top-row px-4 d-flex justify-content-between align-items-center">
             <h2 class="text-left">Migrate to Azure Cosmos DB for MongoDB (vCore based)</h2>
-            <div class="ms-2 small">v0.7.9</div>
+            <div class="ms-2 small">v0.7.11</div>
         </div>
 
         <article class="content px-4">
diff --git a/MongoMigrationWebApp/readme.txt b/MongoMigrationWebApp/readme.txt
@@ -0,0 +1,3 @@
+Version: 0.7.10
+Instructions: https://github.com/AzureCosmosDB/MongoMigrationWebBasedUtility
+Purpose: An online and offline migration utility designed to migrate any MongoDB source to Azure Cosmos DB for Mongo vCore. It operates as an Azure Web App (or on-premises) and offers multiple methods for data migration. It can either use mongodump and mongorestore for data movement or employ the MongoDB driver to read data from the source and write it
diff --git a/OnlineMongoMigrationProcessor/Helpers/Helper.cs b/OnlineMongoMigrationProcessor/Helpers/Helper.cs
@@ -267,6 +267,9 @@ public static bool IsOfflineJobCompleted(MigrationJob migrationJob)
             {
                 if (mu.SourceStatus == CollectionStatus.OK)
                 {
+                    if(mu.DumpComplete && migrationJob.IsSimulatedRun)
+                        return true;
+                    
                     if (!mu.RestoreComplete || !mu.DumpComplete)
                         return false;
                 }
diff --git a/OnlineMongoMigrationProcessor/MigrationWorker.cs b/OnlineMongoMigrationProcessor/MigrationWorker.cs
@@ -146,6 +146,10 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS
                 {
                     _sourceClient = new MongoClient(sourceConnectionString);
                     Log.WriteLine("Source Client Created");
+                    if(job.IsSimulatedRun)
+                    {
+                        Log.WriteLine("Simulated Run. No changes will be made to the target.");
+                    }                   
                     Log.Save();
 
 
@@ -213,7 +217,7 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS
 
 
 
-                                if (!_job.UseMongoDump)
+                                if (!_job.UseMongoDump && !job.IsSimulatedRun)
                                 {
                                     var database = _sourceClient.GetDatabase(unit.DatabaseName);
                                     var collection = database.GetCollection<BsonDocument>(unit.CollectionName);
diff --git a/OnlineMongoMigrationProcessor/Models/DataModels.cs b/OnlineMongoMigrationProcessor/Models/DataModels.cs
@@ -88,6 +88,7 @@ public class MigrationJob
         public bool IsStarted { get; set; }
         public bool CurrentlyActive { get; set; }
         public bool UseMongoDump { get; set; }
+        public bool IsSimulatedRun { get; set; }
         public List<MigrationUnit>? MigrationUnits { get; set; }
     }
 
diff --git a/OnlineMongoMigrationProcessor/Processors/CopyProcessor.cs b/OnlineMongoMigrationProcessor/Processors/CopyProcessor.cs
@@ -138,7 +138,7 @@ public void Migrate(MigrationUnit item, string sourceConnectionString, string ta
 
                                 var documentCopier = new MongoDocumentCopier();
                                 documentCopier.Initialize(_targetClient, collection, dbName, colName, _config.MongoCopyPageSize);
-                                var result = documentCopier.CopyDocumentsAsync(_jobs, item, i, initialPercent, contributionFactor, docCount, filter, _cts.Token).GetAwaiter().GetResult();
+                                var result = documentCopier.CopyDocumentsAsync(_jobs, item, i, initialPercent, contributionFactor, docCount, filter, _cts.Token,_job.IsSimulatedRun).GetAwaiter().GetResult();
 
                                 if (result)
                                 {
diff --git a/OnlineMongoMigrationProcessor/Processors/DumpRestoreProcessor.cs b/OnlineMongoMigrationProcessor/Processors/DumpRestoreProcessor.cs
@@ -179,7 +179,7 @@ public void Migrate(MigrationUnit item, string sourceConnectionString, string ta
                                     _jobs?.Save(); // Persist state
                                     dumpAttempts = 0;
 
-                                    if (!restoreInvoked)
+                                    if (!restoreInvoked )
                                     {
                                         Log.WriteLine($"{dbName}.{colName} Uploader invoked");
 
@@ -282,7 +282,7 @@ private void Upload(MigrationUnit item, string targetConnectionString)
 
             Log.WriteLine($"{dbName}.{colName} Uploader started");
 
-            while (!item.RestoreComplete && Directory.Exists(folder) && !_executionCancelled && _job.CurrentlyActive)
+            while (!item.RestoreComplete && Directory.Exists(folder) && !_executionCancelled && _job.CurrentlyActive && !_job.IsSimulatedRun)
             {
                 int restoredChunks = 0;
                 long restoredDocs = 0;
@@ -314,7 +314,7 @@ private void Upload(MigrationUnit item, string targetConnectionString)
                             backoff = TimeSpan.FromSeconds(2);
                             bool continueProcessing = true;
                             bool skipRestore = false;
-                            while (restoreAttempts < maxRetries && !_executionCancelled && continueProcessing && !item.RestoreComplete && _job.CurrentlyActive)
+                            while (restoreAttempts < maxRetries && !_executionCancelled && continueProcessing && !item.RestoreComplete && _job.CurrentlyActive )
                             {
                                 restoreAttempts++;
                                 skipRestore=false;
@@ -348,7 +348,7 @@ private void Upload(MigrationUnit item, string targetConnectionString)
                                             }
 
                                             // checking if source  and target doc counts are same
-                                            if (item.MigrationChunks[i].DocCountInTarget == item.MigrationChunks[i].DumpResultDocCount)
+                                            if (item.MigrationChunks[i].DocCountInTarget == item.MigrationChunks[i].DumpQueryDocCount)
                                             {
                                                 Log.WriteLine($"Restore for {dbName}.{colName}-{i} No documents missing, count in Target: {item.MigrationChunks[i].DocCountInTarget}");
                                                 Log.Save();
@@ -468,11 +468,14 @@ private void Upload(MigrationUnit item, string targetConnectionString)
                     }
                 }
             }
-            if (item.RestoreComplete && item.DumpComplete)
+            if ((item.RestoreComplete && item.DumpComplete)|| (item.DumpComplete && _job.IsSimulatedRun))
             {
                 try
                 {
-                    Directory.Delete(folder, true);
+
+                    if (Directory.Exists(folder))
+                        Directory.Delete(folder, true);
+
                     // Process change streams
                     if (_job.IsOnline && !_executionCancelled)
                     {
diff --git a/OnlineMongoMigrationProcessor/Processors/MongoChangeStreamProcessor.cs b/OnlineMongoMigrationProcessor/Processors/MongoChangeStreamProcessor.cs
@@ -219,7 +219,7 @@ private bool ProcessCursor(MigrationJob job, ChangeStreamDocument<BsonDocument>
 
                     // Output change details to the console
                     Log.AddVerboseMessage($"{change.OperationType} operation detected in {targetCollection.CollectionNamespace} for _id: {change.DocumentKey["_id"]} having TS (UTC): {MongoHelper.BsonTimestampToUtcDateTime(timestamp)}");
-                    ProcessChange(change, targetCollection);
+                    ProcessChange(change, targetCollection,job.IsSimulatedRun);
                     item.CursorUtcTimestamp = MongoHelper.BsonTimestampToUtcDateTime(timestamp);
                 }
                 else if (!job.SourceServerVersion.StartsWith("3") && change.WallTime != null) //for vcore
@@ -229,14 +229,14 @@ private bool ProcessCursor(MigrationJob job, ChangeStreamDocument<BsonDocument>
 
                     // Output change details to the console
                     Log.AddVerboseMessage($"{change.OperationType} operation detected in {targetCollection.CollectionNamespace} for _id: {change.DocumentKey["_id"]} having TS (UTC): {timestamp.Value}");
-                    ProcessChange(change, targetCollection);
+                    ProcessChange(change, targetCollection, job.IsSimulatedRun);
                     item.CursorUtcTimestamp = timestamp.Value;
                 }
                 else
                 {
                     // Output change details to the console
                     Log.AddVerboseMessage($"{change.OperationType} operation detected in {targetCollection.CollectionNamespace} for _id: {change.DocumentKey["_id"]}");
-                    ProcessChange(change, targetCollection);
+                    ProcessChange(change, targetCollection, job.IsSimulatedRun);
                 }
                 item.ResumeToken = cursor.Current.FirstOrDefault().ResumeToken.ToJson();
                 _jobs?.Save(); // persists state
@@ -262,8 +262,11 @@ private bool ProcessCursor(MigrationJob job, ChangeStreamDocument<BsonDocument>
         }
 
 
-        private void ProcessChange(ChangeStreamDocument<BsonDocument> change, IMongoCollection<BsonDocument> targetCollection)
+        private void ProcessChange(ChangeStreamDocument<BsonDocument> change, IMongoCollection<BsonDocument> targetCollection, bool isWriteSimulated)
         {
+            if (isWriteSimulated)
+                return;
+
             try
             {
                 switch (change.OperationType)
diff --git a/OnlineMongoMigrationProcessor/Processors/MongoDocumentCopier.cs b/OnlineMongoMigrationProcessor/Processors/MongoDocumentCopier.cs
@@ -79,7 +79,8 @@ public async Task<bool> CopyDocumentsAsync(
             double contribFactor,
             long targetCount,
             FilterDefinition<BsonDocument> filter,
-            CancellationToken cancellationToken)
+            CancellationToken cancellationToken,
+            bool isWriteSimulated)
         {
             ConcurrentBag<Exception> errors = new ConcurrentBag<Exception>();
             try
@@ -132,7 +133,7 @@ public async Task<bool> CopyDocumentsAsync(
                         try
                         {
                             
-                            await ProcessSegmentAsync(segment, combinedFilter, jobList, item, migrationChunkIndex, basePercent, contribFactor, targetCount, errors, cancellationToken);
+                            await ProcessSegmentAsync(segment, combinedFilter, jobList, item, migrationChunkIndex, basePercent, contribFactor, targetCount, errors, cancellationToken, isWriteSimulated);
                         }
                         finally
                         {
@@ -181,7 +182,7 @@ public async Task<bool> CopyDocumentsAsync(
                     return false;
                 }
 
-                if (item.MigrationChunks[migrationChunkIndex].DocCountInTarget == item.MigrationChunks[migrationChunkIndex].DumpResultDocCount)
+                if (item.MigrationChunks[migrationChunkIndex].DocCountInTarget == item.MigrationChunks[migrationChunkIndex].DumpQueryDocCount)
                 {
                     Log.WriteLine($"Document copy for chunk [{migrationChunkIndex}], verified no documents missing, count in target: {item.MigrationChunks[migrationChunkIndex].DocCountInTarget}");
                     Log.Save();
@@ -215,7 +216,8 @@ private async Task ProcessSegmentAsync(
             double contribFactor,
             long targetCount,
             ConcurrentBag<Exception> errors,
-            CancellationToken cancellationToken)
+            CancellationToken cancellationToken,
+            bool IsWriteSimulated)
         {
               
             string segmentId = segment.Id;
@@ -295,13 +297,14 @@ private async Task ProcessSegmentAsync(
                             break;
                         }
 
-                        var options = new InsertManyOptions { IsOrdered = false };
-                        // Insert the current batch into the target collection
-                        await _targetCollection.InsertManyAsync(set, options,cancellationToken: cancellationToken);
-
+                        if (!IsWriteSimulated)
+                        {
+                            var options = new InsertManyOptions { IsOrdered = false };
+                            // Insert the current batch into the target collection
+                            await _targetCollection.InsertManyAsync(set, options, cancellationToken: cancellationToken);                           
+                        }
                         Interlocked.Add(ref _successCount, set.Count);
 
-   
                     }
                     catch (OutOfMemoryException ex)
                     {

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+Version: 0.7.10`
	`2`	`+Instructions: https://github.com/AzureCosmosDB/MongoMigrationWebBasedUtility`
	`3`	`+Purpose: An online and offline migration utility designed to migrate any MongoDB source to Azure Cosmos DB for Mongo vCore. It operates as an Azure Web App (or on-premises) and offers multiple methods for data migration. It can either use mongodump and mongorestore for data movement or employ the MongoDB driver to read data from the source and write it`
Original file line number	Diff line number	Diff line change
`@@ -267,6 +267,9 @@ public static bool IsOfflineJobCompleted(MigrationJob migrationJob)`
`267`	`267`	`{`
`268`	`268`	`if (mu.SourceStatus == CollectionStatus.OK)`
`269`	`269`	`{`
	`270`	`+ if(mu.DumpComplete && migrationJob.IsSimulatedRun)`
	`271`	`+ return true;`
	`272`	`+`
`270`	`273`	`if (!mu.RestoreComplete \|\| !mu.DumpComplete)`
`271`	`274`	`return false;`
`272`	`275`	`}`
Original file line number	Diff line number	Diff line change
`@@ -146,6 +146,10 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS`
`146`	`146`	`{`
`147`	`147`	`_sourceClient = new MongoClient(sourceConnectionString);`
`148`	`148`	`Log.WriteLine("Source Client Created");`
	`149`	`+ if(job.IsSimulatedRun)`
	`150`	`+ {`
	`151`	`+ Log.WriteLine("Simulated Run. No changes will be made to the target.");`
	`152`	`+ }`
`149`	`153`	`Log.Save();`
`150`	`154`
`151`	`155`
`@@ -213,7 +217,7 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS`
`213`	`217`
`214`	`218`
`215`	`219`
`216`		`- if (!_job.UseMongoDump)`
	`220`	`+ if (!_job.UseMongoDump && !job.IsSimulatedRun)`
`217`	`221`	`{`
`218`	`222`	`var database = _sourceClient.GetDatabase(unit.DatabaseName);`
`219`	`223`	`var collection = database.GetCollection<BsonDocument>(unit.CollectionName);`
Original file line number	Diff line number	Diff line change
`@@ -88,6 +88,7 @@ public class MigrationJob`
`88`	`88`	`public bool IsStarted { get; set; }`
`89`	`89`	`public bool CurrentlyActive { get; set; }`
`90`	`90`	`public bool UseMongoDump { get; set; }`
	`91`	`+ public bool IsSimulatedRun { get; set; }`
`91`	`92`	`public List<MigrationUnit>? MigrationUnits { get; set; }`
`92`	`93`	`}`
`93`	`94`
Original file line number	Diff line number	Diff line change
`@@ -138,7 +138,7 @@ public void Migrate(MigrationUnit item, string sourceConnectionString, string ta`
`138`	`138`
`139`	`139`	`var documentCopier = new MongoDocumentCopier();`
`140`	`140`	`documentCopier.Initialize(_targetClient, collection, dbName, colName, _config.MongoCopyPageSize);`
`141`		`- var result = documentCopier.CopyDocumentsAsync(_jobs, item, i, initialPercent, contributionFactor, docCount, filter, _cts.Token).GetAwaiter().GetResult();`
	`141`	`+ var result = documentCopier.CopyDocumentsAsync(_jobs, item, i, initialPercent, contributionFactor, docCount, filter, _cts.Token,_job.IsSimulatedRun).GetAwaiter().GetResult();`
`142`	`142`
`143`	`143`	`if (result)`
`144`	`144`	`{`
Original file line number	Diff line number	Diff line change
`@@ -179,7 +179,7 @@ public void Migrate(MigrationUnit item, string sourceConnectionString, string ta`
`179`	`179`	`_jobs?.Save(); // Persist state`
`180`	`180`	`dumpAttempts = 0;`
`181`	`181`
`182`		`- if (!restoreInvoked)`
	`182`	`+ if (!restoreInvoked )`
`183`	`183`	`{`
`184`	`184`	`Log.WriteLine($"{dbName}.{colName} Uploader invoked");`
`185`	`185`
`@@ -282,7 +282,7 @@ private void Upload(MigrationUnit item, string targetConnectionString)`
`282`	`282`
`283`	`283`	`Log.WriteLine($"{dbName}.{colName} Uploader started");`
`284`	`284`
`285`		`- while (!item.RestoreComplete && Directory.Exists(folder) && !_executionCancelled && _job.CurrentlyActive)`
	`285`	`+ while (!item.RestoreComplete && Directory.Exists(folder) && !_executionCancelled && _job.CurrentlyActive && !_job.IsSimulatedRun)`
`286`	`286`	`{`
`287`	`287`	`int restoredChunks = 0;`
`288`	`288`	`long restoredDocs = 0;`
`@@ -314,7 +314,7 @@ private void Upload(MigrationUnit item, string targetConnectionString)`
`314`	`314`	`backoff = TimeSpan.FromSeconds(2);`
`315`	`315`	`bool continueProcessing = true;`
`316`	`316`	`bool skipRestore = false;`
`317`		`- while (restoreAttempts < maxRetries && !_executionCancelled && continueProcessing && !item.RestoreComplete && _job.CurrentlyActive)`
	`317`	`+ while (restoreAttempts < maxRetries && !_executionCancelled && continueProcessing && !item.RestoreComplete && _job.CurrentlyActive )`
`318`	`318`	`{`
`319`	`319`	`restoreAttempts++;`
`320`	`320`	`skipRestore=false;`
`@@ -348,7 +348,7 @@ private void Upload(MigrationUnit item, string targetConnectionString)`
`348`	`348`	`}`
`349`	`349`
`350`	`350`	`// checking if source and target doc counts are same`
`351`		`- if (item.MigrationChunks[i].DocCountInTarget == item.MigrationChunks[i].DumpResultDocCount)`
	`351`	`+ if (item.MigrationChunks[i].DocCountInTarget == item.MigrationChunks[i].DumpQueryDocCount)`
`352`	`352`	`{`
`353`	`353`	`Log.WriteLine($"Restore for {dbName}.{colName}-{i} No documents missing, count in Target: {item.MigrationChunks[i].DocCountInTarget}");`
`354`	`354`	`Log.Save();`
`@@ -468,11 +468,14 @@ private void Upload(MigrationUnit item, string targetConnectionString)`
`468`	`468`	`}`
`469`	`469`	`}`
`470`	`470`	`}`
`471`		`- if (item.RestoreComplete && item.DumpComplete)`
	`471`	`+ if ((item.RestoreComplete && item.DumpComplete)\|\| (item.DumpComplete && _job.IsSimulatedRun))`
`472`	`472`	`{`
`473`	`473`	`try`
`474`	`474`	`{`
`475`		`- Directory.Delete(folder, true);`
	`475`	`+`
	`476`	`+ if (Directory.Exists(folder))`
	`477`	`+ Directory.Delete(folder, true);`
	`478`	`+`
`476`	`479`	`// Process change streams`
`477`	`480`	`if (_job.IsOnline && !_executionCancelled)`
`478`	`481`	`{`
Original file line number	Diff line number	Diff line change
`@@ -219,7 +219,7 @@ private bool ProcessCursor(MigrationJob job, ChangeStreamDocument<BsonDocument>`
`219`	`219`
`220`	`220`	`// Output change details to the console`
`221`	`221`	`Log.AddVerboseMessage($"{change.OperationType} operation detected in {targetCollection.CollectionNamespace} for _id: {change.DocumentKey["_id"]} having TS (UTC): {MongoHelper.BsonTimestampToUtcDateTime(timestamp)}");`
`222`		`- ProcessChange(change, targetCollection);`
	`222`	`+ ProcessChange(change, targetCollection,job.IsSimulatedRun);`
`223`	`223`	`item.CursorUtcTimestamp = MongoHelper.BsonTimestampToUtcDateTime(timestamp);`
`224`	`224`	`}`
`225`	`225`	`else if (!job.SourceServerVersion.StartsWith("3") && change.WallTime != null) //for vcore`
`@@ -229,14 +229,14 @@ private bool ProcessCursor(MigrationJob job, ChangeStreamDocument<BsonDocument>`
`229`	`229`
`230`	`230`	`// Output change details to the console`
`231`	`231`	`Log.AddVerboseMessage($"{change.OperationType} operation detected in {targetCollection.CollectionNamespace} for _id: {change.DocumentKey["_id"]} having TS (UTC): {timestamp.Value}");`
`232`		`- ProcessChange(change, targetCollection);`
	`232`	`+ ProcessChange(change, targetCollection, job.IsSimulatedRun);`
`233`	`233`	`item.CursorUtcTimestamp = timestamp.Value;`
`234`	`234`	`}`
`235`	`235`	`else`
`236`	`236`	`{`
`237`	`237`	`// Output change details to the console`
`238`	`238`	`Log.AddVerboseMessage($"{change.OperationType} operation detected in {targetCollection.CollectionNamespace} for _id: {change.DocumentKey["_id"]}");`
`239`		`- ProcessChange(change, targetCollection);`
	`239`	`+ ProcessChange(change, targetCollection, job.IsSimulatedRun);`
`240`	`240`	`}`
`241`	`241`	`item.ResumeToken = cursor.Current.FirstOrDefault().ResumeToken.ToJson();`
`242`	`242`	`_jobs?.Save(); // persists state`
`@@ -262,8 +262,11 @@ private bool ProcessCursor(MigrationJob job, ChangeStreamDocument<BsonDocument>`
`262`	`262`	`}`
`263`	`263`
`264`	`264`
`265`		`- private void ProcessChange(ChangeStreamDocument<BsonDocument> change, IMongoCollection<BsonDocument> targetCollection)`
	`265`	`+ private void ProcessChange(ChangeStreamDocument<BsonDocument> change, IMongoCollection<BsonDocument> targetCollection, bool isWriteSimulated)`
`266`	`266`	`{`
	`267`	`+ if (isWriteSimulated)`
	`268`	`+ return;`
	`269`	`+`
`267`	`270`	`try`
`268`	`271`	`{`
`269`	`272`	`switch (change.OperationType)`