Skip to content

Commit bca2dcf

Browse files
sandeepsnairmssandeepsnairms
authored
Merge pull request #67 from AzureCosmosDB/v0.7
V0.7
2 parents 994fe67 + 608d80d commit bca2dcf

File tree

6 files changed

+99
-53
lines changed

6 files changed

+99
-53
lines changed

‎MongoMigrationWebApp/Shared/MainLayout.razor

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<main>
77
<div class="top-row px-4 d-flex justify-content-between align-items-center">
88
<h2 class="text-left">Migrate to Azure Cosmos DB for MongoDB (vCore based)</h2>
9-
<div class="ms-2 small">v0.7.5</div>
9+
<div class="ms-2 small">v0.7.6</div>
1010
</div>
1111

1212
<article class="content px-4">

‎OnlineMongoMigrationProcessor/Helpers/Helper.cs

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.IO.Compression;
66
using System.Text.RegularExpressions;
77
using System.Threading.Tasks;
8+
using System.Web;
89

910
#pragma warning disable CS8600 // Converting null literal or possible null value to non-nullable type.
1011
#pragma warning disable CS8602 // Dereference of a possibly null reference.
@@ -198,6 +199,27 @@ public static string GetWorkingFolder()
198199
return _workingFolder;
199200
}
200201

202+
public static string UpdateAppName(string connectionString, string appName)
203+
{
204+
if (string.IsNullOrWhiteSpace(connectionString))
205+
throw new ArgumentException("Connection string cannot be null or empty.", nameof(connectionString));
206+
207+
if (string.IsNullOrWhiteSpace(appName))
208+
throw new ArgumentException("App name cannot be null or empty.", nameof(appName));
209+
210+
var uri = new Uri(connectionString);
211+
var queryParams = HttpUtility.ParseQueryString(uri.Query);
212+
213+
// Set or update the appName parameter
214+
queryParams["appName"] = appName;
215+
216+
// Reconstruct the connection string with updated parameters
217+
var newQuery = queryParams.ToString();
218+
var updatedConnectionString = connectionString.Replace(uri.Query.ToString(),"?"+ newQuery);
219+
220+
return updatedConnectionString;
221+
}
222+
201223
public static Tuple<bool, string> ValidateNamespaceFormat(string input)
202224
{
203225
// Regular expression pattern to match db1.col1, db2.col2, db3.col4 format

‎OnlineMongoMigrationProcessor/Helpers/MongoHelper.cs

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using MongoDB.Bson;
22
using MongoDB.Driver;
33
using System;
4+
using System.Collections;
45
using System.Collections.Generic;
56
using System.Text.RegularExpressions;
67
using System.Threading.Tasks;
@@ -192,11 +193,6 @@ public static async Task<bool> CheckCollectionExists(MongoClient client, string
192193

193194
var database = client.GetDatabase(databaseName);
194195

195-
//var collectionNamesCursor = await database.ListCollectionNamesAsync();
196-
//var collectionNames = await collectionNamesCursor.ToListAsync();
197-
//return collectionNames.Contains(collectionName);
198-
199-
200196
var collection = database.GetCollection<BsonDocument>(collectionName);
201197

202198
// Try to find one document (limit query to 1 for efficiency)
@@ -207,6 +203,33 @@ public static async Task<bool> CheckCollectionExists(MongoClient client, string
207203
return document != null; // If a document is found, collection exists
208204
}
209205

206+
public static async Task<(long CollectionSizeBytes, long DocumentCount)> GetCollectionStatsAsync(MongoClient client, string databaseName, string collectionName)
207+
{
208+
var database = client.GetDatabase(databaseName);
209+
var collection = database.GetCollection<BsonDocument>(collectionName);
210+
211+
var statsCommand = new BsonDocument { { "collStats", collectionName } };
212+
var stats = await database.RunCommandAsync<BsonDocument>(statsCommand);
213+
long totalCollectionSizeBytes = stats.Contains("storageSize") ? stats["storageSize"].ToInt64() : stats["size"].ToInt64();
214+
215+
long documentCount;
216+
if (stats["count"].IsInt32)
217+
{
218+
documentCount = stats["count"].ToInt32();
219+
}
220+
else if (stats["count"].IsInt64)
221+
{
222+
documentCount = stats["count"].ToInt64();
223+
}
224+
else
225+
{
226+
throw new InvalidOperationException("Unexpected data type for document count.");
227+
}
228+
229+
return new (totalCollectionSizeBytes, documentCount);
230+
}
231+
232+
210233
public static async Task<bool> DeleteAndCopyIndexesAsync(string targetConnectionString, IMongoCollection<BsonDocument> sourceCollection)
211234
{
212235
try

‎OnlineMongoMigrationProcessor/MigrationWorker.cs

Lines changed: 16 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Collections;
23
using System.Collections.Generic;
34
using System.IO;
45
using System.Linq;
@@ -66,6 +67,8 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS
6667
sourceConnectionString = Helper.EncodeMongoPasswordInConnectionString(sourceConnectionString);
6768
targetConnectionString = Helper.EncodeMongoPasswordInConnectionString(targetConnectionString);
6869

70+
targetConnectionString = Helper.UpdateAppName(targetConnectionString, "MSFTMongoWebMigration-" + Guid.NewGuid().ToString());
71+
6972
if (Config == null)
7073
{
7174
Config = new MigrationSettings();
@@ -238,7 +241,15 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS
238241
{
239242
if (await MongoHelper.CheckCollectionExists(_sourceClient, migrationUnit.DatabaseName, migrationUnit.CollectionName))
240243
{
244+
var targetClient = new MongoClient(targetConnectionString);
245+
246+
if (await MongoHelper.CheckCollectionExists(targetClient, migrationUnit.DatabaseName, migrationUnit.CollectionName))
247+
{
248+
Log.WriteLine($"{migrationUnit.DatabaseName}.{migrationUnit.CollectionName} already exists on target");
249+
Log.Save();
250+
}
241251
_migrationProcessor.Migrate(migrationUnit, sourceConnectionString, targetConnectionString);
252+
242253
}
243254
else
244255
{
@@ -289,28 +300,14 @@ public async Task StartMigrationAsync(MigrationJob job, string sourceConnectionS
289300

290301
private async Task<List<MigrationChunk>> PartitionCollection(string databaseName, string collectionName, string idField = "_id")
291302
{
292-
var database = _sourceClient.GetDatabase(databaseName);
293-
var collection = database.GetCollection<BsonDocument>(collectionName);
294303

304+
var stas=await MongoHelper.GetCollectionStatsAsync(_sourceClient, databaseName, collectionName);
295305

296-
var statsCommand = new BsonDocument { { "collStats", collectionName } };
297-
var stats = await database.RunCommandAsync<BsonDocument>(statsCommand);
298-
long totalCollectionSizeBytes = stats.Contains("storageSize") ? stats["storageSize"].ToInt64() : stats["size"].ToInt64();
306+
long documentCount = stas.DocumentCount;
307+
long totalCollectionSizeBytes = stas.CollectionSizeBytes;
299308

300-
301-
long documentCount;
302-
if (stats["count"].IsInt32)
303-
{
304-
documentCount = stats["count"].ToInt32();
305-
}
306-
else if (stats["count"].IsInt64)
307-
{
308-
documentCount = stats["count"].ToInt64();
309-
}
310-
else
311-
{
312-
throw new InvalidOperationException("Unexpected data type for document count.");
313-
}
309+
var database = _sourceClient.GetDatabase(databaseName);
310+
var collection = database.GetCollection<BsonDocument>(collectionName);
314311

315312
int totalChunks = 0;
316313
long minDocsInChunk = 0;

‎OnlineMongoMigrationProcessor/Models/DataModels.cs

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,11 @@ public bool Save()
5353
lock (_fileLock)
5454
{
5555
string json = JsonConvert.SerializeObject(this);
56-
File.WriteAllText(_filePath, json);
56+
//File.WriteAllText(_filePath, json);
57+
string tempFile = _filePath + ".tmp";
58+
File.WriteAllText(tempFile, json);
59+
File.Move(tempFile, _filePath, true); // Atomic move on most OSes
60+
5761
}
5862
return true;
5963
}

‎OnlineMongoMigrationProcessor/Processors/MongoDocumentCopier.cs

Lines changed: 27 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -361,33 +361,33 @@ private async Task ProcessSegmentAsync(
361361
}
362362
}
363363

364-
private async Task<long> DeleteInBatchesAsync(IMongoCollection<BsonDocument> collection, FilterDefinition<BsonDocument> filter, int batchSize, string chunkindex)
365-
{
366-
long deletedCount = 0;
367-
int ctr = 1;
368-
while (true)
369-
{
370-
Log.AddVerboseMessage($"Getting page {ctr} to delete from target for segment {chunkindex}");
371-
372-
// Get a batch of document _ids to delete
373-
var batchIds = await collection.Find(filter)
374-
.Limit(batchSize)
375-
.Project(doc => doc["_id"])
376-
.ToListAsync();
377-
378-
if (batchIds.Count == 0)
379-
break; // No more documents to delete
380-
381-
// Delete documents in this batch
382-
var deleteFilter = Builders<BsonDocument>.Filter.In("_id", batchIds);
383-
384-
Log.AddVerboseMessage($"Deleting page {ctr} from target for segment {chunkindex}");
385-
var result = await collection.DeleteManyAsync(deleteFilter);
386-
deletedCount = deletedCount + result.DeletedCount;
387-
ctr++;
388-
}
389-
return deletedCount;
390-
}
364+
//private async Task<long> DeleteInBatchesAsync(IMongoCollection<BsonDocument> collection, FilterDefinition<BsonDocument> filter, int batchSize, string chunkindex)
365+
//{
366+
// long deletedCount = 0;
367+
// int ctr = 1;
368+
// while (true)
369+
// {
370+
// Log.AddVerboseMessage($"Getting page {ctr} to delete from target for segment {chunkindex}");
371+
372+
// // Get a batch of document _ids to delete
373+
// var batchIds = await collection.Find(filter)
374+
// .Limit(batchSize)
375+
// .Project(doc => doc["_id"])
376+
// .ToListAsync();
377+
378+
// if (batchIds.Count == 0)
379+
// break; // No more documents to delete
380+
381+
// // Delete documents in this batch
382+
// var deleteFilter = Builders<BsonDocument>.Filter.In("_id", batchIds);
383+
384+
// Log.AddVerboseMessage($"Deleting page {ctr} from target for segment {chunkindex}");
385+
// var result = await collection.DeleteManyAsync(deleteFilter);
386+
// deletedCount = deletedCount + result.DeletedCount;
387+
// ctr++;
388+
// }
389+
// return deletedCount;
390+
//}
391391

392392
private void LogErrors(MongoBulkWriteException<BsonDocument> ex)
393393
{

0 commit comments

Comments
 (0)