Speed up the loading of Imperator pops

ImperatorToCK3/Imperator/Pops/Pop.cs

@@ -1,13 +1,27 @@
-using commonItems.Collections;
+using commonItems;
+using commonItems.Collections;
 
 namespace ImperatorToCK3.Imperator.Pops;
 
-public partial class Pop : IIdentifiable<ulong> {
+public class Pop : IIdentifiable<ulong> {
 	public ulong Id { get; } = 0;
 	public string Type { get; set; } = "";
 	public string Culture { get; set; } = "";
 	public string Religion { get; set; } = "";
 	public Pop(ulong id) {
 		Id = id;
 	}
+
+	public static Pop Parse(string idString, BufferedReader reader) {
+		var newPop = new Pop(ulong.Parse(idString));
+
+		var parser = new Parser();
+		parser.RegisterKeyword("type", r => newPop.Type = r.GetString());
+		parser.RegisterKeyword("culture", r => newPop.Culture = r.GetString());
+		parser.RegisterKeyword("religion", r => newPop.Religion = r.GetString());
+		parser.RegisterRegex(CommonRegexes.Catchall, ParserHelpers.IgnoreAndLogItem);
+		parser.ParseStream(reader);
+
+		return newPop;
+	}
 }

ImperatorToCK3/Imperator/Pops/PopCollection.cs

@@ -1,30 +1,54 @@
 using commonItems;
 using commonItems.Collections;
+using System.Collections.Generic;
+using System.Threading.Channels;
+using System.Threading.Tasks;
 
 namespace ImperatorToCK3.Imperator.Pops;
 
-public sealed class PopCollection : IdObjectCollection<ulong, Pop> {
+public sealed class PopCollection : ConcurrentIdObjectCollection<ulong, Pop> {
 	public void LoadPopsFromBloc(BufferedReader blocReader) {
 		var blocParser = new Parser();
 		blocParser.RegisterKeyword("population", LoadPops);
 		blocParser.IgnoreAndLogUnregisteredItems();
 		blocParser.ParseStream(blocReader);
 	}
+
 	public void LoadPops(BufferedReader reader) {
-		var parser = new Parser();
-		RegisterKeys(parser);
-		parser.ParseStream(reader);
-	}
-	private void RegisterKeys(Parser parser) {
-		parser.RegisterRegex(CommonRegexes.Integer, (reader, thePopId) => {
-			var popStr = reader.GetStringOfItem().ToString();
-			if (!popStr.Contains('{')) {
-				return;
-			}
-			var tempStream = new BufferedReader(popStr);
-			var pop = Pop.Parse(thePopId, tempStream);
-			Add(pop);
+		// Load pops using the producer-consumer pattern.
+
+		var channel = Channel.CreateUnbounded<KeyValuePair<string, string>>();
+		var channelWriter = channel.Writer;
+		var channelReader = channel.Reader;
+
+		var producerTask = Task.Run(() => {
+			var parser = new Parser();
+			parser.RegisterRegex(CommonRegexes.Integer, (popReader, thePopId) => {
+				var popStr = popReader.GetStringOfItem().ToString();
+				if (!popStr.Contains('{')) {
+					return;
+				}
+
+				if (!channelWriter.TryWrite(new(thePopId, popStr))) {
+					Logger.Warn($"Failed to enqueue pop {thePopId} for processing.");
+				}
+			});
+			parser.RegisterRegex(CommonRegexes.Catchall, ParserHelpers.IgnoreAndLogItem);
+			parser.ParseStream(reader);
+
+			channelWriter.Complete();
 		});
-		parser.RegisterRegex(CommonRegexes.Catchall, ParserHelpers.IgnoreAndLogItem);
+
+		var consumerTasks = new List<Task>();
+		for (var i = 0; i < 5; ++i) {
+			consumerTasks.Add(Task.Run(async () => {
+				await foreach (var (popIdStr, popDataStr) in channelReader.ReadAllAsync()) {
+					var pop = Pop.Parse(popIdStr, new BufferedReader(popDataStr));
+					Add(pop);
+				}
+			}));
+		}
+
+		Task.WaitAll(producerTask, Task.WhenAll(consumerTasks));
 	}
 }