From fc72f6bfbaaf106bc502dfef0e2b900a3fa32af5 Mon Sep 17 00:00:00 2001
From: Seweryn Presnal <IhateTrains@users.noreply.github.com>
Date: Sun, 23 Jun 2024 17:06:19 +0200
Subject: [PATCH] Speed up the loading of Imperator pops

---
 ImperatorToCK3/Imperator/Pops/Pop.cs          | 18 ++++++-
 .../Imperator/Pops/PopCollection.cs           | 54 +++++++++++++------
 ImperatorToCK3/Imperator/Pops/PopFactory.cs   | 20 -------
 3 files changed, 55 insertions(+), 37 deletions(-)
 delete mode 100644 ImperatorToCK3/Imperator/Pops/PopFactory.cs

diff --git a/ImperatorToCK3/Imperator/Pops/Pop.cs b/ImperatorToCK3/Imperator/Pops/Pop.cs
index bee00587d..33a2c67b9 100644
--- a/ImperatorToCK3/Imperator/Pops/Pop.cs
+++ b/ImperatorToCK3/Imperator/Pops/Pop.cs
@@ -1,8 +1,9 @@
-using commonItems.Collections;
+using commonItems;
+using commonItems.Collections;
 
 namespace ImperatorToCK3.Imperator.Pops;
 
-public partial class Pop : IIdentifiable<ulong> {
+public class Pop : IIdentifiable<ulong> {
 	public ulong Id { get; } = 0;
 	public string Type { get; set; } = "";
 	public string Culture { get; set; } = "";
@@ -10,4 +11,17 @@ public partial class Pop : IIdentifiable<ulong> {
 	public Pop(ulong id) {
 		Id = id;
 	}
+	
+	public static Pop Parse(string idString, BufferedReader reader) {
+		var newPop = new Pop(ulong.Parse(idString));
+
+		var parser = new Parser();
+		parser.RegisterKeyword("type", r => newPop.Type = r.GetString());
+		parser.RegisterKeyword("culture", r => newPop.Culture = r.GetString());
+		parser.RegisterKeyword("religion", r => newPop.Religion = r.GetString());
+		parser.RegisterRegex(CommonRegexes.Catchall, ParserHelpers.IgnoreAndLogItem);
+		parser.ParseStream(reader);
+		
+		return newPop;
+	}
 }
\ No newline at end of file
diff --git a/ImperatorToCK3/Imperator/Pops/PopCollection.cs b/ImperatorToCK3/Imperator/Pops/PopCollection.cs
index 68362fd3b..9d74475df 100644
--- a/ImperatorToCK3/Imperator/Pops/PopCollection.cs
+++ b/ImperatorToCK3/Imperator/Pops/PopCollection.cs
@@ -1,30 +1,54 @@
 using commonItems;
 using commonItems.Collections;
+using System.Collections.Generic;
+using System.Threading.Channels;
+using System.Threading.Tasks;
 
 namespace ImperatorToCK3.Imperator.Pops;
 
-public sealed class PopCollection : IdObjectCollection<ulong, Pop> {
+public sealed class PopCollection : ConcurrentIdObjectCollection<ulong, Pop> {
 	public void LoadPopsFromBloc(BufferedReader blocReader) {
 		var blocParser = new Parser();
 		blocParser.RegisterKeyword("population", LoadPops);
 		blocParser.IgnoreAndLogUnregisteredItems();
 		blocParser.ParseStream(blocReader);
 	}
+
 	public void LoadPops(BufferedReader reader) {
-		var parser = new Parser();
-		RegisterKeys(parser);
-		parser.ParseStream(reader);
-	}
-	private void RegisterKeys(Parser parser) {
-		parser.RegisterRegex(CommonRegexes.Integer, (reader, thePopId) => {
-			var popStr = reader.GetStringOfItem().ToString();
-			if (!popStr.Contains('{')) {
-				return;
-			}
-			var tempStream = new BufferedReader(popStr);
-			var pop = Pop.Parse(thePopId, tempStream);
-			Add(pop);
+		// Load pops using the producer-consumer pattern.
+		
+		var channel = Channel.CreateUnbounded<KeyValuePair<string, string>>();
+		var channelWriter = channel.Writer;
+		var channelReader = channel.Reader;
+		
+		var producerTask = Task.Run(() => {
+			var parser = new Parser();
+			parser.RegisterRegex(CommonRegexes.Integer, (popReader, thePopId) => {
+				var popStr = popReader.GetStringOfItem().ToString();
+				if (!popStr.Contains('{')) {
+					return;
+				}
+				
+				if (!channelWriter.TryWrite(new(thePopId, popStr))) {
+					Logger.Warn($"Failed to enqueue pop {thePopId} for processing.");
+				}
+			});
+			parser.RegisterRegex(CommonRegexes.Catchall, ParserHelpers.IgnoreAndLogItem);
+			parser.ParseStream(reader);
+			
+			channelWriter.Complete();
 		});
-		parser.RegisterRegex(CommonRegexes.Catchall, ParserHelpers.IgnoreAndLogItem);
+		
+		var consumerTasks = new List<Task>();
+		for (var i = 0; i < 5; ++i) {
+			consumerTasks.Add(Task.Run(async () => {
+				await foreach (var (popIdStr, popDataStr) in channelReader.ReadAllAsync()) {
+					var pop = Pop.Parse(popIdStr, new BufferedReader(popDataStr));
+					Add(pop);
+				}
+			}));
+		}
+		
+		Task.WaitAll(producerTask, Task.WhenAll(consumerTasks));
 	}
 }
\ No newline at end of file
diff --git a/ImperatorToCK3/Imperator/Pops/PopFactory.cs b/ImperatorToCK3/Imperator/Pops/PopFactory.cs
deleted file mode 100644
index cb1257b78..000000000
--- a/ImperatorToCK3/Imperator/Pops/PopFactory.cs
+++ /dev/null
@@ -1,20 +0,0 @@
-using commonItems;
-
-namespace ImperatorToCK3.Imperator.Pops;
-
-public partial class Pop {
-	static Pop() {
-		popParser.RegisterKeyword("type", reader => tempPop.Type = reader.GetString());
-		popParser.RegisterKeyword("culture", reader => tempPop.Culture = reader.GetString());
-		popParser.RegisterKeyword("religion", reader => tempPop.Religion = reader.GetString());
-		popParser.RegisterRegex(CommonRegexes.Catchall, ParserHelpers.IgnoreAndLogItem);
-	}
-	public static Pop Parse(string idString, BufferedReader reader) {
-		tempPop = new Pop(ulong.Parse(idString));
-		popParser.ParseStream(reader);
-		return tempPop;
-	}
-
-	private static Pop tempPop = new(0);
-	private static readonly Parser popParser = new();
-}
\ No newline at end of file