Purge more characters from CK3

ImperatorToCK3/CK3/Characters/CharacterCollection.cs

@@ -16,6 +16,7 @@
 using ImperatorToCK3.Mappers.Trait;
 using ImperatorToCK3.Mappers.UnitType;
 using Open.Collections;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
@@ -42,24 +43,33 @@ Configuration config
 
 		var unlocalizedImperatorNames = new ConcurrentHashSet<string>();
 
-		Parallel.ForEach(impWorld.Characters, irCharacter => {
-			ImportImperatorCharacter(
-				irCharacter,
-				religionMapper,
-				cultureMapper,
-				traitMapper,
-				nicknameMapper,
-				impWorld.LocDB,
-				ck3LocDB,
-				impWorld.MapData,
-				provinceMapper,
-				deathReasonMapper,
-				dnaFactory,
-				conversionDate,
-				config,
-				unlocalizedImperatorNames
-			);
+		var parallelOptions = new ParallelOptions {
+			MaxDegreeOfParallelism = Environment.ProcessorCount - 1,
+		};
+		Parallel.ForEach(impWorld.Characters, parallelOptions, irCharacter => {
+			try {
+				ImportImperatorCharacter(
+					irCharacter,
+					religionMapper,
+					cultureMapper,
+					traitMapper,
+					nicknameMapper,
+					impWorld.LocDB,
+					ck3LocDB,
+					impWorld.MapData,
+					provinceMapper,
+					deathReasonMapper,
+					dnaFactory,
+					conversionDate,
+					config,
+					unlocalizedImperatorNames
+				);
+			} catch (Exception e) {
+				Logger.Error($"Exception while importing Imperator character {irCharacter.Id}: {e}");
+				Logger.Debug("Exception stack trace: " + e.StackTrace);
+			}
 		});
+
 		if (unlocalizedImperatorNames.Any()) {
 			Logger.Warn("Found unlocalized Imperator names: " + string.Join(", ", unlocalizedImperatorNames));
 		}
@@ -439,13 +449,20 @@ private static IEnumerable<string> LoadCharacterIDsToPreserve() {
 
 	public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection dynasties, HouseCollection houses, Date ck3BookmarkDate) {
 		Logger.Info("Purging unneeded characters...");
-
-		// Characters that hold or held titles should always be kept.
-		var landedCharacterIds = titles.GetAllHolderIds();
+		
+		// Characters from CK3 that hold titles at the bookmark date should be kept.
+		var currentTitleHolderIds = titles.GetHolderIds(ck3BookmarkDate);
 		var landedCharacters = this
-			.Where(character => landedCharacterIds.Contains(character.Id))
+			.Where(character => currentTitleHolderIds.Contains(character.Id))
 			.ToArray();
 		var charactersToCheck = this.Except(landedCharacters);
+
+		// Characters from I:R that held or hold titles should be kept.
+		var allTitleHolderIds = titles.GetAllHolderIds();
+		var imperatorTitleHolders = this
+			.Where(character => character.FromImperator && allTitleHolderIds.Contains(character.Id))
+			.ToArray();
+		charactersToCheck = charactersToCheck.Except(imperatorTitleHolders);
 
 		// Don't purge animation_test or easter egg characters.
 		charactersToCheck = charactersToCheck
@@ -457,12 +474,11 @@ public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection
 
 		// Make some exceptions for characters referenced in game's script files.
 		var characterIdsToKeep = LoadCharacterIDsToPreserve();
-
 		charactersToCheck = charactersToCheck
 			.Where(character => !characterIdsToKeep.Contains(character.Id))
 			.ToArray();
 
-		// Members of landed dynasties will be preserved, unless dead and childless.
+		// I:R members of landed dynasties will be preserved, unless dead and childless.
 		var dynastyIdsOfLandedCharacters = landedCharacters
 			.Select(character => character.GetDynastyId(ck3BookmarkDate))
 			.Distinct()
@@ -493,8 +509,8 @@ public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection
 
 			// See who can be removed.
 			foreach (var character in charactersToCheck) {
-				// Does the character belong to a dynasty that holds or held titles?
-				if (dynastyIdsOfLandedCharacters.Contains(character.GetDynastyId(ck3BookmarkDate))) {
+				// Is the character from Imperator and do they belong to a dynasty that holds or held titles?
+				if (character.FromImperator && dynastyIdsOfLandedCharacters.Contains(character.GetDynastyId(ck3BookmarkDate))) {
 					// Is the character dead and childless? Purge.
 					if (!parentIdsCache.Contains(character.Id)) {
 						charactersToRemove.Add(character);
@@ -517,6 +533,9 @@ public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection
 		houses.PurgeUnneededHouses(this, ck3BookmarkDate);
 		dynasties.PurgeUnneededDynasties(this, houses, ck3BookmarkDate);
 		dynasties.FlattenDynastiesWithNoFounders(this, houses, ck3BookmarkDate);
+
+		// Clean up title history.
+		titles.RemoveInvalidHoldersFromHistory(this);
 	}
 
 	public void RemoveEmployerIdFromLandedCharacters(Title.LandedTitles titles, Date conversionDate) {

ImperatorToCK3/CK3/Titles/LandedTitles.cs

@@ -250,11 +250,26 @@
 			return baronies.FirstOrDefault(b => provinceId == b?.ProvinceId, defaultValue: null);
 		}
 
-		public HashSet<string> GetHolderIds(Date date) {
-			return new HashSet<string>(this.Select(t => t.GetHolderId(date)));
+		public ImmutableHashSet<string> GetHolderIds(Date date) {
+			return this.Select(t => t.GetHolderId(date)).ToImmutableHashSet();
 		}
-		public HashSet<string> GetAllHolderIds() {
-			return this.SelectMany(t => t.GetAllHolderIds()).ToHashSet();
+		public ImmutableHashSet<string> GetAllHolderIds() {
+			return this.SelectMany(t => t.GetAllHolderIds()).ToImmutableHashSet();
+		}
+
+		public void RemoveInvalidHoldersFromHistory(CharacterCollection characters) {
+			Logger.Debug("Removing invalid holders from history...");
+
+			var validIds = characters.Select(c => c.Id).ToImmutableHashSet();
+			foreach (var title in this) {
+				if (!title.History.Fields.TryGetValue("holder", out var holderField)) {
+					continue;
+				}
+
+				holderField.RemoveAllEntries(
+					value => value.ToString() is string valStr && valStr != "0" && !validIds.Contains(valStr)
+				);
+			}
 		}
 
 		public void ImportImperatorCountries(
@@ -517,168 +532,168 @@
 			}
 		}
 
 		public void ImportImperatorHoldings(ProvinceCollection ck3Provinces, Imperator.Characters.CharacterCollection irCharacters, Date conversionDate) {
 			Logger.Info("Importing Imperator holdings...");
 			var counter = 0;
 
 			var highLevelTitlesThatHaveHolders = this
 				.Where(t => t.Rank >= TitleRank.duchy && t.GetHolderId(conversionDate) != "0")
 				.ToImmutableList();
 			var highLevelTitleCapitalBaronyIds = highLevelTitlesThatHaveHolders
 				.Select(t=>t.CapitalCounty?.CapitalBaronyId ?? t.CapitalBaronyId)
 				.ToImmutableHashSet();
 
 			// Dukes and above should be excluded from having their holdings converted.
 			// Otherwise, governors with holdings would own parts of other governorships.
 			var dukeAndAboveIds = highLevelTitlesThatHaveHolders
 				.Where(t => t.Rank >= TitleRank.duchy)
 				.Select(t => t.GetHolderId(conversionDate))
 				.ToImmutableHashSet();
 
 			// We exclude baronies that are capitals of duchies and above.
 			var eligibleBaronies = this
 				.Where(t => t.Rank == TitleRank.barony)
 				.Where(b => !highLevelTitleCapitalBaronyIds.Contains(b.Id))
 				.ToArray();
 
 			var countyCapitalBaronies = eligibleBaronies
 				.Where(b => b.DeJureLiege?.CapitalBaronyId == b.Id)
 				.OrderBy(b => b.Id)
 				.ToArray();
 
 			var nonCapitalBaronies = eligibleBaronies.Except(countyCapitalBaronies).OrderBy(b => b.Id).ToArray();
 
 
 			// In CK3, a county holder shouldn't own baronies in counties that are not their own.
 			// This dictionary tracks what counties are held by what characters.
 			Dictionary<string, HashSet<string>> countiesPerCharacter = []; // characterId -> countyIds
 
 			// Evaluate all capital baronies first (we want to distribute counties first, then baronies).
 			foreach (var barony in countyCapitalBaronies) {
 				var ck3Province = GetBaronyProvince(barony);
 				if (ck3Province is null) {
 					continue;
 				}
 
 				// Skip none holdings and temple holdings.
 				if (ck3Province.GetHoldingType(conversionDate) is "church_holding" or "none") {
 					continue;
 				}
 
 				var irProvince = ck3Province.PrimaryImperatorProvince; // TODO: when the holding owner of the primary I:R province is not able to hold the CK3 equivalent, also check the holding owners from secondary source provinces
 				var ck3Owner = GetEligibleCK3OwnerForImperatorProvince(irProvince);
 				if (ck3Owner is null) {
 					continue;
 				}
 
 				var realm = ck3Owner.ImperatorCharacter?.HomeCountry?.CK3Title;
 				var deFactoLiege = realm;
 				if (realm is not null) {
 					var deJureDuchy = barony.DeJureLiege?.DeJureLiege;
 					if (deJureDuchy is not null && deJureDuchy.GetHolderId(conversionDate) != "0" && deJureDuchy.GetTopRealm(conversionDate) == realm) {
 						deFactoLiege = deJureDuchy;
 					} else {
 						var deJureKingdom = deJureDuchy?.DeJureLiege;
 						if (deJureKingdom is not null && deJureKingdom.GetHolderId(conversionDate) != "0" && deJureKingdom.GetTopRealm(conversionDate) == realm) {
 							deFactoLiege = deJureKingdom;
 						}
 					}
 				}
 
 				// Barony is a county capital, so set the county holder to the holding owner.
 				var county = barony.DeJureLiege;
 				if (county is null) {
 					Logger.Warn($"County capital barony {barony.Id} has no de jure county!");
 					continue;
 				}
 				county.SetHolder(ck3Owner, conversionDate);
 				county.SetDeFactoLiege(deFactoLiege, conversionDate);
 
 				if (!countiesPerCharacter.TryGetValue(ck3Owner.Id, out var countyIds)) {
 					countyIds = [];
 					countiesPerCharacter[ck3Owner.Id] = countyIds;
 				}
 				countyIds.Add(county.Id);
 
 				++counter;
 			}
 
 			// In CK3, a baron that doesn't own counties can only hold a single barony.
 			// This dictionary IDs of such barons that already hold a barony.
 			HashSet<string> baronyHolderIds = [];
 
 			// After all possible county capital baronies are distributed, distribute the rest of the eligible baronies.
 			foreach (var barony in nonCapitalBaronies) {
 				var ck3Province = GetBaronyProvince(barony);
 				if (ck3Province is null) {
 					continue;
 				}
 
 				// Skip none holdings and temple holdings.
 				if (ck3Province.GetHoldingType(conversionDate) is "church_holding" or "none") {
 					continue;
 				}
 
 				var irProvince = ck3Province.PrimaryImperatorProvince; // TODO: when the holding owner of the primary I:R province is not able to hold the CK3 equivalent, also check the holding owners from secondary source provinces
 				var ck3Owner = GetEligibleCK3OwnerForImperatorProvince(irProvince);
 				if (ck3Owner is null) {
 					continue;
 				}
 				if (baronyHolderIds.Contains(ck3Owner.Id)) {
 					continue;
 				}
 
 				var county = barony.DeJureLiege;
 				if (county is null) {
 					Logger.Warn($"Barony {barony.Id} has no de jure county!");
 					continue;
 				}
 				// A non-capital barony cannot be held by a character that owns a county but not the county the barony is in.
 				if (countiesPerCharacter.TryGetValue(ck3Owner.Id, out var countyIds) && !countyIds.Contains(county.Id)) {
 					continue;
 				}
 
 				barony.SetHolder(ck3Owner, conversionDate);
 				// No need to set de facto liege for baronies, they are tied to counties.
 
 				baronyHolderIds.Add(ck3Owner.Id);
 
 				++counter;
 			}
 			Logger.Info($"Imported {counter} holdings from I:R.");
 			Logger.IncrementProgress();
 			return;
 
 			Province? GetBaronyProvince(Title barony) {
 				var ck3ProvinceId = barony.ProvinceId;
 				if (ck3ProvinceId is null) {
 					return null;
 				}
 				if (!ck3Provinces.TryGetValue(ck3ProvinceId.Value, out var ck3Province)) {
 					return null;
 				}
 				return ck3Province;
 			}
 
 			Character? GetEligibleCK3OwnerForImperatorProvince(Imperator.Provinces.Province? irProvince) {
 				var holdingOwnerId = irProvince?.HoldingOwnerId;
 				if (holdingOwnerId is null) {
 					return null;
 				}
 
 				var irOwner = irCharacters[holdingOwnerId.Value];
 				var ck3Owner = irOwner.CK3Character;
 				if (ck3Owner is null) {
 					return null;
 				}
 				if (dukeAndAboveIds.Contains(ck3Owner.Id)) {
 					return null;
 				}
 
 				return ck3Owner;
 			}
 		}
 
 		public void RemoveInvalidLandlessTitles(Date ck3BookmarkDate) {
 			Logger.Info("Removing invalid landless titles...");
 			var removedGeneratedTitles = new HashSet<string>();

ImperatorToCK3/Imperator/World.cs

@@ -174,7 +174,7 @@ private void LaunchImperatorToExportCountryFlags(Configuration config) {
 			Arguments = "-continuelastsave -debug_mode",
 			CreateNoWindow = true,
 			RedirectStandardOutput = true,
-			WindowStyle = ProcessWindowStyle.Hidden
+			WindowStyle = ProcessWindowStyle.Hidden,
 		};
 		var imperatorProcess = Process.Start(processStartInfo);
 		if (imperatorProcess is null) {