Purge more characters from CK3 (#2165)

Author: IhateTrains

ImperatorToCK3/CK3/Characters/CharacterCollection.cs

@@ -16,6 +16,7 @@
 using ImperatorToCK3.Mappers.Trait;
 using ImperatorToCK3.Mappers.UnitType;
 using Open.Collections;
+using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
@@ -42,24 +43,33 @@ Configuration config
 
 		var unlocalizedImperatorNames = new ConcurrentHashSet<string>();
 
-		Parallel.ForEach(impWorld.Characters, irCharacter => {
-			ImportImperatorCharacter(
-				irCharacter,
-				religionMapper,
-				cultureMapper,
-				traitMapper,
-				nicknameMapper,
-				impWorld.LocDB,
-				ck3LocDB,
-				impWorld.MapData,
-				provinceMapper,
-				deathReasonMapper,
-				dnaFactory,
-				conversionDate,
-				config,
-				unlocalizedImperatorNames
-			);
+		var parallelOptions = new ParallelOptions {
+			MaxDegreeOfParallelism = Environment.ProcessorCount - 1,
+		};
+		Parallel.ForEach(impWorld.Characters, parallelOptions, irCharacter => {
+			try {
+				ImportImperatorCharacter(
+					irCharacter,
+					religionMapper,
+					cultureMapper,
+					traitMapper,
+					nicknameMapper,
+					impWorld.LocDB,
+					ck3LocDB,
+					impWorld.MapData,
+					provinceMapper,
+					deathReasonMapper,
+					dnaFactory,
+					conversionDate,
+					config,
+					unlocalizedImperatorNames
+				);
+			} catch (Exception e) {
+				Logger.Error($"Exception while importing Imperator character {irCharacter.Id}: {e}");
+				Logger.Debug("Exception stack trace: " + e.StackTrace);
+			}
 		});
+		
 		if (unlocalizedImperatorNames.Any()) {
 			Logger.Warn("Found unlocalized Imperator names: " + string.Join(", ", unlocalizedImperatorNames));
 		}
@@ -439,13 +449,20 @@ private static IEnumerable<string> LoadCharacterIDsToPreserve() {
 
 	public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection dynasties, HouseCollection houses, Date ck3BookmarkDate) {
 		Logger.Info("Purging unneeded characters...");
-
-		// Characters that hold or held titles should always be kept.
-		var landedCharacterIds = titles.GetAllHolderIds();
+		
+		// Characters from CK3 that hold titles at the bookmark date should be kept.
+		var currentTitleHolderIds = titles.GetHolderIds(ck3BookmarkDate);
 		var landedCharacters = this
-			.Where(character => landedCharacterIds.Contains(character.Id))
+			.Where(character => currentTitleHolderIds.Contains(character.Id))
 			.ToArray();
 		var charactersToCheck = this.Except(landedCharacters);
+		
+		// Characters from I:R that held or hold titles should be kept.
+		var allTitleHolderIds = titles.GetAllHolderIds();
+		var imperatorTitleHolders = this
+			.Where(character => character.FromImperator && allTitleHolderIds.Contains(character.Id))
+			.ToArray();
+		charactersToCheck = charactersToCheck.Except(imperatorTitleHolders);
 
 		// Don't purge animation_test or easter egg characters.
 		charactersToCheck = charactersToCheck
@@ -457,12 +474,11 @@ public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection
 
 		// Make some exceptions for characters referenced in game's script files.
 		var characterIdsToKeep = LoadCharacterIDsToPreserve();
-
 		charactersToCheck = charactersToCheck
 			.Where(character => !characterIdsToKeep.Contains(character.Id))
 			.ToArray();
 
-		// Members of landed dynasties will be preserved, unless dead and childless.
+		// I:R members of landed dynasties will be preserved, unless dead and childless.
 		var dynastyIdsOfLandedCharacters = landedCharacters
 			.Select(character => character.GetDynastyId(ck3BookmarkDate))
 			.Distinct()
@@ -493,8 +509,8 @@ public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection
 
 			// See who can be removed.
 			foreach (var character in charactersToCheck) {
-				// Does the character belong to a dynasty that holds or held titles?
-				if (dynastyIdsOfLandedCharacters.Contains(character.GetDynastyId(ck3BookmarkDate))) {
+				// Is the character from Imperator and do they belong to a dynasty that holds or held titles?
+				if (character.FromImperator && dynastyIdsOfLandedCharacters.Contains(character.GetDynastyId(ck3BookmarkDate))) {
 					// Is the character dead and childless? Purge.
 					if (!parentIdsCache.Contains(character.Id)) {
 						charactersToRemove.Add(character);
@@ -517,6 +533,9 @@ public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection
 		houses.PurgeUnneededHouses(this, ck3BookmarkDate);
 		dynasties.PurgeUnneededDynasties(this, houses, ck3BookmarkDate);
 		dynasties.FlattenDynastiesWithNoFounders(this, houses, ck3BookmarkDate);
+		
+		// Clean up title history.
+		titles.RemoveInvalidHoldersFromHistory(this);
 	}
 
 	public void RemoveEmployerIdFromLandedCharacters(Title.LandedTitles titles, Date conversionDate) {

ImperatorToCK3/CK3/Titles/LandedTitles.cs

@@ -250,11 +250,26 @@ public override void Remove(string name) {
 			return baronies.FirstOrDefault(b => provinceId == b?.ProvinceId, defaultValue: null);
 		}
 
-		public HashSet<string> GetHolderIds(Date date) {
-			return new HashSet<string>(this.Select(t => t.GetHolderId(date)));
+		public ImmutableHashSet<string> GetHolderIds(Date date) {
+			return this.Select(t => t.GetHolderId(date)).ToImmutableHashSet();
 		}
-		public HashSet<string> GetAllHolderIds() {
-			return this.SelectMany(t => t.GetAllHolderIds()).ToHashSet();
+		public ImmutableHashSet<string> GetAllHolderIds() {
+			return this.SelectMany(t => t.GetAllHolderIds()).ToImmutableHashSet();
+		}
+
+		public void RemoveInvalidHoldersFromHistory(CharacterCollection characters) {
+			Logger.Debug("Removing invalid holders from history...");
+			
+			var validIds = characters.Select(c => c.Id).ToImmutableHashSet();
+			foreach (var title in this) {
+				if (!title.History.Fields.TryGetValue("holder", out var holderField)) {
+					continue;
+				}
+
+				holderField.RemoveAllEntries(
+					value => value.ToString() is string valStr && valStr != "0" && !validIds.Contains(valStr)
+				);
+			}
 		}
 
 		public void ImportImperatorCountries(

ImperatorToCK3/Imperator/World.cs

@@ -174,7 +174,7 @@ private void LaunchImperatorToExportCountryFlags(Configuration config) {
 			Arguments = "-continuelastsave -debug_mode",
 			CreateNoWindow = true,
 			RedirectStandardOutput = true,
-			WindowStyle = ProcessWindowStyle.Hidden
+			WindowStyle = ProcessWindowStyle.Hidden,
 		};
 		var imperatorProcess = Process.Start(processStartInfo);
 		if (imperatorProcess is null) {