Purge unneeded dynasty houses from CK3 (#1969)

ImperatorToCK3.UnitTests/CK3/Characters/CK3CharacterTests.cs

@@ -444,7 +444,7 @@ public void UnneededCharactersArePurged() {
 		ck3Characters.Add(unlandedFromCK3);
 
 		var titles = new Title.LandedTitles();
-		ck3Characters.PurgeUnneededCharacters(titles, new DynastyCollection(), ConversionDate);
+		ck3Characters.PurgeUnneededCharacters(titles, new DynastyCollection(), new HouseCollection(), ConversionDate);
 
 		Assert.Empty(ck3Characters);
 	}
@@ -508,7 +508,7 @@ public void DeadLandlessCharactersArePurgedIfChildless() {
 		Assert.Equal(dynasty.Id, fatherOfLandedCharacter.GetDynastyId(ConversionDate));
 		Assert.Equal(dynasty.Id, childlessRelative.GetDynastyId(ConversionDate));
 
-		ck3Characters.PurgeUnneededCharacters(titles, dynasties, ConversionDate);
+		ck3Characters.PurgeUnneededCharacters(titles, dynasties, new HouseCollection(), ConversionDate);
 
 		ck3Characters.Should().BeEquivalentTo(new[] {
 			landedCharacter,

ImperatorToCK3.UnitTests/Outputter/DynastiesOutputterTests.cs

@@ -23,7 +23,7 @@ public class DynastiesOutputterTests {
 
 	[Fact]
 	public void DynastiesAreOutputted() {
-		const string outputModName = "outputMod";
+		const string outputModPath = "output/outputMod";
 		var locDB = new LocDB("english");
 		const string imperatorRoot = "TestFiles/Imperator/root";
 		ModFilesystem irModFS = new(imperatorRoot, Array.Empty<Mod>());
@@ -50,13 +50,13 @@ public void DynastiesAreOutputted() {
 		};
 		dynasties.Add(dynasty2);
 
-		var outputPath = Path.Combine("output", outputModName, "common/dynasties/ir_dynasties.txt");
+		var outputPath = Path.Combine(outputModPath, "common/dynasties/irtock3_all_dynasties.txt");
 		if (File.Exists(outputPath)) {
 			// clean up from previous runs.
 			File.Delete(outputPath);
 		}
 		SystemUtils.TryCreateFolder(CommonFunctions.GetPath(outputPath));
-		DynastiesOutputter.OutputDynasties(outputModName, dynasties);
+		DynastiesOutputter.OutputDynasties(outputModPath, dynasties);
 
 		using var file = File.OpenRead(outputPath);
 		var reader = new StreamReader(file);

ImperatorToCK3/CK3/Characters/Character.cs

@@ -629,6 +629,10 @@ public void SetDynastyId(string dynastyId, Date? date) {
 		return History.GetFieldValue("dynasty", date)?.ToString();
 	}
 
+	public string? GetDynastyHouseId(Date date) {
+		return History.GetFieldValue("dynasty_house", date)?.ToString();
+	}
+
 	private string? jailorId;
 	public void SetEmployer(Character employer, Date? date) {
 		SetEmployerId(employer.Id, date);

ImperatorToCK3/CK3/Characters/CharacterCollection.cs

@@ -436,7 +436,7 @@ private static IEnumerable<string> LoadCharacterIDsToPreserve() {
 		return characterIDsToPreserve;
 	}
 
-	public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection dynasties, Date ck3BookmarkDate) {
+	public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection dynasties, HouseCollection houses, Date ck3BookmarkDate) {
 		Logger.Info("Purging unneeded characters...");
 
 		// Characters that hold or held titles should always be kept.
@@ -509,8 +509,9 @@ public void PurgeUnneededCharacters(Title.LandedTitles titles, DynastyCollection
 			charactersToCheck = charactersToCheck.Except(charactersToRemove).ToList();
 		} while(charactersToRemove.Count > 0);
 
-		// At this point we probably have many imported dynasties with no characters left.
+		// At this point we probably have many dynasties with no characters left.
 		// Let's purge them.
+		houses.PurgeUnneededHouses(this, ck3BookmarkDate);
 		dynasties.PurgeUnneededDynasties(this, ck3BookmarkDate);
 	}
 

ImperatorToCK3/CK3/Dynasties/Dynasty.cs

@@ -7,7 +7,6 @@
 using ImperatorToCK3.Imperator.Cultures;
 using ImperatorToCK3.Imperator.Families;
 using ImperatorToCK3.Mappers.Culture;
-using Open.Collections;
 using System.Collections.Generic;
 using System.Linq;
 
@@ -50,9 +49,31 @@ public Dynasty(CK3.Characters.Character character, string irFamilyName, Cultures
 		SetLocFromImperatorFamilyName(Family.GetMaleForm(irFamilyName, irCulturesDB), locDB);
 	}
 
+	public Dynasty(string dynastyId, BufferedReader dynastyReader) {
+		Id = dynastyId;
+		FromImperator = false;
+
+		var parser = new Parser();
+		parser.RegisterKeyword("prefix", reader => Prefix = reader.GetString());
+		parser.RegisterKeyword("name", reader => Name = reader.GetString());
+		parser.RegisterKeyword("culture", reader => CultureId = reader.GetString());
+		parser.RegisterKeyword("motto", reader => Motto = reader.GetString());
+		parser.RegisterKeyword("forced_coa_religiongroup", reader => ForcedCoaReligionGroup = reader.GetString());
+		parser.IgnoreAndLogUnregisteredItems();
+		parser.ParseStream(dynastyReader);
+
+		if (string.IsNullOrEmpty(Name)) {
+			Logger.Warn($"Dynasty {Id} has no name! Setting fallback unlocalized name.");
+			Name = Id;
+		}
+	}
+
 	[NonSerialized] public string Id { get; }
-	[SerializedName("name")] public string Name { get; }
+	[SerializedName("prefix")] public string? Prefix { get; private set; }
+	[SerializedName("name")] public string Name { get; private set; }
 	[SerializedName("culture")] public string? CultureId { get; set; }
+	[SerializedName("motto")] public string? Motto { get; set; }
+	[SerializedName("forced_coa_religiongroup")] public string? ForcedCoaReligionGroup { get; set; }
 
 	[NonSerialized] public LocBlock? LocalizedName { get; private set; }
 	[NonSerialized] public StringOfItem? CoA { get; set; }

ImperatorToCK3/CK3/Dynasties/DynastyCollection.cs

@@ -1,6 +1,7 @@
 using commonItems;
 using commonItems.Collections;
 using commonItems.Localization;
+using commonItems.Mods;
 using ImperatorToCK3.CK3.Characters;
 using ImperatorToCK3.CK3.Titles;
 using ImperatorToCK3.Mappers.Culture;
@@ -34,6 +35,18 @@ public void ImportImperatorFamilies(Imperator.World irWorld, CultureMapper cultu
 		Logger.IncrementProgress();
 	}
 
+	public void LoadCK3Dynasties(ModFilesystem ck3ModFS) {
+		Logger.Info("Loading dynasties from CK3...");
+
+		var parser = new Parser();
+		parser.RegisterRegex(CommonRegexes.String, (reader, dynastyId) => {
+			var dynasty = new Dynasty(dynastyId, reader);
+			AddOrReplace(dynasty);
+		});
+		parser.IgnoreAndLogUnregisteredItems();
+		parser.ParseGameFolder("common/dynasties", ck3ModFS, "txt", recursive: true, parallel: true);
+	}
+
 	private void CreateDynastiesForCharactersFromMinorFamilies(Imperator.World irWorld, LocDB locDB, Date date) {
 		Logger.Info("Creating dynasties for characters from minor families...");
 
@@ -97,7 +110,7 @@ public void PurgeUnneededDynasties(CharacterCollection characters, Date date) {
 		}
 
 		int removedCount = 0;
-		foreach (var dynasty in this.Where(d => d.FromImperator).ToList()) {
+		foreach (var dynasty in this.ToList()) {
 			if (!dynastiesToKeep.Contains(dynasty.Id)) {
 				Remove(dynasty.Id);
 				++removedCount;

ImperatorToCK3/CK3/Dynasties/House.cs

@@ -0,0 +1,30 @@
+using commonItems;
+using commonItems.Collections;
+using commonItems.Serialization;
+using commonItems.SourceGenerators;
+
+namespace ImperatorToCK3.CK3.Dynasties;
+
+[SerializationByProperties]
+public partial class House : IPDXSerializable, IIdentifiable<string> {
+	[NonSerialized] public string Id { get; }
+
+	[SerializedName("prefix")] public string? Prefix { get; private set; }
+	[SerializedName("name")] public string? Name { get; private set; }
+	[SerializedName("dynasty")] public string? DynastyId { get; private set; }
+	[SerializedName("motto")] public string? Motto { get; private set; }
+	[SerializedName("forced_coa_religiongroup")] public string? ForcedCoaReligionGroup { get; private set; }
+
+	public House(string id, BufferedReader houseReader) {
+		Id = id;
+
+		var parser = new Parser();
+		parser.RegisterKeyword("prefix", reader => Prefix = reader.GetString());
+		parser.RegisterKeyword("name", reader => Name = reader.GetString());
+		parser.RegisterKeyword("dynasty", reader => DynastyId = reader.GetString());
+		parser.RegisterKeyword("motto", reader => Motto = reader.GetString());
+		parser.RegisterKeyword("forced_coa_religiongroup", reader => ForcedCoaReligionGroup = reader.GetString());
+		parser.IgnoreAndLogUnregisteredItems();
+		parser.ParseStream(houseReader);
+	}
+}

ImperatorToCK3/CK3/Dynasties/HouseCollection.cs

@@ -0,0 +1,43 @@
+using commonItems;
+using commonItems.Collections;
+using commonItems.Mods;
+using ImperatorToCK3.CK3.Characters;
+using System.Linq;
+
+namespace ImperatorToCK3.CK3.Dynasties;
+
+public class HouseCollection : IdObjectCollection<string, House> {
+	public void LoadCK3Houses(ModFilesystem ck3ModFS) {
+		Logger.Info("Loading dynasty houses from CK3...");
+
+		var parser = new Parser();
+		parser.RegisterRegex(CommonRegexes.String, (reader, houseId) => {
+			var house = new House(houseId, reader);
+			AddOrReplace(house);
+		});
+		parser.IgnoreAndLogUnregisteredItems();
+		parser.ParseGameFolder("common/dynasty_houses", ck3ModFS, "txt", recursive: true, parallel: true);
+	}
+
+	public void PurgeUnneededHouses(CharacterCollection ck3Characters, Date date) {
+		Logger.Info("Purging unneeded dynasty houses...");
+
+		var houseIdsToKeep = ck3Characters
+			.Select(c => c.GetDynastyHouseId(date))
+			.Where(id => id is not null)
+			.Distinct()
+			.Cast<string>()
+			.ToHashSet();
+
+		int removedCount = 0;
+		foreach (var house in this.ToList()) {
+			if (houseIdsToKeep.Contains(house.Id)) {
+				continue;
+			}
+
+			Remove(house.Id);
+			++removedCount;
+		}
+		Logger.Info($"Purged {removedCount} unneeded dynasty houses.");
+	}
+}

ImperatorToCK3/CK3/World.cs

@@ -44,7 +44,8 @@ public class World {
 	private ScriptValueCollection ScriptValues { get; } = new();
 	public NamedColorCollection NamedColors { get; } = new();
 	public CharacterCollection Characters { get; } = new();
-	public DynastyCollection Dynasties { get; } = new();
+	public DynastyCollection Dynasties { get; } = [];
+	public HouseCollection DynastyHouses { get; } = [];
 	public ProvinceCollection Provinces { get; } = new();
 	public Title.LandedTitles LandedTitles { get; } = new();
 	public PillarCollection CulturalPillars { get; }
@@ -203,7 +204,9 @@ public World(Imperator.World impWorld, Configuration config) {
 		);
 		ClearFeaturedCharactersDescriptions(config.CK3BookmarkDate);
 
+		Dynasties.LoadCK3Dynasties(ModFS);
 		Dynasties.ImportImperatorFamilies(impWorld, cultureMapper, impWorld.LocDB, CorrectedDate);
+		DynastyHouses.LoadCK3Houses(ModFS);
 
 		// Load existing CK3 government IDs.
 		Logger.Info("Loading CK3 government IDs...");
@@ -283,7 +286,7 @@ public World(Imperator.World impWorld, Configuration config) {
 		Characters.ImportLegions(LandedTitles, impWorld.Units, impWorld.Characters, CorrectedDate, unitTypeMapper, MenAtArmsTypes, provinceMapper, config);
 
 		Characters.RemoveEmployerIdFromLandedCharacters(LandedTitles, CorrectedDate);
-		Characters.PurgeUnneededCharacters(LandedTitles, Dynasties, config.CK3BookmarkDate);
+		Characters.PurgeUnneededCharacters(LandedTitles, Dynasties, DynastyHouses, config.CK3BookmarkDate);
 
 		// Check if any muslim religion exists in Imperator. Otherwise, remove Islam from the entire CK3 map.
 		var possibleMuslimReligionNames = new List<string> { "muslim", "islam", "sunni", "shiite" };

ImperatorToCK3/Data_Files/blankMod/output/localization/english/CONVERTER_cultures_l_english.yml

@@ -326,9 +326,7 @@ l_english:
  galatian: "Galatian"
  galatian_collective_noun: "Galatians"
  galatian_prefix: "Galato" # found 'Galato–Lycaonian'
- hunnic: "Hunnic"
- hunnic_collective_noun: "Huns"
- hunnic_prefix: "Hunno" # found 'Hunno-Germanic'
+ #hunnic_prefix: "Hunno" # found 'Hunno-Germanic'
  urslavic: "Slavic"
  urslavic_collective_noun: "Slavs"
  urslavic_prefix: "Slavo" # found 'Slavo-Germanic'

ImperatorToCK3/Data_Files/blankMod/output/localization/english/CONVERTER_names_l_english.yml

@@ -212,7 +212,7 @@
  Alcimus: "Alcimus"
  Guendolen: "Guendolen"
  Dynwen: "Dynwen"
- Kawng_Tai: "Kawng Tai"
+ #Kawng_Tai: "Kawng Tai"
 
 # Oscan
  Aulis: "Aulis"

ImperatorToCK3/Data_Files/blankMod/output/localization/english/sea_character_names_l_english.yml

@@ -49,7 +49,7 @@
   Abis:0 "Abis"
   Abish:0 "Abish"
   Abishqa:0 "Abishqa"
-  Abiyasa:0 "Abiyasa"
+  #Abiyasa:0 "Abiyasa"
   Abli:0 "Abli"
   Abo:0 "Abo"
   Abog:0 "Abog"
@@ -818,8 +818,8 @@
   Batei:0 "Batei"
   Bati:0 "Bati"
   Batik:0 "Batik"
-  Batis:0 "Batis"
-  Bato:0 "Bato"
+  #Batis:0 "Batis"
+  #Bato:0 "Bato"
   Batommavong:0 "Batommavong"
   Batongbacal:0 "Batongbacal"
   Batongmalaque:0 "Batongmalaque"
@@ -1501,7 +1501,7 @@
   Cheongja:0 "Cheongja"
   Cheonmyeong:0 "Cheonmyeong"
   Cheorin:0 "Cheorin"
-  Chert:0 "Chert"
+  #Chert:0 "Chert"
   Chervang:0 "Chervang"
   Cherzong:0 "Cherzong"
   Chesa:0 "Chesa"
@@ -3201,7 +3201,7 @@
   Isini:0 "Isini"
   Isip:0 "Isip"
   Ismah:0 "Ismah"
-  Ismail:0 "İsmail"
+  #Ismail:0 "İsmail"
   Isog:0 "Isog"
   Isonashi:0 "Isonashi"
   Isot:0 "Isot"

ImperatorToCK3/Data_Files/blankMod/output/localization/french/CONVERTER_cultures_l_french.yml

@@ -348,9 +348,9 @@
  galatian: "galatienne"
  galatian_collective_noun: "Galates"
  galatian_prefix: "galato"
- hunnic: "hunnique"
- hunnic_collective_noun: "Huns"
- hunnic_prefix: "hunno"
+ #hunnic: "hunnique"
+ #hunnic_collective_noun: "Huns"
+ #hunnic_prefix: "hunno"
  urslavic: "slave"
  urslavic_collective_noun: "Slaves"
  urslavic_prefix: "slavo"

ImperatorToCK3/Data_Files/blankMod/output/localization/german/CONVERTER_cultures_l_german.yml

@@ -347,9 +347,9 @@
  galatian: "galatisch"
  galatian_collective_noun: "Galater"
  galatian_prefix: "galato"
- hunnic: "hunnisch"
- hunnic_collective_noun: "Hunnen"
- hunnic_prefix: "hunno"
+ #hunnic: "hunnisch"
+ #hunnic_collective_noun: "Hunnen"
+ #hunnic_prefix: "hunno"
  urslavic: "slawisch"
  urslavic_collective_noun: "Slawen"
  urslavic_prefix: "slawo"

ImperatorToCK3/Data_Files/blankMod/output/localization/korean/CONVERTER_cultures_l_korean.yml

@@ -349,9 +349,9 @@
  galatian: "갈라디아"
  galatian_collective_noun: "갈라디아"
  galatian_prefix: "갈라디아"
- hunnic: "훈"
- hunnic_collective_noun: "훈"
- hunnic_prefix: "훈"
+ #hunnic: "훈"
+ #hunnic_collective_noun: "훈"
+ #hunnic_prefix: "훈"
  urslavic: "슬라브어"
  urslavic_collective_noun: "슬라브어"
  urslavic_prefix: "슬라브어"

ImperatorToCK3/Data_Files/blankMod/output/localization/russian/CONVERTER_cultures_l_russian.yml

@@ -353,9 +353,9 @@
  galatian: "Галаты"
  galatian_collective_noun: "Галатск"
  galatian_prefix: "Галатo"
- hunnic: "Гунны"
- hunnic_collective_noun: "Гуннск"
- hunnic_prefix: "Гуннo"
+ #hunnic: "Гунны"
+ #hunnic_collective_noun: "Гуннск"
+ #hunnic_prefix: "Гуннo"
  urslavic: "Эсты"
  urslavic_collective_noun: "Эстийск"
  urslavic_prefix: "Эсто"

ImperatorToCK3/Data_Files/blankMod/output/localization/simp_chinese/CONVERTER_cultures_l_simp_chinese.yml

@@ -266,8 +266,8 @@
 
  galatian: "加拉太"
  galatian_collective_noun: "加拉太人"
- hunnic: "匈奴"
- hunnic_collective_noun: "匈奴人"
+ #hunnic: "匈奴"
+ #hunnic_collective_noun: "匈奴人"
  urslavic: "斯拉夫语"
  urslavic_collective_noun: "斯拉夫语人"
  oscan: "奥斯坎"

ImperatorToCK3/Data_Files/blankMod/output/localization/spanish/CONVERTER_cultures_l_spanish.yml

@@ -266,8 +266,8 @@
 
  galatian: "gálata"
  galatian_collective_noun: "Gálatas"
- hunnic: "huna"
- hunnic_collective_noun: "Hunos"
+ #hunnic: "huna"
+ #hunnic_collective_noun: "Hunos"
  urslavic: "eslava"
  urslavic_collective_noun: "Eslavos"
  oscan: "osca"