Skip to content

Memory usage optimizations #2618

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
May 25, 2025
Merged

Memory usage optimizations #2618

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c083118
Memory usage optimizations
IhateTrains May 25, 2025
948d07c
More string interning
IhateTrains May 25, 2025
f38f7b5
Intern I:R nickname strings
IhateTrains May 25, 2025
027dea4
Fix GetAndInternStrings
IhateTrains May 25, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ImperatorToCK3/CK3/Dynasties/Dynasty.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ public Dynasty(string dynastyId, BufferedReader dynastyReader) {
var parser = new Parser();
parser.RegisterKeyword("prefix", reader => Prefix = reader.GetString());
parser.RegisterKeyword("name", reader => Name = reader.GetString());
parser.RegisterKeyword("culture", reader => CultureId = reader.GetString());
parser.RegisterKeyword("culture", reader => CultureId = string.Intern(reader.GetString()));
parser.RegisterKeyword("motto", reader => Motto = reader.GetString());
parser.RegisterKeyword("forced_coa_religiongroup", reader => ForcedCoaReligionGroup = reader.GetString());
parser.IgnoreAndLogUnregisteredItems();
Expand Down
8 changes: 8 additions & 0 deletions ImperatorToCK3/CommonUtils/BufferedReaderExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,12 @@ internal static Dictionary<string, string> GetAssignmentsAsDict(this BufferedRea
.GroupBy(a => a.Key)
.ToDictionary(g => g.Key, g => g.Last().Value);
}

internal static List<string> GetAndInternStrings(this BufferedReader reader) {
var strings = new List<string>();
foreach (var str in reader.GetStrings()) {
strings.Add(string.Intern(str));
}
return strings;
}
}
6 changes: 3 additions & 3 deletions ImperatorToCK3/Imperator/Armies/Subunit.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,14 @@ public Subunit(ulong id, BufferedReader subunitReader) {
Id = id;

var parser = new Parser();
parser.RegisterKeyword("category", reader => Category = reader.GetString());
parser.RegisterKeyword("type", reader => Type = reader.GetString());
parser.RegisterKeyword("category", reader => Category = string.Intern(reader.GetString()));
parser.RegisterKeyword("type", reader => Type = string.Intern(reader.GetString()));
parser.RegisterKeyword("country", reader => CountryId = reader.GetULong());
parser.RegisterKeyword("strength", reader => Strength = reader.GetDouble());
parser.IgnoreAndStoreUnregisteredItems(IgnoredTokens);

parser.ParseStream(subunitReader);
}

public static IgnoredKeywordsSet IgnoredTokens { get; } = new();
public static IgnoredKeywordsSet IgnoredTokens { get; } = [];
}
12 changes: 6 additions & 6 deletions ImperatorToCK3/Imperator/Characters/Character.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,12 +120,12 @@
parser.RegisterKeyword("country", reader => character.parsedCountryId = reader.GetULong());
parser.RegisterKeyword("home_country", reader => character.parsedHomeCountryId = reader.GetULong());
parser.RegisterKeyword("province", reader => character.ProvinceId = reader.GetULong());
parser.RegisterKeyword("culture", reader => character.culture = reader.GetString());
parser.RegisterKeyword("religion", reader => character.Religion = reader.GetString());
parser.RegisterKeyword("culture", reader => character.culture = string.Intern(reader.GetString()));
parser.RegisterKeyword("religion", reader => character.Religion = string.Intern(reader.GetString()));
parser.RegisterKeyword("fertility", reader => character.Fertility = reader.GetDouble());
parser.RegisterKeyword("health", reader => character.Health = reader.GetDouble());
parser.RegisterKeyword("family", reader => character.parsedFamilyId = reader.GetULong());
parser.RegisterKeyword("traits", reader => character.Traits = reader.GetStrings());
parser.RegisterKeyword("traits", reader => character.Traits = reader.GetAndInternStrings());
parser.RegisterKeyword("female", reader => character.Female = reader.GetBool());
parser.RegisterKeyword("children", reader => character.parsedChildrenIds = [.. reader.GetULongs()]);
parser.RegisterKeyword("spouse", reader => character.parsedSpouseIds = [.. reader.GetULongs()]);
Expand All @@ -146,9 +146,9 @@
var dateStr = reader.GetString();
character.DeathDate = new Date(dateStr, AUC: true); // converted to AD
});
parser.RegisterKeyword("death", reader => character.DeathReason = reader.GetString());
parser.RegisterKeyword("death", reader => character.DeathReason = string.Intern(reader.GetString()));
parser.RegisterKeyword("attributes", reader => character.Attributes = CharacterAttributes.Parse(reader));
parser.RegisterKeyword("nickname", reader => character.Nickname = reader.GetString());
parser.RegisterKeyword("nickname", reader => character.Nickname = string.Intern(reader.GetString()));
parser.RegisterKeyword("dna", reader => character.DNA = reader.GetString());
parser.RegisterKeyword("mother", reader => character.parsedMotherId = reader.GetULong());
parser.RegisterKeyword("father", reader => character.parsedFatherId = reader.GetULong());
Expand All @@ -171,7 +171,7 @@
var variablesParser = new Parser();
variablesParser.RegisterKeyword("data", dataReader => {
var blobParser = new Parser();
blobParser.RegisterKeyword("flag", blobReader => variables.Add(blobReader.GetString()));
blobParser.RegisterKeyword("flag", blobReader => variables.Add(string.Intern(blobReader.GetString())));
blobParser.IgnoreUnregisteredItems();

foreach (var blob in new BlobList(dataReader).Blobs) {
Expand Down Expand Up @@ -305,43 +305,43 @@
/// </summary>
/// <param name="irMapData">Imperator map data.</param>
/// <returns></returns>
public ulong? GetSourceLandProvince(MapData irMapData) {
HashSet<ulong> rejectedProvinceIds = [];

if (ProvinceId.HasValue) {
if (!irMapData.ProvinceDefinitions.TryGetValue(ProvinceId.Value, out var provinceDef)) {
Logger.Warn($"Potential source province {ProvinceId.Value} for character {Id} has no definition!");
} else if (provinceDef.IsLand) {
return ProvinceId;
}
rejectedProvinceIds.Add(ProvinceId.Value);
}

var homeCountryCapital = HomeCountry?.CapitalProvinceId;
if (homeCountryCapital.HasValue && !rejectedProvinceIds.Contains(homeCountryCapital.Value)) {
if (!irMapData.ProvinceDefinitions.TryGetValue(homeCountryCapital.Value, out var homeCountryCapitalDef)) {
Logger.Warn($"Potential source province {homeCountryCapital.Value} for character {Id} has no definition!");
} else if (homeCountryCapitalDef.IsLand) {
return homeCountryCapital;
}
rejectedProvinceIds.Add(homeCountryCapital.Value);
}

var countryCapital = Country?.CapitalProvinceId;
if (countryCapital.HasValue && !rejectedProvinceIds.Contains(countryCapital.Value)) {
if (!irMapData.ProvinceDefinitions.TryGetValue(countryCapital.Value, out var countryCapitalDef)) {
Logger.Warn($"Potential source province {countryCapital.Value} for character {Id} has no definition!");
} else if (countryCapitalDef.IsLand) {
return countryCapital;
}
rejectedProvinceIds.Add(countryCapital.Value);
}

var fatherProvince = Father?.GetSourceLandProvince(irMapData);
if (fatherProvince.HasValue) {
return fatherProvince;
}

var motherProvince = Mother?.GetSourceLandProvince(irMapData);
return motherProvince;
}

Check notice on line 347 in ImperatorToCK3/Imperator/Characters/Character.cs

View check run for this annotation

codefactor.io / CodeFactor

ImperatorToCK3/Imperator/Characters/Character.cs#L308-L347 

Complex Method
Expand Down
17 changes: 10 additions & 7 deletions ImperatorToCK3/Imperator/Countries/CountryFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,8 @@ private static void RegisterCountryKeywords(Parser parser, Country parsedCountry
.GroupBy(a => a.Key)
.ToFrozenDictionary(g => g.Key, g => int.Parse(g.Last().Value));
});
parser.RegisterKeyword("primary_culture", reader => parsedCountry.PrimaryCulture = reader.GetString());
parser.RegisterKeyword("religion", reader => parsedCountry.Religion = reader.GetString());
parser.RegisterKeyword("primary_culture", reader => parsedCountry.PrimaryCulture = string.Intern(reader.GetString()));
parser.RegisterKeyword("religion", reader => parsedCountry.Religion = string.Intern(reader.GetString()));
parser.RegisterKeyword("government_key", reader => SetGovernmentType(reader, parsedCountry));
parser.RegisterKeyword("family", reader => parsedCountry.parsedFamilyIds.Add(reader.GetULong()));
parser.RegisterKeyword("minor_family", reader => parsedCountry.parsedFamilyIds.Add(reader.GetULong()));
Expand All @@ -59,7 +59,7 @@ private static void RegisterCountryKeywords(Parser parser, Country parsedCountry
var variablesParser = new Parser();
variablesParser.RegisterKeyword("data", dataReader => {
var blobParser = new Parser();
blobParser.RegisterKeyword("flag", blobReader => variables.Add(blobReader.GetString()));
blobParser.RegisterKeyword("flag", blobReader => variables.Add(string.Intern(blobReader.GetString())));
blobParser.IgnoreUnregisteredItems();

foreach (var blob in new BlobList(dataReader).Blobs) {
Expand All @@ -78,9 +78,12 @@ private static void RegisterCountryKeywords(Parser parser, Country parsedCountry
});
parser.RegisterKeyword("mark_invention", ParserHelpers.IgnoreItem);
parser.RegisterKeyword("ruler_term", reader => parsedCountry.RulerTerms.Add(RulerTerm.Parse(reader)));
parser.RegisterRegex(monarchyLawRegexStr, reader => parsedCountry.monarchyLaws.Add(reader.GetString()));
parser.RegisterRegex(republicLawRegexStr, reader => parsedCountry.republicLaws.Add(reader.GetString()));
parser.RegisterRegex(tribalLawRegexStr, reader => parsedCountry.tribalLaws.Add(reader.GetString()));
parser.RegisterRegex(monarchyLawRegexStr, reader =>
parsedCountry.monarchyLaws.Add(string.Intern(reader.GetString())));
parser.RegisterRegex(republicLawRegexStr, reader =>
parsedCountry.republicLaws.Add(string.Intern(reader.GetString())));
parser.RegisterRegex(tribalLawRegexStr, reader =>
parsedCountry.tribalLaws.Add(string.Intern(reader.GetString())));
parser.RegisterKeyword("is_antagonist", ParserHelpers.IgnoreItem);
parser.RegisterKeyword("has_senior_ally", ParserHelpers.IgnoreItem);
parser.RegisterKeyword("cached_happiness_for_owned", ParserHelpers.IgnoreItem);
Expand All @@ -92,7 +95,7 @@ private static void RegisterCountryKeywords(Parser parser, Country parsedCountry
}

private static void SetGovernmentType(BufferedReader reader, Country parsedCountry) {
var governmentStr = reader.GetString();
var governmentStr = string.Intern(reader.GetString());
parsedCountry.Government = governmentStr;
// set government type
if (monarchyGovernments.Contains(governmentStr)) {
Expand Down
8 changes: 4 additions & 4 deletions ImperatorToCK3/Imperator/Countries/RulerTerm.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public static RulerTerm Parse(BufferedReader reader) {
var dateString = r.GetString();
newTerm.StartDate = new Date(dateString, AUC: true);
});
parser.RegisterKeyword("government", r => newTerm.Government = r.GetString());
parser.RegisterKeyword("government", r => newTerm.Government = string.Intern(r.GetString()));
parser.RegisterRegex(CommonRegexes.Catchall, (r, token) => {
IgnoredTokens.Add(token);
ParserHelpers.IgnoreItem(r);
Expand Down Expand Up @@ -59,9 +59,9 @@ public RulerTerm(BufferedReader prehistoryRulerReader, CountryCollection countri
var dateStr = reader.GetString();
StartDate = new Date(dateStr, AUC: true);
});
prehistoryParser.RegisterKeyword("religion", reader => PreImperatorRuler.Religion = reader.GetString());
prehistoryParser.RegisterKeyword("culture", reader => PreImperatorRuler.Culture = reader.GetString());
prehistoryParser.RegisterKeyword("nickname", reader => PreImperatorRuler.Nickname = reader.GetString());
prehistoryParser.RegisterKeyword("religion", reader => PreImperatorRuler.Religion = string.Intern(reader.GetString()));
prehistoryParser.RegisterKeyword("culture", reader => PreImperatorRuler.Culture = string.Intern(reader.GetString()));
prehistoryParser.RegisterKeyword("nickname", reader => PreImperatorRuler.Nickname = string.Intern(reader.GetString()));
prehistoryParser.RegisterKeyword("country", reader => {
var tag = reader.GetString();
if (tagToCountryCache.TryGetValue(tag, out var cachedCountry)) {
Expand Down
2 changes: 1 addition & 1 deletion ImperatorToCK3/Imperator/Families/Family.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ static FamilyFactory() {
family.PrestigeRatio = reader.GetDouble()
);
parser.RegisterKeyword("culture", reader =>
family.Culture = reader.GetString()
family.Culture = string.Intern(reader.GetString())
);
parser.RegisterKeyword("minor_family", reader =>
family.Minor = reader.GetBool()
Expand Down
6 changes: 3 additions & 3 deletions ImperatorToCK3/Imperator/Pops/Pop.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ public static Pop Parse(string idString, BufferedReader reader) {
var newPop = new Pop(ulong.Parse(idString));

var parser = new Parser();
parser.RegisterKeyword("type", r => newPop.Type = r.GetString());
parser.RegisterKeyword("culture", r => newPop.Culture = r.GetString());
parser.RegisterKeyword("religion", r => newPop.Religion = r.GetString());
parser.RegisterKeyword("type", r => newPop.Type = string.Intern(r.GetString()));
parser.RegisterKeyword("culture", r => newPop.Culture = string.Intern(r.GetString()));
parser.RegisterKeyword("religion", r => newPop.Religion = string.Intern(r.GetString()));
parser.RegisterRegex(CommonRegexes.Catchall, ParserHelpers.IgnoreAndLogItem);
parser.ParseStream(reader);

Expand Down
4 changes: 2 additions & 2 deletions ImperatorToCK3/Imperator/Provinces/ProvinceFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ static Province() {
parsedProvince.Controller = reader.GetULong()
);
provinceParser.RegisterKeyword("culture", reader =>
parsedProvince.Culture = reader.GetString()
parsedProvince.Culture = string.Intern(reader.GetString())
);
provinceParser.RegisterKeyword("religion", reader =>
parsedProvince.ReligionId = reader.GetString()
parsedProvince.ReligionId = string.Intern(reader.GetString())
);
provinceParser.RegisterKeyword("pop", reader =>
parsedProvince.parsedPopIds.Add(reader.GetULong())
Expand Down
Loading