From acb1511feecba511e67e127aa9651ee8d8eea27a Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 17:11:08 +1200 Subject: [PATCH 01/13] Improve map serialization error logging --- .../EntitySerialization/EntityDeserializer.cs | 8 +- .../EntitySerialization/EntitySerializer.cs | 97 +++++++++++-------- Robust.Shared/EntitySerialization/Options.cs | 9 +- .../EntitySerialization/SerializationEnums.cs | 24 +++++ 4 files changed, 96 insertions(+), 42 deletions(-) diff --git a/Robust.Shared/EntitySerialization/EntityDeserializer.cs b/Robust.Shared/EntitySerialization/EntityDeserializer.cs index ad7eb21adcf..baf500e10bf 100644 --- a/Robust.Shared/EntitySerialization/EntityDeserializer.cs +++ b/Robust.Shared/EntitySerialization/EntityDeserializer.cs @@ -1152,6 +1152,7 @@ EntityUid ITypeReader.Read( ISerializationContext? context, ISerializationManager.InstantiationDelegate? _) { + string msg; if (node.Value == "invalid") { if (CurrentComponent == "Transform") @@ -1160,7 +1161,7 @@ EntityUid ITypeReader.Read( if (!Options.LogInvalidEntities) return EntityUid.Invalid; - var msg = CurrentReadingEntity is not { } curr + msg = CurrentReadingEntity is not { } curr ? $"Encountered invalid EntityUid reference" : $"Encountered invalid EntityUid reference wile reading entity {curr.YamlId}, component: {CurrentComponent}"; _log.Error(msg); @@ -1170,7 +1171,10 @@ EntityUid ITypeReader.Read( if (int.TryParse(node.Value, out var val) && UidMap.TryGetValue(val, out var entity)) return entity; - _log.Error($"Invalid yaml entity id: '{val}'"); + msg = CurrentReadingEntity is not { } ent + ? "Encountered unknown EntityUid reference" + : $"Encountered unknown EntityUid reference wile reading entity {ent.YamlId}, component: {CurrentComponent}"; + _log.Error(msg); return EntityUid.Invalid; } diff --git a/Robust.Shared/EntitySerialization/EntitySerializer.cs b/Robust.Shared/EntitySerialization/EntitySerializer.cs index 2b9b6fa1249..00f0f6f1d08 100644 --- a/Robust.Shared/EntitySerialization/EntitySerializer.cs +++ b/Robust.Shared/EntitySerialization/EntitySerializer.cs @@ -412,7 +412,7 @@ private void SerializeEntityInternal(EntityUid uid) // It might be possible that something could cause an entity to be included twice. // E.g., if someone serializes a grid w/o its map, and then tries to separately include the map and all its children. - // In that case, the grid would already have been serialized as a orphan. + // In that case, the grid would already have been serialized as an orphan. // uhhh.... I guess its fine? if (EntityData.ContainsKey(saveId)) return; @@ -489,45 +489,22 @@ private void SerializeEntityInternal(EntityUid uid) xform._localRotation = 0; } - foreach (var component in EntMan.GetComponentsInternal(uid)) + try { - var compType = component.GetType(); - - var reg = _factory.GetRegistration(compType); - if (reg.Unsaved) - continue; - - CurrentComponent = reg.Name; - MappingDataNode? compMapping; - MappingDataNode? protoMapping = null; - if (cache != null && cache.TryGetValue(reg.Name, out protoMapping)) - { - // If this has a prototype, we need to use alwaysWrite: true. - // E.g., an anchored prototype might have anchored: true. If we we are saving an un-anchored - // instance of this entity, and if we have alwaysWrite: false, then compMapping would not include - // the anchored data-field (as false is the default for this bool data field), so the entity would - // implicitly be saved as anchored. - compMapping = _serialization.WriteValueAs(compType, component, alwaysWrite: true, context: this); - - // This will not recursively call Except() on the values of the mapping. It will only remove - // key-value pairs if both the keys and values are equal. - compMapping = compMapping.Except(protoMapping); - if(compMapping == null) - continue; - } - else - { - compMapping = _serialization.WriteValueAs(compType, component, alwaysWrite: false, context: this); - } + SerializeComponents(uid, cache, components); + } + catch + { + _log.Error($"Caught exception while serializing component {CurrentComponent} of entity {EntMan.ToPrettyString(uid)}"); + if (Options.EntityExceptionBehaviour == EntityExceptionBehaviour.Rethrow) + throw; - // Don't need to write it if nothing was written! Note that if this entity has no associated - // prototype, we ALWAYS want to write the component, because merely the fact that it exists is - // information that needs to be written. - if (compMapping.Children.Count != 0 || protoMapping == null) - { - compMapping.InsertAt(0, "type", new ValueDataNode(reg.Name)); - components.Add(compMapping); - } + Prototypes[protoId].Remove(saveId); + EntityData.Remove(saveId); + CurrentEntityYamlUid = 0; + CurrentEntity = null; + CurrentComponent = null; + return; } CurrentComponent = null; @@ -567,6 +544,50 @@ private void SerializeEntityInternal(EntityUid uid) CurrentEntity = null; } + private void SerializeComponents(EntityUid uid, Dictionary? cache, SequenceDataNode components) + { + foreach (var component in EntMan.GetComponentsInternal(uid)) + { + var compType = component.GetType(); + + var reg = _factory.GetRegistration(compType); + if (reg.Unsaved) + continue; + + CurrentComponent = reg.Name; + MappingDataNode? compMapping; + MappingDataNode? protoMapping = null; + if (cache != null && cache.TryGetValue(reg.Name, out protoMapping)) + { + // If this has a prototype, we need to use alwaysWrite: true. + // E.g., an anchored prototype might have anchored: true. If we we are saving an un-anchored + // instance of this entity, and if we have alwaysWrite: false, then compMapping would not include + // the anchored data-field (as false is the default for this bool data field), so the entity would + // implicitly be saved as anchored. + compMapping = _serialization.WriteValueAs(compType, component, alwaysWrite: true, context: this); + + // This will not recursively call Except() on the values of the mapping. It will only remove + // key-value pairs if both the keys and values are equal. + compMapping = compMapping.Except(protoMapping); + if(compMapping == null) + continue; + } + else + { + compMapping = _serialization.WriteValueAs(compType, component, alwaysWrite: false, context: this); + } + + // Don't need to write it if nothing was written! Note that if this entity has no associated + // prototype, we ALWAYS want to write the component, because merely the fact that it exists is + // information that needs to be written. + if (compMapping.Children.Count == 0 && protoMapping != null) + continue; + + compMapping.InsertAt(0, "type", new ValueDataNode(reg.Name)); + components.Add(compMapping); + } + } + private Dictionary? GetProtoCache(EntityPrototype? proto) { if (proto == null) diff --git a/Robust.Shared/EntitySerialization/Options.cs b/Robust.Shared/EntitySerialization/Options.cs index 0237f3f9c7a..1c343d204a1 100644 --- a/Robust.Shared/EntitySerialization/Options.cs +++ b/Robust.Shared/EntitySerialization/Options.cs @@ -1,5 +1,4 @@ using System.Numerics; -using JetBrains.Annotations; using Robust.Shared.EntitySerialization.Components; using Robust.Shared.GameObjects; using Robust.Shared.Log; @@ -21,7 +20,13 @@ public record struct SerializationOptions public MissingEntityBehaviour MissingEntityBehaviour = MissingEntityBehaviour.IncludeNullspace; /// - /// Whether or not to log an error when serializing an entity without its parent. + /// What to do when an exception is thrown while trying to serialize an entity. The default behaviour is to abort + /// the serialization. + /// + public EntityExceptionBehaviour EntityExceptionBehaviour = EntityExceptionBehaviour.Rethrow; + + /// + /// Whether to log an error when serializing an entity without its parent. /// public bool ErrorOnOrphan = true; diff --git a/Robust.Shared/EntitySerialization/SerializationEnums.cs b/Robust.Shared/EntitySerialization/SerializationEnums.cs index f64fe4c3980..ae4b4323c08 100644 --- a/Robust.Shared/EntitySerialization/SerializationEnums.cs +++ b/Robust.Shared/EntitySerialization/SerializationEnums.cs @@ -86,3 +86,27 @@ public enum MissingEntityBehaviour /// AutoInclude, } + + +public enum EntityExceptionBehaviour +{ + /// + /// Re-throw the exception, interrupting the serialization. + /// + Rethrow, + + /// + /// Continue serializing and simply skip/ignore this entity. May result in broken maps that log errors or simply + /// fail to load. + /// + IgnoreEntity, + + // TODO SERIALIZATION + /* + /// + /// Continue the serialization while skipping over the component that caused the exception to be thrown. May result + /// in broken maps that log errors or simply fail to load. + /// + IgnoreComponent, + */ +} From 377920199a1cf917e3558c445911e7d3be8d64b6 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 19:36:40 +1200 Subject: [PATCH 02/13] Prevent remove children of erroring entities --- .../EntitySerialization/EntitySerializer.cs | 50 +++++++++++++++++-- .../EntitySerialization/SerializationEnums.cs | 6 +++ 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/Robust.Shared/EntitySerialization/EntitySerializer.cs b/Robust.Shared/EntitySerialization/EntitySerializer.cs index 00f0f6f1d08..9a789fe2eb0 100644 --- a/Robust.Shared/EntitySerialization/EntitySerializer.cs +++ b/Robust.Shared/EntitySerialization/EntitySerializer.cs @@ -110,6 +110,11 @@ public sealed class EntitySerializer : ISerializationContext, /// public readonly Dictionary> Prototypes = new(); + /// + /// Set of entities that have encountered issues during serialization and are now being ignored. + /// + public HashSet ErroringEntities = new(); + /// /// Yaml ids of all serialized map entities. /// @@ -499,11 +504,10 @@ private void SerializeEntityInternal(EntityUid uid) if (Options.EntityExceptionBehaviour == EntityExceptionBehaviour.Rethrow) throw; - Prototypes[protoId].Remove(saveId); - EntityData.Remove(saveId); CurrentEntityYamlUid = 0; CurrentEntity = null; CurrentComponent = null; + RemoveErroringEntity(uid); return; } @@ -524,7 +528,7 @@ private void SerializeEntityInternal(EntityUid uid) return; } - // an entity may have less components than the original prototype, so we need to check if any are missing. + // an entity may have fewer components than the original prototype, so we need to check if any are missing. SequenceDataNode? missingComponents = null; foreach (var (name, comp) in meta.EntityPrototype.Components) { @@ -544,6 +548,32 @@ private void SerializeEntityInternal(EntityUid uid) CurrentEntity = null; } + /// + /// Remove an exception throwing entity (and possibly its children) from the serialized data. + /// + private void RemoveErroringEntity(EntityUid uid) + { + ErroringEntities.Add(uid); + if (YamlUidMap.TryGetValue(uid, out var yamlId)) + { + EntityData.Remove(yamlId); + if (_metaQuery.TryGetComponent(uid, out var meta) + && meta.EntityPrototype != null + && Prototypes.TryGetValue(meta.EntityPrototype.ID, out var proto)) + { + proto.Remove(yamlId); + } + } + + if (Options.EntityExceptionBehaviour != EntityExceptionBehaviour.IgnoreEntityAndChildren) + return; + + foreach (var child in _xformQuery.GetComponent(uid)._children) + { + RemoveErroringEntity(child); + } + } + private void SerializeComponents(EntityUid uid, Dictionary? cache, SequenceDataNode components) { foreach (var component in EntMan.GetComponentsInternal(uid)) @@ -677,7 +707,10 @@ public MappingDataNode WriteTileMap() public SequenceDataNode WriteEntitySection() { - if (YamlIds.Count != YamlUidMap.Count || YamlIds.Count != EntityData.Count) + // Check that EntityData contains the expected number of entities. + if (Options.EntityExceptionBehaviour != EntityExceptionBehaviour.IgnoreEntity + && Options.EntityExceptionBehaviour != EntityExceptionBehaviour.IgnoreEntityAndChildren + && (YamlIds.Count != YamlUidMap.Count || YamlIds.Count != EntityData.Count)) { // Maybe someone reserved a yaml id with ReserveYamlId() or implicitly with GetId() without actually // ever serializing the entity, This can lead to references to non-existent entities. @@ -899,6 +932,7 @@ public DataNode Write( if (YamlUidMap.TryGetValue(value, out var yamlId)) return new ValueDataNode(yamlId.ToString(CultureInfo.InvariantCulture)); + if (CurrentComponent == _xformName) { if (value == EntityUid.Invalid) @@ -907,12 +941,18 @@ public DataNode Write( DebugTools.Assert(!Orphans.Contains(CurrentEntityYamlUid)); Orphans.Add(CurrentEntityYamlUid); - if (Options.ErrorOnOrphan && CurrentEntity != null && value != Truncate) + if (Options.ErrorOnOrphan && CurrentEntity != null && value != Truncate && !ErroringEntities.Contains(value)) _log.Error($"Serializing entity {EntMan.ToPrettyString(CurrentEntity)} without including its parent {EntMan.ToPrettyString(value)}"); return new ValueDataNode("invalid"); } + if (ErroringEntities.Contains(value)) + { + // Referenced entity already logged an error, so we just silently fail. + return new ValueDataNode("invalid"); + } + if (value == EntityUid.Invalid) { if (Options.MissingEntityBehaviour != MissingEntityBehaviour.Ignore) diff --git a/Robust.Shared/EntitySerialization/SerializationEnums.cs b/Robust.Shared/EntitySerialization/SerializationEnums.cs index ae4b4323c08..47c77619af0 100644 --- a/Robust.Shared/EntitySerialization/SerializationEnums.cs +++ b/Robust.Shared/EntitySerialization/SerializationEnums.cs @@ -101,6 +101,12 @@ public enum EntityExceptionBehaviour /// IgnoreEntity, + /// + /// Continue serializing and simply skip/ignore this entity and all ofits children. + /// May result in broken maps that log errors or simply fail to load. + /// + IgnoreEntityAndChildren, + // TODO SERIALIZATION /* /// From c5b887a02e2730fdbc61e8f637d2c8f99a23a033 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 19:59:07 +1200 Subject: [PATCH 03/13] better logging --- Robust.Shared/EntitySerialization/EntitySerializer.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Robust.Shared/EntitySerialization/EntitySerializer.cs b/Robust.Shared/EntitySerialization/EntitySerializer.cs index 9a789fe2eb0..b2e47a344f4 100644 --- a/Robust.Shared/EntitySerialization/EntitySerializer.cs +++ b/Robust.Shared/EntitySerialization/EntitySerializer.cs @@ -498,12 +498,15 @@ private void SerializeEntityInternal(EntityUid uid) { SerializeComponents(uid, cache, components); } - catch + catch(Exception e) { - _log.Error($"Caught exception while serializing component {CurrentComponent} of entity {EntMan.ToPrettyString(uid)}"); if (Options.EntityExceptionBehaviour == EntityExceptionBehaviour.Rethrow) + { + _log.Error($"Caught exception while serializing component {CurrentComponent} of entity {EntMan.ToPrettyString(uid)}"); throw; + } + _log.Error($"Caught exception while serializing component {CurrentComponent} of entity {EntMan.ToPrettyString(uid)}:\n{e}"); CurrentEntityYamlUid = 0; CurrentEntity = null; CurrentComponent = null; From e089da285362030d760c39aece39f35cf2562a8d Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 20:10:02 +1200 Subject: [PATCH 04/13] Improve error tolerance --- .../EntitySerialization/EntitySerializer.cs | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/Robust.Shared/EntitySerialization/EntitySerializer.cs b/Robust.Shared/EntitySerialization/EntitySerializer.cs index b2e47a344f4..422fd0c672e 100644 --- a/Robust.Shared/EntitySerialization/EntitySerializer.cs +++ b/Robust.Shared/EntitySerialization/EntitySerializer.cs @@ -556,24 +556,28 @@ private void SerializeEntityInternal(EntityUid uid) /// private void RemoveErroringEntity(EntityUid uid) { - ErroringEntities.Add(uid); - if (YamlUidMap.TryGetValue(uid, out var yamlId)) + if (Options.EntityExceptionBehaviour == EntityExceptionBehaviour.IgnoreEntityAndChildren) { - EntityData.Remove(yamlId); - if (_metaQuery.TryGetComponent(uid, out var meta) - && meta.EntityPrototype != null - && Prototypes.TryGetValue(meta.EntityPrototype.ID, out var proto)) + foreach (var child in _xformQuery.GetComponent(uid)._children) { - proto.Remove(yamlId); + RemoveErroringEntity(child); } } - if (Options.EntityExceptionBehaviour != EntityExceptionBehaviour.IgnoreEntityAndChildren) + ErroringEntities.Add(uid); + if (!YamlUidMap.TryGetValue(uid, out var yamlId)) return; - foreach (var child in _xformQuery.GetComponent(uid)._children) + Nullspace.Remove(yamlId); + Orphans.Remove(yamlId); + Maps.Remove(yamlId); + Grids.Remove(yamlId); + EntityData.Remove(yamlId); + if (_metaQuery.TryGetComponent(uid, out var meta) + && meta.EntityPrototype != null + && Prototypes.TryGetValue(meta.EntityPrototype.ID, out var proto)) { - RemoveErroringEntity(child); + proto.Remove(yamlId); } } From 2ce4b5b7d7904a8c43e9a27268ddf3f6c0c8e9a5 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 20:21:18 +1200 Subject: [PATCH 05/13] Even more exception tolerance --- .../EntitySerialization/EntityDeserializer.cs | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Robust.Shared/EntitySerialization/EntityDeserializer.cs b/Robust.Shared/EntitySerialization/EntityDeserializer.cs index baf500e10bf..b6af9911ee0 100644 --- a/Robust.Shared/EntitySerialization/EntityDeserializer.cs +++ b/Robust.Shared/EntitySerialization/EntityDeserializer.cs @@ -685,38 +685,38 @@ private void GetRootEntities() foreach (var yamlId in MapYamlIds) { - var uid = UidMap[yamlId]; - if (_mapQuery.TryComp(uid, out var map)) + if (UidMap.TryGetValue(yamlId, out var uid) && _mapQuery.TryComp(uid, out var map)) { Result.Maps.Add((uid, map)); EntMan.EnsureComponent(uid); } else - _log.Error($"Missing map entity: {EntMan.ToPrettyString(uid)}"); + _log.Error($"Missing map entity: {EntMan.ToPrettyString(uid)}. YamlId: {yamlId}"); } foreach (var yamlId in GridYamlIds) { - var uid = UidMap[yamlId]; - if (_gridQuery.TryComp(uid, out var grid)) + if (UidMap.TryGetValue(yamlId, out var uid) && _gridQuery.TryComp(uid, out var grid)) Result.Grids.Add((uid, grid)); else - _log.Error($"Missing grid entity: {EntMan.ToPrettyString(uid)}"); + _log.Error($"Missing grid entity: {EntMan.ToPrettyString(uid)}. YamlId: {yamlId}"); } foreach (var yamlId in OrphanYamlIds) { - var uid = UidMap[yamlId]; - if (_mapQuery.HasComponent(uid) || _xformQuery.Comp(uid).ParentUid.IsValid()) - _log.Error($"Entity {EntMan.ToPrettyString(uid)} was incorrectly labelled as an orphan?"); + if (UidMap.TryGetValue(yamlId, out var uid)) + _log.Error($"Missing orphan entity with YamlId: {yamlId}"); + else if (_mapQuery.HasComponent(uid) || _xformQuery.Comp(uid).ParentUid.IsValid()) + _log.Error($"Entity {EntMan.ToPrettyString(uid)} was incorrectly labelled as an orphan? YamlId: {yamlId}"); else Result.Orphans.Add(uid); } foreach (var yamlId in NullspaceYamlIds) { - var uid = UidMap[yamlId]; - if (_mapQuery.HasComponent(uid) || _xformQuery.Comp(uid).ParentUid.IsValid()) + if (UidMap.TryGetValue(yamlId, out var uid)) + _log.Error($"Missing nullspace entity with YamlId: {yamlId}"); + else if (_mapQuery.HasComponent(uid) || _xformQuery.Comp(uid).ParentUid.IsValid()) _log.Error($"Entity {EntMan.ToPrettyString(uid)} was incorrectly labelled as a null-space entity?"); else Result.NullspaceEntities.Add(uid); @@ -1172,8 +1172,8 @@ EntityUid ITypeReader.Read( return entity; msg = CurrentReadingEntity is not { } ent - ? "Encountered unknown EntityUid reference" - : $"Encountered unknown EntityUid reference wile reading entity {ent.YamlId}, component: {CurrentComponent}"; + ? "Encountered unknown entity yaml uid" + : $"Encountered unknown entity yaml uid wile reading entity {ent.YamlId}, component: {CurrentComponent}"; _log.Error(msg); return EntityUid.Invalid; } From d0884c94f57b2c86d46ba42911598786b0f7282d Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 20:25:57 +1200 Subject: [PATCH 06/13] missing ! --- Robust.Shared/EntitySerialization/EntityDeserializer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Robust.Shared/EntitySerialization/EntityDeserializer.cs b/Robust.Shared/EntitySerialization/EntityDeserializer.cs index b6af9911ee0..11da552364b 100644 --- a/Robust.Shared/EntitySerialization/EntityDeserializer.cs +++ b/Robust.Shared/EntitySerialization/EntityDeserializer.cs @@ -704,7 +704,7 @@ private void GetRootEntities() foreach (var yamlId in OrphanYamlIds) { - if (UidMap.TryGetValue(yamlId, out var uid)) + if (!UidMap.TryGetValue(yamlId, out var uid)) _log.Error($"Missing orphan entity with YamlId: {yamlId}"); else if (_mapQuery.HasComponent(uid) || _xformQuery.Comp(uid).ParentUid.IsValid()) _log.Error($"Entity {EntMan.ToPrettyString(uid)} was incorrectly labelled as an orphan? YamlId: {yamlId}"); @@ -714,7 +714,7 @@ private void GetRootEntities() foreach (var yamlId in NullspaceYamlIds) { - if (UidMap.TryGetValue(yamlId, out var uid)) + if (!UidMap.TryGetValue(yamlId, out var uid)) _log.Error($"Missing nullspace entity with YamlId: {yamlId}"); else if (_mapQuery.HasComponent(uid) || _xformQuery.Comp(uid).ParentUid.IsValid()) _log.Error($"Entity {EntMan.ToPrettyString(uid)} was incorrectly labelled as a null-space entity?"); From 016cf8669a60066ba1d0fc5147034890763a1830 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 20:28:17 +1200 Subject: [PATCH 07/13] Add WriteYaml and WriteObject to IReplayFileWriter --- .../Replays/IReplayRecordingManager.cs | 29 +++++++++++++++++++ .../SharedReplayRecordingManager.Write.cs | 16 +++++++--- .../Replays/SharedReplayRecordingManager.cs | 23 +++++++++++++++ 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/Robust.Shared/Replays/IReplayRecordingManager.cs b/Robust.Shared/Replays/IReplayRecordingManager.cs index 0a6b0adb70a..a307573bf5d 100644 --- a/Robust.Shared/Replays/IReplayRecordingManager.cs +++ b/Robust.Shared/Replays/IReplayRecordingManager.cs @@ -6,7 +6,9 @@ using System.Threading.Tasks; using Robust.Shared.ContentPack; using Robust.Shared.GameStates; +using Robust.Shared.Serialization; using Robust.Shared.Utility; +using YamlDotNet.RepresentationModel; namespace Robust.Shared.Replays; @@ -202,6 +204,33 @@ void WriteBytes( ResPath path, ReadOnlyMemory bytes, CompressionLevel compressionLevel = CompressionLevel.Optimal); + + /// + /// Writes a yaml document into a file in the replay. + /// + /// The file path to write to. + /// The yaml document to write to the file. + /// How much to compress the file. + void WriteYaml( + ResPath path, + YamlDocument yaml, + CompressionLevel compressionLevel = CompressionLevel.Optimal); + + /// + /// Serializes an object using and write it into a file in the replay. + /// + /// + /// As these objects can't really be deserialized without launching a server/client with a matching game version, + /// you should consider using some other means of serializing data if you want it to be parseable outside of a + /// replay client. + /// + /// The file path to write to. + /// The object to serialize and write to the file. + /// How much to compress the file. + void WriteObject( + ResPath path, + T obj, + CompressionLevel compressionLevel = CompressionLevel.Optimal); } /// diff --git a/Robust.Shared/Replays/SharedReplayRecordingManager.Write.cs b/Robust.Shared/Replays/SharedReplayRecordingManager.Write.cs index cc3a12b0c23..00dd9b50ef9 100644 --- a/Robust.Shared/Replays/SharedReplayRecordingManager.Write.cs +++ b/Robust.Shared/Replays/SharedReplayRecordingManager.Write.cs @@ -26,22 +26,30 @@ internal abstract partial class SharedReplayRecordingManager // and even then not for much longer than a couple hundred ms at most. private readonly List _finalizingWriteTasks = new(); - private void WriteYaml(RecordingState state, ResPath path, YamlDocument data) + private void WriteYaml( + RecordingState state, + ResPath path, + YamlDocument data, + CompressionLevel level = CompressionLevel.Optimal) { var memStream = new MemoryStream(); using var writer = new StreamWriter(memStream); var yamlStream = new YamlStream { data }; yamlStream.Save(new YamlMappingFix(new Emitter(writer)), false); writer.Flush(); - WriteBytes(state, path, memStream.AsMemory()); + WriteBytes(state, path, memStream.AsMemory(), level); } - private void WriteSerializer(RecordingState state, ResPath path, T obj) + private void WriteSerializer( + RecordingState state, + ResPath path, + T obj, + CompressionLevel level = CompressionLevel.Optimal) { var memStream = new MemoryStream(); _serializer.SerializeDirect(memStream, obj); - WriteBytes(state, path, memStream.AsMemory()); + WriteBytes(state, path, memStream.AsMemory(), level); } private void WritePooledBytes( diff --git a/Robust.Shared/Replays/SharedReplayRecordingManager.cs b/Robust.Shared/Replays/SharedReplayRecordingManager.cs index 656e9ff83b5..aabba0e9ae6 100644 --- a/Robust.Shared/Replays/SharedReplayRecordingManager.cs +++ b/Robust.Shared/Replays/SharedReplayRecordingManager.cs @@ -375,6 +375,11 @@ private void WriteInitialMetadata(string name, RecordingState recState) private void WriteFinalMetadata(RecordingState recState) { var yamlMetadata = new MappingDataNode(); + + // TODO REPLAYS + // Why are these separate events? + // I assume it was for backwards compatibility / avoiding breaking changes? + // But eventually RecordingStopped2 will probably be renamed and there'll just be more breaking changes. RecordingStopped?.Invoke(yamlMetadata); RecordingStopped2?.Invoke(new ReplayRecordingStopped { @@ -552,6 +557,24 @@ public void WriteBytes(ResPath path, ReadOnlyMemory bytes, CompressionLeve manager.WriteBytes(state, path, bytes, compressionLevel); } + /// + /// Write a yaml document to a file. + /// + public void WriteYaml(ResPath path, YamlDocument document, CompressionLevel compressionLevel) + { + CheckDisposed(); + manager.WriteYaml(state, path, document, compressionLevel); + } + + /// + /// Write a binary object to a file using + /// + public void WriteObject(ResPath path, T obj, CompressionLevel compressionLevel) + { + CheckDisposed(); + manager.WriteSerializer(state, path, obj, compressionLevel); + } + private void CheckDisposed() { if (state.Done) From ebfa4bd5feecf51d7bf6f9e6143346ca3345d7f4 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 20:36:43 +1200 Subject: [PATCH 08/13] Add MapLoaderSystem.TrySaveAllEntities() --- .../EntitySerialization/EntitySerializer.cs | 22 +++-- .../Systems/MapLoaderSystem.Save.cs | 97 ++++++++++++++++++- .../Map/Events/MapSerializationEvents.cs | 5 +- 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/Robust.Shared/EntitySerialization/EntitySerializer.cs b/Robust.Shared/EntitySerialization/EntitySerializer.cs index 422fd0c672e..a80395a1871 100644 --- a/Robust.Shared/EntitySerialization/EntitySerializer.cs +++ b/Robust.Shared/EntitySerialization/EntitySerializer.cs @@ -60,6 +60,7 @@ public sealed class EntitySerializer : ISerializationContext, private readonly ISawmill _log; public readonly Dictionary YamlUidMap = new(); public readonly HashSet YamlIds = new(); + public readonly ValueDataNode InvalidNode = new("invalid"); public string? CurrentComponent { get; private set; } @@ -221,6 +222,7 @@ public void SerializeEntity(EntityUid uid) /// setting of it may auto-include additional entities /// aside from the one provided. /// + /// The set of entities to serialize public void SerializeEntities(HashSet entities) { foreach (var uid in entities) @@ -292,7 +294,12 @@ private bool FindSavedTileMap(EntityUid root, [NotNullWhen(true)] out Dictionary return true; } - // iterate over all of its children and grab the first grid with a mapping + map = null; + + // if this is a map, iterate over all of its children and grab the first grid with a mapping + if (!_mapQuery.HasComponent(root)) + return false; + var xform = _xformQuery.GetComponent(root); foreach (var child in xform._children) { @@ -302,7 +309,6 @@ private bool FindSavedTileMap(EntityUid root, [NotNullWhen(true)] out Dictionary return true; } - map = null; return false; } @@ -943,7 +949,7 @@ public DataNode Write( if (CurrentComponent == _xformName) { if (value == EntityUid.Invalid) - return new ValueDataNode("invalid"); + return InvalidNode; DebugTools.Assert(!Orphans.Contains(CurrentEntityYamlUid)); Orphans.Add(CurrentEntityYamlUid); @@ -951,13 +957,13 @@ public DataNode Write( if (Options.ErrorOnOrphan && CurrentEntity != null && value != Truncate && !ErroringEntities.Contains(value)) _log.Error($"Serializing entity {EntMan.ToPrettyString(CurrentEntity)} without including its parent {EntMan.ToPrettyString(value)}"); - return new ValueDataNode("invalid"); + return InvalidNode; } if (ErroringEntities.Contains(value)) { // Referenced entity already logged an error, so we just silently fail. - return new ValueDataNode("invalid"); + return InvalidNode; } if (value == EntityUid.Invalid) @@ -965,7 +971,7 @@ public DataNode Write( if (Options.MissingEntityBehaviour != MissingEntityBehaviour.Ignore) _log.Error($"Encountered an invalid entityUid reference."); - return new ValueDataNode("invalid"); + return InvalidNode; } if (value == Truncate) @@ -980,9 +986,9 @@ public DataNode Write( _log.Error(EntMan.Deleted(value) ? $"Encountered a reference to a deleted entity {value} while serializing {EntMan.ToPrettyString(CurrentEntity)}." : $"Encountered a reference to a missing entity: {value} while serializing {EntMan.ToPrettyString(CurrentEntity)}."); - return new ValueDataNode("invalid"); + return InvalidNode; case MissingEntityBehaviour.Ignore: - return new ValueDataNode("invalid"); + return InvalidNode; case MissingEntityBehaviour.IncludeNullspace: if (!EntMan.TryGetComponent(value, out TransformComponent? xform) || xform.ParentUid != EntityUid.Invalid diff --git a/Robust.Shared/EntitySerialization/Systems/MapLoaderSystem.Save.cs b/Robust.Shared/EntitySerialization/Systems/MapLoaderSystem.Save.cs index a0d6488e1d0..4ffe8dbc9b1 100644 --- a/Robust.Shared/EntitySerialization/Systems/MapLoaderSystem.Save.cs +++ b/Robust.Shared/EntitySerialization/Systems/MapLoaderSystem.Save.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using Robust.Shared.GameObjects; using Robust.Shared.Map; @@ -16,8 +17,12 @@ public sealed partial class MapLoaderSystem public event EntitySerializer.IsSerializableDelegate? OnIsSerializable; /// - /// Recursively serialize the given entity and its children. + /// Recursively serialize the given entities and all of their children. /// + /// + /// This method is not optimized for being given a large set of entities. I.e., this should be a small handful of + /// maps or grids, not something like . + /// public (MappingDataNode Node, FileCategory Category) SerializeEntitiesRecursive( HashSet entities, SerializationOptions? options = null) @@ -29,8 +34,6 @@ public sealed partial class MapLoaderSystem Log.Info($"Serializing entities: {string.Join(", ", entities.Select(x => ToPrettyString(x).ToString()))}"); var maps = entities.Select(x => Transform(x).MapID).ToHashSet(); - var ev = new BeforeSerializationEvent(entities, maps); - RaiseLocalEvent(ev); // In case no options were provided, we assume that if all of the starting entities are pre-init, we should // expect that **all** entities that get serialized should be pre-init. @@ -39,6 +42,9 @@ public sealed partial class MapLoaderSystem ExpectPreInit = (entities.All(x => LifeStage(x) < EntityLifeStage.MapInitialized)) }; + var ev = new BeforeSerializationEvent(entities, maps, opts.Category); + RaiseLocalEvent(ev); + var serializer = new EntitySerializer(_dependency, opts); serializer.OnIsSerializeable += OnIsSerializable; @@ -235,4 +241,89 @@ public bool TrySaveGeneric( Write(path, data); return true; } + + /// + public bool TrySaveAllEntities(ResPath path, SerializationOptions? options = null) + { + if (!TrySerializeAllEntities(out var data, options)) + return false; + + Write(path, data); + return true; + } + + /// + /// Attempt to serialize all entities. + /// + /// + /// Note that this alone is not sufficient for a proper full-game save, as the game may contain things like chat + /// logs or resources and prototypes that were uploaded mid-game. + /// + public bool TrySerializeAllEntities([NotNullWhen(true)] out MappingDataNode? data, SerializationOptions? options = null) + { + data = null; + var opts = options ?? SerializationOptions.Default with + { + MissingEntityBehaviour = MissingEntityBehaviour.Error + }; + + opts.Category = FileCategory.Save; + _stopwatch.Restart(); + Log.Info($"Serializing all entities"); + + var entities = EntityManager.GetEntities().ToHashSet(); + var maps = _mapSystem.Maps.Keys.ToHashSet(); + var ev = new BeforeSerializationEvent(entities, maps, FileCategory.Save); + var serializer = new EntitySerializer(_dependency, opts); + + // Remove any non-serializable entities and their children (prevent error spam) + var toRemove = new Queue(); + foreach (var entity in entities) + { + // TODO SERIALIZATION Perf + // IsSerializable gets called again by serializer.SerializeEntities() + if (!serializer.IsSerializable(entity)) + toRemove.Enqueue(entity); + } + + if (toRemove.Count > 0) + { + if (opts.MissingEntityBehaviour == MissingEntityBehaviour.Error) + { + // The save will probably contain references to the non-serializable entities, and we avoid spamming errors. + opts.MissingEntityBehaviour = MissingEntityBehaviour.Ignore; + Log.Error($"Attempted to serialize one or more non-serializable entities"); + } + + while (toRemove.TryDequeue(out var next)) + { + entities.Remove(next); + foreach (var uid in Transform(next)._children) + { + toRemove.Enqueue(uid); + } + } + } + + try + { + RaiseLocalEvent(ev); + serializer.OnIsSerializeable += OnIsSerializable; + serializer.SerializeEntities(entities); + data = serializer.Write(); + var cat = serializer.GetCategory(); + DebugTools.AssertEqual(cat, FileCategory.Save); + var ev2 = new AfterSerializationEvent(entities, data, cat); + RaiseLocalEvent(ev2); + + Log.Debug($"Serialized {serializer.EntityData.Count} entities in {_stopwatch.Elapsed}"); + } + catch (Exception e) + { + Log.Error($"Caught exception while trying to serialize all entities:\n{e}"); + return false; + } + + return true; + } } diff --git a/Robust.Shared/Map/Events/MapSerializationEvents.cs b/Robust.Shared/Map/Events/MapSerializationEvents.cs index c2721c1bcf0..477161ed5da 100644 --- a/Robust.Shared/Map/Events/MapSerializationEvents.cs +++ b/Robust.Shared/Map/Events/MapSerializationEvents.cs @@ -34,7 +34,10 @@ public sealed class BeforeEntityReadEvent /// For convenience, the event also contains a set with all the maps that the entities are on. This does not /// necessarily mean that the maps are themselves getting serialized. /// -public readonly record struct BeforeSerializationEvent(HashSet Entities, HashSet MapIds); +public readonly record struct BeforeSerializationEvent( + HashSet Entities, + HashSet MapIds, + FileCategory Category = FileCategory.Unknown); /// /// This event is broadcast just after entities (and their children) have been serialized, but before it gets written to a yaml file. From ecba27bc843c96ed61a24bef98cb93270fd00880 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 9 Sep 2025 20:48:22 +1200 Subject: [PATCH 09/13] On second thought, WriteObject will just be abused --- Robust.Shared/Replays/IReplayRecordingManager.cs | 16 ---------------- .../Replays/SharedReplayRecordingManager.cs | 12 ------------ 2 files changed, 28 deletions(-) diff --git a/Robust.Shared/Replays/IReplayRecordingManager.cs b/Robust.Shared/Replays/IReplayRecordingManager.cs index a307573bf5d..6f356774b98 100644 --- a/Robust.Shared/Replays/IReplayRecordingManager.cs +++ b/Robust.Shared/Replays/IReplayRecordingManager.cs @@ -215,22 +215,6 @@ void WriteYaml( ResPath path, YamlDocument yaml, CompressionLevel compressionLevel = CompressionLevel.Optimal); - - /// - /// Serializes an object using and write it into a file in the replay. - /// - /// - /// As these objects can't really be deserialized without launching a server/client with a matching game version, - /// you should consider using some other means of serializing data if you want it to be parseable outside of a - /// replay client. - /// - /// The file path to write to. - /// The object to serialize and write to the file. - /// How much to compress the file. - void WriteObject( - ResPath path, - T obj, - CompressionLevel compressionLevel = CompressionLevel.Optimal); } /// diff --git a/Robust.Shared/Replays/SharedReplayRecordingManager.cs b/Robust.Shared/Replays/SharedReplayRecordingManager.cs index aabba0e9ae6..11514d0d5f3 100644 --- a/Robust.Shared/Replays/SharedReplayRecordingManager.cs +++ b/Robust.Shared/Replays/SharedReplayRecordingManager.cs @@ -557,24 +557,12 @@ public void WriteBytes(ResPath path, ReadOnlyMemory bytes, CompressionLeve manager.WriteBytes(state, path, bytes, compressionLevel); } - /// - /// Write a yaml document to a file. - /// public void WriteYaml(ResPath path, YamlDocument document, CompressionLevel compressionLevel) { CheckDisposed(); manager.WriteYaml(state, path, document, compressionLevel); } - /// - /// Write a binary object to a file using - /// - public void WriteObject(ResPath path, T obj, CompressionLevel compressionLevel) - { - CheckDisposed(); - manager.WriteSerializer(state, path, obj, compressionLevel); - } - private void CheckDisposed() { if (state.Done) From 6d4bc2cfb80a14242836e5dc26ec457e3a696654 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Wed, 10 Sep 2025 13:55:02 +1200 Subject: [PATCH 10/13] I forgot to commit --- Resources/EnginePrototypes/Audio/audio_entities.yml | 2 +- Robust.Shared/Player/ActorComponent.cs | 2 +- Robust.Shared/Replays/IReplayRecordingManager.cs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Resources/EnginePrototypes/Audio/audio_entities.yml b/Resources/EnginePrototypes/Audio/audio_entities.yml index 5de22856c2f..e9449c5ed26 100644 --- a/Resources/EnginePrototypes/Audio/audio_entities.yml +++ b/Resources/EnginePrototypes/Audio/audio_entities.yml @@ -2,7 +2,7 @@ id: Audio name: Audio description: Audio entity used by engine - save: false + save: false # TODO PERSISTENCE what about looping or long sounds? components: - type: Transform gridTraversal: false diff --git a/Robust.Shared/Player/ActorComponent.cs b/Robust.Shared/Player/ActorComponent.cs index 234defbbf75..0c245116eb6 100644 --- a/Robust.Shared/Player/ActorComponent.cs +++ b/Robust.Shared/Player/ActorComponent.cs @@ -3,7 +3,7 @@ namespace Robust.Shared.Player; -[RegisterComponent] +[RegisterComponent, UnsavedComponent] public sealed partial class ActorComponent : Component { [ViewVariables] diff --git a/Robust.Shared/Replays/IReplayRecordingManager.cs b/Robust.Shared/Replays/IReplayRecordingManager.cs index 6f356774b98..11a84caeaec 100644 --- a/Robust.Shared/Replays/IReplayRecordingManager.cs +++ b/Robust.Shared/Replays/IReplayRecordingManager.cs @@ -6,7 +6,6 @@ using System.Threading.Tasks; using Robust.Shared.ContentPack; using Robust.Shared.GameStates; -using Robust.Shared.Serialization; using Robust.Shared.Utility; using YamlDotNet.RepresentationModel; From b74b51895f6fc91566e79510029b4aa482467203 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Thu, 11 Sep 2025 19:23:05 +1200 Subject: [PATCH 11/13] Add default implementation to avoid breaking changes --- Robust.Shared/Replays/IReplayRecordingManager.cs | 13 ++++++++++++- .../Replays/SharedReplayRecordingManager.cs | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/Robust.Shared/Replays/IReplayRecordingManager.cs b/Robust.Shared/Replays/IReplayRecordingManager.cs index 11a84caeaec..6a2a7b89c09 100644 --- a/Robust.Shared/Replays/IReplayRecordingManager.cs +++ b/Robust.Shared/Replays/IReplayRecordingManager.cs @@ -2,11 +2,14 @@ using Robust.Shared.Serialization.Markdown.Mapping; using System; using System.Collections.Generic; +using System.IO; using System.IO.Compression; using System.Threading.Tasks; using Robust.Shared.ContentPack; using Robust.Shared.GameStates; +using Robust.Shared.Serialization; using Robust.Shared.Utility; +using YamlDotNet.Core; using YamlDotNet.RepresentationModel; namespace Robust.Shared.Replays; @@ -213,7 +216,15 @@ void WriteBytes( void WriteYaml( ResPath path, YamlDocument yaml, - CompressionLevel compressionLevel = CompressionLevel.Optimal); + CompressionLevel compressionLevel = CompressionLevel.Optimal) + { + var memStream = new MemoryStream(); + using var writer = new StreamWriter(memStream); + var yamlStream = new YamlStream {yaml}; + yamlStream.Save(new YamlMappingFix(new Emitter(writer)), false); + writer.Flush(); + WriteBytes(path, memStream.AsMemory(), compressionLevel); + } } /// diff --git a/Robust.Shared/Replays/SharedReplayRecordingManager.cs b/Robust.Shared/Replays/SharedReplayRecordingManager.cs index 11514d0d5f3..40cb32de0be 100644 --- a/Robust.Shared/Replays/SharedReplayRecordingManager.cs +++ b/Robust.Shared/Replays/SharedReplayRecordingManager.cs @@ -557,7 +557,7 @@ public void WriteBytes(ResPath path, ReadOnlyMemory bytes, CompressionLeve manager.WriteBytes(state, path, bytes, compressionLevel); } - public void WriteYaml(ResPath path, YamlDocument document, CompressionLevel compressionLevel) + void IReplayFileWriter.WriteYaml(ResPath path, YamlDocument document, CompressionLevel compressionLevel) { CheckDisposed(); manager.WriteYaml(state, path, document, compressionLevel); From 02866f11a5d877c97e2e4a7772c66b03f3fe7d02 Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Thu, 11 Sep 2025 19:24:59 +1200 Subject: [PATCH 12/13] release notes --- RELEASE-NOTES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 6ff69becdb4..8ab21564c12 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -42,10 +42,12 @@ END TEMPLATE--> * `Control.OrderedChildCollection` (gotten from `.Children`) now implements `IReadOnlyList`, allowing it to be indexed directly. * `System.WeakReference` is now available in the sandbox. * `IClydeViewport` now has an `Id` and `ClearCachedResources` event. Together, these allow you to properly cache rendering resources per viewport. +* Added `IReplayFileWriter.WriteYaml()`, for writing yaml documents to a replay zip file. ### Bugfixes -*None yet* +* `ActorComponent` now has the `UnsavedComponentAttribute` + * Previously it was unintentionally get serialized to yaml, which could result in NREs when deserializing. ### Other From 973ebec68efee7e16b557f2e58abfc1f0dc569cf Mon Sep 17 00:00:00 2001 From: ElectroJr Date: Tue, 23 Sep 2025 16:16:05 +1200 Subject: [PATCH 13/13] fix merge issues --- Robust.Shared/EntitySerialization/EntitySerializer.cs | 6 ------ Robust.Shared/EntitySerialization/SerializationEnums.cs | 2 +- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/Robust.Shared/EntitySerialization/EntitySerializer.cs b/Robust.Shared/EntitySerialization/EntitySerializer.cs index 1ea5d2cddce..a80395a1871 100644 --- a/Robust.Shared/EntitySerialization/EntitySerializer.cs +++ b/Robust.Shared/EntitySerialization/EntitySerializer.cs @@ -966,12 +966,6 @@ public DataNode Write( return InvalidNode; } - if (ErroringEntities.Contains(value)) - { - // Referenced entity already logged an error, so we just silently fail. - return new ValueDataNode("invalid"); - } - if (value == EntityUid.Invalid) { if (Options.MissingEntityBehaviour != MissingEntityBehaviour.Ignore) diff --git a/Robust.Shared/EntitySerialization/SerializationEnums.cs b/Robust.Shared/EntitySerialization/SerializationEnums.cs index 7a588bfa983..bfc6ae1c7b9 100644 --- a/Robust.Shared/EntitySerialization/SerializationEnums.cs +++ b/Robust.Shared/EntitySerialization/SerializationEnums.cs @@ -101,7 +101,7 @@ public enum EntityExceptionBehaviour IgnoreEntity, /// - /// Continue serializing and simply skip/ignore this entity and all ofits children. + /// Continue serializing and simply skip/ignore this entity and all of its children. /// May result in broken maps that log errors or simply fail to load. /// IgnoreEntityAndChildren,