Add support for including map/game saves in replays.

RELEASE-NOTES.md

@@ -40,11 +40,14 @@ END TEMPLATE-->
 ### New features
 
 * Sprites and Sprite layers have a new `Loop` data field that can be set to false to automatically pause animations once they have finished.
+* Added `IReplayFileWriter.WriteYaml()`, for writing yaml documents to a replay zip file.
 
 ### Bugfixes
 
 * Fixed `CollectionExtensions.TryGetValue` throwing an exception when given a negative list index.
-
+* `ActorComponent` now has the `UnsavedComponentAttribute`
+  * Previously it was unintentionally get serialized to yaml, which could result in NREs when deserializing.
+
 ### Other
 
 *None yet*

Resources/EnginePrototypes/Audio/audio_entities.yml

@@ -2,7 +2,7 @@
   id: Audio
   name: Audio
   description: Audio entity used by engine
-  save: false
+  save: false # TODO PERSISTENCE what about looping or long sounds?
   components:
     - type: Transform
       gridTraversal: false

Robust.Shared/EntitySerialization/EntitySerializer.cs

@@ -60,6 +60,7 @@ public sealed class EntitySerializer : ISerializationContext,
     private readonly ISawmill _log;
     public readonly Dictionary<EntityUid, int> YamlUidMap = new();
     public readonly HashSet<int> YamlIds = new();
+    public readonly ValueDataNode InvalidNode = new("invalid");
 
 
     public string? CurrentComponent { get; private set; }
@@ -221,6 +222,7 @@ public void SerializeEntity(EntityUid uid)
     /// setting of <see cref="SerializationOptions.MissingEntityBehaviour"/> it may auto-include additional entities
     /// aside from the one provided.
     /// </summary>
+    /// <param name="entities">The set of entities to serialize</param>
     public void SerializeEntities(HashSet<EntityUid> entities)
     {
         foreach (var uid in entities)
@@ -292,7 +294,12 @@ private bool FindSavedTileMap(EntityUid root, [NotNullWhen(true)] out Dictionary
             return true;
         }
 
-        // iterate over all of its children and grab the first grid with a mapping
+        map = null;
+
+        // if this is a map, iterate over all of its children and grab the first grid with a mapping
+        if (!_mapQuery.HasComponent(root))
+            return false;
+
         var xform = _xformQuery.GetComponent(root);
         foreach (var child in xform._children)
         {
@@ -302,7 +309,6 @@ private bool FindSavedTileMap(EntityUid root, [NotNullWhen(true)] out Dictionary
             return true;
         }
 
-        map = null;
         return false;
     }
 
@@ -943,29 +949,29 @@ public DataNode Write(
         if (CurrentComponent == _xformName)
         {
             if (value == EntityUid.Invalid)
-                return new ValueDataNode("invalid");
+                return InvalidNode;
 
             DebugTools.Assert(!Orphans.Contains(CurrentEntityYamlUid));
             Orphans.Add(CurrentEntityYamlUid);
 
             if (Options.ErrorOnOrphan && CurrentEntity != null && value != Truncate && !ErroringEntities.Contains(value))
                 _log.Error($"Serializing entity {EntMan.ToPrettyString(CurrentEntity)} without including its parent {EntMan.ToPrettyString(value)}");
 
-            return new ValueDataNode("invalid");
+            return InvalidNode;
         }
 
         if (ErroringEntities.Contains(value))
         {
             // Referenced entity already logged an error, so we just silently fail.
-            return new ValueDataNode("invalid");
+            return InvalidNode;
         }
 
         if (value == EntityUid.Invalid)
         {
             if (Options.MissingEntityBehaviour != MissingEntityBehaviour.Ignore)
                 _log.Error($"Encountered an invalid entityUid reference.");
 
-            return new ValueDataNode("invalid");
+            return InvalidNode;
         }
 
         if (value == Truncate)
@@ -980,9 +986,9 @@ public DataNode Write(
                 _log.Error(EntMan.Deleted(value)
                     ? $"Encountered a reference to a deleted entity {value} while serializing {EntMan.ToPrettyString(CurrentEntity)}."
                     : $"Encountered a reference to a missing entity: {value} while serializing {EntMan.ToPrettyString(CurrentEntity)}.");
-                return new ValueDataNode("invalid");
+                return InvalidNode;
             case MissingEntityBehaviour.Ignore:
-                return new ValueDataNode("invalid");
+                return InvalidNode;
             case MissingEntityBehaviour.IncludeNullspace:
                 if (!EntMan.TryGetComponent(value, out TransformComponent? xform)
                     || xform.ParentUid != EntityUid.Invalid

Robust.Shared/EntitySerialization/SerializationEnums.cs

@@ -87,7 +87,6 @@ public enum MissingEntityBehaviour
     AutoInclude,
 }
 
-
 public enum EntityExceptionBehaviour
 {
     /// <summary>

Robust.Shared/EntitySerialization/Systems/MapLoaderSystem.Save.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 using Robust.Shared.GameObjects;
 using Robust.Shared.Map;
@@ -16,8 +17,12 @@ public sealed partial class MapLoaderSystem
     public event EntitySerializer.IsSerializableDelegate? OnIsSerializable;
 
     /// <summary>
-    /// Recursively serialize the given entity and its children.
+    /// Recursively serialize the given entities and all of their children.
     /// </summary>
+    /// <remarks>
+    /// This method is not optimized for being given a large set of entities. I.e., this should be a small handful of
+    /// maps or grids, not something like <see cref="EntityManager.AllEntityUids"/>.
+    /// </remarks>
     public (MappingDataNode Node, FileCategory Category) SerializeEntitiesRecursive(
         HashSet<EntityUid> entities,
         SerializationOptions? options = null)
@@ -29,8 +34,6 @@ public sealed partial class MapLoaderSystem
         Log.Info($"Serializing entities: {string.Join(", ", entities.Select(x => ToPrettyString(x).ToString()))}");
 
         var maps = entities.Select(x => Transform(x).MapID).ToHashSet();
-        var ev = new BeforeSerializationEvent(entities, maps);
-        RaiseLocalEvent(ev);
 
         // In case no options were provided, we assume that if all of the starting entities are pre-init, we should
         // expect that **all** entities that get serialized should be pre-init.
@@ -39,6 +42,9 @@ public sealed partial class MapLoaderSystem
             ExpectPreInit = (entities.All(x => LifeStage(x) < EntityLifeStage.MapInitialized))
         };
 
+        var ev = new BeforeSerializationEvent(entities, maps, opts.Category);
+        RaiseLocalEvent(ev);
+
         var serializer = new EntitySerializer(_dependency, opts);
         serializer.OnIsSerializeable += OnIsSerializable;
 
@@ -235,4 +241,89 @@ public bool TrySaveGeneric(
         Write(path, data);
         return true;
     }
+
+    /// <inheritdoc cref="TrySerializeAllEntities(out MappingDataNode, SerializationOptions?)"/>
+    public bool TrySaveAllEntities(ResPath path, SerializationOptions? options = null)
+    {
+        if (!TrySerializeAllEntities(out var data, options))
+            return false;
+
+        Write(path, data);
+        return true;
+    }
+
+    /// <summary>
+    /// Attempt to serialize all entities.
+    /// </summary>
+    /// <remarks>
+    /// Note that this alone is not sufficient for a proper full-game save, as the game may contain things like chat
+    /// logs or resources and prototypes that were uploaded mid-game.
+    /// </remarks>
+    public bool TrySerializeAllEntities([NotNullWhen(true)] out MappingDataNode? data, SerializationOptions? options = null)
+    {
+        data  = null;
+        var opts = options ?? SerializationOptions.Default with
+        {
+            MissingEntityBehaviour = MissingEntityBehaviour.Error
+        };
+
+        opts.Category = FileCategory.Save;
+        _stopwatch.Restart();
+        Log.Info($"Serializing all entities");
+
+        var entities = EntityManager.GetEntities().ToHashSet();
+        var maps = _mapSystem.Maps.Keys.ToHashSet();
+        var ev = new BeforeSerializationEvent(entities, maps, FileCategory.Save);
+        var serializer = new EntitySerializer(_dependency, opts);
+
+        // Remove any non-serializable entities and their children (prevent error spam)
+        var toRemove = new Queue<EntityUid>();
+        foreach (var entity in entities)
+        {
+            // TODO SERIALIZATION Perf
+            // IsSerializable gets called again by serializer.SerializeEntities()
+            if (!serializer.IsSerializable(entity))
+                toRemove.Enqueue(entity);
+        }
+
+        if (toRemove.Count > 0)
+        {
+            if (opts.MissingEntityBehaviour == MissingEntityBehaviour.Error)
+            {
+                // The save will probably contain references to the non-serializable entities, and we avoid spamming errors.
+                opts.MissingEntityBehaviour = MissingEntityBehaviour.Ignore;
+                Log.Error($"Attempted to serialize one or more non-serializable entities");
+            }
+
+            while (toRemove.TryDequeue(out var next))
+            {
+                entities.Remove(next);
+                foreach (var uid in Transform(next)._children)
+                {
+                    toRemove.Enqueue(uid);
+                }
+            }
+        }
+
+        try
+        {
+            RaiseLocalEvent(ev);
+            serializer.OnIsSerializeable += OnIsSerializable;
+            serializer.SerializeEntities(entities);
+            data = serializer.Write();
+            var cat = serializer.GetCategory();
+            DebugTools.AssertEqual(cat, FileCategory.Save);
+            var ev2 = new AfterSerializationEvent(entities, data, cat);
+            RaiseLocalEvent(ev2);
+
+            Log.Debug($"Serialized {serializer.EntityData.Count} entities in {_stopwatch.Elapsed}");
+        }
+        catch (Exception e)
+        {
+            Log.Error($"Caught exception while trying to serialize all entities:\n{e}");
+            return false;
+        }
+
+        return true;
+    }
 }

Robust.Shared/Map/Events/MapSerializationEvents.cs

@@ -34,7 +34,10 @@ public sealed class BeforeEntityReadEvent
 /// For convenience, the event also contains a set with all the maps that the entities are on. This does not
 /// necessarily mean that the maps are themselves getting serialized.
 /// </summary>
-public readonly record struct BeforeSerializationEvent(HashSet<EntityUid> Entities, HashSet<MapId> MapIds);
+public readonly record struct BeforeSerializationEvent(
+    HashSet<EntityUid> Entities,
+    HashSet<MapId> MapIds,
+    FileCategory Category = FileCategory.Unknown);
 
 /// <summary>
 /// This event is broadcast just after entities (and their children) have been serialized, but before it gets written to a yaml file.

Robust.Shared/Player/ActorComponent.cs

@@ -3,7 +3,7 @@
 
 namespace Robust.Shared.Player;
 
-[RegisterComponent]
+[RegisterComponent, UnsavedComponent]
 public sealed partial class ActorComponent : Component
 {
     [ViewVariables]

Robust.Shared/Replays/IReplayRecordingManager.cs

@@ -2,11 +2,15 @@
 using Robust.Shared.Serialization.Markdown.Mapping;
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.IO.Compression;
 using System.Threading.Tasks;
 using Robust.Shared.ContentPack;
 using Robust.Shared.GameStates;
+using Robust.Shared.Serialization;
 using Robust.Shared.Utility;
+using YamlDotNet.Core;
+using YamlDotNet.RepresentationModel;
 
 namespace Robust.Shared.Replays;
 
@@ -202,6 +206,25 @@ void WriteBytes(
         ResPath path,
         ReadOnlyMemory<byte> bytes,
         CompressionLevel compressionLevel = CompressionLevel.Optimal);
+
+    /// <summary>
+    /// Writes a yaml document into a file in the replay.
+    /// </summary>
+    /// <param name="path">The file path to write to.</param>
+    /// <param name="yaml">The yaml document to write to the file.</param>
+    /// <param name="compressionLevel">How much to compress the file.</param>
+    void WriteYaml(
+        ResPath path,
+        YamlDocument yaml,
+        CompressionLevel compressionLevel = CompressionLevel.Optimal)
+    {
+        var memStream = new MemoryStream();
+        using var writer = new StreamWriter(memStream);
+        var yamlStream = new YamlStream {yaml};
+        yamlStream.Save(new YamlMappingFix(new Emitter(writer)), false);
+        writer.Flush();
+        WriteBytes(path, memStream.AsMemory(), compressionLevel);
+    }
 }
 
 /// <summary>

Robust.Shared/Replays/SharedReplayRecordingManager.Write.cs

@@ -26,22 +26,30 @@ internal abstract partial class SharedReplayRecordingManager
     // and even then not for much longer than a couple hundred ms at most.
     private readonly List<Task> _finalizingWriteTasks = new();
 
-    private void WriteYaml(RecordingState state, ResPath path, YamlDocument data)
+    private void WriteYaml(
+        RecordingState state,
+        ResPath path,
+        YamlDocument data,
+        CompressionLevel level = CompressionLevel.Optimal)
     {
         var memStream = new MemoryStream();
         using var writer = new StreamWriter(memStream);
         var yamlStream = new YamlStream { data };
         yamlStream.Save(new YamlMappingFix(new Emitter(writer)), false);
         writer.Flush();
-        WriteBytes(state, path, memStream.AsMemory());
+        WriteBytes(state, path, memStream.AsMemory(), level);
     }
 
-    private void WriteSerializer<T>(RecordingState state, ResPath path, T obj)
+    private void WriteSerializer<T>(
+        RecordingState state,
+        ResPath path,
+        T obj,
+        CompressionLevel level = CompressionLevel.Optimal)
     {
         var memStream = new MemoryStream();
         _serializer.SerializeDirect(memStream, obj);
 
-        WriteBytes(state, path, memStream.AsMemory());
+        WriteBytes(state, path, memStream.AsMemory(), level);
     }
 
     private void WritePooledBytes(

Robust.Shared/Replays/SharedReplayRecordingManager.cs

@@ -375,6 +375,11 @@ private void WriteInitialMetadata(string name, RecordingState recState)
     private void WriteFinalMetadata(RecordingState recState)
     {
         var yamlMetadata = new MappingDataNode();
+
+        // TODO REPLAYS
+        // Why are these separate events?
+        // I assume it was for backwards compatibility / avoiding breaking changes?
+        // But eventually RecordingStopped2 will probably be renamed and there'll just be more breaking changes.
         RecordingStopped?.Invoke(yamlMetadata);
         RecordingStopped2?.Invoke(new ReplayRecordingStopped
         {
@@ -552,6 +557,12 @@ public void WriteBytes(ResPath path, ReadOnlyMemory<byte> bytes, CompressionLeve
             manager.WriteBytes(state, path, bytes, compressionLevel);
         }
 
+        void IReplayFileWriter.WriteYaml(ResPath path, YamlDocument document, CompressionLevel compressionLevel)
+        {
+            CheckDisposed();
+            manager.WriteYaml(state, path, document, compressionLevel);
+        }
+
         private void CheckDisposed()
         {
             if (state.Done)