Fix Docker and tarball publishing by making sure layer digests get into the manifest json

Author: baronfel

src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs

@@ -217,7 +217,8 @@ internal void AddLayer(Layer l)
         _rootFsLayers.Add(l.Descriptor.UncompressedDigest!);
     }
 
-    internal void SetUser(string user, bool isUserInteraction = false) {
+    internal void SetUser(string user, bool isUserInteraction = false)
+    {
         // we don't let automatic/inferred user settings overwrite an explicit user request
         if (_userHasBeenExplicitlySet && !isUserInteraction)
         {

src/Containers/Microsoft.NET.Build.Containers/Layer.cs

@@ -49,7 +49,13 @@ internal Layer(FileInfo backingFile, Descriptor descriptor)
 
     public static Layer FromDescriptor(Descriptor descriptor, ContentStore store)
     {
-        return new(new(store.PathForDescriptor(descriptor)), descriptor);
+        FileInfo path = new(store.PathForDescriptor(descriptor));
+        return FromBackingFile(path, descriptor);
+    }
+
+    public static Layer FromBackingFile(FileInfo backingFile, Descriptor descriptor)
+    {
+        return new(backingFile, descriptor);
     }
 
     public static async Task<Layer> FromFiles((string absPath, string relPath)[] inputFiles, string containerPath, bool isWindowsLayer, string manifestMediaType, ContentStore store, FileInfo layerWritePath, CancellationToken ct)
@@ -219,13 +225,16 @@ private sealed class HashDigestGZipStream : Stream
         private readonly IncrementalHash sha256Hash;
         private readonly GZipStream compressionStream;
 
-        public HashDigestGZipStream(Stream writeStream, bool leaveOpen)
+        public HashDigestGZipStream(Stream writeStream, bool leaveOpen, CompressionMode compressionMode = CompressionMode.Compress)
+            : base()
         {
             sha256Hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
-            compressionStream = new GZipStream(writeStream, CompressionMode.Compress, leaveOpen);
+            compressionStream = new GZipStream(writeStream, compressionMode, leaveOpen);
         }
 
-        public override bool CanWrite => true;
+        public override bool CanWrite => compressionStream.CanWrite;
+        public override bool CanRead => compressionStream.CanRead;
+        public override bool CanSeek => compressionStream.CanSeek;
 
         public override void Write(byte[] buffer, int offset, int count)
         {
@@ -271,12 +280,30 @@ public override ValueTask WriteAsync(ReadOnlyMemory<byte> buffer, CancellationTo
             return compressionStream.WriteAsync(buffer, cancellationToken);
         }
 
-        public override bool CanRead => false;
-        public override bool CanSeek => false;
         public override long Length => throw new NotImplementedException();
         public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); }
 
-        public override int Read(byte[] buffer, int offset, int count) => throw new NotImplementedException();
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            var read = compressionStream.Read(buffer, offset, count);
+            sha256Hash.AppendData(buffer.AsSpan(offset, read));
+            return read;
+        }
+
+        public override Task<int> ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken)
+        {
+            var read = compressionStream.ReadAsync(buffer, offset, count, cancellationToken);
+            sha256Hash.AppendData(buffer.AsSpan(offset, read.Result));
+            return read;
+        }
+
+        public override ValueTask<int> ReadAsync(Memory<byte> buffer, CancellationToken cancellationToken = default)
+        {
+            var read = compressionStream.ReadAsync(buffer, cancellationToken);
+            sha256Hash.AppendData(buffer.Span.Slice(0, read.Result));
+            return read;
+        }
+
         public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException();
         public override void SetLength(long value) => throw new NotImplementedException();
     }

src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/DockerCli.cs

@@ -363,15 +363,16 @@ CancellationToken cancellationToken
         cancellationToken.ThrowIfCancellationRequested();
         using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true);
         // Feed each layer tarball into the stream
-        JsonArray layerTarballPaths = new();
-        await WriteImageLayers(writer, pushData.layers, l => $"{l.Descriptor.Digest.Substring("sha256:".Length)}/layer.tar", cancellationToken)
+        var layerTarballPathFunc = (Layer l) => $"{l.Descriptor.Digest.Substring("sha256:".Length)}/layer.tar";
+        await WriteImageLayers(writer, pushData.layers, layerTarballPathFunc, cancellationToken)
             .ConfigureAwait(false);
 
         string configTarballPath = $"{pushData.configDigest.Split(':')[1]!}.json";
         await WriteImageConfig(writer, pushData.imageConfig, configTarballPath, cancellationToken)
             .ConfigureAwait(false);
 
         // Add manifest
+        JsonArray layerTarballPaths = new JsonArray([.. pushData.layers.Select(layerTarballPathFunc).Select(s => JsonValue.Create(s))]);
         await WriteManifestForDockerImage(writer, pushData.repository, pushData.tags, configTarballPath, layerTarballPaths, cancellationToken)
             .ConfigureAwait(false);
     }

src/Containers/Microsoft.NET.Build.Containers/Tasks/MakeContainerTarball.cs

@@ -49,7 +49,7 @@ public async Task<bool> ExecuteAsync()
         (long manifestSize, string manifestDigest, ManifestV2 manifestStructure) = await ReadManifest();
         var configDigest = manifestStructure.Config.digest;
         var config = await JsonSerializer.DeserializeAsync<JsonObject>(File.OpenRead(Configuration.ItemSpec), cancellationToken: _cts.Token);
-        var layers = Layers.Select(l => new Layer(new(l.ItemSpec), GetDescriptor(l))).ToArray();
+        var layers = Layers.Select(l => Layer.FromBackingFile(new(l.ItemSpec), GetDescriptor(l))).ToArray();
         var filePath = DetermineFilePath();
         await using var fileStream = File.Create(filePath);
         GeneratedArchiveFilePath = filePath;

src/Containers/Microsoft.NET.Build.Containers/Tasks/PushContainerToLocal.cs

@@ -55,7 +55,7 @@ public async Task<bool> ExecuteAsync()
             return false;
         }
 
-        var layers = Layers.Select(l => new Layer(new(l.ItemSpec), GetDescriptor(l))).ToArray();
+        var layers = Layers.Select(l => Layer.FromBackingFile(new(l.ItemSpec), GetDescriptor(l))).ToArray();
         try
         {
             await containerCli.LoadAsync((Repository, Tags, configDigest, config!, layers), DockerCli.WriteDockerImageToStreamAsync, _cts.Token);