Draft of incremental container generation

src/Containers/Microsoft.NET.Build.Containers/BuiltImage.cs

@@ -1,6 +1,9 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text.Json;
+using System.Text.Json.Nodes;
+
 namespace Microsoft.NET.Build.Containers;
 
 /// <summary>
@@ -11,32 +14,22 @@ internal readonly struct BuiltImage
     /// <summary>
     /// Gets image configuration in JSON format.
     /// </summary>
-    internal required string Config { get; init; }
-
-    /// <summary>
-    /// Gets image digest.
-    /// </summary>
-    internal string? ImageDigest { get; init; }
-
-    /// <summary>
-    /// Gets image SHA.
-    /// </summary>
-    internal string? ImageSha { get; init; }
+    internal required JsonObject Config { get; init; }
 
     /// <summary>
     /// Gets image manifest.
     /// </summary>
-    internal required string Manifest { get; init; } 
+    internal required ManifestV2 Manifest { get; init; }
 
     /// <summary>
     /// Gets manifest digest.
     /// </summary>
-    internal required string ManifestDigest { get; init; }
+    internal string ManifestDigest => Manifest.GetDigest();
 
     /// <summary>
     /// Gets manifest mediaType.
     /// </summary>
-    internal required string ManifestMediaType { get; init; }
+    internal string ManifestMediaType => Manifest.MediaType!;
 
     /// <summary>
     /// Gets image layers.

src/Containers/Microsoft.NET.Build.Containers/ContainerBuilder.cs

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Text.Json;
+using System.Text.Json.Nodes;
 using Microsoft.Extensions.Logging;
 using Microsoft.NET.Build.Containers.Resources;
 
@@ -15,7 +17,7 @@ internal enum KnownImageFormats
 internal static class ContainerBuilder
 {
     internal static async Task<int> ContainerizeAsync(
-        DirectoryInfo publishDirectory,
+        (string absolutePath, string relativePath)[] inputFiles,
         string workingDir,
         string baseRegistry,
         string baseImageName,
@@ -33,22 +35,22 @@ internal static async Task<int> ContainerizeAsync(
         Dictionary<string, string> labels,
         Port[]? exposedPorts,
         Dictionary<string, string> envVars,
-        string containerRuntimeIdentifier,
-        string ridGraphPath,
         string localRegistry,
         string? containerUser,
         string? archiveOutputPath,
         bool generateLabels,
         bool generateDigestLabel,
         KnownImageFormats? imageFormat,
+        string contentStoreRoot,
+        FileInfo baseImageManifestPath,
+        FileInfo baseImageConfigPath,
+        FileInfo generatedConfigPath,
+        FileInfo generatedManifestPath,
+        FileInfo generatedLayerPath,
         ILoggerFactory loggerFactory,
         CancellationToken cancellationToken)
     {
         cancellationToken.ThrowIfCancellationRequested();
-        if (!publishDirectory.Exists)
-        {
-            throw new ArgumentException(string.Format(Resource.GetString(nameof(Strings.PublishDirectoryDoesntExist)), nameof(publishDirectory), publishDirectory.FullName));
-        }
         ILogger logger = loggerFactory.CreateLogger("Containerize");
         logger.LogTrace("Trace logging: enabled.");
 
@@ -70,13 +72,7 @@ internal static async Task<int> ContainerizeAsync(
         {
             try
             {
-                var ridGraphPicker = new RidGraphManifestPicker(ridGraphPath);
-                imageBuilder = await registry.GetImageManifestAsync(
-                    baseImageName,
-                    sourceImageReference.Reference,
-                    containerRuntimeIdentifier,
-                    ridGraphPicker,
-                    cancellationToken).ConfigureAwait(false);
+                imageBuilder = await LoadFromManifestAndConfig(baseImageManifestPath.FullName, imageFormat, baseImageConfigPath.FullName, logger);
             }
             catch (RepositoryNotFoundException)
             {
@@ -98,11 +94,6 @@ internal static async Task<int> ContainerizeAsync(
         {
             throw new NotSupportedException(Resource.GetString(nameof(Strings.ImagePullNotSupported)));
         }
-        if (imageBuilder is null)
-        {
-            Console.WriteLine(Resource.GetString(nameof(Strings.BaseImageNotFound)), sourceImageReference, containerRuntimeIdentifier);
-            return 1;
-        }
         logger.LogInformation(Strings.ContainerBuilder_StartBuildingImage, imageName, string.Join(",", imageName), sourceImageReference);
         cancellationToken.ThrowIfCancellationRequested();
 
@@ -115,7 +106,14 @@ internal static async Task<int> ContainerizeAsync(
             _ => imageBuilder.ManifestMediaType // should be impossible unless we add to the enum
         };
 
-        Layer newLayer = Layer.FromDirectory(publishDirectory.FullName, workingDir, imageBuilder.IsWindows, imageBuilder.ManifestMediaType);
+        var storePath = new DirectoryInfo(contentStoreRoot);
+        if (!storePath.Exists)
+        {
+            throw new ArgumentException($"The content store path '{contentStoreRoot}' does not exist.");
+        }
+        var store = new ContentStore(storePath);
+
+        Layer newLayer = await Layer.FromFiles(inputFiles, workingDir, imageBuilder.IsWindows, imageBuilder.ManifestMediaType, store, generatedLayerPath, cancellationToken);
         imageBuilder.AddLayer(newLayer);
         imageBuilder.SetWorkingDirectory(workingDir);
 
@@ -174,6 +172,16 @@ internal static async Task<int> ContainerizeAsync(
         BuiltImage builtImage = imageBuilder.Build();
         cancellationToken.ThrowIfCancellationRequested();
 
+        // at this point we're done with modifications and are just pushing the data other places
+
+        var serializedManifest = JsonSerializer.Serialize(builtImage.Manifest);
+        var manifestWriteTask = File.WriteAllTextAsync(generatedManifestPath.FullName, serializedManifest, DigestUtils.UTF8);
+
+        var serializedConfig = JsonSerializer.Serialize(builtImage.Config);
+        var configWriteTask = File.WriteAllTextAsync(generatedConfigPath.FullName, serializedConfig, DigestUtils.UTF8);
+
+        await Task.WhenAll(manifestWriteTask, configWriteTask).ConfigureAwait(false);
+
         int exitCode;
         switch (destinationImageReference.Kind)
         {
@@ -256,4 +264,24 @@ private static async Task<int> PushToRemoteRegistryAsync(ILogger logger, BuiltIm
 
         return 0;
     }
+
+
+    public static async Task<ImageBuilder> LoadFromManifestAndConfig(string manifestPath, KnownImageFormats? desiredImageFormat, string configPath, ILogger logger)
+    {
+        var baseImageManifest = await JsonSerializer.DeserializeAsync<ManifestV2>(File.OpenRead(manifestPath));
+        var baseImageConfig = await JsonNode.ParseAsync(File.OpenRead(configPath));
+        if (baseImageConfig is null || baseImageManifest is null) throw new ArgumentException($"Expected to load manifest from {manifestPath} and config from {configPath}");
+        // forcibly change the media type if required from that of the base image
+        var mediaType = baseImageManifest.MediaType;
+        if (desiredImageFormat is not null)
+        {
+            mediaType = desiredImageFormat switch
+            {
+                KnownImageFormats.Docker => SchemaTypes.DockerManifestV2,
+                KnownImageFormats.OCI => SchemaTypes.OciManifestV1,
+                _ => mediaType // should be impossible unless we add to the enum
+            };
+        }
+        return new ImageBuilder(baseImageManifest, mediaType!, new ImageConfig(baseImageConfig), logger);
+    }
 }

src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs

@@ -6,34 +6,67 @@
 
 namespace Microsoft.NET.Build.Containers;
 
-internal static class ContentStore
+/// <summary>
+/// Structured access to the content store for manifests and blobs at a given root path.
+/// </summary>
+/// <param name="root"></param>
+internal class ContentStore(DirectoryInfo root)
 {
-    public static string ArtifactRoot { get; set; } = Path.Combine(Path.GetTempPath(), "Containers");
-    public static string ContentRoot
+    public string ArtifactRoot
+    {
+        get
+        {
+            string artifactPath = Path.Join(root.FullName, "Containers");
+            Directory.CreateDirectory(artifactPath);
+            return artifactPath;
+        }
+    }
+
+    /// <summary>
+    /// Where all the blobs are stored in this ContentStore - these will be addressed purely by digest. The contents may be JSON blobs, 
+    /// layer tarballs, or other - you need to know the media type to interpret the contents. 
+    /// </summary>
+    public string ContentRoot
     {
         get
         {
             string contentPath = Path.Join(ArtifactRoot, "Content");
-
             Directory.CreateDirectory(contentPath);
-
             return contentPath;
         }
     }
 
-    public static string TempPath
+    /// <summary>
+    /// Where all the reference pointers are stored in this ContentStore. These will be addressed by logical reference - registry, repository, tag.
+    /// The contents will be a digest and media type, which can then be looked up in the <see cref="ContentRoot"/>. 
+    /// </summary>
+    public string ReferenceRoot
     {
         get
         {
-            string tempPath = Path.Join(ArtifactRoot, "Temp");
+            string referencePath = Path.Combine(ArtifactRoot, "Manifests");
+            Directory.CreateDirectory(referencePath);
+            return referencePath;
+        }
+    }
 
+    public string TempPath
+    {
+        get
+        {
+            string tempPath = Path.Join(ArtifactRoot, "Temp");
             Directory.CreateDirectory(tempPath);
-
             return tempPath;
         }
     }
 
-    public static string PathForDescriptor(Descriptor descriptor)
+    /// <summary>
+    /// A safety valve on top of <see cref="GetPathForHash"/> that also validates that we know/understand the media type of the Descriptor
+    /// </summary>
+    /// <param name="descriptor"></param>
+    /// <returns></returns>
+    /// <exception cref="ArgumentException">If the Descriptor isn't a layer mediatype</exception>
+    public string PathForDescriptor(Descriptor descriptor)
     {
         string digest = descriptor.Digest;
 
@@ -50,20 +83,32 @@ public static string PathForDescriptor(Descriptor descriptor)
             "application/vnd.docker.image.rootfs.diff.tar"
             or "application/vnd.oci.image.layer.v1.tar"
                 => ".tar",
+            SchemaTypes.DockerManifestListV2
+            or SchemaTypes.DockerManifestV2
+            or SchemaTypes.OciImageIndexV1 
+            or SchemaTypes.OciManifestV1
+            or SchemaTypes.DockerContainerV1 => string.Empty,
             _ => throw new ArgumentException(Resource.FormatString(nameof(Strings.UnrecognizedMediaType), descriptor.MediaType))
         };
 
         return GetPathForHash(contentHash) + extension;
     }
 
-
-    public static string GetPathForHash(string contentHash)
-    {
-        return Path.Combine(ContentRoot, contentHash);
-    }
-
-    public static string GetTempFile()
-    {
-        return Path.Join(TempPath, Path.GetRandomFileName());
-    }
+    /// <summary>
+    /// Returns the path in the <see cref="ReferenceRoot"/> for the manifest reference for this registry/repository/tag.
+    /// </summary>
+    /// <param name="registry"></param>
+    /// <param name="repository"></param>
+    /// <param name="tag"></param>
+    /// <returns></returns>
+    public string PathForManifestByReferenceOrDigest(string registry, string repository, string tag) => Path.Combine(ReferenceRoot, registry, repository, tag);
+
+    /// <summary>
+    /// Returns the path to the content store for a given content hash (<c>algo</c>:<c>digest</c>) pair.
+    /// </summary>
+    /// <param name="contentHash"></param>
+    /// <returns></returns>
+    public string GetPathForHash(string contentHash) => Path.Combine(ContentRoot, contentHash);
+
+    public string GetTempFile() => Path.Join(TempPath, Path.GetRandomFileName());
 }

src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs

@@ -2,20 +2,27 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using System.Security.Cryptography;
+using System.Text.Json;
+using System.Text.Json.Serialization;
 
 namespace Microsoft.NET.Build.Containers;
 
 internal sealed class DigestUtils
 {
+    /// <summary>
+    /// UTF8 encoding without BOM.
+    /// </summary>
+    internal static Encoding UTF8 = new UTF8Encoding(false);
+
     /// <summary>
     /// Gets digest for string <paramref name="str"/>.
     /// </summary>
-    internal static string GetDigest(string str) => GetDigestFromSha(GetSha(str));
+    internal static string GetDigest<T>(T content) => GetDigestFromSha(GetSha(content));
 
     /// <summary>
-    /// Formats digest based on ready SHA <paramref name="sha"/>.
+    /// Formats digest based on ready SHA <paramref name="sha256"/>.
     /// </summary>
-    internal static string GetDigestFromSha(string sha) => $"sha256:{sha}";
+    internal static string GetDigestFromSha(string sha256) => $"sha256:{sha256}";
 
     internal static string GetShaFromDigest(string digest)
     {
@@ -30,11 +37,20 @@ internal static string GetShaFromDigest(string digest)
     /// <summary>
     /// Gets the SHA of <paramref name="str"/>.
     /// </summary>
-    internal static string GetSha(string str)
+    internal static (long size, string sha256) GetSha(string str)
     {
         Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
-        SHA256.HashData(Encoding.UTF8.GetBytes(str), hash);
+        var bytes = UTF8.GetBytes(str);
+        SHA256.HashData(bytes, hash);
 
-        return Convert.ToHexStringLower(hash);
+        return (bytes.LongLength, Convert.ToHexStringLower(hash));
     }
+
+    internal static string GetSha<T>(T content)
+    {
+        var jsonstring = JsonSerializer.Serialize(content);
+        return GetSha(jsonstring).sha256;
+    }
+
+    internal static long GetUtf8Length(string content) => UTF8.GetBytes(content).LongLength;
 }

src/Containers/Microsoft.NET.Build.Containers/Exceptions/UnableToDownloadFromRepositoryException.cs

@@ -5,8 +5,8 @@ namespace Microsoft.NET.Build.Containers;
 
 internal sealed class UnableToDownloadFromRepositoryException : Exception
 {
-    public UnableToDownloadFromRepositoryException(string repository)
-        : base($"The download of the image from repository { repository } has failed.")
+    public UnableToDownloadFromRepositoryException(string repository, Exception? innerException = null)
+        : base($"The download of the image from repository {repository} has failed.", innerException)
     {
     }
 }