diff --git a/src/Containers/Microsoft.NET.Build.Containers/BuiltImage.cs b/src/Containers/Microsoft.NET.Build.Containers/BuiltImage.cs index 91f47193ce66..ea0694eea097 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/BuiltImage.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/BuiltImage.cs @@ -1,6 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; +using System.Text.Json.Nodes; + namespace Microsoft.NET.Build.Containers; /// @@ -11,32 +14,22 @@ internal readonly struct BuiltImage /// /// Gets image configuration in JSON format. /// - internal required string Config { get; init; } - - /// - /// Gets image digest. - /// - internal string? ImageDigest { get; init; } - - /// - /// Gets image SHA. - /// - internal string? ImageSha { get; init; } + internal required JsonObject Config { get; init; } /// /// Gets image manifest. /// - internal required string Manifest { get; init; } + internal required ManifestV2 Manifest { get; init; } /// /// Gets manifest digest. /// - internal required string ManifestDigest { get; init; } + internal string ManifestDigest => Manifest.GetDigest(); /// /// Gets manifest mediaType. /// - internal required string ManifestMediaType { get; init; } + internal string ManifestMediaType => Manifest.MediaType!; /// /// Gets image layers. diff --git a/src/Containers/Microsoft.NET.Build.Containers/ContainerBuilder.cs b/src/Containers/Microsoft.NET.Build.Containers/ContainerBuilder.cs index ff496c844e70..4ce4ef82a199 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ContainerBuilder.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ContainerBuilder.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; +using System.Text.Json.Nodes; using Microsoft.Extensions.Logging; using Microsoft.NET.Build.Containers.Resources; @@ -15,7 +17,7 @@ internal enum KnownImageFormats internal static class ContainerBuilder { internal static async Task ContainerizeAsync( - DirectoryInfo publishDirectory, + (string absolutePath, string relativePath)[] inputFiles, string workingDir, string baseRegistry, string baseImageName, @@ -33,22 +35,22 @@ internal static async Task ContainerizeAsync( Dictionary labels, Port[]? exposedPorts, Dictionary envVars, - string containerRuntimeIdentifier, - string ridGraphPath, string localRegistry, string? containerUser, string? archiveOutputPath, bool generateLabels, bool generateDigestLabel, KnownImageFormats? imageFormat, + string contentStoreRoot, + FileInfo baseImageManifestPath, + FileInfo baseImageConfigPath, + FileInfo generatedConfigPath, + FileInfo generatedManifestPath, + FileInfo generatedLayerPath, ILoggerFactory loggerFactory, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - if (!publishDirectory.Exists) - { - throw new ArgumentException(string.Format(Resource.GetString(nameof(Strings.PublishDirectoryDoesntExist)), nameof(publishDirectory), publishDirectory.FullName)); - } ILogger logger = loggerFactory.CreateLogger("Containerize"); logger.LogTrace("Trace logging: enabled."); @@ -70,13 +72,7 @@ internal static async Task ContainerizeAsync( { try { - var ridGraphPicker = new RidGraphManifestPicker(ridGraphPath); - imageBuilder = await registry.GetImageManifestAsync( - baseImageName, - sourceImageReference.Reference, - containerRuntimeIdentifier, - ridGraphPicker, - cancellationToken).ConfigureAwait(false); + imageBuilder = await LoadFromManifestAndConfig(baseImageManifestPath.FullName, imageFormat, baseImageConfigPath.FullName, logger); } catch (RepositoryNotFoundException) { @@ -98,11 +94,6 @@ internal static async Task ContainerizeAsync( { throw new NotSupportedException(Resource.GetString(nameof(Strings.ImagePullNotSupported))); } - if (imageBuilder is null) - { - Console.WriteLine(Resource.GetString(nameof(Strings.BaseImageNotFound)), sourceImageReference, containerRuntimeIdentifier); - return 1; - } logger.LogInformation(Strings.ContainerBuilder_StartBuildingImage, imageName, string.Join(",", imageName), sourceImageReference); cancellationToken.ThrowIfCancellationRequested(); @@ -115,7 +106,14 @@ internal static async Task ContainerizeAsync( _ => imageBuilder.ManifestMediaType // should be impossible unless we add to the enum }; - Layer newLayer = Layer.FromDirectory(publishDirectory.FullName, workingDir, imageBuilder.IsWindows, imageBuilder.ManifestMediaType); + var storePath = new DirectoryInfo(contentStoreRoot); + if (!storePath.Exists) + { + throw new ArgumentException($"The content store path '{contentStoreRoot}' does not exist."); + } + var store = new ContentStore(storePath); + + Layer newLayer = await Layer.FromFiles(inputFiles, workingDir, imageBuilder.IsWindows, imageBuilder.ManifestMediaType, store, generatedLayerPath, cancellationToken); imageBuilder.AddLayer(newLayer); imageBuilder.SetWorkingDirectory(workingDir); @@ -174,6 +172,16 @@ internal static async Task ContainerizeAsync( BuiltImage builtImage = imageBuilder.Build(); cancellationToken.ThrowIfCancellationRequested(); + // at this point we're done with modifications and are just pushing the data other places + + var serializedManifest = JsonSerializer.Serialize(builtImage.Manifest); + var manifestWriteTask = File.WriteAllTextAsync(generatedManifestPath.FullName, serializedManifest, DigestUtils.UTF8); + + var serializedConfig = JsonSerializer.Serialize(builtImage.Config); + var configWriteTask = File.WriteAllTextAsync(generatedConfigPath.FullName, serializedConfig, DigestUtils.UTF8); + + await Task.WhenAll(manifestWriteTask, configWriteTask).ConfigureAwait(false); + int exitCode; switch (destinationImageReference.Kind) { @@ -256,4 +264,24 @@ private static async Task PushToRemoteRegistryAsync(ILogger logger, BuiltIm return 0; } + + + public static async Task LoadFromManifestAndConfig(string manifestPath, KnownImageFormats? desiredImageFormat, string configPath, ILogger logger) + { + var baseImageManifest = await JsonSerializer.DeserializeAsync(File.OpenRead(manifestPath)); + var baseImageConfig = await JsonNode.ParseAsync(File.OpenRead(configPath)); + if (baseImageConfig is null || baseImageManifest is null) throw new ArgumentException($"Expected to load manifest from {manifestPath} and config from {configPath}"); + // forcibly change the media type if required from that of the base image + var mediaType = baseImageManifest.MediaType; + if (desiredImageFormat is not null) + { + mediaType = desiredImageFormat switch + { + KnownImageFormats.Docker => SchemaTypes.DockerManifestV2, + KnownImageFormats.OCI => SchemaTypes.OciManifestV1, + _ => mediaType // should be impossible unless we add to the enum + }; + } + return new ImageBuilder(baseImageManifest, mediaType!, new ImageConfig(baseImageConfig), logger); + } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs b/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs index 2fb16be3ec5d..46f3263c7f04 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ContentStore.cs @@ -6,34 +6,67 @@ namespace Microsoft.NET.Build.Containers; -internal static class ContentStore +/// +/// Structured access to the content store for manifests and blobs at a given root path. +/// +/// +internal class ContentStore(DirectoryInfo root) { - public static string ArtifactRoot { get; set; } = Path.Combine(Path.GetTempPath(), "Containers"); - public static string ContentRoot + public string ArtifactRoot + { + get + { + string artifactPath = Path.Join(root.FullName, "Containers"); + Directory.CreateDirectory(artifactPath); + return artifactPath; + } + } + + /// + /// Where all the blobs are stored in this ContentStore - these will be addressed purely by digest. The contents may be JSON blobs, + /// layer tarballs, or other - you need to know the media type to interpret the contents. + /// + public string ContentRoot { get { string contentPath = Path.Join(ArtifactRoot, "Content"); - Directory.CreateDirectory(contentPath); - return contentPath; } } - public static string TempPath + /// + /// Where all the reference pointers are stored in this ContentStore. These will be addressed by logical reference - registry, repository, tag. + /// The contents will be a digest and media type, which can then be looked up in the . + /// + public string ReferenceRoot { get { - string tempPath = Path.Join(ArtifactRoot, "Temp"); + string referencePath = Path.Combine(ArtifactRoot, "Manifests"); + Directory.CreateDirectory(referencePath); + return referencePath; + } + } + public string TempPath + { + get + { + string tempPath = Path.Join(ArtifactRoot, "Temp"); Directory.CreateDirectory(tempPath); - return tempPath; } } - public static string PathForDescriptor(Descriptor descriptor) + /// + /// A safety valve on top of that also validates that we know/understand the media type of the Descriptor + /// + /// + /// + /// If the Descriptor isn't a layer mediatype + public string PathForDescriptor(Descriptor descriptor) { string digest = descriptor.Digest; @@ -50,20 +83,32 @@ public static string PathForDescriptor(Descriptor descriptor) "application/vnd.docker.image.rootfs.diff.tar" or "application/vnd.oci.image.layer.v1.tar" => ".tar", + SchemaTypes.DockerManifestListV2 + or SchemaTypes.DockerManifestV2 + or SchemaTypes.OciImageIndexV1 + or SchemaTypes.OciManifestV1 + or SchemaTypes.DockerContainerV1 => string.Empty, _ => throw new ArgumentException(Resource.FormatString(nameof(Strings.UnrecognizedMediaType), descriptor.MediaType)) }; return GetPathForHash(contentHash) + extension; } - - public static string GetPathForHash(string contentHash) - { - return Path.Combine(ContentRoot, contentHash); - } - - public static string GetTempFile() - { - return Path.Join(TempPath, Path.GetRandomFileName()); - } + /// + /// Returns the path in the for the manifest reference for this registry/repository/tag. + /// + /// + /// + /// + /// + public string PathForManifestByReferenceOrDigest(string registry, string repository, string tag) => Path.Combine(ReferenceRoot, registry, repository, tag); + + /// + /// Returns the path to the content store for a given content hash (algo:digest) pair. + /// + /// + /// + public string GetPathForHash(string contentHash) => Path.Combine(ContentRoot, contentHash); + + public string GetTempFile() => Path.Join(TempPath, Path.GetRandomFileName()); } diff --git a/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs b/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs index c0c7e16e6793..db01b3ef6f28 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/DigestUtils.cs @@ -2,20 +2,27 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; namespace Microsoft.NET.Build.Containers; internal sealed class DigestUtils { + /// + /// UTF8 encoding without BOM. + /// + internal static Encoding UTF8 = new UTF8Encoding(false); + /// /// Gets digest for string . /// - internal static string GetDigest(string str) => GetDigestFromSha(GetSha(str)); + internal static string GetDigest(T content) => GetDigestFromSha(GetSha(content)); /// - /// Formats digest based on ready SHA . + /// Formats digest based on ready SHA . /// - internal static string GetDigestFromSha(string sha) => $"sha256:{sha}"; + internal static string GetDigestFromSha(string sha256) => $"sha256:{sha256}"; internal static string GetShaFromDigest(string digest) { @@ -30,11 +37,20 @@ internal static string GetShaFromDigest(string digest) /// /// Gets the SHA of . /// - internal static string GetSha(string str) + internal static (long size, string sha256) GetSha(string str) { Span hash = stackalloc byte[SHA256.HashSizeInBytes]; - SHA256.HashData(Encoding.UTF8.GetBytes(str), hash); + var bytes = UTF8.GetBytes(str); + SHA256.HashData(bytes, hash); - return Convert.ToHexStringLower(hash); + return (bytes.LongLength, Convert.ToHexStringLower(hash)); } + + internal static string GetSha(T content) + { + var jsonstring = JsonSerializer.Serialize(content); + return GetSha(jsonstring).sha256; + } + + internal static long GetUtf8Length(string content) => UTF8.GetBytes(content).LongLength; } diff --git a/src/Containers/Microsoft.NET.Build.Containers/Exceptions/UnableToDownloadFromRepositoryException.cs b/src/Containers/Microsoft.NET.Build.Containers/Exceptions/UnableToDownloadFromRepositoryException.cs index 30718be88ed6..bff2fc364dd4 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Exceptions/UnableToDownloadFromRepositoryException.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Exceptions/UnableToDownloadFromRepositoryException.cs @@ -5,8 +5,8 @@ namespace Microsoft.NET.Build.Containers; internal sealed class UnableToDownloadFromRepositoryException : Exception { - public UnableToDownloadFromRepositoryException(string repository) - : base($"The download of the image from repository { repository } has failed.") + public UnableToDownloadFromRepositoryException(string repository, Exception? innerException = null) + : base($"The download of the image from repository {repository} has failed.", innerException) { } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions.cs b/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions.cs new file mode 100644 index 000000000000..08292a366ee6 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions.cs @@ -0,0 +1,39 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.Tasks.DownloadContainerManifest")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.IMultiImageManifest")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.ISingleImageManifest")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.IPlatformSpecificManifestData.digest")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.IPlatformSpecificManifestData.mediaType")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.IPlatformSpecificManifestData.platform")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.IPlatformSpecificManifestData.size")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.IPlatformSpecificManifestData")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.ImageIndexV1")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.ManifestListV2")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.IManifest")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.RidMapping.CreateRidForPlatform(Microsoft.NET.Build.Containers.PlatformInformation)~System.String")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.RidMapping.GetManifestsByRid(Microsoft.NET.Build.Containers.PlatformSpecificManifest[])~System.Collections.Generic.IReadOnlyDictionary{System.String,Microsoft.NET.Build.Containers.PlatformSpecificManifest}")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.RidMapping.GetManifestsByRid(Microsoft.NET.Build.Containers.PlatformSpecificOciManifest[])~System.Collections.Generic.IReadOnlyDictionary{System.String,Microsoft.NET.Build.Containers.PlatformSpecificOciManifest}")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.RidMapping")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.Tasks.DownloadLayers.Cancel")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.Tasks.DownloadLayers.Execute~System.Boolean")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.Tasks.DownloadLayers.ExecuteAsync~System.Threading.Tasks.Task{System.Boolean}")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.DownloadLayers.ContentStore")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.DownloadLayers.Registry")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.DownloadLayers.Repository")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.Tasks.DownloadLayers")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.Tasks.SelectRuntimeIdentifierSpecificItems.Cancel")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.Tasks.SelectRuntimeIdentifierSpecificItems.Execute~System.Boolean")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~M:Microsoft.NET.Build.Containers.Tasks.SelectRuntimeIdentifierSpecificItems.ExecuteAsync~System.Threading.Tasks.Task{System.Boolean}")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.SelectRuntimeIdentifierSpecificItems.Items")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.SelectRuntimeIdentifierSpecificItems.RuntimeIdentifierGraphPath")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.SelectRuntimeIdentifierSpecificItems.SelectedItems")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.SelectRuntimeIdentifierSpecificItems.TargetRuntimeIdentifier")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "type", Target = "~T:Microsoft.NET.Build.Containers.Tasks.SelectRuntimeIdentifierSpecificItems")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageConfigurationPath")] +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageManifestPath")] diff --git a/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions2.cs b/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions2.cs new file mode 100644 index 000000000000..d49df66f7808 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions2.cs @@ -0,0 +1,8 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContentStoreRoot")] diff --git a/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions3.cs b/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions3.cs new file mode 100644 index 000000000000..9c78f67755fd --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/GlobalSuppressions3.cs @@ -0,0 +1,8 @@ +// This file is used by Code Analysis to maintain SuppressMessage +// attributes that are applied to this project. +// Project-level suppressions either have no target or are given +// a specific target and scoped to a namespace, type, member, etc. + +using System.Diagnostics.CodeAnalysis; + +[assembly: SuppressMessage("ApiDesign", "RS0016:Add public types and members to the declared API", Justification = "", Scope = "member", Target = "~P:Microsoft.NET.Build.Containers.Tasks.CreateNewImage.PublishFiles")] diff --git a/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs b/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs index e97a8151a897..1ceb2caa5542 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ImageBuilder.cs @@ -6,6 +6,7 @@ using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.NET.Build.Containers.Resources; +using System.Text.Json.Nodes; namespace Microsoft.NET.Build.Containers; @@ -61,22 +62,23 @@ internal BuiltImage Build() AssignUserFromEnvironment(); AssignPortsFromEnvironment(); - string imageJsonStr = _baseImageConfig.BuildConfig(); - string imageSha = DigestUtils.GetSha(imageJsonStr); + JsonObject config = _baseImageConfig.BuildConfig(); + string configAsString = JsonSerializer.Serialize(config); + (long length, string imageSha) = DigestUtils.GetSha(configAsString); string imageDigest = DigestUtils.GetDigestFromSha(imageSha); - long imageSize = Encoding.UTF8.GetBytes(imageJsonStr).Length; - ManifestConfig newManifestConfig = _manifest.Config with - { - digest = imageDigest, - size = imageSize, - mediaType = ManifestMediaType switch + ManifestConfig newManifestConfig = + new() { - SchemaTypes.OciManifestV1 => SchemaTypes.OciImageConfigV1, - SchemaTypes.DockerManifestV2 => SchemaTypes.DockerContainerV1, - _ => SchemaTypes.OciImageConfigV1 // opinion - defaulting to modern here, but really this should never happen - } - }; + digest = imageDigest, + size = length, + mediaType = ManifestMediaType switch + { + SchemaTypes.OciManifestV1 => SchemaTypes.OciImageConfigV1, + SchemaTypes.DockerManifestV2 => SchemaTypes.DockerContainerV1, + _ => SchemaTypes.OciImageConfigV1 // opinion - defaulting to modern here, but really this should never happen + } + }; ManifestV2 newManifest = new ManifestV2() { @@ -88,12 +90,8 @@ internal BuiltImage Build() return new BuiltImage() { - Config = imageJsonStr, - ImageDigest = imageDigest, - ImageSha = imageSha, - Manifest = JsonSerializer.SerializeToNode(newManifest)?.ToJsonString() ?? "", - ManifestDigest = newManifest.GetDigest(), - ManifestMediaType = ManifestMediaType, + Config = config, + Manifest = newManifest, Layers = _manifest.Layers }; } diff --git a/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs b/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs index d16563c052db..1dcdaae9c221 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ImageConfig.cs @@ -73,7 +73,7 @@ internal ImageConfig(JsonNode config) /// /// Builds in additional configuration and returns updated image configuration in JSON format as string. /// - internal string BuildConfig() + internal JsonObject BuildConfig() { var newConfig = new JsonObject(); @@ -151,7 +151,7 @@ internal string BuildConfig() ["history"] = new JsonArray(_history.Select(CreateHistory).ToArray()) }; - return configContainer.ToJsonString(); + return configContainer; static JsonArray ToJsonArray(IEnumerable items) => new(items.Where(s => !string.IsNullOrEmpty(s)).Select(s => JsonValue.Create(s)).ToArray()); } @@ -217,7 +217,8 @@ internal void AddLayer(Layer l) _rootFsLayers.Add(l.Descriptor.UncompressedDigest!); } - internal void SetUser(string user, bool isUserInteraction = false) { + internal void SetUser(string user, bool isUserInteraction = false) + { // we don't let automatic/inferred user settings overwrite an explicit user request if (_userHasBeenExplicitlySet && !isUserInteraction) { diff --git a/src/Containers/Microsoft.NET.Build.Containers/ImageIndexGenerator.cs b/src/Containers/Microsoft.NET.Build.Containers/ImageIndexGenerator.cs index 64a00b4236cb..2451285ec4de 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ImageIndexGenerator.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ImageIndexGenerator.cs @@ -17,7 +17,7 @@ internal static class ImageIndexGenerator /// Returns json string of image index and image index mediaType. /// /// - internal static (string, string) GenerateImageIndex(BuiltImage[] images) + internal static IMultiImageManifest GenerateImageIndex(BuiltImage[] images) { if (images.Length == 0) { @@ -33,11 +33,11 @@ internal static (string, string) GenerateImageIndex(BuiltImage[] images) if (manifestMediaType == SchemaTypes.DockerManifestV2) { - return (GenerateImageIndex(images, SchemaTypes.DockerManifestV2, SchemaTypes.DockerManifestListV2), SchemaTypes.DockerManifestListV2); + return GenerateDockerManifestList(images, SchemaTypes.DockerManifestV2, SchemaTypes.DockerManifestListV2); } else if (manifestMediaType == SchemaTypes.OciManifestV1) { - return (GenerateImageIndex(images, SchemaTypes.OciManifestV1, SchemaTypes.OciImageIndexV1), SchemaTypes.OciImageIndexV1); + return GenerateDockerManifestList(images, SchemaTypes.OciManifestV1, SchemaTypes.OciImageIndexV1); } else { @@ -54,7 +54,7 @@ internal static (string, string) GenerateImageIndex(BuiltImage[] images) /// Returns json string of image index and image index mediaType. /// /// - internal static string GenerateImageIndex(BuiltImage[] images, string manifestMediaType, string imageIndexMediaType) + internal static ManifestListV2 GenerateDockerManifestList(BuiltImage[] images, string manifestMediaType, string imageIndexMediaType) { if (images.Length == 0) { @@ -70,7 +70,7 @@ internal static string GenerateImageIndex(BuiltImage[] images, string manifestMe manifests[i] = new PlatformSpecificManifest { mediaType = manifestMediaType, - size = images[i].Manifest.Length, + size = JsonSerializer.SerializeToNode(images[i].Manifest)!.ToJsonString().Length, digest = images[i].ManifestDigest, platform = new PlatformInformation { @@ -87,10 +87,10 @@ internal static string GenerateImageIndex(BuiltImage[] images, string manifestMe manifests = manifests }; - return GetJsonStringFromImageIndex(imageIndex); + return imageIndex; } - internal static string GenerateImageIndexWithAnnotations(string manifestMediaType, string manifestDigest, long manifestSize, string repository, string[] tags) + internal static ImageIndexV1 GenerateImageIndexWithAnnotations(string manifestMediaType, string manifestDigest, long manifestSize, string repository, string[] tags) { string containerdImageNamePrefix = repository.Contains('/') ? "docker.io/" : "docker.io/library/"; @@ -118,21 +118,6 @@ internal static string GenerateImageIndexWithAnnotations(string manifestMediaTyp manifests = manifests }; - return GetJsonStringFromImageIndex(index); - } - - private static string GetJsonStringFromImageIndex(T imageIndex) - { - var nullIgnoreOptions = new JsonSerializerOptions - { - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull - }; - // To avoid things like \u002B for '+' especially in media types ("application/vnd.oci.image.manifest.v1\u002Bjson"), we use UnsafeRelaxedJsonEscaping. - var escapeOptions = new JsonSerializerOptions - { - Encoder = JavaScriptEncoder.UnsafeRelaxedJsonEscaping - }; - - return JsonSerializer.SerializeToNode(imageIndex, nullIgnoreOptions)?.ToJsonString(escapeOptions) ?? ""; + return index; } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/Layer.cs b/src/Containers/Microsoft.NET.Build.Containers/Layer.cs index fc97038017c2..4c25377b8313 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Layer.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Layer.cs @@ -1,11 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Diagnostics; using System.Formats.Tar; using System.IO.Compression; using System.IO.Enumeration; using System.Security.Cryptography; +using System.Threading.Tasks; using Microsoft.NET.Build.Containers.Resources; namespace Microsoft.NET.Build.Containers; @@ -32,29 +34,35 @@ internal class Layer public virtual Descriptor Descriptor { get; } - public string BackingFile { get; } + public FileInfo BackingFile { get; } internal Layer() { Descriptor = new Descriptor(); - BackingFile = ""; + BackingFile = null!; } - internal Layer(string backingFile, Descriptor descriptor) + internal Layer(FileInfo backingFile, Descriptor descriptor) { BackingFile = backingFile; Descriptor = descriptor; } - public static Layer FromDescriptor(Descriptor descriptor) + public static Layer FromDescriptor(Descriptor descriptor, ContentStore store) { - return new(ContentStore.PathForDescriptor(descriptor), descriptor); + FileInfo path = new(store.PathForDescriptor(descriptor)); + return FromBackingFile(path, descriptor); } - public static Layer FromDirectory(string directory, string containerPath, bool isWindowsLayer, string manifestMediaType) + public static Layer FromBackingFile(FileInfo backingFile, Descriptor descriptor) + { + return new(backingFile, descriptor); + } + + public static async Task FromFiles((string absPath, string relPath)[] inputFiles, string containerPath, bool isWindowsLayer, string manifestMediaType, ContentStore store, FileInfo layerWritePath, CancellationToken ct) { long fileSize; - Span hash = stackalloc byte[SHA256.HashSizeInBytes]; - Span uncompressedHash = stackalloc byte[SHA256.HashSizeInBytes]; + var hash = MemoryPool.Shared.Rent(SHA256.HashSizeInBytes); + var uncompressedHash = MemoryPool.Shared.Rent(SHA256.HashSizeInBytes); // Docker treats a COPY instruction that copies to a path like `/app` by // including `app/` as a directory, with no leading slash. Emulate that here. @@ -86,7 +94,7 @@ public static Layer FromDirectory(string directory, string containerPath, bool i entryAttributes["MSWINDOWS.rawsd"] = BuiltinUsersSecurityDescriptor; } - string tempTarballPath = ContentStore.GetTempFile(); + string tempTarballPath = store.GetTempFile(); using (FileStream fs = File.Create(tempTarballPath)) { using (HashDigestGZipStream gz = new(fs, leaveOpen: true)) @@ -97,58 +105,49 @@ public static Layer FromDirectory(string directory, string containerPath, bool i if (isWindowsLayer) { var entry = new PaxTarEntry(TarEntryType.Directory, "Files", entryAttributes); - writer.WriteEntry(entry); + await writer.WriteEntryAsync(entry, ct); } - // Write an entry for the application directory. - WriteTarEntryForFile(writer, new DirectoryInfo(directory), containerPath, entryAttributes); + // Write an entry for the container working directory. + await writer.WriteEntryAsync( + new PaxTarEntry(TarEntryType.Directory, containerPath, entryAttributes) + { + Mode = UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute | + UnixFileMode.GroupRead | UnixFileMode.GroupExecute | + UnixFileMode.OtherRead | UnixFileMode.OtherExecute + }, ct); // Write entries for the application directory contents. - var fileList = new FileSystemEnumerable<(FileSystemInfo file, string containerPath)>( - directory: directory, - transform: (ref FileSystemEntry entry) => - { - FileSystemInfo fsi = entry.ToFileSystemInfo(); - string relativePath = Path.GetRelativePath(directory, fsi.FullName); - if (OperatingSystem.IsWindows()) - { - // Use only '/' directory separators. - relativePath = relativePath.Replace('\\', '/'); - } - return (fsi, $"{containerPath}/{relativePath}"); - }, - options: new EnumerationOptions() - { - AttributesToSkip = FileAttributes.System, // Include hidden files - RecurseSubdirectories = true - }); - foreach (var item in fileList) + foreach ((string absolutePath, string containerRelativePath) in inputFiles) { - WriteTarEntryForFile(writer, item.file, item.containerPath, entryAttributes); + var file = new FileInfo(absolutePath); + var adjustedRelativePath = OperatingSystem.IsWindows() ? containerRelativePath.Replace('\\', '/') : containerRelativePath; + var finalRelativePath = $"{containerPath}/{adjustedRelativePath}"; + await WriteTarEntryForFile(writer, file, finalRelativePath, entryAttributes, ct); } // Windows layers need a Hives folder, we do not need to create any Registry Hive deltas inside if (isWindowsLayer) { var entry = new PaxTarEntry(TarEntryType.Directory, "Hives", entryAttributes); - writer.WriteEntry(entry); + await writer.WriteEntryAsync(entry, ct); } } // Dispose of the TarWriter before getting the hash so the final data get written to the tar stream - int bytesWritten = gz.GetCurrentUncompressedHash(uncompressedHash); - Debug.Assert(bytesWritten == uncompressedHash.Length); + int bytesWritten = gz.GetCurrentUncompressedHash(uncompressedHash.Memory); + Debug.Assert(bytesWritten == uncompressedHash.Memory.Length); } fileSize = fs.Length; fs.Position = 0; - int bW = SHA256.HashData(fs, hash); - Debug.Assert(bW == hash.Length); + int bW = await SHA256.HashDataAsync(fs, hash.Memory, ct); + Debug.Assert(bW == hash.Memory.Length); // Writes a tar entry corresponding to the file system item. - static void WriteTarEntryForFile(TarWriter writer, FileSystemInfo file, string containerPath, IEnumerable> entryAttributes) + static async Task WriteTarEntryForFile(TarWriter writer, FileSystemInfo file, string containerPath, IEnumerable> entryAttributes, CancellationToken ct) { UnixFileMode mode = DetermineFileMode(file); @@ -160,7 +159,7 @@ static void WriteTarEntryForFile(TarWriter writer, FileSystemInfo file, string c Mode = mode, DataStream = fileStream }; - writer.WriteEntry(entry); + await writer.WriteEntryAsync(entry, ct); } else { @@ -168,7 +167,7 @@ static void WriteTarEntryForFile(TarWriter writer, FileSystemInfo file, string c { Mode = mode }; - writer.WriteEntry(entry); + await writer.WriteEntryAsync(entry, ct); } static UnixFileMode DetermineFileMode(FileSystemInfo file) @@ -185,12 +184,12 @@ static UnixFileMode DetermineFileMode(FileSystemInfo file) } } - string contentHash = Convert.ToHexStringLower(hash); - string uncompressedContentHash = Convert.ToHexStringLower(uncompressedHash); + string contentHash = Convert.ToHexStringLower(hash.Memory.Span); + string uncompressedContentHash = Convert.ToHexStringLower(uncompressedHash.Memory.Span); string layerMediaType = manifestMediaType switch { - // TODO: configurable? gzip always? + // TODO: configurable? gzip always? SchemaTypes.DockerManifestV2 => SchemaTypes.DockerLayerGzip, SchemaTypes.OciManifestV1 => SchemaTypes.OciLayerGzipV1, _ => throw new ArgumentException(Resource.FormatString(nameof(Strings.UnrecognizedMediaType), manifestMediaType)) @@ -204,18 +203,19 @@ static UnixFileMode DetermineFileMode(FileSystemInfo file) UncompressedDigest = $"sha256:{uncompressedContentHash}", }; - string storedContent = ContentStore.PathForDescriptor(descriptor); - - Directory.CreateDirectory(ContentStore.ContentRoot); - - File.Move(tempTarballPath, storedContent, overwrite: true); + string storedContent = store.PathForDescriptor(descriptor); + var _ = store.ContentRoot; + // TODO: the publish side of things requires that the layer exists in the content root (because we look it up by digest), + // but we should ideally store these in the msbuild intermediate path so that we can clean it nicely. + File.Copy(tempTarballPath, storedContent, overwrite: true); + File.Move(tempTarballPath, layerWritePath.FullName, overwrite: true); - return new(storedContent, descriptor); + return new(layerWritePath, descriptor); } - internal virtual Stream OpenBackingFile() => File.OpenRead(BackingFile); + internal virtual Stream OpenBackingFile() => BackingFile.OpenRead(); - private static readonly char[] PathSeparators = new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; + private static readonly char[] PathSeparators = [Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar]; /// /// A stream capable of computing the hash digest of raw uncompressed data while also compressing it. @@ -225,13 +225,16 @@ private sealed class HashDigestGZipStream : Stream private readonly IncrementalHash sha256Hash; private readonly GZipStream compressionStream; - public HashDigestGZipStream(Stream writeStream, bool leaveOpen) + public HashDigestGZipStream(Stream writeStream, bool leaveOpen, CompressionMode compressionMode = CompressionMode.Compress) + : base() { sha256Hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); - compressionStream = new GZipStream(writeStream, CompressionMode.Compress, leaveOpen); + compressionStream = new GZipStream(writeStream, compressionMode, leaveOpen); } - public override bool CanWrite => true; + public override bool CanWrite => compressionStream.CanWrite; + public override bool CanRead => compressionStream.CanRead; + public override bool CanSeek => compressionStream.CanSeek; public override void Write(byte[] buffer, int offset, int count) { @@ -251,6 +254,7 @@ public override void Flush() } internal int GetCurrentUncompressedHash(Span buffer) => sha256Hash.GetCurrentHash(buffer); + internal int GetCurrentUncompressedHash(Memory buffer) => sha256Hash.GetCurrentHash(buffer.Span); protected override void Dispose(bool disposing) { @@ -265,18 +269,41 @@ protected override void Dispose(bool disposing) } } - // This class is never used with async writes, but if it ever is, implement these overrides public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) - => throw new NotImplementedException(); + { + sha256Hash.AppendData(buffer, offset, count); + return compressionStream.WriteAsync(buffer, offset, count, cancellationToken); + } public override ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken) - => throw new NotImplementedException(); + { + sha256Hash.AppendData(buffer.Span); + return compressionStream.WriteAsync(buffer, cancellationToken); + } - public override bool CanRead => false; - public override bool CanSeek => false; public override long Length => throw new NotImplementedException(); public override long Position { get => throw new NotImplementedException(); set => throw new NotImplementedException(); } - public override int Read(byte[] buffer, int offset, int count) => throw new NotImplementedException(); + public override int Read(byte[] buffer, int offset, int count) + { + var read = compressionStream.Read(buffer, offset, count); + sha256Hash.AppendData(buffer.AsSpan(offset, read)); + return read; + } + + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + var read = compressionStream.ReadAsync(buffer, offset, count, cancellationToken); + sha256Hash.AppendData(buffer.AsSpan(offset, read.Result)); + return read; + } + + public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + var read = compressionStream.ReadAsync(buffer, cancellationToken); + sha256Hash.AppendData(buffer.Span.Slice(0, read.Result)); + return read; + } + public override long Seek(long offset, SeekOrigin origin) => throw new NotImplementedException(); public override void SetLength(long value) => throw new NotImplementedException(); } diff --git a/src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/ArchiveFileRegistry.cs b/src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/ArchiveFileRegistry.cs index d127635ba584..78a67fb22724 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/ArchiveFileRegistry.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/ArchiveFileRegistry.cs @@ -14,9 +14,9 @@ public ArchiveFileRegistry(string archiveOutputPath) ArchiveOutputPath = archiveOutputPath; } - internal async Task LoadAsync(T image, SourceImageReference sourceReference, + internal async Task LoadAsync(T pushData, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken, - Func writeStreamFunc) + Func<(T, SourceImageReference, DestinationImageReference), Stream, CancellationToken, Task> writeStreamFunc) { var fullPath = Path.GetFullPath(ArchiveOutputPath); @@ -25,7 +25,7 @@ internal async Task LoadAsync(T image, SourceImageReference sourceReference, // if doesn't end with a file extension, assume it's a directory if (!Path.HasExtension(fullPath)) { - fullPath += Path.DirectorySeparatorChar; + fullPath += Path.DirectorySeparatorChar; } // pointing to a directory? -> append default name @@ -45,18 +45,18 @@ internal async Task LoadAsync(T image, SourceImageReference sourceReference, await using var fileStream = File.Create(fullPath); // Call the delegate to write the image to the stream - await writeStreamFunc(image, sourceReference, destinationReference, fileStream, cancellationToken).ConfigureAwait(false); + await writeStreamFunc((pushData, sourceReference, destinationReference), fileStream, cancellationToken).ConfigureAwait(false); } public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReference, DestinationImageReference destinationReference, - CancellationToken cancellationToken) + CancellationToken cancellationToken) => await LoadAsync(image, sourceReference, destinationReference, cancellationToken, - DockerCli.WriteImageToStreamAsync); + (tup, a, b) => DockerCli.WriteImageToStreamAsync(tup.Item1, tup.Item2, tup.Item3, a, b)); public async Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference, DestinationImageReference destinationReference, - CancellationToken cancellationToken) + CancellationToken cancellationToken) => await LoadAsync(multiArchImage, sourceReference, destinationReference, cancellationToken, DockerCli.WriteMultiArchOciImageToStreamAsync); diff --git a/src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/DockerCli.cs b/src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/DockerCli.cs index ebc8064f8e41..2bc2f37fda4a 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/DockerCli.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/LocalDaemons/DockerCli.cs @@ -51,6 +51,9 @@ public DockerCli(string? command, ILoggerFactory loggerFactory) public DockerCli(ILoggerFactory loggerFactory) : this(null, loggerFactory) { } + public bool IsDocker => _command == DockerCommand; + public bool IsPodman => _command == PodmanCommand; + private static string FindFullPathFromPath(string command) { foreach (string directory in (Environment.GetEnvironmentVariable("PATH") ?? string.Empty).Split(Path.PathSeparator)) @@ -85,10 +88,8 @@ private async ValueTask FindFullCommandPath(CancellationToken cancellati } private async Task LoadAsync( - T image, - SourceImageReference sourceReference, - DestinationImageReference destinationReference, - Func writeStreamFunc, + T pushData, + Func writeStreamFunc, CancellationToken cancellationToken, bool checkContainerdStore = false) { @@ -100,7 +101,7 @@ private async Task LoadAsync( } string commandPath = await FindFullCommandPath(cancellationToken); - + _logger.LogInformation($"Streaming image to local {commandPath}..."); // call `docker load` and get it ready to receive input ProcessStartInfo loadInfo = new(commandPath, $"load") { @@ -113,7 +114,7 @@ private async Task LoadAsync( throw new NotImplementedException(Resource.FormatString(Strings.ContainerRuntimeProcessCreationFailed, commandPath)); // Call the delegate to write the image to the stream - await writeStreamFunc(image, sourceReference, destinationReference, loadProcess.StandardInput.BaseStream, cancellationToken) + await writeStreamFunc(pushData, loadProcess.StandardInput.BaseStream, cancellationToken) .ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); @@ -122,6 +123,8 @@ await writeStreamFunc(image, sourceReference, destinationReference, loadProcess. await loadProcess.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + _logger.LogInformation($"Image streaming completed"); + cancellationToken.ThrowIfCancellationRequested(); if (loadProcess.ExitCode != 0) @@ -130,13 +133,19 @@ await writeStreamFunc(image, sourceReference, destinationReference, loadProcess. } } - public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken) + public async Task LoadAsync(BuiltImage image, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken) // For loading to the local registry, we use the Docker format. Two reasons: one - compatibility with previous behavior before oci formatted publishing was available, two - Podman cannot load multi tag oci image tarball. - => await LoadAsync(image, sourceReference, destinationReference, WriteDockerImageToStreamAsync, cancellationToken); + => await LoadAsync((image, sourceReference, destinationReference), WriteDockerImageToStreamAsync, cancellationToken); + + public async Task LoadAsync( + (string repository, string[] tags, string configDigest, JsonObject config, Layer[] layers) imageData, + Func<(string repository, string[] tags, string configDigest, JsonObject config, Layer[] layers), Stream, CancellationToken, Task> writeStreamFunc, + CancellationToken cancellationToken) + => await LoadAsync(imageData, writeStreamFunc, cancellationToken, checkContainerdStore: false); + + public async Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken) + => await LoadAsync((multiArchImage, sourceReference, destinationReference), WriteMultiArchOciImageToStreamAsync, cancellationToken, checkContainerdStore: true); - public async Task LoadAsync(MultiArchImage multiArchImage, SourceImageReference sourceReference, DestinationImageReference destinationReference, CancellationToken cancellationToken) - => await LoadAsync(multiArchImage, sourceReference, destinationReference, WriteMultiArchOciImageToStreamAsync, cancellationToken, checkContainerdStore: true); - public async Task IsAvailableAsync(CancellationToken cancellationToken) { bool commandPathWasUnknown = _command is null; // avoid running the version command twice. @@ -289,11 +298,11 @@ public static async Task WriteImageToStreamAsync(BuiltImage image, SourceImageRe { if (image.ManifestMediaType == SchemaTypes.DockerManifestV2) { - await WriteDockerImageToStreamAsync(image, sourceReference, destinationReference, imageStream, cancellationToken); + await WriteDockerImageToStreamAsync((image, sourceReference, destinationReference), imageStream, cancellationToken); } else if (image.ManifestMediaType == SchemaTypes.OciManifestV1) { - await WriteOciImageToStreamAsync(image, sourceReference, destinationReference, imageStream, cancellationToken); + await WriteOciImageToStreamAsync((image, sourceReference, destinationReference), imageStream, cancellationToken); } else { @@ -301,10 +310,26 @@ public static async Task WriteImageToStreamAsync(BuiltImage image, SourceImageRe } } + public static async Task WriteImageToStreamAsync(string repository, string[] tags, JsonObject config, Layer[] layers, ManifestV2 manifest, Stream imageStream, CancellationToken cancellationToken) + { + if (manifest.MediaType == SchemaTypes.DockerManifestV2) + { + await WriteDockerImageToStreamAsync((repository, tags, DigestUtils.GetDigest(config), config, layers), imageStream, cancellationToken); + } + else if (manifest.MediaType == SchemaTypes.OciManifestV1) + { + await WriteOciImageToStreamAsync((repository, tags, DigestUtils.GetDigest(config), config, layers, manifest), imageStream, cancellationToken); + } + else + { + throw new ArgumentException(Resource.FormatString(nameof(Strings.UnsupportedMediaTypeForTarball), manifest.MediaType)); + } + } + private static async Task WriteDockerImageToStreamAsync( - BuiltImage image, + (BuiltImage image, SourceImageReference sourceReference, - DestinationImageReference destinationReference, + DestinationImageReference destinationReference) pushData, Stream imageStream, CancellationToken cancellationToken) { @@ -313,15 +338,42 @@ private static async Task WriteDockerImageToStreamAsync( // Feed each layer tarball into the stream JsonArray layerTarballPaths = new(); - await WriteImageLayers(writer, image, sourceReference, d => $"{d.Substring("sha256:".Length)}/layer.tar", cancellationToken, layerTarballPaths) + await WriteImageLayers(writer, pushData.image, pushData.sourceReference, d => $"{d.Substring("sha256:".Length)}/layer.tar", cancellationToken, layerTarballPaths) + .ConfigureAwait(false); + + string configTarballPath = $"{pushData.image.Manifest.Config.digest.Split(':')[1]!}.json"; + await WriteImageConfig(writer, pushData.image, configTarballPath, cancellationToken) + .ConfigureAwait(false); + + // Add manifest + await WriteManifestForDockerImage(writer, pushData.destinationReference.Repository, pushData.destinationReference.Tags, configTarballPath, layerTarballPaths, cancellationToken) + .ConfigureAwait(false); + } + + public static async Task WriteDockerImageToStreamAsync( + (string repository, + string[] tags, + string configDigest, + JsonObject imageConfig, + Layer[] layers) pushData, + Stream imageStream, + CancellationToken cancellationToken + ) + { + cancellationToken.ThrowIfCancellationRequested(); + using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true); + // Feed each layer tarball into the stream + var layerTarballPathFunc = (Layer l) => $"{l.Descriptor.Digest.Substring("sha256:".Length)}/layer.tar"; + await WriteImageLayers(writer, pushData.layers, layerTarballPathFunc, cancellationToken) .ConfigureAwait(false); - string configTarballPath = $"{image.ImageSha!}.json"; - await WriteImageConfig(writer, image, configTarballPath, cancellationToken) + string configTarballPath = $"{pushData.configDigest.Split(':')[1]!}.json"; + await WriteImageConfig(writer, pushData.imageConfig, configTarballPath, cancellationToken) .ConfigureAwait(false); // Add manifest - await WriteManifestForDockerImage(writer, destinationReference, configTarballPath, layerTarballPaths, cancellationToken) + JsonArray layerTarballPaths = new JsonArray([.. pushData.layers.Select(layerTarballPathFunc).Select(s => JsonValue.Create(s))]); + await WriteManifestForDockerImage(writer, pushData.repository, pushData.tags, configTarballPath, layerTarballPaths, cancellationToken) .ConfigureAwait(false); } @@ -358,6 +410,22 @@ private static async Task WriteImageLayers( } } + private static async Task WriteImageLayers( + TarWriter writer, + Layer[] layers, + Func layerTarballPathFunc, + CancellationToken cancellationToken + ) + { + cancellationToken.ThrowIfCancellationRequested(); + + foreach (var layer in layers) + { + string layerTarballPath = layerTarballPathFunc(layer); + await writer.WriteEntryAsync(layer.BackingFile.FullName, layerTarballPath, cancellationToken).ConfigureAwait(false); + } + } + private static async Task WriteImageConfig( TarWriter writer, BuiltImage image, @@ -365,7 +433,24 @@ private static async Task WriteImageConfig( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - using (MemoryStream configStream = new(Encoding.UTF8.GetBytes(image.Config))) + using (MemoryStream configStream = new(DigestUtils.UTF8.GetBytes(image.Config.ToJsonString()))) + { + PaxTarEntry configEntry = new(TarEntryType.RegularFile, configPath) + { + DataStream = configStream + }; + await writer.WriteEntryAsync(configEntry, cancellationToken).ConfigureAwait(false); + } + } + + private static async Task WriteImageConfig( + TarWriter writer, + JsonObject config, + string configPath, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + using (MemoryStream configStream = new(DigestUtils.UTF8.GetBytes(config.ToJsonString()))) { PaxTarEntry configEntry = new(TarEntryType.RegularFile, configPath) { @@ -377,15 +462,16 @@ private static async Task WriteImageConfig( private static async Task WriteManifestForDockerImage( TarWriter writer, - DestinationImageReference destinationReference, + string repository, + string[] tags, string configTarballPath, JsonArray layerTarballPaths, CancellationToken cancellationToken) { JsonArray tagsNode = new(); - foreach (string tag in destinationReference.Tags) + foreach (string tag in tags) { - tagsNode.Add($"{destinationReference.Repository}:{tag}"); + tagsNode.Add($"{repository}:{tag}"); } JsonNode manifestNode = new JsonArray(new JsonObject @@ -396,7 +482,7 @@ private static async Task WriteManifestForDockerImage( }); cancellationToken.ThrowIfCancellationRequested(); - using (MemoryStream manifestStream = new(Encoding.UTF8.GetBytes(manifestNode.ToJsonString()))) + using (MemoryStream manifestStream = new(DigestUtils.UTF8.GetBytes(manifestNode.ToJsonString()))) { PaxTarEntry manifestEntry = new(TarEntryType.RegularFile, "manifest.json") { @@ -408,9 +494,9 @@ private static async Task WriteManifestForDockerImage( } private static async Task WriteOciImageToStreamAsync( - BuiltImage image, + (BuiltImage image, SourceImageReference sourceReference, - DestinationImageReference destinationReference, + DestinationImageReference destinationReference) pushData, Stream imageStream, CancellationToken cancellationToken) { @@ -418,10 +504,34 @@ private static async Task WriteOciImageToStreamAsync( using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true); - await WriteOciImageToBlobs(writer, image, sourceReference, cancellationToken) + await WriteOciImageToBlobs(writer, pushData.image, pushData.sourceReference, cancellationToken) .ConfigureAwait(false); - await WriteIndexJsonForOciImage(writer, image, destinationReference, cancellationToken) + await WriteIndexJsonForOciImage(writer, pushData.image, pushData.destinationReference, cancellationToken) + .ConfigureAwait(false); + + await WriteOciLayout(writer, cancellationToken) + .ConfigureAwait(false); + } + + private static async Task WriteOciImageToStreamAsync( + (string repository, + string[] tags, + string configDigest, + JsonObject imageConfig, + Layer[] layers, + ManifestV2 manifest) pushData, + Stream imageStream, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true); + + await WriteOciImageToBlobs(writer, pushData.configDigest, pushData.imageConfig, pushData.layers, pushData.manifest, cancellationToken) + .ConfigureAwait(false); + + await WriteIndexJsonForOciImage(writer, pushData.repository, pushData.tags, pushData.manifest, cancellationToken) .ConfigureAwait(false); await WriteOciLayout(writer, cancellationToken) @@ -434,7 +544,7 @@ private static async Task WriteOciLayout(TarWriter writer, CancellationToken can string ociLayoutPath = "oci-layout"; var ociLayoutContent = "{\"imageLayoutVersion\": \"1.0.0\"}"; - using (MemoryStream ociLayoutStream = new MemoryStream(Encoding.UTF8.GetBytes(ociLayoutContent))) + using (MemoryStream ociLayoutStream = new MemoryStream(DigestUtils.UTF8.GetBytes(ociLayoutContent))) { PaxTarEntry layoutEntry = new(TarEntryType.RegularFile, ociLayoutPath) { @@ -452,7 +562,25 @@ private static async Task WriteManifestForOciImage( cancellationToken.ThrowIfCancellationRequested(); string manifestPath = $"{_blobsPath}/{image.ManifestDigest.Substring("sha256:".Length)}"; - using (MemoryStream manifestStream = new MemoryStream(Encoding.UTF8.GetBytes(image.Manifest))) + using (MemoryStream manifestStream = new MemoryStream(DigestUtils.UTF8.GetBytes(JsonSerializer.Serialize(image.Manifest)))) + { + PaxTarEntry manifestEntry = new(TarEntryType.RegularFile, manifestPath) + { + DataStream = manifestStream + }; + await writer.WriteEntryAsync(manifestEntry, cancellationToken).ConfigureAwait(false); + } + } + + private static async Task WriteManifestForOciImage( + TarWriter writer, + ManifestV2 manifest, + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + string manifestPath = $"{_blobsPath}/{manifest.GetDigest().Substring("sha256:".Length)}"; + using (MemoryStream manifestStream = new MemoryStream(DigestUtils.UTF8.GetBytes(JsonSerializer.Serialize(manifest)))) { PaxTarEntry manifestEntry = new(TarEntryType.RegularFile, manifestPath) { @@ -470,14 +598,41 @@ private static async Task WriteIndexJsonForOciImage( { cancellationToken.ThrowIfCancellationRequested(); - string indexJson = ImageIndexGenerator.GenerateImageIndexWithAnnotations( + var index = ImageIndexGenerator.GenerateImageIndexWithAnnotations( SchemaTypes.OciManifestV1, image.ManifestDigest, - image.Manifest.Length, + (DigestUtils.UTF8.GetBytes(JsonSerializer.Serialize(image.Manifest))).Length, destinationReference.Repository, destinationReference.Tags); - using (MemoryStream indexStream = new(Encoding.UTF8.GetBytes(indexJson))) + using (MemoryStream indexStream = new(DigestUtils.UTF8.GetBytes(JsonSerializer.Serialize(index)))) + { + PaxTarEntry indexEntry = new(TarEntryType.RegularFile, "index.json") + { + DataStream = indexStream + }; + await writer.WriteEntryAsync(indexEntry, cancellationToken).ConfigureAwait(false); + } + } + + private static async Task WriteIndexJsonForOciImage( + TarWriter writer, + string repository, + string[] tags, + ManifestV2 manifest, + + CancellationToken cancellationToken) + { + cancellationToken.ThrowIfCancellationRequested(); + + var index = ImageIndexGenerator.GenerateImageIndexWithAnnotations( + SchemaTypes.OciManifestV1, + manifest.GetDigest(), + DigestUtils.UTF8.GetBytes(JsonSerializer.Serialize(manifest)).Length, + repository, + tags); + + using (MemoryStream indexStream = new(DigestUtils.UTF8.GetBytes(JsonSerializer.Serialize(index)))) { PaxTarEntry indexEntry = new(TarEntryType.RegularFile, "index.json") { @@ -496,17 +651,35 @@ private static async Task WriteOciImageToBlobs( await WriteImageLayers(writer, image, sourceReference, d => $"{_blobsPath}/{d.Substring("sha256:".Length)}", cancellationToken) .ConfigureAwait(false); - await WriteImageConfig(writer, image, $"{_blobsPath}/{image.ImageSha!}", cancellationToken) + await WriteImageConfig(writer, image, $"{_blobsPath}/{image.Manifest.Config.digest.Split(':')[1]!}", cancellationToken) .ConfigureAwait(false); await WriteManifestForOciImage(writer, image, cancellationToken) .ConfigureAwait(false); } + private static async Task WriteOciImageToBlobs( + TarWriter writer, + string configDigest, + JsonObject imageConfig, + Layer[] layers, + ManifestV2 manifest, + CancellationToken cancellationToken) + { + await WriteImageLayers(writer, layers, l => $"{_blobsPath}/{l.Descriptor.Digest.Substring("sha256:".Length)}", cancellationToken) + .ConfigureAwait(false); + + await WriteImageConfig(writer, imageConfig, $"{_blobsPath}/{configDigest.Split(':')[1]!}", cancellationToken) + .ConfigureAwait(false); + + await WriteManifestForOciImage(writer, manifest, cancellationToken) + .ConfigureAwait(false); + } + public static async Task WriteMultiArchOciImageToStreamAsync( - MultiArchImage multiArchImage, + (MultiArchImage multiArchImage, SourceImageReference sourceReference, - DestinationImageReference destinationReference, + DestinationImageReference destinationReference) pushData, Stream imageStream, CancellationToken cancellationToken) { @@ -514,13 +687,13 @@ public static async Task WriteMultiArchOciImageToStreamAsync( using TarWriter writer = new(imageStream, TarEntryFormat.Pax, leaveOpen: true); - foreach (var image in multiArchImage.Images!) + foreach (var image in pushData.multiArchImage.Images!) { - await WriteOciImageToBlobs(writer, image, sourceReference, cancellationToken) + await WriteOciImageToBlobs(writer, image, pushData.sourceReference, cancellationToken) .ConfigureAwait(false); } - await WriteIndexJsonForMultiArchOciImage(writer, multiArchImage, destinationReference, cancellationToken) + await WriteIndexJsonForMultiArchOciImage(writer, pushData.multiArchImage, pushData.destinationReference, cancellationToken) .ConfigureAwait(false); await WriteOciLayout(writer, cancellationToken) @@ -539,8 +712,8 @@ private static async Task WriteIndexJsonForMultiArchOciImage( var manifestListDigest = DigestUtils.GetDigest(multiArchImage.ImageIndex); var manifestListSha = DigestUtils.GetShaFromDigest(manifestListDigest); var manifestListPath = $"{_blobsPath}/{manifestListSha}"; - - using (MemoryStream indexStream = new(Encoding.UTF8.GetBytes(multiArchImage.ImageIndex))) + var manifestBytes = DigestUtils.UTF8.GetBytes(JsonSerializer.Serialize(multiArchImage.ImageIndex)); + using (MemoryStream indexStream = new(manifestBytes)) { PaxTarEntry indexEntry = new(TarEntryType.RegularFile, manifestListPath) { @@ -552,14 +725,14 @@ private static async Task WriteIndexJsonForMultiArchOciImage( // 2. create index.json that points to manifest list in the blobs cancellationToken.ThrowIfCancellationRequested(); - string indexJson = ImageIndexGenerator.GenerateImageIndexWithAnnotations( - multiArchImage.ImageIndexMediaType, - manifestListDigest, - multiArchImage.ImageIndex.Length, - destinationReference.Repository, + var index = ImageIndexGenerator.GenerateImageIndexWithAnnotations( + multiArchImage.ImageIndex.MediaType!, + manifestListDigest, + manifestBytes.Length, + destinationReference.Repository, destinationReference.Tags); - using (MemoryStream indexStream = new(Encoding.UTF8.GetBytes(indexJson))) + using (MemoryStream indexStream = new(DigestUtils.UTF8.GetBytes(JsonSerializer.Serialize(index)))) { PaxTarEntry indexEntry = new(TarEntryType.RegularFile, "index.json") { diff --git a/src/Containers/Microsoft.NET.Build.Containers/Logging/MSBuildLogger.cs b/src/Containers/Microsoft.NET.Build.Containers/Logging/MSBuildLogger.cs index e6b96093f53b..44d05b0c6ace 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Logging/MSBuildLogger.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Logging/MSBuildLogger.cs @@ -5,45 +5,94 @@ using Microsoft.Build.Framework; using Microsoft.Build.Utilities; using Microsoft.Extensions.Logging; +using Microsoft.NET.StringTools; using ILogger = Microsoft.Extensions.Logging.ILogger; namespace Microsoft.NET.Build.Containers.Logging; +/// +/// a struct that maps to the parameters of the MSBuild LogX methods. We'll extract this from M.E.ILogger state/scope information so that we can be maximally compatible with the MSBuild logging system. +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// +internal record struct MSBuildMessageParameters(string? subcategory, + string? code, + string? helpKeyword, + string? file, + int? lineNumber, + int? columnNumber, + int? endLineNumber, + int? endColumnNumber, + string message); + /// /// Implements an ILogger that passes the logs to the wrapped TaskLoggingHelper. /// +/// +/// This logger is designed to be used with MSBuild tasks, allowing logs to be written in a way that integrates with the MSBuild logging system. +/// It looks for specific property names in the state/scope parts of the message and maps them to the parameters of the MSBuild LogX methods. +/// Those specific keys are: +/// +/// Subcategory +/// Code +/// HelpKeyword +/// File +/// LineNumber +/// ColumnNumber +/// EndLineNumber +/// EndColumnNumber +/// {OriginalFormat}(usually provided by the underlying logging framework) +/// +/// +/// So if you add these to the scope (e.g. via _logger.BeginScope(new Dictionary{ ... })) or on the message format itself, +/// they will be extracted and used to format the message correctly for MSBuild. +/// internal sealed class MSBuildLogger : ILogger { private static readonly IDisposable Scope = new DummyDisposable(); private readonly TaskLoggingHelper _loggingHelper; + private readonly string _category; + private IExternalScopeProvider? _scopeProvider; - public MSBuildLogger(string category, TaskLoggingHelper loggingHelperToWrap) + public MSBuildLogger(string category, TaskLoggingHelper loggingHelperToWrap, IExternalScopeProvider? scopeProvider = null) { + _category = category; _loggingHelper = loggingHelperToWrap; + _scopeProvider = scopeProvider; } - IDisposable ILogger.BeginScope(TState state) => Scope; + IDisposable ILogger.BeginScope(TState state) => _scopeProvider?.Push(state) ?? Scope; public bool IsEnabled(LogLevel logLevel) => true; public void Log(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func formatter) { + var message = FormatMessage(_category, state, exception, formatter, _scopeProvider); switch (logLevel) { case LogLevel.Trace: - _loggingHelper.LogMessage(MessageImportance.Low, formatter(state, exception)); + _loggingHelper.LogMessage(message.subcategory, message.code, message.helpKeyword, message.file, message.lineNumber ?? 0, message.columnNumber ?? 0, message.endLineNumber ?? 0, message.endColumnNumber ?? 0, MessageImportance.Low, message.message); break; case LogLevel.Debug: + _loggingHelper.LogMessage(message.subcategory, message.code, message.helpKeyword, message.file, message.lineNumber ?? 0, message.columnNumber ?? 0, message.endLineNumber ?? 0, message.endColumnNumber ?? 0, MessageImportance.Normal, message.message); + break; case LogLevel.Information: - _loggingHelper.LogMessage(MessageImportance.High, formatter(state, exception)); + _loggingHelper.LogMessage(message.subcategory, message.code, message.helpKeyword, message.file, message.lineNumber ?? 0, message.columnNumber ?? 0, message.endLineNumber ?? 0, message.endColumnNumber ?? 0, MessageImportance.High, message.message); break; case LogLevel.Warning: - _loggingHelper.LogWarning(formatter(state, exception)); + _loggingHelper.LogWarning(message.subcategory, message.code, message.helpKeyword, message.file, message.lineNumber ?? 0, message.columnNumber ?? 0, message.endLineNumber ?? 0, message.endColumnNumber ?? 0, message.message); break; case LogLevel.Error: case LogLevel.Critical: - _loggingHelper.LogError(formatter(state, exception)); + _loggingHelper.LogError(message.subcategory, message.code, message.helpKeyword, message.file, message.lineNumber ?? 0, message.columnNumber ?? 0, message.endLineNumber ?? 0, message.endColumnNumber ?? 0, message.message); break; case LogLevel.None: break; @@ -52,6 +101,148 @@ public void Log(LogLevel logLevel, EventId eventId, TState state, Except } } + public static MSBuildMessageParameters FormatMessage(string category, TState state, Exception? exception, Func formatter, IExternalScopeProvider? scopeProvider) + { + MSBuildMessageParameters message = default; + using var builder = new SpanBasedStringBuilder(); + var categoryBlock = string.Concat("[".AsSpan(), category.AsSpan(), "] ".AsSpan()); + builder.Append(categoryBlock); + var formatted = formatter(state, exception); + builder.Append(formatted); + + if (scopeProvider is not null) + { + // state will be a FormattedLogValues instance + // scope will be our dictionary thing we need to probe into + scopeProvider.ForEachScope((scope, state) => + { + var stateItems = (state as IReadOnlyList>)!; + string originalFormat = null!; + + foreach (var kvp in stateItems) + { + switch (kvp.Key) + { + case "{OriginalFormat}": + // If the key is {OriginalFormat}, we will use it to set the originalFormat variable. + // This is used to avoid appending the same key again in the message. + if (kvp.Value is string format) + { + originalFormat = format; + } + continue; + case "Subcategory": + message.subcategory = kvp.Value as string; + continue; + case "Code": + message.code = kvp.Value as string; + continue; + case "HelpKeyword": + message.helpKeyword = kvp.Value as string; + continue; + case "File": + message.file = kvp.Value as string; + continue; + case "LineNumber": + if (kvp.Value is int lineNumber) + message.lineNumber = lineNumber; + continue; + case "ColumnNumber": + if (kvp.Value is int columnNumber) + message.columnNumber = columnNumber; + continue; + case "EndLineNumber": + if (kvp.Value is int endLineNumber) + message.endLineNumber = endLineNumber; + continue; + case "EndColumnNumber": + if (kvp.Value is int endColumnNumber) + message.endColumnNumber = endColumnNumber; + continue; + default: + var wrappedKey = "{" + kvp.Key + "}"; + if (originalFormat.Contains(wrappedKey)) + { + // If the key is part of the format string of the original format, we don't need to append it again. + continue; + } + + // Otherwise, append the key and value to the message. + // if MSbuild had a property bag concept on the message APIs, + // we could use that instead of appending to the message. + + builder.Append($" {kvp.Key}={kvp.Value}"); + continue; + } + } + + if (scope is IDictionary dict) + { + foreach (var kvp in dict) + { + switch (kvp.Key) + { + // map all of the keys we decide are special and map to MSbuild message concepts + case "{OriginalFormat}": + continue; + case "Subcategory": + message.subcategory = kvp.Value as string; + continue; + case "Code": + message.code = kvp.Value as string; + continue; + case "HelpKeyword": + message.helpKeyword = kvp.Value as string; + continue; + case "File": + message.file = kvp.Value as string; + continue; + case "LineNumber": + if (kvp.Value is int lineNumber) + message.lineNumber = lineNumber; + continue; + case "ColumnNumber": + if (kvp.Value is int columnNumber) + message.columnNumber = columnNumber; + continue; + case "EndLineNumber": + if (kvp.Value is int endLineNumber) + message.endLineNumber = endLineNumber; + continue; + case "EndColumnNumber": + if (kvp.Value is int endColumnNumber) + message.endColumnNumber = endColumnNumber; + continue; + default: + var wrappedKey = "{" + kvp.Key + "}"; + if (originalFormat.Contains(wrappedKey)) + { + // If the key is part of the format string of the original format, we don't need to append it again. + continue; + } + + // Otherwise, append the key and value to the message. + // if MSbuild had a property bag concept on the message APIs, + // we could use that instead of appending to the message. + + builder.Append($" {kvp.Key}={kvp.Value}"); + continue; + } + } + } + else if (scope is string s) + { + builder.Append($" {s}"); + } + + + }, state); + } + + message.message = builder.ToString(); + return message; + } + /// /// A simple disposable to describe scopes with . /// @@ -59,4 +250,9 @@ private sealed class DummyDisposable : IDisposable { public void Dispose() { } } + + internal void SetScopeProvider(IExternalScopeProvider scopeProvider) + { + _scopeProvider = scopeProvider; + } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/Logging/MSBuildLoggerProvider.cs b/src/Containers/Microsoft.NET.Build.Containers/Logging/MSBuildLoggerProvider.cs index 8db4a7b2ae51..2991b1f5119d 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Logging/MSBuildLoggerProvider.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Logging/MSBuildLoggerProvider.cs @@ -11,9 +11,11 @@ namespace Microsoft.NET.Build.Containers.Logging; /// An that creates s which passes /// all the logs to MSBuild's . /// -internal class MSBuildLoggerProvider : ILoggerProvider +internal class MSBuildLoggerProvider : ILoggerProvider, ISupportExternalScope { private readonly TaskLoggingHelper _loggingHelper; + private List _loggers = new List(); + private IExternalScopeProvider? _scopeProvider; public MSBuildLoggerProvider(TaskLoggingHelper loggingHelperToWrap) { @@ -22,8 +24,19 @@ public MSBuildLoggerProvider(TaskLoggingHelper loggingHelperToWrap) public ILogger CreateLogger(string categoryName) { - return new MSBuildLogger(categoryName, _loggingHelper); + var logger = new MSBuildLogger(categoryName, _loggingHelper, _scopeProvider); + _loggers.Add(logger); + return logger; } public void Dispose() { } + + public void SetScopeProvider(IExternalScopeProvider scopeProvider) + { + _scopeProvider = scopeProvider; + foreach (var logger in _loggers) + { + logger.SetScopeProvider(scopeProvider); + } + } } diff --git a/src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs b/src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs index 471f450f42d6..7e47b11aa21e 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ManifestListV2.cs @@ -5,11 +5,22 @@ namespace Microsoft.NET.Build.Containers; -public record struct ManifestListV2(int schemaVersion, string mediaType, PlatformSpecificManifest[] manifests); +/// +/// Marker interface that signals that this contains sub-manifests +/// +public interface IMultiImageManifest: IManifest; + +public record struct ManifestListV2(int schemaVersion, string mediaType, PlatformSpecificManifest[] manifests) : IMultiImageManifest +{ + public string? MediaType => mediaType; +} public record struct PlatformInformation(string architecture, string os, string? variant, string[] features, [property: JsonPropertyName("os.version")][field: JsonPropertyName("os.version")] string? version); public record struct PlatformSpecificManifest(string mediaType, long size, string digest, PlatformInformation platform); -public record struct ImageIndexV1(int schemaVersion, string mediaType, PlatformSpecificOciManifest[] manifests); +public record struct ImageIndexV1(int schemaVersion, string mediaType, PlatformSpecificOciManifest[] manifests) : IMultiImageManifest +{ + public string? MediaType => mediaType; +} public record struct PlatformSpecificOciManifest(string mediaType, long size, string digest, PlatformInformation platform, Dictionary annotations); diff --git a/src/Containers/Microsoft.NET.Build.Containers/ManifestV2.cs b/src/Containers/Microsoft.NET.Build.Containers/ManifestV2.cs index b4334c5e26db..8c17699511aa 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/ManifestV2.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/ManifestV2.cs @@ -6,13 +6,21 @@ namespace Microsoft.NET.Build.Containers; +/// +/// Marker interface so we can return polymorphic manifests of all kinds +/// +public interface IManifest +{ + public string? MediaType { get; } +}; + /// /// The struct represents image manifest specification. /// /// /// https://github.com/opencontainers/image-spec/blob/main/manifest.md /// -public class ManifestV2 +public class ManifestV2 : IManifest { [JsonIgnore] public string? KnownDigest { get; set; } @@ -50,7 +58,7 @@ public class ManifestV2 /// /// Gets the digest for this manifest. /// - public string GetDigest() => KnownDigest ??= DigestUtils.GetDigest(JsonSerializer.SerializeToNode(this)?.ToJsonString() ?? string.Empty); + public string GetDigest() => KnownDigest ??= DigestUtils.GetDigest(this); } public record struct ManifestConfig(string mediaType, long size, string digest); diff --git a/src/Containers/Microsoft.NET.Build.Containers/MultiArchImage.cs b/src/Containers/Microsoft.NET.Build.Containers/MultiArchImage.cs index b24fbf65e87f..99c2e014f14b 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/MultiArchImage.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/MultiArchImage.cs @@ -8,9 +8,7 @@ namespace Microsoft.NET.Build.Containers; /// internal readonly struct MultiArchImage { - internal required string ImageIndex { get; init; } - - internal required string ImageIndexMediaType { get; init; } + internal required IMultiImageManifest ImageIndex { get; init; } internal BuiltImage[]? Images { get; init; } } \ No newline at end of file diff --git a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net10.0/PublicAPI.Unshipped.txt b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net10.0/PublicAPI.Unshipped.txt index 4d92cb103d8d..726f0395ec29 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net10.0/PublicAPI.Unshipped.txt +++ b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net10.0/PublicAPI.Unshipped.txt @@ -29,6 +29,8 @@ Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.BaseImageDigest.get -> str Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.BaseImageDigest.set -> void Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.ArchiveOutputPath.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.ArchiveOutputPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.GeneratedManifestPath.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.GeneratedManifestPath.set -> void Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.LocalRegistry.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.LocalRegistry.set -> void Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.GeneratedArchiveOutputPath.get -> string! @@ -46,7 +48,72 @@ Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.OutputRegistry.get -> stri Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.OutputRegistry.set -> void Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.Repository.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.Repository.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerLayer.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerLayer.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerConfig.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerConfig.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerManifest.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerManifest.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedConfigurationPath.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedConfigurationPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedLayerPath.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedLayerPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedManifestPath.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedManifestPath.set -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.ArchivePath.get -> string! +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.ArchivePath.set -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Cancel() -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Configuration.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Configuration.set -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.ExecuteAsync() -> System.Threading.Tasks.Task! +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.GeneratedArchiveFilePath.get -> string! +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.GeneratedArchiveFilePath.set -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Layers.get -> Microsoft.Build.Framework.ITaskItem![]! +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Layers.set -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.MakeContainerTarball() -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Manifest.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Manifest.set -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Repository.get -> string! +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Repository.set -> void +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Tags.get -> string![]! +Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Tags.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Cancel() -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Configuration.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Configuration.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.ExecuteAsync() -> System.Threading.Tasks.Task! +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Layers.get -> Microsoft.Build.Framework.ITaskItem![]! +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Layers.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.LocalRegistry.get -> string? +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.LocalRegistry.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Manifest.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Manifest.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.PushContainerToLocal() -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Repository.get -> string! +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Repository.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Tags.get -> string![]! +Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Tags.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Cancel() -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Configuration.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Configuration.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.ExecuteAsync() -> System.Threading.Tasks.Task! +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Layers.get -> Microsoft.Build.Framework.ITaskItem![]! +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Layers.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Manifest.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Manifest.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.PushContainerToRemoteRegistry() -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Registry.get -> string! +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Registry.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Repository.get -> string! +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Repository.set -> void +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Tags.get -> string![]! +Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Tags.set -> void override Microsoft.NET.Build.Containers.Tasks.CreateImageIndex.Execute() -> bool +override Microsoft.NET.Build.Containers.Tasks.MakeContainerTarball.Execute() -> bool +override Microsoft.NET.Build.Containers.Tasks.PushContainerToLocal.Execute() -> bool +override Microsoft.NET.Build.Containers.Tasks.PushContainerToRemoteRegistry.Execute() -> bool static readonly Microsoft.NET.Build.Containers.Constants.Version -> string! Microsoft.NET.Build.Containers.ContainerHelpers Microsoft.NET.Build.Containers.ContainerHelpers.ParsePortError @@ -194,15 +261,19 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageDigest.get -> strin Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageDigest.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseRegistry.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseRegistry.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageConfigurationPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageConfigurationPath.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageManifestPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageManifestPath.get -> Microsoft.Build.Framework.ITaskItem! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Cancel() -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerEnvironmentVariables.get -> Microsoft.Build.Framework.ITaskItem![]! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerEnvironmentVariables.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerizeDirectory.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerizeDirectory.set -> void -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.get -> string! -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerUser.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerUser.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContentStoreRoot.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContentStoreRoot.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.CreateNewImage() -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Dispose() -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.Entrypoint.get -> Microsoft.Build.Framework.ITaskItem![]! @@ -241,10 +312,6 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.OutputRegistry.get -> string Microsoft.NET.Build.Containers.Tasks.CreateNewImage.OutputRegistry.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ArchiveOutputPath.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ArchiveOutputPath.set -> void -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.PublishDirectory.get -> string! -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.PublishDirectory.set -> void -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.RuntimeIdentifierGraphPath.get -> string! -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.RuntimeIdentifierGraphPath.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ToolExe.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ToolExe.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ToolPath.get -> string! @@ -255,8 +322,6 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.get -> bool Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.get -> bool Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.set -> void -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SkipPublishing.get -> bool -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SkipPublishing.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedContainerNames.get -> Microsoft.Build.Framework.ITaskItem![]! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedContainerNames.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ImageFormat.get -> string? diff --git a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net472/PublicAPI.Unshipped.txt b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net472/PublicAPI.Unshipped.txt index 607f906e2676..f5732d45457d 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net472/PublicAPI.Unshipped.txt +++ b/src/Containers/Microsoft.NET.Build.Containers/PublicAPI/net472/PublicAPI.Unshipped.txt @@ -16,6 +16,14 @@ Microsoft.NET.Build.Containers.Port.Type.set -> void Microsoft.NET.Build.Containers.Tasks.ComputeDotnetBaseImageAndTag.ContainerFamily.get -> string? Microsoft.NET.Build.Containers.Tasks.ComputeDotnetBaseImageAndTag.UserBaseImage.get -> string? Microsoft.NET.Build.Containers.Tasks.ComputeDotnetBaseImageAndTag.UserBaseImage.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContentStoreRoot.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContentStoreRoot.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.PublishFiles.get -> Microsoft.Build.Framework.ITaskItem![]! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.PublishFiles.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageConfigurationPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageConfigurationPath.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageManifestPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.BaseImageManifestPath.get -> Microsoft.Build.Framework.ITaskItem! ~override Microsoft.NET.Build.Containers.Port.ToString() -> string static Microsoft.NET.Build.Containers.Port.operator !=(Microsoft.NET.Build.Containers.Port left, Microsoft.NET.Build.Containers.Port right) -> bool static Microsoft.NET.Build.Containers.Port.operator ==(Microsoft.NET.Build.Containers.Port left, Microsoft.NET.Build.Containers.Port right) -> bool @@ -37,8 +45,6 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerEnvironmentVariable Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerEnvironmentVariables.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerizeDirectory.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerizeDirectory.set -> void -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.get -> string! -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerRuntimeIdentifier.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerUser.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ContainerUser.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.CreateNewImage() -> void @@ -78,24 +84,30 @@ Microsoft.NET.Build.Containers.Tasks.CreateNewImage.OutputRegistry.get -> string Microsoft.NET.Build.Containers.Tasks.CreateNewImage.OutputRegistry.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ArchiveOutputPath.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ArchiveOutputPath.set -> void -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.PublishDirectory.get -> string! -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.PublishDirectory.set -> void -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.RuntimeIdentifierGraphPath.get -> string! -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.RuntimeIdentifierGraphPath.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.get -> string! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.WorkingDirectory.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.get -> bool Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateLabels.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.get -> bool Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateDigestLabel.set -> void -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SkipPublishing.get -> bool -Microsoft.NET.Build.Containers.Tasks.CreateNewImage.SkipPublishing.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedContainerNames.get -> Microsoft.Build.Framework.ITaskItem![]! Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedContainerNames.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ImageFormat.get -> string? Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ImageFormat.set -> void Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedDigestLabel.get -> Microsoft.Build.Framework.ITaskItem? Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedDigestLabel.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedConfigurationPath.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedConfigurationPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedLayerPath.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedLayerPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedManifestPath.get -> string! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedManifestPath.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerLayer.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerLayer.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerConfig.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerConfig.set -> void +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerManifest.get -> Microsoft.Build.Framework.ITaskItem! +Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GeneratedAppContainerManifest.set -> void override Microsoft.NET.Build.Containers.Tasks.CreateNewImage.ToolName.get -> string! override Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateCommandLineCommands() -> string! override Microsoft.NET.Build.Containers.Tasks.CreateNewImage.GenerateFullPathToTool() -> string! diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/DefaultManifestOperations.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/DefaultManifestOperations.cs index a8b2248d8996..119aacabb1ea 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/DefaultManifestOperations.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/DefaultManifestOperations.cs @@ -3,6 +3,7 @@ using System.Net; using System.Net.Http.Headers; +using System.Net.Http.Json; using System.Text.Json; using Microsoft.Extensions.Logging; using Microsoft.NET.Build.Containers.Resources; @@ -38,11 +39,9 @@ public async Task GetAsync(string repositoryName, string re }; } - public async Task PutAsync(string repositoryName, string reference, string manifestJson, string mediaType, CancellationToken cancellationToken) + public async Task PutAsync(string repositoryName, string reference, T manifest, CancellationToken cancellationToken) where T : IManifest { - HttpContent manifestUploadContent = new StringContent(manifestJson); - manifestUploadContent.Headers.ContentType = new MediaTypeHeaderValue(mediaType); - + JsonContent manifestUploadContent = JsonContent.Create(manifest, mediaType: new MediaTypeHeaderValue(manifest.MediaType!)); HttpResponseMessage putResponse = await _client.PutAsync(new Uri(_baseUri, $"/v2/{repositoryName}/manifests/{reference}"), manifestUploadContent, cancellationToken).ConfigureAwait(false); if (!putResponse.IsSuccessStatusCode) diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/IManifestOperations.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/IManifestOperations.cs index 23381179403d..86256fd9a8db 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/IManifestOperations.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/IManifestOperations.cs @@ -14,5 +14,5 @@ internal interface IManifestOperations { public Task GetAsync(string repositoryName, string reference, CancellationToken cancellationToken); - public Task PutAsync(string repositoryName, string reference, string manifestListJson, string mediaType, CancellationToken cancellationToken); + public Task PutAsync(string repositoryName, string reference, T manifest, CancellationToken cancellationToken) where T : IManifest; } diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs index 100ff11d24cb..b87a425e98b6 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/Registry.cs @@ -8,61 +8,12 @@ using Microsoft.Extensions.Logging; using Microsoft.NET.Build.Containers.Resources; using NuGet.RuntimeModel; +using System.Windows.Markup; +using System.Text.RegularExpressions; +using System.Text.Json; namespace Microsoft.NET.Build.Containers; -internal interface IManifestPicker -{ - public PlatformSpecificManifest? PickBestManifestForRid(IReadOnlyDictionary manifestList, string runtimeIdentifier); - public PlatformSpecificOciManifest? PickBestManifestForRid(IReadOnlyDictionary manifestList, string runtimeIdentifier); -} - -internal sealed class RidGraphManifestPicker : IManifestPicker -{ - private readonly RuntimeGraph _runtimeGraph; - - public RidGraphManifestPicker(string runtimeIdentifierGraphPath) - { - _runtimeGraph = GetRuntimeGraphForDotNet(runtimeIdentifierGraphPath); - } - public PlatformSpecificManifest? PickBestManifestForRid(IReadOnlyDictionary ridManifestDict, string runtimeIdentifier) - { - var bestManifestRid = GetBestMatchingRid(_runtimeGraph, runtimeIdentifier, ridManifestDict.Keys); - if (bestManifestRid is null) - { - return null; - } - return ridManifestDict[bestManifestRid]; - } - - public PlatformSpecificOciManifest? PickBestManifestForRid(IReadOnlyDictionary ridManifestDict, string runtimeIdentifier) - { - var bestManifestRid = GetBestMatchingRid(_runtimeGraph, runtimeIdentifier, ridManifestDict.Keys); - if (bestManifestRid is null) - { - return null; - } - return ridManifestDict[bestManifestRid]; - } - - private static string? GetBestMatchingRid(RuntimeGraph runtimeGraph, string runtimeIdentifier, IEnumerable availableRuntimeIdentifiers) - { - HashSet availableRids = new HashSet(availableRuntimeIdentifiers, StringComparer.Ordinal); - foreach (var candidateRuntimeIdentifier in runtimeGraph.ExpandRuntime(runtimeIdentifier)) - { - if (availableRids.Contains(candidateRuntimeIdentifier)) - { - return candidateRuntimeIdentifier; - } - } - - return null; - } - - private static RuntimeGraph GetRuntimeGraphForDotNet(string ridGraphPath) => JsonRuntimeFormat.ReadRuntimeGraph(ridGraphPath); - -} - internal enum RegistryMode { Push, @@ -79,6 +30,7 @@ internal sealed class Registry private readonly ILogger _logger; private readonly IRegistryAPI _registryAPI; private readonly RegistrySettings _settings; + private readonly ContentStore _store; /// /// The name of the registry, which is the host name, optionally followed by a colon and the port number. @@ -87,24 +39,24 @@ internal sealed class Registry /// public string RegistryName { get; } - internal Registry(string registryName, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null) : - this(new Uri($"https://{registryName}"), logger, registryAPI, settings) + internal Registry(string registryName, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null, ContentStore? store = null) : + this(new Uri($"https://{registryName}"), logger, registryAPI, settings, store) { } - internal Registry(string registryName, ILogger logger, RegistryMode mode, RegistrySettings? settings = null) : - this(new Uri($"https://{registryName}"), logger, new RegistryApiFactory(mode), settings) + internal Registry(string registryName, ILogger logger, RegistryMode mode, RegistrySettings? settings = null, ContentStore? store = null) : + this(new Uri($"https://{registryName}"), logger, new RegistryApiFactory(mode), settings, store) { } - internal Registry(Uri baseUri, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null) : - this(baseUri, logger, new RegistryApiFactory(registryAPI), settings) + internal Registry(Uri baseUri, ILogger logger, IRegistryAPI registryAPI, RegistrySettings? settings = null, ContentStore? store = null) : + this(baseUri, logger, new RegistryApiFactory(registryAPI), settings, store) { } - internal Registry(Uri baseUri, ILogger logger, RegistryMode mode, RegistrySettings? settings = null) : - this(baseUri, logger, new RegistryApiFactory(mode), settings) + internal Registry(Uri baseUri, ILogger logger, RegistryMode mode, RegistrySettings? settings = null, ContentStore? store = null) : + this(baseUri, logger, new RegistryApiFactory(mode), settings, store) { } - private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, RegistrySettings? settings = null) + private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, RegistrySettings? settings = null, ContentStore? store = null) { RegistryName = DeriveRegistryName(baseUri); @@ -118,6 +70,7 @@ private Registry(Uri baseUri, ILogger logger, RegistryApiFactory factory, Regist _logger = logger; _settings = settings ?? new RegistrySettings(RegistryName); _registryAPI = factory.Create(RegistryName, BaseUri, logger, _settings.IsInsecure); + _store = store ?? new ContentStore(new(Path.GetTempPath())); } private static string DeriveRegistryName(Uri baseUri) @@ -183,30 +136,109 @@ public bool IsGoogleArtifactRegistry /// private bool SupportsParallelUploads => !IsAmazonECRRegistry && _settings.ParallelUploadEnabled; + /// + /// Fetches the data for a given manifest tag or digest from the local content store if present. + /// If not present, fetches it from the remote and caches it in the local content store. + /// + /// + /// + /// + /// + /// + public async Task GetManifestCore(string repositoryName, string referenceOrDigest, CancellationToken cTok, bool skipCache = true) + { + cTok.ThrowIfCancellationRequested(); + // check if we have the reference in the ContentStore's reference area already. + // if so, read it from there. + var referencePath = _store.PathForManifestByReferenceOrDigest(RegistryName, repositoryName, referenceOrDigest); + if (!skipCache && File.Exists(referencePath)) + { + var lines = await File.ReadAllLinesAsync(referencePath, cTok); + (var digest, var mediaType, var size) = (lines[0], lines[1], lines[2]); + using var contentStream = File.OpenRead(_store.PathForDescriptor(new(mediaType, digest, long.Parse(size)))); + return await ParseManifest(mediaType, contentStream, digest); + } + // if not, make a remote call and add it to the ContentStore's reference area. + else + { + using var response = await _registryAPI.Manifest.GetAsync(repositoryName, referenceOrDigest, cTok).ConfigureAwait(false); + response.Headers.TryGetValues("Docker-Content-Digest", out var knownDigests); + var digest = knownDigests?.FirstOrDefault()!; + var mediaType = response.Content.Headers.ContentType?.MediaType!; + long size = response.Content.Headers.ContentLength ?? 0; + var descriptor = new Descriptor(mediaType, digest, size); + // write the manifest contents to the durable store + var storagePath = _store.PathForDescriptor(descriptor); + // if the file already exists at this digest then we can skip the download + if (File.Exists(storagePath)) + { + using var fs = File.OpenRead(storagePath); + return await ParseManifest(mediaType, fs, digest); + } + else + { + using var storageStream = File.OpenWrite(storagePath); + using var responseStream = await response.Content.ReadAsStreamAsync(cTok); + await responseStream.CopyToAsync(storageStream); + responseStream.Position = 0; + // write the marker file for the reference + // IMPORTANT: must stay in sync with the lines read in the if block above + var parentDir = Path.GetDirectoryName(referencePath)!; + // we're creating a multi-level directory structure, so ensure the parent directories exist before writing + Directory.CreateDirectory(parentDir); + await File.WriteAllLinesAsync(referencePath, [ + digest, + mediaType, + size.ToString() + ], DigestUtils.UTF8, cTok); + // now that the data is all set for next time, return the manifest + return await ParseManifest(mediaType, responseStream, digest); + } + } + + async Task ParseManifest(string? mediaType, Stream content, string? digest) + { + IManifest? manifest = mediaType switch + { + SchemaTypes.DockerManifestV2 or SchemaTypes.OciManifestV1 => await JsonSerializer.DeserializeAsync(content, cancellationToken: cTok), + SchemaTypes.DockerManifestListV2 => await JsonSerializer.DeserializeAsync(content, cancellationToken: cTok), + SchemaTypes.OciImageIndexV1 => await JsonSerializer.DeserializeAsync(content, cancellationToken: cTok), + null => throw new ArgumentException($"No media type found for manifest {RegistryName}/{repositoryName}@{referenceOrDigest}"), + _ => throw new ArgumentException($"Unknown manifest media type {mediaType}") + }; + if (manifest is ManifestV2 v) + { + v.KnownDigest = digest; + return v; + } + else + { + return manifest!; + } + } + } + public async Task GetImageManifestAsync(string repositoryName, string reference, string runtimeIdentifier, IManifestPicker manifestPicker, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - using HttpResponseMessage initialManifestResponse = await _registryAPI.Manifest.GetAsync(repositoryName, reference, cancellationToken).ConfigureAwait(false); - return initialManifestResponse.Content.Headers.ContentType?.MediaType switch + var manifest = await GetManifestCore(repositoryName, reference, cancellationToken); + + return manifest switch { - SchemaTypes.DockerManifestV2 or SchemaTypes.OciManifestV1 => await ReadSingleImageAsync( - repositoryName, - await ReadManifest().ConfigureAwait(false), - initialManifestResponse.Content.Headers.ContentType.MediaType, - cancellationToken).ConfigureAwait(false), - SchemaTypes.DockerManifestListV2 => await PickBestImageFromManifestListAsync( + ManifestV2 singleArchManifest => await ReadSingleImageAsync(repositoryName, singleArchManifest, cancellationToken), + ManifestListV2 multiArchDockerManifest => await PickBestImageFromManifestListAsync( repositoryName, reference, - await initialManifestResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false), + multiArchDockerManifest, runtimeIdentifier, manifestPicker, cancellationToken).ConfigureAwait(false), - SchemaTypes.OciImageIndexV1 => + ImageIndexV1 multiArchOciIndex => await PickBestImageFromImageIndexAsync( repositoryName, reference, - await initialManifestResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false), + multiArchOciIndex, runtimeIdentifier, manifestPicker, cancellationToken).ConfigureAwait(false), @@ -217,17 +249,6 @@ await initialManifestResponse.Content.ReadFromJsonAsync(cancellati BaseUri, unknownMediaType)) }; - - async Task ReadManifest() - { - initialManifestResponse.Headers.TryGetValues("Docker-Content-Digest", out var knownDigest); - var manifest = (await initialManifestResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false))!; - if (knownDigest?.FirstOrDefault() is string knownDigestValue) - { - manifest.KnownDigest = knownDigestValue; - } - return manifest; - } } internal async Task GetManifestListAsync(string repositoryName, string reference, CancellationToken cancellationToken) @@ -242,79 +263,40 @@ async Task ReadManifest() }; } - private async Task ReadSingleImageAsync(string repositoryName, ManifestV2 manifest, string manifestMediaType, CancellationToken cancellationToken) + public async Task GetJsonBlobCore(string repositoryName, string digest, string mediaType, CancellationToken cancellationToken) { + // check if digest is available locally and serialize it, otherwise download from registry and store locally cancellationToken.ThrowIfCancellationRequested(); - ManifestConfig config = manifest.Config; - string configSha = config.digest; - - JsonNode configDoc = await _registryAPI.Blob.GetJsonAsync(repositoryName, configSha, cancellationToken).ConfigureAwait(false); - - cancellationToken.ThrowIfCancellationRequested(); - // ManifestV2.MediaType can be null, so we also provide manifest mediaType from http response - return new ImageBuilder(manifest, manifest.MediaType ?? manifestMediaType, new ImageConfig(configDoc), _logger); - } - - - private static IReadOnlyDictionary GetManifestsByRid(PlatformSpecificManifest[] manifestList) - { - var ridDict = new Dictionary(); - foreach (var manifest in manifestList) + var descriptor = new Descriptor(mediaType, digest, 0); + var storagePath = _store.PathForDescriptor(descriptor); + if (File.Exists(storagePath)) { - if (CreateRidForPlatform(manifest.platform) is { } rid) - { - ridDict.TryAdd(rid, manifest); - } + using var fs = File.OpenRead(storagePath); + return (await JsonNode.ParseAsync(fs, cancellationToken: cancellationToken))!; } - - return ridDict; - } - - private static IReadOnlyDictionary GetManifestsByRid(PlatformSpecificOciManifest[] manifestList) - { - var ridDict = new Dictionary(); - foreach (var manifest in manifestList) + else { - if (CreateRidForPlatform(manifest.platform) is { } rid) - { - ridDict.TryAdd(rid, manifest); - } - } - return ridDict; + using var jsonStream = await _registryAPI.Blob.GetStreamAsync(repositoryName, digest, cancellationToken); + using var fsStream = File.OpenWrite(storagePath); + await jsonStream.CopyToAsync(fsStream, cancellationToken); + // note: cannot just use the jsonStream here, as it may not be seekable + fsStream.Position = 0; + return (await JsonNode.ParseAsync(fsStream, cancellationToken: cancellationToken))!; + } } - private static string? CreateRidForPlatform(PlatformInformation platform) + private async Task ReadSingleImageAsync(string repositoryName, ManifestV2 manifest, CancellationToken cancellationToken) { - // we only support linux and windows containers explicitly, so anything else we should skip past. - var osPart = platform.os switch - { - "linux" => "linux", - "windows" => "win", - _ => null - }; - // TODO: this part needs a lot of work, the RID graph isn't super precise here and version numbers (especially on windows) are _whack_ - // TODO: we _may_ need OS-specific version parsing. Need to do more research on what the field looks like across more manifest lists. - var versionPart = platform.version?.Split('.') switch - { - [var major, ..] => major, - _ => null - }; - var platformPart = platform.architecture switch - { - "amd64" => "x64", - "x386" => "x86", - "arm" => $"arm{(platform.variant != "v7" ? platform.variant : "")}", - "arm64" => "arm64", - "ppc64le" => "ppc64le", - "s390x" => "s390x", - "riscv64" => "riscv64", - "loongarch64" => "loongarch64", - _ => null - }; + cancellationToken.ThrowIfCancellationRequested(); + ManifestConfig config = manifest.Config; + string configSha = config.digest; + + JsonNode configDoc = await GetJsonBlobCore(repositoryName, configSha, manifest.Config.mediaType, cancellationToken); - if (osPart is null || platformPart is null) return null; - return $"{osPart}{versionPart ?? ""}-{platformPart}"; + cancellationToken.ThrowIfCancellationRequested(); + // ManifestV2.MediaType can be null, so we also provide manifest mediaType from http response + return new ImageBuilder(manifest, manifest.MediaType ?? SchemaTypes.DockerManifestV2, new ImageConfig(configDoc), _logger); } @@ -327,14 +309,13 @@ private async Task PickBestImageFromManifestListAsync( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var ridManifestDict = GetManifestsByRid(manifestList.manifests); + var ridManifestDict = RidMapping.GetManifestsByRid(manifestList.manifests); if (manifestPicker.PickBestManifestForRid(ridManifestDict, runtimeIdentifier) is PlatformSpecificManifest matchingManifest) { - return await ReadImageFromManifest( + return await ReadSingleImageFromManifest( repositoryName, reference, matchingManifest.digest, - matchingManifest.mediaType, runtimeIdentifier, ridManifestDict.Keys, cancellationToken); @@ -354,14 +335,13 @@ private async Task PickBestImageFromImageIndexAsync( CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - var ridManifestDict = GetManifestsByRid(index.manifests); + var ridManifestDict = RidMapping.GetManifestsByRid(index.manifests); if (manifestPicker.PickBestManifestForRid(ridManifestDict, runtimeIdentifier) is PlatformSpecificOciManifest matchingManifest) { - return await ReadImageFromManifest( + return await ReadSingleImageFromManifest( repositoryName, reference, matchingManifest.digest, - matchingManifest.mediaType, runtimeIdentifier, ridManifestDict.Keys, cancellationToken); @@ -372,25 +352,32 @@ private async Task PickBestImageFromImageIndexAsync( } } - private async Task ReadImageFromManifest( + /// + /// Reads a manifest at the given digest, assuming it is a single-arch manifest + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + private async Task ReadSingleImageFromManifest( string repositoryName, string reference, string manifestDigest, - string mediaType, string runtimeIdentifier, IEnumerable rids, CancellationToken cancellationToken) { - using HttpResponseMessage manifestResponse = await _registryAPI.Manifest.GetAsync(repositoryName, manifestDigest, cancellationToken).ConfigureAwait(false); - - cancellationToken.ThrowIfCancellationRequested(); - var manifest = await manifestResponse.Content.ReadFromJsonAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + var manifest = await GetManifestCore(repositoryName, manifestDigest, cancellationToken); if (manifest is null) throw new BaseImageNotFoundException(runtimeIdentifier, repositoryName, reference, rids); - manifest.KnownDigest = manifestDigest; + if (manifest is not ManifestV2 singleArchManifest) throw new ArgumentException("Only supports single-arch manifests in this pathway"); return await ReadSingleImageAsync( repositoryName, - manifest, - mediaType, + singleArchManifest, cancellationToken).ConfigureAwait(false); } @@ -403,7 +390,7 @@ private async Task ReadImageFromManifest( public async Task DownloadBlobAsync(string repository, Descriptor descriptor, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); - string localPath = ContentStore.PathForDescriptor(descriptor); + string localPath = _store.PathForDescriptor(descriptor); if (File.Exists(localPath)) { @@ -411,10 +398,11 @@ public async Task DownloadBlobAsync(string repository, Descriptor descri return localPath; } - string tempTarballPath = ContentStore.GetTempFile(); + string tempTarballPath = _store.GetTempFile(); try { + _logger.LogInformation($"Downloading layer {descriptor.Digest} from {repository} to content store."); // No local copy, so download one using Stream responseStream = await _registryAPI.Blob.GetStreamAsync(repository, descriptor.Digest, cancellationToken).ConfigureAwait(false); @@ -423,9 +411,9 @@ public async Task DownloadBlobAsync(string repository, Descriptor descri await responseStream.CopyToAsync(fs, cancellationToken).ConfigureAwait(false); } } - catch (Exception) + catch (Exception e) { - throw new UnableToDownloadFromRepositoryException(repository); + throw new UnableToDownloadFromRepositoryException(repository, e); } cancellationToken.ThrowIfCancellationRequested(); @@ -508,7 +496,15 @@ private Task UploadBlobContentsAsync(Stream contents, } } - private async Task UploadBlobAsync(string repository, string digest, Stream contents, CancellationToken cancellationToken) + /// + /// Uploads an opaque blob to the registry, checking for existence first. + /// + /// + /// + /// + /// + /// + public async Task UploadBlobAsync(string repository, string digest, Stream contents, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -545,9 +541,9 @@ public async Task PushManifestListAsync( foreach (var tag in destinationImageReference.Tags) { _logger.LogInformation(Strings.Registry_TagUploadStarted, tag, RegistryName); - await _registryAPI.Manifest.PutAsync(destinationImageReference.Repository, tag, multiArchImage.ImageIndex, multiArchImage.ImageIndexMediaType, cancellationToken).ConfigureAwait(false); + await _registryAPI.Manifest.PutAsync(destinationImageReference.Repository, tag, multiArchImage.ImageIndex, cancellationToken).ConfigureAwait(false); _logger.LogInformation(Strings.Registry_TagUploaded, tag, RegistryName); - } + } } public Task PushAsync(BuiltImage builtImage, SourceImageReference source, DestinationImageReference destination, CancellationToken cancellationToken) @@ -580,7 +576,7 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source, // Ensure the blob is available locally await sourceRegistry.DownloadBlobAsync(source.Repository, descriptor, cancellationToken).ConfigureAwait(false); // Then push it to the destination registry - await destinationRegistry.PushLayerAsync(Layer.FromDescriptor(descriptor), destination.Repository, cancellationToken).ConfigureAwait(false); + await destinationRegistry.PushLayerAsync(Layer.FromDescriptor(descriptor, _store), destination.Repository, cancellationToken).ConfigureAwait(false); _logger.LogInformation(Strings.Registry_LayerUploaded, digest, destinationRegistry.RegistryName); } else @@ -603,9 +599,9 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source, } cancellationToken.ThrowIfCancellationRequested(); - using (MemoryStream stringStream = new(Encoding.UTF8.GetBytes(builtImage.Config))) + using (MemoryStream stringStream = new(DigestUtils.UTF8.GetBytes(builtImage.Config.ToJsonString()))) { - var configDigest = builtImage.ImageDigest!; + var configDigest = builtImage.Manifest.Config.digest; _logger.LogInformation(Strings.Registry_ConfigUploadStarted, configDigest); await UploadBlobAsync(destination.Repository, configDigest, stringStream, cancellationToken).ConfigureAwait(false); _logger.LogInformation(Strings.Registry_ConfigUploaded); @@ -620,18 +616,26 @@ private async Task PushAsync(BuiltImage builtImage, SourceImageReference source, foreach (string tag in destination.Tags) { _logger.LogInformation(Strings.Registry_TagUploadStarted, tag, RegistryName); - await _registryAPI.Manifest.PutAsync(destination.Repository, tag, builtImage.Manifest, builtImage.ManifestMediaType, cancellationToken).ConfigureAwait(false); + await _registryAPI.Manifest.PutAsync(destination.Repository, tag, builtImage.Manifest, cancellationToken).ConfigureAwait(false); _logger.LogInformation(Strings.Registry_TagUploaded, tag, RegistryName); } } else { _logger.LogInformation(Strings.Registry_ManifestUploadStarted, RegistryName, builtImage.ManifestDigest); - await _registryAPI.Manifest.PutAsync(destination.Repository, builtImage.ManifestDigest, builtImage.Manifest, builtImage.ManifestMediaType, cancellationToken).ConfigureAwait(false); + await _registryAPI.Manifest.PutAsync(destination.Repository, builtImage.ManifestDigest, builtImage.Manifest, cancellationToken).ConfigureAwait(false); _logger.LogInformation(Strings.Registry_ManifestUploaded, RegistryName); } } + public async Task UploadManifestAsync(string repository, string tagOrDigest, T manifest, CancellationToken cancellationToken) where T : IManifest + { + cancellationToken.ThrowIfCancellationRequested(); + _logger.LogInformation(Strings.Registry_ManifestUploadStarted, RegistryName, tagOrDigest); + await _registryAPI.Manifest.PutAsync(repository, tagOrDigest, manifest, cancellationToken).ConfigureAwait(false); + _logger.LogInformation(Strings.Registry_ManifestUploaded, RegistryName); + } + private readonly ref struct RegistryApiFactory { private readonly IRegistryAPI? _registryApi; diff --git a/src/Containers/Microsoft.NET.Build.Containers/Registry/RidMapping.cs b/src/Containers/Microsoft.NET.Build.Containers/Registry/RidMapping.cs new file mode 100644 index 000000000000..6a178e872ad4 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Registry/RidMapping.cs @@ -0,0 +1,119 @@ +using NuGet.RuntimeModel; + +namespace Microsoft.NET.Build.Containers; + +internal interface IManifestPicker +{ + public PlatformSpecificManifest? PickBestManifestForRid(IReadOnlyDictionary manifestList, string runtimeIdentifier); + public PlatformSpecificOciManifest? PickBestManifestForRid(IReadOnlyDictionary manifestList, string runtimeIdentifier); +} + +internal sealed class RidGraphManifestPicker : IManifestPicker +{ + private readonly RuntimeGraph _runtimeGraph; + + public RidGraphManifestPicker(string runtimeIdentifierGraphPath) + { + _runtimeGraph = GetRuntimeGraphForDotNet(runtimeIdentifierGraphPath); + } + public PlatformSpecificManifest? PickBestManifestForRid(IReadOnlyDictionary ridManifestDict, string runtimeIdentifier) + { + var bestManifestRid = GetBestMatchingRid(_runtimeGraph, runtimeIdentifier, ridManifestDict.Keys); + if (bestManifestRid is null) + { + return null; + } + return ridManifestDict[bestManifestRid]; + } + + public PlatformSpecificOciManifest? PickBestManifestForRid(IReadOnlyDictionary ridManifestDict, string runtimeIdentifier) + { + var bestManifestRid = GetBestMatchingRid(_runtimeGraph, runtimeIdentifier, ridManifestDict.Keys); + if (bestManifestRid is null) + { + return null; + } + return ridManifestDict[bestManifestRid]; + } + + private static string? GetBestMatchingRid(RuntimeGraph runtimeGraph, string runtimeIdentifier, IEnumerable availableRuntimeIdentifiers) + { + HashSet availableRids = new HashSet(availableRuntimeIdentifiers, StringComparer.Ordinal); + foreach (var candidateRuntimeIdentifier in runtimeGraph.ExpandRuntime(runtimeIdentifier)) + { + if (availableRids.Contains(candidateRuntimeIdentifier)) + { + return candidateRuntimeIdentifier; + } + } + + return null; + } + + private static RuntimeGraph GetRuntimeGraphForDotNet(string ridGraphPath) => JsonRuntimeFormat.ReadRuntimeGraph(ridGraphPath); +} + +public static class RidMapping +{ + public static IReadOnlyDictionary GetManifestsByRid(PlatformSpecificManifest[] manifestList) + { + var ridDict = new Dictionary(); + foreach (var manifest in manifestList) + { + if (CreateRidForPlatform(manifest.platform) is { } rid) + { + ridDict.TryAdd(rid, manifest); + } + } + + return ridDict; + } + + public static IReadOnlyDictionary GetManifestsByRid(PlatformSpecificOciManifest[] manifestList) + { + var ridDict = new Dictionary(); + foreach (var manifest in manifestList) + { + if (CreateRidForPlatform(manifest.platform) is { } rid) + { + ridDict.TryAdd(rid, manifest); + } + } + + return ridDict; + } + + public static string? CreateRidForPlatform(PlatformInformation platform) + { + // we only support linux and windows containers explicitly, so anything else we should skip past. + var osPart = platform.os switch + { + "linux" => "linux", + "windows" => "win", + _ => null + }; + // TODO: this part needs a lot of work, the RID graph isn't super precise here and version numbers (especially on windows) are _whack_ + // TODO: we _may_ need OS-specific version parsing. Need to do more research on what the field looks like across more manifest lists. + var versionPart = platform.version?.Split('.') switch + { + [var major, ..] => major, + _ => null + }; + var platformPart = platform.architecture switch + { + "amd64" => "x64", + "x386" => "x86", + "arm" => $"arm{(platform.variant != "v7" ? platform.variant : "")}", + "arm64" => "arm64", + "ppc64le" => "ppc64le", + "s390x" => "s390x", + "riscv64" => "riscv64", + "loongarch64" => "loongarch64", + _ => null + }; + + if (osPart is null || platformPart is null) return null; + return $"{osPart}{versionPart ?? ""}-{platformPart}"; + } + +} \ No newline at end of file diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/Strings.resx b/src/Containers/Microsoft.NET.Build.Containers/Resources/Strings.resx index 05e8baba0154..6479e7d1b188 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/Strings.resx +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/Strings.resx @@ -382,7 +382,7 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.cs.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.cs.xlf index d5ed687ee973..28815712b707 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.cs.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.cs.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - Index image nejde vytvořit, protože zadané image nejsou platné. Položky musí mít metadata Config, Manifest, ManifestMediaType a ManifestDigest. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + Index image nejde vytvořit, protože zadané image nejsou platné. Položky musí mít metadata Config, Manifest, ManifestMediaType a ManifestDigest. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.de.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.de.xlf index ba76f860056f..40210d696e84 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.de.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.de.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - Der Imageindex kann nicht erstellt werden, da die angegebenen Bilder ungültig sind. Elemente müssen die Metadaten "Config", "Manifest", "ManifestMediaType" und "ManifestDigest" aufweisen. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + Der Imageindex kann nicht erstellt werden, da die angegebenen Bilder ungültig sind. Elemente müssen die Metadaten "Config", "Manifest", "ManifestMediaType" und "ManifestDigest" aufweisen. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.es.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.es.xlf index 164c9d447a5d..1517ab2fa3ab 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.es.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.es.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - No se puede crear el índice de imágenes porque las imágenes proporcionadas no son válidas. Los elementos deben tener metadatos 'Config', 'Manifest', 'ManifestMediaType' y 'ManifestDigest'. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + No se puede crear el índice de imágenes porque las imágenes proporcionadas no son válidas. Los elementos deben tener metadatos 'Config', 'Manifest', 'ManifestMediaType' y 'ManifestDigest'. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.fr.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.fr.xlf index f0d3a647de7b..5d771137d92e 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.fr.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.fr.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - Impossible de créer l’index d’images, car les images fournies ne sont pas valides. Les éléments doivent avoir les métadonnées 'Config', 'Manifest', 'ManifestMediaType' et 'ManifestDigest'. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + Impossible de créer l’index d’images, car les images fournies ne sont pas valides. Les éléments doivent avoir les métadonnées 'Config', 'Manifest', 'ManifestMediaType' et 'ManifestDigest'. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.it.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.it.xlf index eaf97ac4648f..688d426b95fb 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.it.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.it.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - Non è possibile creare l'indice dell'immagine perché le immagini specificate non sono valide. Gli elementi devono contenere i metadati 'Config', 'Manifest', 'ManifestMediaType' e 'ManifestDigest'. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + Non è possibile creare l'indice dell'immagine perché le immagini specificate non sono valide. Gli elementi devono contenere i metadati 'Config', 'Manifest', 'ManifestMediaType' e 'ManifestDigest'. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ja.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ja.xlf index 8c99589574ad..e9c3a46b3b71 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ja.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ja.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - 指定されたイメージが無効なため、イメージ インデックスを作成できません。項目には、'Config'、'Manifest'、'ManifestMediaType'、'ManifestDigest' のメタデータが必要です。 + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + 指定されたイメージが無効なため、イメージ インデックスを作成できません。項目には、'Config'、'Manifest'、'ManifestMediaType'、'ManifestDigest' のメタデータが必要です。 diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ko.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ko.xlf index bff24bae4b17..ac2cf37e21c7 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ko.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ko.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - 제공된 이미지가 잘못되었으므로 이미지 인덱스를 만들 수 없습니다. 항목에는 'Config', 'Manifest', 'ManifestMediaType' 및 'ManifestDigest' 메타데이터가 있어야 합니다. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + 제공된 이미지가 잘못되었으므로 이미지 인덱스를 만들 수 없습니다. 항목에는 'Config', 'Manifest', 'ManifestMediaType' 및 'ManifestDigest' 메타데이터가 있어야 합니다. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.pl.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.pl.xlf index 801b6190de72..5c4ef7a1e50a 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.pl.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.pl.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - Nie można utworzyć indeksu obrazu, ponieważ podane obrazy są nieprawidłowe. Elementy muszą mieć metadane "Config", "Manifest", "ManifestMediaType" i "ManifestDigest". + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + Nie można utworzyć indeksu obrazu, ponieważ podane obrazy są nieprawidłowe. Elementy muszą mieć metadane "Config", "Manifest", "ManifestMediaType" i "ManifestDigest". diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.pt-BR.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.pt-BR.xlf index 971dda801778..e387ee411f54 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.pt-BR.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.pt-BR.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - Não é possível criar o índice de imagem porque as imagens fornecidas são inválidas. Os itens devem ter metadados 'Config', 'Manifest', 'ManifestMediaType' e 'ManifestDigest'. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + Não é possível criar o índice de imagem porque as imagens fornecidas são inválidas. Os itens devem ter metadados 'Config', 'Manifest', 'ManifestMediaType' e 'ManifestDigest'. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ru.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ru.xlf index 1f4ef89cfdfa..9c0e7175d6fc 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ru.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.ru.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - Не удается создать индекс образа, так как предоставленные образы недопустимы. Элементы должны иметь метаданные "Config", "Manifest", "ManifestMediaType" и "ManifestDigest". + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + Не удается создать индекс образа, так как предоставленные образы недопустимы. Элементы должны иметь метаданные "Config", "Manifest", "ManifestMediaType" и "ManifestDigest". diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.tr.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.tr.xlf index faad0ded04f1..4dbf739e866e 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.tr.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.tr.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - Sağlanan görüntüler geçersiz olduğundan görüntü dizini oluşturulamıyor. Öğeler 'Config', 'Manifest', 'ManifestMediaType' ve 'ManifestDigest' meta verilerine sahip olmalıdır. + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + Sağlanan görüntüler geçersiz olduğundan görüntü dizini oluşturulamıyor. Öğeler 'Config', 'Manifest', 'ManifestMediaType' ve 'ManifestDigest' meta verilerine sahip olmalıdır. diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.zh-Hans.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.zh-Hans.xlf index f75697ff279f..111671db67fb 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.zh-Hans.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.zh-Hans.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - 无法创建映像索引，因为提供的映像无效。项必须具有 “Config”、“Manifest”、“ManifestMediaType” 和 “ManifestDigest” 元数据。 + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + 无法创建映像索引，因为提供的映像无效。项必须具有 “Config”、“Manifest”、“ManifestMediaType” 和 “ManifestDigest” 元数据。 diff --git a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.zh-Hant.xlf b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.zh-Hant.xlf index f9bdce96c863..03e5e76a1b3a 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.zh-Hant.xlf +++ b/src/Containers/Microsoft.NET.Build.Containers/Resources/xlf/Strings.zh-Hant.xlf @@ -225,8 +225,8 @@ - Cannot create image index because provided images are invalid. Items must have 'Config', 'Manifest', 'ManifestMediaType' and 'ManifestDigest' metadata. - 無法建立映射索引，因為提供的映像無效。項目必須有 'Config'、'Manifest'、'ManifestMediaType' 和 'ManifestDigest' 元數據。 + Cannot create image index because provided images are invalid. Items must have 'ConfigurationPath' and 'ManifestPath' metadata. + 無法建立映射索引，因為提供的映像無效。項目必須有 'Config'、'Manifest'、'ManifestMediaType' 和 'ManifestDigest' 元數據。 diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.Interface.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.Interface.cs index 538240841c6b..e91e288e843a 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.Interface.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.Interface.cs @@ -34,7 +34,7 @@ partial class CreateImageIndex public string BaseImageDigest { get; set; } /// - /// Manifests to include in the image index. + /// Manifests to include in the image index. Should have an ManifestPath and a ConfigurationPath metadata pointing to those relevant files. /// [Required] public ITaskItem[] GeneratedContainers { get; set; } @@ -66,6 +66,12 @@ partial class CreateImageIndex [Required] public string[] ImageTags { get; set; } + /// + /// Where to write the generated image index to + /// + [Required] + public string GeneratedManifestPath { get; set; } + /// /// The generated archive output path. /// @@ -92,7 +98,8 @@ public CreateImageIndex() ImageTags = Array.Empty(); GeneratedArchiveOutputPath = string.Empty; GeneratedImageIndex = string.Empty; + GeneratedManifestPath = string.Empty; TaskResources = Resource.Manager; } -} \ No newline at end of file +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.cs index 16e5b8392372..746d00767325 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateImageIndex.cs @@ -68,105 +68,86 @@ internal async Task ExecuteAsync(CancellationToken cancellationToken) OutputRegistry, LocalRegistry); - var images = ParseImages(destinationImageReference.Kind); + var images = await ParseImages(GeneratedContainers, cancellationToken); if (Log.HasLoggedErrors) { return false; } var multiArchImage = CreateMultiArchImage(images, destinationImageReference.Kind); + using var fileStream = File.OpenWrite(GeneratedManifestPath); + await JsonSerializer.SerializeAsync(fileStream, multiArchImage.ImageIndex); - GeneratedImageIndex = multiArchImage.ImageIndex; + GeneratedImageIndex = JsonSerializer.Serialize(multiArchImage.ImageIndex); GeneratedArchiveOutputPath = ArchiveOutputPath; logger.LogInformation(Strings.BuildingImageIndex, destinationImageReference, string.Join(", ", images.Select(i => i.ManifestDigest))); var telemetry = new Telemetry(sourceImageReference, destinationImageReference, Log); - + // TODO: remove this push and extract to another Task await ImagePublisher.PublishImageAsync(multiArchImage, sourceImageReference, destinationImageReference, Log, telemetry, cancellationToken) - .ConfigureAwait(false); + .ConfigureAwait(false); return !Log.HasLoggedErrors; } - private BuiltImage[] ParseImages(DestinationImageReferenceKind destinationKind) + private async Task ParseImages(ITaskItem[] containers, CancellationToken ctok) { - var images = new BuiltImage[GeneratedContainers.Length]; + var images = await Task.WhenAll(containers.Select(itemDescription => ParseBuiltImage(itemDescription))); + var validImages = images.Where(image => image is not null).Cast().ToArray()!; + return validImages; - for (int i = 0; i < GeneratedContainers.Length; i++) + async Task ParseBuiltImage(ITaskItem itemDescription) { - var unparsedImage = GeneratedContainers[i]; - - string config = unparsedImage.GetMetadata("Configuration"); - string manifestDigest = unparsedImage.GetMetadata("ManifestDigest"); - string manifest = unparsedImage.GetMetadata("Manifest"); - string manifestMediaType = unparsedImage.GetMetadata("ManifestMediaType"); + var configFile = new FileInfo(itemDescription.GetMetadata("ConfigurationPath")); + var manifestFile = new FileInfo(itemDescription.GetMetadata("ManifestPath")); - if (string.IsNullOrEmpty(config) || string.IsNullOrEmpty(manifestDigest) || string.IsNullOrEmpty(manifest) || string.IsNullOrEmpty(manifestMediaType)) + if (!configFile.Exists || !manifestFile.Exists) { Log.LogError(Strings.InvalidImageMetadata); - break; + return null; } - (string architecture, string os) = GetArchitectureAndOsFromConfig(config); - - // We don't need ImageDigest, ImageSha, Layers for remote registry, as the individual images should be pushed already - string? imageDigest = null; - string? imageSha = null; - List? layers = null; - - if (destinationKind == DestinationImageReferenceKind.LocalRegistry) + if (await GetArchitectureAndOsFromConfig(configFile, ctok) is not (var config, var architecture, var os)) { - var manifestV2 = JsonSerializer.Deserialize(manifest); - if (manifestV2 == null) - { - Log.LogError(Strings.InvalidImageManifest); - break; - } - - imageDigest = manifestV2.Config.digest; - imageSha = DigestUtils.GetShaFromDigest(imageDigest); - layers = manifestV2.Layers; - } + Log.LogError(Strings.InvalidImageConfig); + return null; + } - images[i] = new BuiltImage() + ManifestV2 manifestV2 = (await JsonSerializer.DeserializeAsync(manifestFile.OpenRead()))!; + return new BuiltImage() { Config = config, - ImageDigest = imageDigest, - ImageSha = imageSha, - Manifest = manifest, - ManifestDigest = manifestDigest, - ManifestMediaType = manifestMediaType, - Layers = layers, + Manifest = manifestV2, + Layers = manifestV2.Layers, OS = os, Architecture = architecture }; } - - return images; } - private (string, string) GetArchitectureAndOsFromConfig(string config) + private async Task<(JsonObject, string, string)?> GetArchitectureAndOsFromConfig(FileInfo config, CancellationToken cTok) { - var configJson = JsonNode.Parse(config) as JsonObject; + using var fileStream = config.OpenRead(); + var configJson = await JsonNode.ParseAsync(fileStream, cancellationToken: cTok) as JsonObject; if (configJson is null) { Log.LogError(Strings.InvalidImageConfig); - return (string.Empty, string.Empty); + return null; } var architecture = configJson["architecture"]?.ToString(); if (architecture is null) { Log.LogError(Strings.ImageConfigMissingArchitecture); - return (string.Empty, string.Empty); - } + return null; + } var os = configJson["os"]?.ToString(); if (os is null) { Log.LogError(Strings.ImageConfigMissingOs); - return (string.Empty, string.Empty); + return null; } - return (architecture, os); + return (configJson, architecture, os); } private static MultiArchImage CreateMultiArchImage(BuiltImage[] images, DestinationImageReferenceKind destinationImageKind) @@ -177,16 +158,14 @@ private static MultiArchImage CreateMultiArchImage(BuiltImage[] images, Destinat return new MultiArchImage() { // For multi-arch we publish only oci-formatted image tarballs. - ImageIndex = ImageIndexGenerator.GenerateImageIndex(images, SchemaTypes.OciManifestV1, SchemaTypes.OciImageIndexV1), - ImageIndexMediaType = SchemaTypes.OciImageIndexV1, + ImageIndex = ImageIndexGenerator.GenerateDockerManifestList(images, SchemaTypes.OciManifestV1, SchemaTypes.OciImageIndexV1), Images = images }; case DestinationImageReferenceKind.RemoteRegistry: - (string imageIndex, string mediaType) = ImageIndexGenerator.GenerateImageIndex(images); + var imageIndex = ImageIndexGenerator.GenerateImageIndex(images); return new MultiArchImage() { ImageIndex = imageIndex, - ImageIndexMediaType = mediaType, // For remote registry we don't need individual images, as they should be pushed already }; default: diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs index ebbd1de29901..d3b093c7303b 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.Interface.cs @@ -41,6 +41,12 @@ partial class CreateNewImage /// public string BaseImageDigest { get; set; } + [Required] + public ITaskItem BaseImageManifestPath { get; set; } + + [Required] + public ITaskItem BaseImageConfigurationPath { get; set; } + /// /// The registry to push to. /// @@ -69,11 +75,11 @@ partial class CreateNewImage public string[] ImageTags { get; set; } /// - /// The directory for the build outputs to be published. - /// Constructed from "$(MSBuildProjectDirectory)\$(PublishDir)" + /// The files to be published to the container. + /// MUST have RelativePath metadata.. /// [Required] - public string PublishDirectory { get; set; } + public ITaskItem[] PublishFiles { get; set; } /// /// The working directory of the container. @@ -129,18 +135,6 @@ partial class CreateNewImage /// public ITaskItem[] ContainerEnvironmentVariables { get; set; } - /// - /// The RID to use to determine the host manifest if the parent container is a manifest list - /// - [Required] - public string ContainerRuntimeIdentifier { get; set; } - - /// - /// The path to the runtime identifier graph file. This is used to compute RID compatibility for Image Manifest List entries. - /// - [Required] - public string RuntimeIdentifierGraphPath { get; set; } - /// /// The username or UID which is a platform-specific structure that allows specific control over which user the process run as. /// This acts as a default value to use when the value is not specified when creating a container. @@ -170,9 +164,25 @@ partial class CreateNewImage /// public string? ImageFormat { get; set; } - /// If true, the tooling will skip the publishing step. + public string ContentStoreRoot { get; set; } + + /// + /// Where to write the generated manifest file. /// - public bool SkipPublishing { get; set; } + [Required] + public string GeneratedManifestPath { get; set; } = ""; + + /// + /// Where to write the generated configuration file. + /// + [Required] + public string GeneratedConfigurationPath { get; set; } = ""; + + /// + /// Where to write the generated layer tarball. + /// + [Required] + public string GeneratedLayerPath { get; set; } = ""; [Output] public string GeneratedContainerManifest { get; set; } @@ -192,6 +202,15 @@ partial class CreateNewImage [Output] public ITaskItem[] GeneratedContainerNames { get; set; } + [Output] + public ITaskItem GeneratedAppContainerLayer { get; set; } + + [Output] + public ITaskItem GeneratedAppContainerConfig { get; set; } + + [Output] + public ITaskItem GeneratedAppContainerManifest { get; set; } + [Output] public ITaskItem? GeneratedDigestLabel { get; set; } @@ -204,11 +223,13 @@ public CreateNewImage() BaseImageName = ""; BaseImageTag = ""; BaseImageDigest = ""; + BaseImageManifestPath = null!; + BaseImageConfigurationPath = null!; OutputRegistry = ""; ArchiveOutputPath = ""; Repository = ""; ImageTags = Array.Empty(); - PublishDirectory = ""; + PublishFiles = []; WorkingDirectory = ""; Entrypoint = Array.Empty(); EntrypointArgs = Array.Empty(); @@ -219,10 +240,9 @@ public CreateNewImage() Labels = Array.Empty(); ExposedPorts = Array.Empty(); ContainerEnvironmentVariables = Array.Empty(); - ContainerRuntimeIdentifier = ""; - RuntimeIdentifierGraphPath = ""; LocalRegistry = ""; ContainerUser = ""; + ContentStoreRoot = ""; GeneratedContainerConfiguration = ""; GeneratedContainerManifest = ""; @@ -231,6 +251,9 @@ public CreateNewImage() GeneratedContainerMediaType = ""; GeneratedContainerNames = Array.Empty(); GeneratedDigestLabel = null; + GeneratedAppContainerLayer = null!; + GeneratedAppContainerConfig = null!; + GeneratedAppContainerManifest = null!; GenerateLabels = false; GenerateDigestLabel = false; diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs index 4b9993e06cf7..83483dd760d9 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImage.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; +using System.Text.Json.Nodes; using Microsoft.Build.Framework; using Microsoft.Extensions.Logging; using Microsoft.NET.Build.Containers.Logging; @@ -52,12 +54,6 @@ internal async Task ExecuteAsync(CancellationToken cancellationToken) ILoggerFactory msbuildLoggerFactory = new LoggerFactory(new[] { loggerProvider }); ILogger logger = msbuildLoggerFactory.CreateLogger(); - if (!Directory.Exists(PublishDirectory)) - { - Log.LogErrorWithCodeFromResources(nameof(Strings.PublishDirectoryDoesntExist), nameof(PublishDirectory), PublishDirectory); - return !Log.HasLoggedErrors; - } - RegistryMode sourceRegistryMode = BaseRegistry.Equals(OutputRegistry, StringComparison.InvariantCultureIgnoreCase) ? RegistryMode.PullFromOutput : RegistryMode.Pull; Registry? sourceRegistry = IsLocalPull ? null : new Registry(BaseRegistry, logger, sourceRegistryMode); SourceImageReference sourceImageReference = new(sourceRegistry, BaseImageName, BaseImageTag, BaseImageDigest); @@ -72,96 +68,47 @@ internal async Task ExecuteAsync(CancellationToken cancellationToken) var telemetry = new Telemetry(sourceImageReference, destinationImageReference, Log); - ImageBuilder? imageBuilder; - if (sourceRegistry is { } registry) + KnownImageFormats? format = null; + if (ImageFormat is not null) { - try + if (Enum.TryParse(ImageFormat, out KnownImageFormats knownFormat)) { - var picker = new RidGraphManifestPicker(RuntimeIdentifierGraphPath); - imageBuilder = await registry.GetImageManifestAsync( - BaseImageName, - sourceImageReference.Reference, - ContainerRuntimeIdentifier, - picker, - cancellationToken).ConfigureAwait(false); + format = knownFormat; } - catch (RepositoryNotFoundException) - { - telemetry.LogUnknownRepository(); - Log.LogErrorWithCodeFromResources(nameof(Strings.RepositoryNotFound), BaseImageName, BaseImageTag, BaseImageDigest, registry.RegistryName); - return !Log.HasLoggedErrors; - } - catch (UnableToAccessRepositoryException) - { - telemetry.LogCredentialFailure(sourceImageReference); - Log.LogErrorWithCodeFromResources(nameof(Strings.UnableToAccessRepository), BaseImageName, registry.RegistryName); - return !Log.HasLoggedErrors; - } - catch (ContainerHttpException e) - { - Log.LogErrorFromException(e, showStackTrace: false, showDetail: true, file: null); - return !Log.HasLoggedErrors; - } - catch (BaseImageNotFoundException e) + else { - telemetry.LogRidMismatch(e.RequestedRuntimeIdentifier, e.AvailableRuntimeIdentifiers.ToArray()); - Log.LogErrorFromException(e, showStackTrace: false, showDetail: true, file: null); - return !Log.HasLoggedErrors; + Log.LogErrorWithCodeFromResources(nameof(Strings.InvalidContainerImageFormat), ImageFormat, string.Join(",", Enum.GetNames(typeof(KnownImageFormats)))); + return false; } } - else + + ImageBuilder? imageBuilder; + if (sourceRegistry is { } registry) { - throw new NotSupportedException(Resource.GetString(nameof(Strings.ImagePullNotSupported))); + imageBuilder = await ContainerBuilder.LoadFromManifestAndConfig(BaseImageManifestPath.ItemSpec, format, BaseImageConfigurationPath.ItemSpec, logger); } - - if (imageBuilder is null) + else { - Log.LogErrorWithCodeFromResources(nameof(Strings.BaseImageNotFound), sourceImageReference, ContainerRuntimeIdentifier); - return !Log.HasLoggedErrors; + throw new NotSupportedException(Resource.GetString(nameof(Strings.ImagePullNotSupported))); } - (string message, object[] parameters) = SkipPublishing ? - (Strings.ContainerBuilder_StartBuildingImageForRid, new object[] { Repository, ContainerRuntimeIdentifier, sourceImageReference }) : + (string message, object[] parameters) = (Strings.ContainerBuilder_StartBuildingImage, new object[] { Repository, String.Join(",", ImageTags), sourceImageReference }); Log.LogMessage(MessageImportance.High, message, parameters); - // forcibly change the media type if required - if (ImageFormat is not null) - { - if (Enum.TryParse(ImageFormat, out var imageFormat)) - { - imageBuilder.ManifestMediaType = imageFormat switch - { - KnownImageFormats.Docker => SchemaTypes.DockerManifestV2, - KnownImageFormats.OCI => SchemaTypes.OciManifestV1, - _ => imageBuilder.ManifestMediaType // should be impossible unless we add to the enum - }; - } - else - { - Log.LogErrorWithCodeFromResources(nameof(Strings.InvalidContainerImageFormat), ImageFormat, string.Join(",", Enum.GetValues())); - } - } - - // forcibly change the media type if required - if (ImageFormat is not null) + var storePath = new DirectoryInfo(ContentStoreRoot); + if (!storePath.Exists) { - if (Enum.TryParse(ImageFormat, out var imageFormat)) - { - imageBuilder.ManifestMediaType = imageFormat switch - { - KnownImageFormats.Docker => SchemaTypes.DockerManifestV2, - KnownImageFormats.OCI => SchemaTypes.OciManifestV1, - _ => imageBuilder.ManifestMediaType // should be impossible unless we add to the enum - }; - } - else - { - Log.LogErrorWithCodeFromResources(nameof(Strings.InvalidContainerImageFormat), ImageFormat, string.Join(",", Enum.GetValues())); - } + throw new ArgumentException($"The content store path '{ContentStoreRoot}' does not exist."); } - - Layer newLayer = Layer.FromDirectory(PublishDirectory, WorkingDirectory, imageBuilder.IsWindows, imageBuilder.ManifestMediaType); + var store = new ContentStore(storePath); + + (string absolutefilePath, string relativeContainerPath)[] filesWithRelativePaths = + PublishFiles + .Select(f => (f.ItemSpec, f.GetMetadata("RelativePath"))) + .Where(x => !string.IsNullOrWhiteSpace(x.ItemSpec) && !string.IsNullOrWhiteSpace(x.Item2)) + .ToArray(); + Layer newLayer = await Layer.FromFiles(filesWithRelativePaths, WorkingDirectory, imageBuilder.IsWindows, imageBuilder.ManifestMediaType, store, new(GeneratedLayerPath), cancellationToken); imageBuilder.AddLayer(newLayer); imageBuilder.SetWorkingDirectory(WorkingDirectory); @@ -209,12 +156,42 @@ internal async Task ExecuteAsync(CancellationToken cancellationToken) cancellationToken.ThrowIfCancellationRequested(); // at this point we're done with modifications and are just pushing the data other places - GeneratedContainerManifest = builtImage.Manifest; - GeneratedContainerConfiguration = builtImage.Config; + + var serializedManifest = JsonSerializer.Serialize(builtImage.Manifest); + var manifestWriteTask = File.WriteAllTextAsync(GeneratedManifestPath, serializedManifest, DigestUtils.UTF8); + + var serializedConfig = JsonSerializer.Serialize(builtImage.Config); + var configWriteTask = File.WriteAllTextAsync(GeneratedConfigurationPath, serializedConfig, DigestUtils.UTF8); + + await Task.WhenAll(manifestWriteTask, configWriteTask).ConfigureAwait(false); + + GeneratedContainerManifest = serializedManifest; + GeneratedContainerConfiguration = serializedConfig; GeneratedContainerDigest = builtImage.ManifestDigest; GeneratedArchiveOutputPath = ArchiveOutputPath; GeneratedContainerMediaType = builtImage.ManifestMediaType; GeneratedContainerNames = destinationImageReference.FullyQualifiedImageNames().Select(name => new Microsoft.Build.Utilities.TaskItem(name)).ToArray(); + GeneratedAppContainerLayer = new Microsoft.Build.Utilities.TaskItem(GeneratedLayerPath, new Dictionary(4) + { + ["Size"] = newLayer.Descriptor.Size.ToString(), + ["MediaType"] = newLayer.Descriptor.MediaType, + ["Digest"] = newLayer.Descriptor.Digest, + }); + + GeneratedAppContainerConfig = new Microsoft.Build.Utilities.TaskItem(GeneratedConfigurationPath, new Dictionary(2) + { + ["Size"] = builtImage.Manifest.Config.size.ToString(), + ["MediaType"] = builtImage.Manifest.Config.mediaType, + ["Digest"] = builtImage.Manifest.Config.digest, + }); + + GeneratedAppContainerManifest = new Microsoft.Build.Utilities.TaskItem(GeneratedManifestPath, new Dictionary(2) + { + ["Size"] = new FileInfo(GeneratedManifestPath).Length.ToString(), + ["MediaType"] = builtImage.Manifest.MediaType!, + ["Digest"] = builtImage.Manifest.GetDigest(), + }); + if (baseImageLabel is not null && baseImageDigest is not null) { var labelItem = new Microsoft.Build.Utilities.TaskItem(baseImageLabel); @@ -222,12 +199,6 @@ internal async Task ExecuteAsync(CancellationToken cancellationToken) GeneratedDigestLabel = labelItem; } - if (!SkipPublishing) - { - await ImagePublisher.PublishImageAsync(builtImage, sourceImageReference, destinationImageReference, Log, telemetry, cancellationToken) - .ConfigureAwait(false); - } - return !Log.HasLoggedErrors; } diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImageToolTask.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImageToolTask.cs index 90bf398c900b..0cb7992a761c 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImageToolTask.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/CreateNewImageToolTask.cs @@ -71,10 +71,6 @@ protected override ProcessStartInfo GetProcessStartInfo(string pathToTool, strin /// internal string GenerateCommandLineCommandsInt() { - if (string.IsNullOrWhiteSpace(PublishDirectory)) - { - throw new InvalidOperationException(Resource.FormatString(nameof(Strings.RequiredPropertyNotSetOrEmpty), nameof(PublishDirectory))); - } if (string.IsNullOrWhiteSpace(BaseRegistry)) { throw new InvalidOperationException(Resource.FormatString(nameof(Strings.RequiredPropertyNotSetOrEmpty), nameof(BaseRegistry))); @@ -96,7 +92,11 @@ internal string GenerateCommandLineCommandsInt() //mandatory options builder.AppendFileNameIfNotNull(Path.Combine(ContainerizeDirectory, "containerize.dll")); - builder.AppendFileNameIfNotNull(PublishDirectory.TrimEnd(new char[] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar })); + foreach (ITaskItem inputFile in PublishFiles) + { + var relPath = inputFile.GetMetadata("TargetPath") is string tp && !string.IsNullOrEmpty(tp) ? tp : inputFile.GetMetadata("RelativePath"); + builder.AppendSwitchIfNotNull("--input-file", $"{inputFile.ItemSpec}={relPath}"); + } builder.AppendSwitchIfNotNull("--baseregistry ", BaseRegistry); builder.AppendSwitchIfNotNull("--baseimagename ", BaseImageName); builder.AppendSwitchIfNotNull("--repository ", Repository); @@ -180,16 +180,6 @@ internal string GenerateCommandLineCommandsInt() string[] readyEnvVariables = sanitizedEnvVariables.Select(i => i.ItemSpec + "=" + i.GetMetadata("Value")).ToArray(); builder.AppendSwitchIfNotNull("--environmentvariables ", readyEnvVariables, delimiter: " "); - if (!string.IsNullOrWhiteSpace(ContainerRuntimeIdentifier)) - { - builder.AppendSwitchIfNotNull("--rid ", ContainerRuntimeIdentifier); - } - - if (!string.IsNullOrWhiteSpace(RuntimeIdentifierGraphPath)) - { - builder.AppendSwitchIfNotNull("--ridgraphpath ", RuntimeIdentifierGraphPath); - } - if (!string.IsNullOrWhiteSpace(ContainerUser)) { builder.AppendSwitchIfNotNull("--container-user ", ContainerUser); @@ -210,6 +200,13 @@ internal string GenerateCommandLineCommandsInt() builder.AppendSwitch("--generate-digest-label"); } + builder.AppendSwitchIfNotNull("--base-image-manifest", BaseImageManifestPath); + builder.AppendSwitchIfNotNull("--base-image-config", BaseImageConfigurationPath); + + builder.AppendSwitchIfNotNull("--generated-manifest", GeneratedManifestPath); + builder.AppendSwitchIfNotNull("--generated-configuration", GeneratedConfigurationPath); + builder.AppendSwitchIfNotNull("--generated-layer", GeneratedLayerPath); + return builder.ToString(); void AppendSwitchIfNotNullSanitized(CommandLineBuilder builder, string commandArgName, string propertyName, ITaskItem[] value) diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/DownloadContainerManifest.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/DownloadContainerManifest.cs new file mode 100644 index 000000000000..832b0854efe4 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/DownloadContainerManifest.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Threading.Tasks; +using Microsoft.Build.Framework; +using Microsoft.Extensions.Logging; +using Microsoft.NET.Build.Containers.Logging; +using NuGet.RuntimeModel; +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace Microsoft.NET.Build.Containers.Tasks; + +public class DownloadContainerManifest : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource _cts = new(); + + [Required] + public string Registry { get; set; } = string.Empty; + + [Required] + public string Repository { get; set; } = string.Empty; + + public string? Tag { get; set; } + + public string? Digest { get; set; } + + [Required] + public string ContentStore { get; set; } = string.Empty; + + + [Output] + public ITaskItem[] Manifests { get; set; } = Array.Empty(); + + [Output] + public ITaskItem[] Configs { get; set; } = Array.Empty(); + + [Output] + public ITaskItem[] Layers { get; set; } = Array.Empty(); + + + public void Cancel() => _cts.Cancel(); + + public override bool Execute() => ExecuteAsync().GetAwaiter().GetResult(); + + public async Task ExecuteAsync() + { + if (Tag is null && Digest is null) + { + throw new ArgumentException("Must provide one of Tag and Digest"); + } + + using MSBuildLoggerProvider loggerProvider = new(Log); + ILoggerFactory msbuildLoggerFactory = new LoggerFactory(new[] { loggerProvider }); + ILogger logger = msbuildLoggerFactory.CreateLogger(); + var store = new ContentStore(new(ContentStore)); + var registry = new Registry(Registry, logger, RegistryMode.Pull, store: store); + + // download the manifest from the registry (maybe), and download child manifests if it's a multi-image manifest + // we accept a nullable coercion here for tag/digest because we just checked that at least one is present + // TODO: set skipCache to false when we have some kind of user-settable semantic for Docker's ímage pull policy concept + var outerManifest = await registry.GetManifestCore(Repository, (Digest ?? Tag)!, _cts.Token, skipCache: true); + if (outerManifest is ManifestV2 singleArchManifest) + { + Log.LogMessage($"Found single-arch manifest for {Repository} with digest {singleArchManifest.KnownDigest}"); + var platformData = await DownloadConfigForManifest(registry, singleArchManifest); + SetOutputs([(singleArchManifest, platformData)], store); + return true; + } + else if (outerManifest is IMultiImageManifest multiArchManifest) + { + Log.LogMessage($"Found multi-arch manifest for {Repository}, fetching child manifests"); ; + if (multiArchManifest is ManifestListV2 manifestList) + { + var manifests = await Task.WhenAll(manifestList.manifests.Select(GetPlatformDataForManifest)); + SetOutputs(manifests, store); + return true; + } + else if (multiArchManifest is ImageIndexV1 imageIndex) + { + var manifests = await Task.WhenAll(imageIndex.manifests.Select(GetOciPlatformDataForManifest)); + SetOutputs(manifests, store); + return true; + } + else + { + throw new InvalidOperationException("Unknown multi-arch manifest type"); + } + } + else + { + throw new InvalidOperationException("Unknown manifest type"); + } + async Task<(ManifestV2, PlatformInformation)> GetPlatformDataForManifest(PlatformSpecificManifest p) + { + var manifest = await registry.GetManifestCore(Repository, p.digest, _cts.Token); + if (manifest is not ManifestV2 manifestV2) + { + throw new InvalidOperationException("Expected single-arch manifest"); + } + Log.LogMessage($"Found child manifest for platform {p.platform} with digest {manifestV2.KnownDigest}"); + var platformData = await DownloadConfigForManifest(registry, manifestV2); + return (manifestV2, platformData); + } + + async Task<(ManifestV2, PlatformInformation)> GetOciPlatformDataForManifest(PlatformSpecificOciManifest p) + { + var manifest = await registry.GetManifestCore(Repository, p.digest, _cts.Token); + if (manifest is not ManifestV2 manifestV2) + { + throw new InvalidOperationException("Expected single-arch manifest"); + } + Log.LogMessage($"Found child manifest for platform {p.platform} with digest {manifestV2.KnownDigest}"); + var platformData = await DownloadConfigForManifest(registry, manifestV2); + return (manifestV2, platformData); + } + } + + /// + /// Downloads the configuration for a manifest, which contains platform information. + /// This ensures that the per-RID configs are present for future build steps + /// + /// + /// + /// + async Task DownloadConfigForManifest(Registry registry, ManifestV2 manifest) + { + var configJson = await registry.GetJsonBlobCore(Repository, manifest.Config.digest, manifest.Config.mediaType, _cts.Token); + return configJson.Deserialize(); + } + + void SetOutputs(IReadOnlyList<(ManifestV2 manifest, PlatformInformation platformInfo)> manifests, ContentStore store) + { + var manifestItems = new List(manifests.Count); + var configItems = new List(manifests.Count); + var layerItems = new List(manifests.Count * manifests[0].manifest.Layers.Count); //estimate + foreach (var (manifest, platform) in manifests) + { + var manifestDescriptor = new Descriptor(manifest.MediaType!, manifest.KnownDigest!, 0); + var manifestLocalPath = store.PathForDescriptor(manifestDescriptor); + + var itemRid = RidMapping.CreateRidForPlatform(platform); + var manifestItem = new Microsoft.Build.Utilities.TaskItem(manifestLocalPath); + manifestItem.SetMetadata("MediaType", manifest.MediaType ?? string.Empty); + manifestItem.SetMetadata("ConfigDigest", manifest.Config.digest); + manifestItem.SetMetadata("RuntimeIdentifier", itemRid); + manifestItem.SetMetadata("Registry", Registry); + manifestItem.SetMetadata("Repository", Repository); + manifestItems.Add(manifestItem); + + var configDescriptor = new Descriptor(manifest.Config.mediaType, manifest.Config.digest, manifest.Config.size); + var configLocalPath = store.PathForDescriptor(configDescriptor); + var configItem = new Microsoft.Build.Utilities.TaskItem(configLocalPath); + configItem.SetMetadata("MediaType", manifest.Config.mediaType ?? string.Empty); + configItem.SetMetadata("Size", manifest.Config.size.ToString()); + configItem.SetMetadata("Digest", manifest.Config.digest); + configItem.SetMetadata("RuntimeIdentifier", itemRid); + configItem.SetMetadata("Registry", Registry); + configItem.SetMetadata("Repository", Repository); + configItems.Add(configItem); + + foreach (var layer in manifest.Layers) + { + var layerDescriptor = new Descriptor(layer.mediaType!, layer.digest, layer.size); + var layerLocalPath = store.PathForDescriptor(layerDescriptor); + var layerItem = new Microsoft.Build.Utilities.TaskItem(layerLocalPath); + layerItem.SetMetadata("MediaType", layer.mediaType ?? string.Empty); + layerItem.SetMetadata("Size", layer.size.ToString()); + layerItem.SetMetadata("Digest", layer.digest); + layerItem.SetMetadata("ConfigDigest", manifest.Config.digest); + layerItem.SetMetadata("RuntimeIdentifier", itemRid); + layerItem.SetMetadata("Registry", Registry); + layerItem.SetMetadata("Repository", Repository); + layerItems.Add(layerItem); + } + } + + Manifests = manifestItems.ToArray(); + Configs = configItems.ToArray(); + Layers = layerItems.ToArray(); + } +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/DownloadLayers.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/DownloadLayers.cs new file mode 100644 index 000000000000..8e5b30c044c4 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/DownloadLayers.cs @@ -0,0 +1,73 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Build.Framework; +using Microsoft.Extensions.Logging; +using Microsoft.NET.Build.Containers.Logging; +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace Microsoft.NET.Build.Containers.Tasks; + +public class DownloadLayers : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource _cts = new(); + + [Required] + public string Registry { get; set; } = string.Empty; + + [Required] + public string Repository { get; set; } = string.Empty; + + [Required] + public string ContentStore { get; set; } = string.Empty; + + /// + /// should have the same data model as the output layers from + /// + [Required] + public ITaskItem[] Layers { get; set; } = Array.Empty(); + + public void Cancel() => _cts.Cancel(); + + public override bool Execute() => ExecuteAsync().GetAwaiter().GetResult(); + + public async Task ExecuteAsync() + { + using MSBuildLoggerProvider loggerProvider = new(Log); + ILoggerFactory msbuildLoggerFactory = new LoggerFactory(new[] { loggerProvider }); + ILogger logger = msbuildLoggerFactory.CreateLogger(); + var store = new ContentStore(new(ContentStore)); + var registry = new Registry(Registry, logger, RegistryMode.Pull, store: store); + + var layerDownloadTasks = new List(Layers.Length); + foreach (var layer in Layers) + { + var storagePath = layer.ItemSpec; + var digest = layer.GetMetadata("Digest"); + var size = layer.GetMetadata("Size"); + var mediaType = layer.GetMetadata("MediaType"); + + if (string.IsNullOrEmpty(digest) || string.IsNullOrEmpty(size) || string.IsNullOrEmpty(mediaType)) + { + logger.LogError(new EventId(123456, "layer_item_validation_failure"), $"Layer {layer.ItemSpec} must have Digest, Size, and MediaType metadata"); + } + + var descriptor = new Descriptor(mediaType, digest, long.Parse(size)); + layerDownloadTasks.Add( + registry.DownloadBlobAsync(Repository, descriptor, _cts.Token).ContinueWith(t => + { + if (t.IsFaulted) + { + logger.LogError(new EventId(123457, "layer_download_failure"), exception: t.Exception, $"Failed to download layer {digest} from {Registry}/{Repository}"); + } + else if (t.IsCanceled) + { + logger.LogError(new EventId(123457, "layer_download_failure"), exception: t.Exception, $"Failed to download layer {digest} from {Registry}/{Repository}"); + } + }) + ); + } + await Task.WhenAll(layerDownloadTasks); + return !Log.HasLoggedErrors; + } +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/MakeContainerTarball.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/MakeContainerTarball.cs new file mode 100644 index 000000000000..62ca318a1e49 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/MakeContainerTarball.cs @@ -0,0 +1,111 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Build.Framework; +using Microsoft.Extensions.Logging; +using Microsoft.NET.Build.Containers.Logging; +using Microsoft.NET.Build.Containers.Resources; +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace Microsoft.NET.Build.Containers.Tasks; + +public class MakeContainerTarball : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource _cts = new(); + + [Required] + public string ArchivePath { get; set; } = null!; + + [Required] + public string Repository { get; set; } = string.Empty; + + [Required] + public string[] Tags { get; set; } = []; + + [Required] + public ITaskItem Manifest { get; set; } = null!; + + [Required] + public ITaskItem Configuration { get; set; } = null!; + + [Required] + public ITaskItem[] Layers { get; set; } = []; + + [Output] + public string GeneratedArchiveFilePath { get; set; } = null!; + + + public void Cancel() => _cts.Cancel(); + + public override bool Execute() => ExecuteAsync().GetAwaiter().GetResult(); + + public async Task ExecuteAsync() + { + using MSBuildLoggerProvider loggerProvider = new(Log); + ILoggerFactory msbuildLoggerFactory = new LoggerFactory(new[] { loggerProvider }); + ILogger logger = msbuildLoggerFactory.CreateLogger(); + (long manifestSize, string manifestDigest, ManifestV2 manifestStructure) = await ReadManifest(); + var configDigest = manifestStructure.Config.digest; + var config = await JsonSerializer.DeserializeAsync(File.OpenRead(Configuration.ItemSpec), cancellationToken: _cts.Token); + var layers = Layers.Select(l => Layer.FromBackingFile(new(l.ItemSpec), GetDescriptor(l))).ToArray(); + var filePath = DetermineFilePath(); + await using var fileStream = File.Create(filePath); + GeneratedArchiveFilePath = filePath; + var telemetry = new Telemetry(new(null, null, null, Telemetry.LocalStorageType.Tarball), Log); + await DockerCli.WriteImageToStreamAsync(Repository, Tags, config!, layers, manifestStructure, fileStream, _cts.Token); + telemetry.LogPublishSuccess(); + return true; + } + + private Descriptor GetDescriptor(ITaskItem item) + { + var mediaType = item.GetMetadata("MediaType"); + var digest = item.GetMetadata("Digest"); + var size = long.Parse(item.GetMetadata("Size")!); + return new Descriptor + { + MediaType = mediaType, + Digest = digest, + Size = size + }; + } + + private async Task<(long size, string digest, ManifestV2 manifest)> ReadManifest() + { + var size = long.Parse(Manifest.GetMetadata("Size")!); + var digest = Manifest.GetMetadata("Digest")!; + var manifestStructure = await JsonSerializer.DeserializeAsync(File.OpenRead(Manifest.ItemSpec), cancellationToken: _cts.Token); + return (size, digest, manifestStructure!); + } + + private string DetermineFilePath() + { + + var fullPath = Path.GetFullPath(ArchivePath); + + var directorySeparatorChar = Path.DirectorySeparatorChar; + + // if doesn't end with a file extension, assume it's a directory + if (!Path.HasExtension(fullPath)) + { + fullPath += Path.DirectorySeparatorChar; + } + + // pointing to a directory? -> append default name + if (fullPath.EndsWith(directorySeparatorChar)) + { + fullPath = Path.Combine(fullPath, Repository + ".tar.gz"); + } + + // create parent directory if required. + var parentDirectory = Path.GetDirectoryName(fullPath); + if (parentDirectory != null && !Directory.Exists(parentDirectory)) + { + Directory.CreateDirectory(parentDirectory); + } + + return fullPath; + } +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/PushContainerToLocal.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/PushContainerToLocal.cs new file mode 100644 index 000000000000..ac1938573191 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/PushContainerToLocal.cs @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Build.Framework; +using Microsoft.Extensions.Logging; +using Microsoft.NET.Build.Containers.Logging; +using Microsoft.NET.Build.Containers.Resources; +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace Microsoft.NET.Build.Containers.Tasks; + +public class PushContainerToLocal : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource _cts = new(); + + public string? LocalRegistry { get; set; } + + [Required] + public string Repository { get; set; } = string.Empty; + + [Required] + public string[] Tags { get; set; } = []; + + [Required] + public ITaskItem Manifest { get; set; } = null!; + + [Required] + public ITaskItem Configuration { get; set; } = null!; + + [Required] + public ITaskItem[] Layers { get; set; } = []; + + + public void Cancel() => _cts.Cancel(); + + public override bool Execute() => ExecuteAsync().GetAwaiter().GetResult(); + + public async Task ExecuteAsync() + { + using MSBuildLoggerProvider loggerProvider = new(Log); + ILoggerFactory msbuildLoggerFactory = new LoggerFactory(new[] { loggerProvider }); + ILogger logger = msbuildLoggerFactory.CreateLogger(); + (long manifestSize, string manifestDigest, ManifestV2 manifestStructure) = await ReadManifest(); + var configDigest = manifestStructure.Config.digest; + var config = await JsonSerializer.DeserializeAsync(File.OpenRead(Configuration.ItemSpec), cancellationToken: _cts.Token); + var containerCli = new DockerCli(LocalRegistry, msbuildLoggerFactory); + + var telemetry = new Telemetry(new(null, null, null, containerCli.IsDocker ? Telemetry.LocalStorageType.Docker : Telemetry.LocalStorageType.Podman), Log); + if (!await containerCli.IsAvailableAsync(_cts.Token).ConfigureAwait(false)) + { + telemetry.LogMissingLocalBinary(); + Log.LogErrorWithCodeFromResources(nameof(Strings.LocalRegistryNotAvailable)); + return false; + } + + var layers = Layers.Select(l => Layer.FromBackingFile(new(l.ItemSpec), GetDescriptor(l))).ToArray(); + try + { + await containerCli.LoadAsync((Repository, Tags, configDigest, config!, layers), DockerCli.WriteDockerImageToStreamAsync, _cts.Token); + } + catch (AggregateException ex) when (ex.InnerException is DockerLoadException dle) + { + telemetry.LogLocalLoadError(); + Log.LogErrorFromException(dle, showStackTrace: false); + } + catch (ArgumentException argEx) + { + Log.LogErrorFromException(argEx, showStackTrace: false); + } + catch (DockerLoadException dle) + { + telemetry.LogLocalLoadError(); + Log.LogErrorFromException(dle, showStackTrace: false); + } + return true; + } + + private Descriptor GetDescriptor(ITaskItem item) + { + var mediaType = item.GetMetadata("MediaType"); + var digest = item.GetMetadata("Digest"); + var size = long.Parse(item.GetMetadata("Size")!); + return new Descriptor + { + MediaType = mediaType, + Digest = digest, + Size = size + }; + } + + private async Task<(long size, string digest, ManifestV2 manifest)> ReadManifest() + { + var size = long.Parse(Manifest.GetMetadata("Size")!); + var digest = Manifest.GetMetadata("Digest")!; + var manifestStructure = await JsonSerializer.DeserializeAsync(File.OpenRead(Manifest.ItemSpec), cancellationToken: _cts.Token); + return (size, digest, manifestStructure!); + } +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/PushContainerToRemoteRegistry.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/PushContainerToRemoteRegistry.cs new file mode 100644 index 000000000000..a37e99640604 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/PushContainerToRemoteRegistry.cs @@ -0,0 +1,152 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json; +using Microsoft.Build.Framework; +using Microsoft.Extensions.Logging; +using Microsoft.NET.Build.Containers.Logging; +using Microsoft.NET.Build.Containers.Resources; +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace Microsoft.NET.Build.Containers.Tasks; + +public class PushContainerToRemoteRegistry : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource _cts = new(); + + [Required] + public string Registry { get; set; } = string.Empty; + + [Required] + public string Repository { get; set; } = string.Empty; + + [Required] + public string[] Tags { get; set; } = []; + + [Required] + public ITaskItem Manifest { get; set; } = null!; + + [Required] + public ITaskItem Configuration { get; set; } = null!; + + [Required] + public ITaskItem[] Layers { get; set; } = []; + + public void Cancel() => _cts.Cancel(); + + public override bool Execute() => ExecuteAsync().GetAwaiter().GetResult(); + + public async Task ExecuteAsync() + { + using MSBuildLoggerProvider loggerProvider = new(Log); + ILoggerFactory msbuildLoggerFactory = new LoggerFactory(new[] { loggerProvider }); + ILogger logger = msbuildLoggerFactory.CreateLogger(nameof(PushContainerToRemoteRegistry)); + var destinationRegistry = new Registry(Registry, msbuildLoggerFactory.CreateLogger(Registry), RegistryMode.Push); + + var telemetry = new Telemetry(new(null, null, Telemetry.GetRegistryType(destinationRegistry), null), Log); + // functionally, we need to + // * upload the layers + var layerUploadTasks = Layers.Select(l => new Layer(new(l.ItemSpec), GetDescriptor(l))).Select(async l => + { + using var _layerScope = logger.BeginScope(new Dictionary + { + ["Layer"] = l.Descriptor.Digest, + ["Repository"] = Repository, + ["MediaType"] = l.Descriptor.MediaType, + ["Digest"] = l.Descriptor.Digest, + ["Size"] = l.Descriptor.Size + }); + logger.LogTrace($"Pushing layer to {Registry}."); + await destinationRegistry.PushLayerAsync(l, Repository, _cts.Token); + }).ToArray(); + await Task.WhenAll(layerUploadTasks); + + // * upload the config + var (size, digest, manifestStructure) = await ReadManifest().ConfigureAwait(false); + _cts.Token.ThrowIfCancellationRequested(); + using (logger.BeginScope(new Dictionary + { + ["Repository"] = Repository, + ["MediaType"] = Configuration.GetMetadata("MediaType")!, + ["Digest"] = Configuration.GetMetadata("Digest")!, + ["Size"] = Configuration.GetMetadata("Size")! + })) + { + logger.LogTrace($"Pushing config to {Registry}."); + var configText = await File.ReadAllTextAsync(Configuration.ItemSpec, _cts.Token); + var configBytes = DigestUtils.UTF8.GetBytes(configText); + var configDigest = DigestUtils.GetDigest(configText); + var msbuildConfigDigest = Configuration.GetMetadata("Digest")!; + + using (MemoryStream configStream = new(configBytes)) + { + logger.LogInformation(Strings.Registry_ConfigUploadStarted, manifestStructure.Config.digest); + await destinationRegistry.UploadBlobAsync(Repository, manifestStructure.Config.digest, configStream, _cts.Token); + logger.LogInformation(Strings.Registry_ConfigUploaded); + } + } + + using (logger.BeginScope(new Dictionary + { + ["Repository"] = Repository, + ["MediaType"] = Manifest.GetMetadata("MediaType")!, + ["Digest"] = Manifest.GetMetadata("Digest")!, + ["Size"] = Manifest.GetMetadata("Size")! + })) + { + if (manifestStructure.GetDigest() != Manifest.GetMetadata("Digest")) + { + logger.LogError($"Manifest digest {Manifest.GetMetadata("Digest")} does not match the computed digest {manifestStructure.GetDigest()} from the manifest file itself."); + } + if (manifestStructure.GetDigest() != DigestUtils.GetDigest(manifestStructure)) + { + logger.LogError($"Manifest digest {manifestStructure.GetDigest()} does not match the computed digest {DigestUtils.GetDigest(manifestStructure)} from the manifest structure itself."); + } + // * upload the manifest as a digest + _cts.Token.ThrowIfCancellationRequested(); + logger.LogInformation(Strings.Registry_ManifestUploadStarted, Registry, manifestStructure.GetDigest()); + await destinationRegistry.UploadManifestAsync(Repository, Manifest.GetMetadata("Digest"), manifestStructure, _cts.Token); + logger.LogInformation(Strings.Registry_ManifestUploaded, Registry); + } + + // * upload the manifest as tags + foreach (var tag in Tags) + { + using var _manifestTagScope = logger.BeginScope(new Dictionary + { + ["Repository"] = Repository, + ["MediaType"] = Manifest.GetMetadata("MediaType")!, + ["Digest"] = Manifest.GetMetadata("Digest")!, + ["Size"] = Manifest.GetMetadata("Size")!, + ["Tag"] = tag + }); + _cts.Token.ThrowIfCancellationRequested(); + logger.LogInformation(Strings.Registry_TagUploadStarted, tag, Registry); + await destinationRegistry.UploadManifestAsync(Repository, tag, manifestStructure, _cts.Token); + logger.LogInformation(Strings.Registry_TagUploaded, tag, Registry); + } + telemetry.LogPublishSuccess(); + return true; + } + + private Descriptor GetDescriptor(ITaskItem item) + { + var mediaType = item.GetMetadata("MediaType"); + var digest = item.GetMetadata("Digest"); + var size = long.Parse(item.GetMetadata("Size")!); + return new Descriptor + { + MediaType = mediaType, + Digest = digest, + Size = size + }; + } + + private async Task<(long size, string digest, ManifestV2 manifest)> ReadManifest() + { + var size = long.Parse(Manifest.GetMetadata("Size")!); + var digest = Manifest.GetMetadata("Digest")!; + var manifestStructure = await JsonSerializer.DeserializeAsync(File.OpenRead(Manifest.ItemSpec), cancellationToken: _cts.Token); + return (size, digest, manifestStructure!); + } +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Tasks/SelectRuntimeIdentifierSpecificItems.cs b/src/Containers/Microsoft.NET.Build.Containers/Tasks/SelectRuntimeIdentifierSpecificItems.cs new file mode 100644 index 000000000000..72288f6155c2 --- /dev/null +++ b/src/Containers/Microsoft.NET.Build.Containers/Tasks/SelectRuntimeIdentifierSpecificItems.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Build.Framework; +using Microsoft.Extensions.Logging; +using Microsoft.NET.Build.Containers.Logging; +using ILogger = Microsoft.Extensions.Logging.ILogger; + +namespace Microsoft.NET.Build.Containers.Tasks; + +public class SelectRuntimeIdentifierSpecificItems : Microsoft.Build.Utilities.Task, ICancelableTask +{ + private CancellationTokenSource _cts = new(); + + [Required] + public string TargetRuntimeIdentifier { get; set; } = string.Empty; + + [Required] + public ITaskItem[] Items { get; set; } = []; + + [Required] + public string RuntimeIdentifierGraphPath { get; set; } = string.Empty; + + [Output] + public ITaskItem[] SelectedItems { get; set; } = []; + + public void Cancel() => _cts.Cancel(); + + public override bool Execute() + { + using MSBuildLoggerProvider loggerProvider = new(Log); + ILoggerFactory msbuildLoggerFactory = new LoggerFactory(new[] { loggerProvider }); + ILogger logger = msbuildLoggerFactory.CreateLogger(); + var graph = NuGet.RuntimeModel.JsonRuntimeFormat.ReadRuntimeGraph(RuntimeIdentifierGraphPath); + + var selectedItems = new List(Items.Length); + foreach (var item in Items) + { + if (item.GetMetadata("RuntimeIdentifier") is string ridValue && + graph.AreCompatible(TargetRuntimeIdentifier, ridValue)) + { + selectedItems.Add(item); + } + } + + // TODO: log if no items were selected + // TODO: log telemetry.LogRidMismatch + + SelectedItems = selectedItems.ToArray(); + return true; + } + + +} diff --git a/src/Containers/Microsoft.NET.Build.Containers/Telemetry.cs b/src/Containers/Microsoft.NET.Build.Containers/Telemetry.cs index 009b93d7717c..ef86289fe46d 100644 --- a/src/Containers/Microsoft.NET.Build.Containers/Telemetry.cs +++ b/src/Containers/Microsoft.NET.Build.Containers/Telemetry.cs @@ -15,9 +15,9 @@ internal class Telemetry /// If the base image came from a local store of some kind, what kind of store was it? /// If the new image is being pushed to a remote registry, what kind of registry is it? /// If the new image is being stored in a local store of some kind, what kind of store is it? - private record class PublishTelemetryContext(RegistryType? RemotePullType, LocalStorageType? LocalPullType, RegistryType? RemotePushType, LocalStorageType? LocalPushType); - private enum RegistryType { Azure, AWS, Google, GitHub, DockerHub, MCR, Other } - private enum LocalStorageType { Docker, Podman, Tarball } + internal record class PublishTelemetryContext(RegistryType? RemotePullType, LocalStorageType? LocalPullType, RegistryType? RemotePushType, LocalStorageType? LocalPushType); + internal enum RegistryType { Azure, AWS, Google, GitHub, DockerHub, MCR, Other } + internal enum LocalStorageType { Docker, Podman, Tarball } private readonly Microsoft.Build.Utilities.TaskLoggingHelper Log; private readonly PublishTelemetryContext Context; @@ -25,9 +25,9 @@ private enum LocalStorageType { Docker, Podman, Tarball } internal Telemetry( SourceImageReference source, DestinationImageReference destination, - Microsoft.Build.Utilities.TaskLoggingHelper Log) + Microsoft.Build.Utilities.TaskLoggingHelper log) { - this.Log = Log; + Log = log; Context = new PublishTelemetryContext( source.Registry is not null ? GetRegistryType(source.Registry) : null, null, // we don't support local pull yet, but we may in the future @@ -35,7 +35,13 @@ internal Telemetry( destination.LocalRegistry is not null ? GetLocalStorageType(destination.LocalRegistry) : null); } - private RegistryType GetRegistryType(Registry r) + internal Telemetry(PublishTelemetryContext context, Microsoft.Build.Utilities.TaskLoggingHelper log) + { + Log = log; + Context = context; + } + + internal static RegistryType GetRegistryType(Registry r) { if (r.IsMcr) return RegistryType.MCR; if (r.IsGithubPackageRegistry) return RegistryType.GitHub; @@ -113,4 +119,4 @@ public void LogLocalLoadError() props.Add("error", "local_load"); Log.LogTelemetry("sdk/container/publish/error", props); } -} \ No newline at end of file +} diff --git a/src/Containers/containerize/ContainerizeCommand.cs b/src/Containers/containerize/ContainerizeCommand.cs index 8da04309a921..32a8d6e90771 100644 --- a/src/Containers/containerize/ContainerizeCommand.cs +++ b/src/Containers/containerize/ContainerizeCommand.cs @@ -12,10 +12,42 @@ namespace containerize; internal class ContainerizeCommand : RootCommand { - internal Argument PublishDirectoryArgument { get; } = new Argument("PublishDirectory") + // internal Argument PublishDirectoryArgument { get; } = new Argument("PublishDirectory") + // { + // Description = "The directory for the build outputs to be published." + // }.AcceptExistingOnly(); + + internal Option<(string absolutefilePath, string relativePath)[]> InputFilesOption { get; } = new("--input-file") { - Description = "The directory for the build outputs to be published." - }.AcceptExistingOnly(); + Description = "Specify once per file in the container, in the format '='", + Required = true, + Arity = ArgumentArity.OneOrMore, + CustomParser = result => + { + var maps = new List<(string absolutefilePath, string relativePath)>(result.Tokens.Count); + foreach (var token in result.Tokens) + { + var parts = token.Value.Split('=', StringSplitOptions.TrimEntries); + if (parts.Length != 2) + { + result.AddError($"Invalid input file format: '{token.Value}'. Expected format is '='."); + continue; + } + if (!Path.IsPathRooted(parts[0])) + { + result.AddError($"The absolute path '{parts[0]}' is not rooted. Please provide a valid absolute path."); + continue; + } + if (string.IsNullOrWhiteSpace(parts[1])) + { + result.AddError("The relative path inside the container working directory cannot be empty."); + continue; + } + maps.Add((absolutefilePath: parts[0], relativePath: parts[1])); + } + return maps.ToArray(); + } + }; internal Option BaseRegistryOption { get; } = new("--baseregistry") { @@ -185,10 +217,6 @@ internal class ContainerizeCommand : RootCommand AllowMultipleArgumentsPerToken = true }; - internal Option RidOption { get; } = new("--rid") { Description = "Runtime Identifier of the generated container." }; - - internal Option RidGraphPathOption { get; } = new("--ridgraphpath") { Description = "Path to the RID graph file." }; - internal Option ContainerUserOption { get; } = new("--container-user") { Description = "User to run the container as." }; internal Option GenerateLabelsOption { get; } = new("--generate-labels") @@ -208,10 +236,50 @@ internal class ContainerizeCommand : RootCommand Description = "If set to OCI or Docker will force the generated image to be that format. If unset, the base images format will be used." }; + internal Option ContentStoreRootOption { get; } = new("--content-store-root") + { + Description = "The path to the content store root. This is used to compute RID compatibility for Image Manifest List entries.", + Required = true + }; + + internal Option BaseImageManifestFileOption { get; } = new("--base-image-manifest") + { + Description = "The path to the local manifest of the base image, selected earlier on in the build process", + Required = true, + Arity = ArgumentArity.ExactlyOne + }; + + internal Option BaseImageConfigFileOption { get; } = new("--base-image-config") + { + Description = "The path to the local container configuration of the base image, selected earlier on in the build process", + Required = true, + Arity = ArgumentArity.ExactlyOne + }; + + internal Option GeneratedManifestPathOption { get; } = new("--generated-manifest") + { + Description = "The path to the generated manifest file.", + Required = true, + Arity = ArgumentArity.ExactlyOne + }; + + internal Option GeneratedConfigurationPathOption { get; } = new("--generated-configuration") + { + Description = "The path to the generated configuration file.", + Required = true, + Arity = ArgumentArity.ExactlyOne + }; + + internal Option GeneratedLayerPathOption { get; } = new("--generated-layer") + { + Description = "The path to the generated layer file.", + Required = true, + Arity = ArgumentArity.ExactlyOne + }; + internal ContainerizeCommand() : base("Containerize an application without Docker.") { - PublishDirectoryArgument.AcceptLegalFilePathsOnly(); - Arguments.Add(PublishDirectoryArgument); + Options.Add(InputFilesOption); Options.Add(BaseRegistryOption); Options.Add(BaseImageNameOption); Options.Add(BaseImageTagOption); @@ -230,18 +298,19 @@ internal ContainerizeCommand() : base("Containerize an application without Docke Options.Add(LabelsOption); Options.Add(PortsOption); Options.Add(EnvVarsOption); - Options.Add(RidOption); - Options.Add(RidGraphPathOption); LocalRegistryOption.AcceptOnlyFromAmong(KnownLocalRegistryTypes.SupportedLocalRegistryTypes); Options.Add(LocalRegistryOption); Options.Add(ContainerUserOption); Options.Add(GenerateLabelsOption); Options.Add(GenerateDigestLabelOption); Options.Add(ImageFormatOption); + Options.Add(ContentStoreRootOption); + Options.Add(BaseImageManifestFileOption); + Options.Add(BaseImageConfigFileOption); SetAction(async (parseResult, cancellationToken) => { - DirectoryInfo _publishDir = parseResult.GetValue(PublishDirectoryArgument)!; + var _inputFiles = parseResult.GetValue(InputFilesOption)!; string _baseReg = parseResult.GetValue(BaseRegistryOption)!; string _baseName = parseResult.GetValue(BaseImageNameOption)!; string _baseTag = parseResult.GetValue(BaseImageTagOption)!; @@ -260,13 +329,17 @@ internal ContainerizeCommand() : base("Containerize an application without Docke Dictionary _labels = parseResult.GetValue(LabelsOption) ?? new Dictionary(); Port[]? _ports = parseResult.GetValue(PortsOption); Dictionary _envVars = parseResult.GetValue(EnvVarsOption) ?? new Dictionary(); - string _rid = parseResult.GetValue(RidOption)!; - string _ridGraphPath = parseResult.GetValue(RidGraphPathOption)!; string _localContainerDaemon = parseResult.GetValue(LocalRegistryOption)!; string? _containerUser = parseResult.GetValue(ContainerUserOption); bool _generateLabels = parseResult.GetValue(GenerateLabelsOption); bool _generateDigestLabel = parseResult.GetValue(GenerateDigestLabelOption); KnownImageFormats? _imageFormat = parseResult.GetValue(ImageFormatOption); + string _contentStoreRoot = parseResult.GetValue(ContentStoreRootOption)!; + FileInfo _baseImageManifestFile = parseResult.GetValue(BaseImageManifestFileOption)!; + FileInfo _baseImageConfigFile = parseResult.GetValue(BaseImageConfigFileOption)!; + FileInfo _generatedManifestPath = parseResult.GetValue(GeneratedManifestPathOption)!; + FileInfo _generatedConfigPath = parseResult.GetValue(GeneratedConfigurationPathOption)!; + FileInfo _generatedLayerPath = parseResult.GetValue(GeneratedLayerPathOption)!; //setup basic logging bool traceEnabled = Env.GetEnvironmentVariableAsBool("CONTAINERIZE_TRACE_LOGGING_ENABLED"); @@ -274,7 +347,7 @@ internal ContainerizeCommand() : base("Containerize an application without Docke using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole(c => c.ColorBehavior = LoggerColorBehavior.Disabled).SetMinimumLevel(verbosity)); await ContainerBuilder.ContainerizeAsync( - _publishDir, + _inputFiles, _workingDir, _baseReg, _baseName, @@ -292,14 +365,18 @@ await ContainerBuilder.ContainerizeAsync( _labels, _ports, _envVars, - _rid, - _ridGraphPath, _localContainerDaemon, _containerUser, _archiveOutputPath, _generateLabels, _generateDigestLabel, _imageFormat, + _contentStoreRoot, + _baseImageManifestFile, + _baseImageConfigFile, + _generatedManifestPath, + _generatedConfigPath, + _generatedLayerPath, loggerFactory, cancellationToken).ConfigureAwait(false); }); diff --git a/src/Containers/packaging/build/Microsoft.NET.Build.Containers.props b/src/Containers/packaging/build/Microsoft.NET.Build.Containers.props index 82178233188c..1503622fdd10 100644 --- a/src/Containers/packaging/build/Microsoft.NET.Build.Containers.props +++ b/src/Containers/packaging/build/Microsoft.NET.Build.Containers.props @@ -14,8 +14,14 @@ - + + + + + - + + + diff --git a/src/Containers/packaging/build/Microsoft.NET.Build.Containers.targets b/src/Containers/packaging/build/Microsoft.NET.Build.Containers.targets index ae67c5c24fce..1d92d4e018c4 100644 --- a/src/Containers/packaging/build/Microsoft.NET.Build.Containers.targets +++ b/src/Containers/packaging/build/Microsoft.NET.Build.Containers.targets @@ -17,6 +17,7 @@ <_ContainerIsSelfContained Condition="'$(SelfContained)' == 'true' or '$(PublishSelfContained)' == 'true'">true true + $([System.IO.Path]::GetTempPath()) @@ -121,7 +122,8 @@ true - true + + false true true true @@ -183,10 +185,11 @@ _ContainerVerifySDKVersion; ComputeContainerConfig; _CheckContainersPackage; + _EnsureManifestsAvailable; - @@ -249,9 +252,116 @@ Text="The $(_ContainersPackageIdentity) NuGet package is explicitly referenced but the current SDK can natively publish the project as a container. Consider removing the package reference to $(_ContainersPackageIdentity) because it is no longer needed." /> + + + + <_ContainerBaseImageManifestDataCacheFile>$(IntermediateOutputPath)$(MSBuildProjectName).ContainerManifestData + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_ContainerGeneratedConfigTrackingFile>$(IntermediateOutputPath)$(MSBuildProjectName).GeneratedContainerConfig + + + + + + + + + + <_ContainerGeneratedManifestPath>$(IntermediateOutputPath)container.manifest.json + <_ContainerGeneratedConfigurationPath>$(IntermediateOutputPath)container.config.json + <_ContainerGeneratedLayerPath>$(IntermediateOutputPath)container.layer.tar.gz + + + + + + + <_ContainerPublishItemBase Include="$(PublishDir)/**/*.*" /> + + <_ContainerPublishItem Include="@(_ContainerPublishItemBase)"> + $([MSBuild]::MakeRelative($(PublishDir), %(_ContainerPublishItemBase.FullPath))) + + + + - + + $(NetCoreRoot) dotnet @@ -265,13 +375,15 @@ BaseImageName="$(ContainerBaseName)" BaseImageTag="$(ContainerBaseTag)" BaseImageDigest="$(ContainerBaseDigest)" + BaseImageManifestPath="@(_BaseContainerThisArchManifest)" + BaseImageConfigurationPath="@(_BaseContainerThisArchConfig)" ImageFormat="$(ContainerImageFormat)" LocalRegistry="$(LocalRegistry)" OutputRegistry="$(ContainerRegistry)" ArchiveOutputPath="$(ContainerArchiveOutputPath)" Repository="$(ContainerRepository)" ImageTags="@(ContainerImageTags)" - PublishDirectory="$(PublishDir)" + PublishFiles="@(_ContainerPublishItem)" WorkingDirectory="$(ContainerWorkingDirectory)" Entrypoint="@(ContainerEntrypoint)" EntrypointArgs="@(ContainerEntrypointArgs)" @@ -282,33 +394,100 @@ Labels="@(ContainerLabel)" ExposedPorts="@(ContainerPort)" ContainerEnvironmentVariables="@(ContainerEnvironmentVariables)" - ContainerRuntimeIdentifier="$(ContainerRuntimeIdentifier)" ContainerUser="$(ContainerUser)" - RuntimeIdentifierGraphPath="$(RuntimeIdentifierGraphPath)" - SkipPublishing="$(_SkipContainerPublishing)" GenerateLabels="$(ContainerGenerateLabels)" - GenerateDigestLabel="$(ContainerGenerateLabelsImageBaseDigest)"> + GenerateDigestLabel="$(ContainerGenerateLabelsImageBaseDigest)" + ContentStoreRoot="$(ContainerContentStoreRoot)" + GeneratedManifestPath="$(_ContainerGeneratedManifestPath)" + GeneratedConfigurationPath="$(_ContainerGeneratedConfigurationPath)" + GeneratedLayerPath="$(_ContainerGeneratedLayerPath)"> - - - - + + + + + $(RuntimeIdentifier) + + + $(RuntimeIdentifier) + + + $(RuntimeIdentifier) + + + + + - $(GeneratedContainerManifest) - $(GeneratedContainerConfiguration) - $(GeneratedContainerDigest) - $(GeneratedContainerMediaType) + $(_ContainerGeneratedManifestPath) + $(_ContainerGeneratedConfigurationPath) + $(_ContainerGeneratedLayerPath) - + + + <_ContainerSinglePublishTrackingFile>$(IntermediateOutputPath)container.SingleContainerPublish + + + + + + + + + + + + + + + + + + + + + + @@ -326,7 +505,9 @@ <_SingleImageContainerFormat Condition="$(_SkipContainerPublishing) == 'true' ">OCI + + <_rids Include="$(ContainerRuntimeIdentifiers)" Condition="'$(ContainerRuntimeIdentifiers)' != ''" /> <_rids Include="$(RuntimeIdentifiers)" Condition="'$(ContainerRuntimeIdentifiers)' == '' and '$(RuntimeIdentifiers)' != ''" /> @@ -347,7 +528,6 @@ _ContainerEnvironmentVariables=@(ContainerEnvironmentVariable->'%(Identity):%(Value)'); ContainerGenerateLabels=$(ContainerGenerateLabels); ContainerGenerateLabelsImageBaseDigest=$(ContainerGenerateLabelsImageBaseDigest); - _SkipContainerPublishing=$(_SkipContainerPublishing); ContainerImageFormat=$(_SingleImageContainerFormat); _IsMultiRIDBuild=false; _IsSingleRIDBuild=true; @@ -356,13 +536,35 @@ <_rids Remove ="$(_rids)" /> + + <_InnerContainerBuildTargets>Publish;_ParseItemsForPublishingSingleContainer;_ComputeContainerExecutionArgs; + <_InnerContainerBuildTargets Condition="'$(_SkipContainerPublishing)' == 'true'">$(_InnerContainerBuildTargets);_PublishSingleContainer + <_InnerContainerBuildTargets Condition="'$(_SkipContainerPublishing)' != 'true'">$(_InnerContainerBuildTargets);_CreateSingleContainer + + + + + + + <_MultiArchManifestInputs Include="@(GeneratedContainer->'%(ManifestPath)');@(GeneratedContainer->'%(ConfigurationPath)')" /> + + + <_MultiArchManifestFilePath>$(IntermediateOutputPath)multiarchcontainer.manifest + + + + + BaseImageDigest="$(ContainerBaseDigest)" + GeneratedManifestPath="$(_MultiArchManifestFilePath)"> + + + @@ -431,8 +637,8 @@ Condition="'$(IsPublishable)' == 'true' AND '$(EnableSdkContainerSupport)' == 'true'" Returns="@(GeneratedContainer)" > - - + + diff --git a/test/Microsoft.NET.Build.Containers.IntegrationTests/ArchiveFileRegistryTests.cs b/test/Microsoft.NET.Build.Containers.IntegrationTests/ArchiveFileRegistryTests.cs index e78fb6854005..afbbf10f72cf 100644 --- a/test/Microsoft.NET.Build.Containers.IntegrationTests/ArchiveFileRegistryTests.cs +++ b/test/Microsoft.NET.Build.Containers.IntegrationTests/ArchiveFileRegistryTests.cs @@ -14,9 +14,9 @@ public async Task ArchiveOutputPathIsExistingDirectory_CreatesFileWithRepository string expectedCreatedFilePath = Path.Combine(TestSettings.TestArtifactsDirectory, "repository.tar.gz"); await CreateRegistryAndCallLoadAsync(archiveOutputPath); - - Assert.True(File.Exists(expectedCreatedFilePath)); - } + + Assert.True(File.Exists(expectedCreatedFilePath)); + } [Theory] [InlineData(true)] @@ -29,8 +29,8 @@ public async Task ArchiveOutputPathIsNonExistingDirectory_CreatesDirectoryAndFil string expectedCreatedFilePath = Path.Combine(archiveOutputPath, "repository.tar.gz"); await CreateRegistryAndCallLoadAsync(archiveOutputPath); - - Assert.True(File.Exists(expectedCreatedFilePath)); + + Assert.True(File.Exists(expectedCreatedFilePath)); } [Fact] @@ -40,8 +40,8 @@ public async Task ArchiveOutputPathIsCustomFileNameInExistingDirectory_CreatesFi string expectedCreatedFilePath = archiveOutputPath; await CreateRegistryAndCallLoadAsync(archiveOutputPath); - - Assert.True(File.Exists(expectedCreatedFilePath)); + + Assert.True(File.Exists(expectedCreatedFilePath)); } [Fact] @@ -51,8 +51,8 @@ public async Task ArchiveOutputPathIsCustomFileNameInNonExistingDirectory_Create string expectedCreatedFilePath = archiveOutputPath; await CreateRegistryAndCallLoadAsync(archiveOutputPath); - - Assert.True(File.Exists(expectedCreatedFilePath)); + + Assert.True(File.Exists(expectedCreatedFilePath)); } private async Task CreateRegistryAndCallLoadAsync(string archiveOutputPath) @@ -65,9 +65,10 @@ await registry.LoadAsync( new SourceImageReference(), destinationImageReference, CancellationToken.None, - async (img, srcRef, destRef, stream, token) => + async ((string img, SourceImageReference srcRef, DestinationImageReference destRef) pushData, Stream stream, CancellationToken token) => + { await Task.CompletedTask; }); } -} \ No newline at end of file +} diff --git a/test/Microsoft.NET.Build.Containers.IntegrationTests/CreateImageIndexTests.cs b/test/Microsoft.NET.Build.Containers.IntegrationTests/CreateImageIndexTests.cs index 2615b2e178ee..5b390e97dfd2 100644 --- a/test/Microsoft.NET.Build.Containers.IntegrationTests/CreateImageIndexTests.cs +++ b/test/Microsoft.NET.Build.Containers.IntegrationTests/CreateImageIndexTests.cs @@ -84,6 +84,20 @@ private DirectoryInfo CreateNewProject() return newProjectDir; } + private static ITaskItem[] MakeItemsForPublishDir(string publishDir) + { + var files = Directory.GetFiles(publishDir, "*", new EnumerationOptions() + { + RecurseSubdirectories=true + }); + + return files.Select(f => new TaskItem(f, new Dictionary + { + ["RelativePath"] = Path.GetRelativePath(publishDir, f) + })).ToArray(); + } + + private TaskItem PublishAndCreateNewImage( string rid, string outputRegistry, @@ -108,13 +122,11 @@ private TaskItem PublishAndCreateNewImage( cni.OutputRegistry = outputRegistry; cni.LocalRegistry = DockerAvailableFactAttribute.LocalRegistry; - cni.PublishDirectory = Path.Combine(newProjectDir.FullName, "bin", "Release", ToolsetInfo.CurrentTargetFramework, rid, "publish"); + cni.PublishFiles = MakeItemsForPublishDir(Path.Combine(newProjectDir.FullName, "bin", "Release", ToolsetInfo.CurrentTargetFramework, rid, "publish")); cni.Repository = repository; cni.ImageTags = tags.Select(t => $"{t}-{rid}").ToArray(); cni.WorkingDirectory = "app/"; - cni.ContainerRuntimeIdentifier = rid; cni.Entrypoint = new TaskItem[] { new("dotnet"), new("build") }; - cni.RuntimeIdentifierGraphPath = ToolsetUtils.GetRuntimeGraphFilePath(); Assert.True(cni.Execute(), FormatBuildMessages(errors)); diff --git a/test/Microsoft.NET.Build.Containers.IntegrationTests/CreateNewImageTests.cs b/test/Microsoft.NET.Build.Containers.IntegrationTests/CreateNewImageTests.cs index b00de74ceaa0..8f8bc8a27bbf 100644 --- a/test/Microsoft.NET.Build.Containers.IntegrationTests/CreateNewImageTests.cs +++ b/test/Microsoft.NET.Build.Containers.IntegrationTests/CreateNewImageTests.cs @@ -53,18 +53,30 @@ public void CreateNewImage_Baseline() task.OutputRegistry = "localhost:5010"; task.LocalRegistry = DockerAvailableFactAttribute.LocalRegistry; - task.PublishDirectory = Path.Combine(newProjectDir.FullName, "bin", "Release", ToolsetInfo.CurrentTargetFramework, "linux-arm64", "publish"); + var baseDir = Path.Combine(newProjectDir.FullName, "bin", "Release"); + task.PublishFiles = MakeItemsForPublishDir(baseDir); task.Repository = "dotnet/create-new-image-baseline"; task.ImageTags = new[] { "latest" }; task.WorkingDirectory = "app/"; - task.ContainerRuntimeIdentifier = "linux-arm64"; task.Entrypoint = new TaskItem[] { new("dotnet"), new("build") }; - task.RuntimeIdentifierGraphPath = ToolsetUtils.GetRuntimeGraphFilePath(); - + Assert.True(task.Execute(), FormatBuildMessages(errors)); newProjectDir.Delete(true); } + private static ITaskItem[] MakeItemsForPublishDir(string publishDir) + { + var files = Directory.GetFiles(publishDir, "*", new EnumerationOptions() + { + RecurseSubdirectories=true + }); + + return files.Select(f => new TaskItem(f, new Dictionary + { + ["RelativePath"] = Path.GetRelativePath(publishDir, f) + })).ToArray(); + } + private static ImageConfig GetImageConfigFromTask(CreateNewImage task) { return new(task.GeneratedContainerConfiguration); @@ -119,12 +131,10 @@ public void ParseContainerProperties_EndToEnd() cni.BaseImageTag = pcp.ParsedContainerTag; cni.Repository = pcp.NewContainerRepository; cni.OutputRegistry = "localhost:5010"; - cni.PublishDirectory = Path.Combine(newProjectDir.FullName, "bin", "release", ToolsetInfo.CurrentTargetFramework); + cni.PublishFiles = MakeItemsForPublishDir(Path.Combine(newProjectDir.FullName, "bin", "release", ToolsetInfo.CurrentTargetFramework)); cni.WorkingDirectory = "app/"; cni.Entrypoint = new TaskItem[] { new(newProjectDir.Name) }; cni.ImageTags = pcp.NewContainerTags; - cni.ContainerRuntimeIdentifier = "linux-x64"; - cni.RuntimeIdentifierGraphPath = ToolsetUtils.GetRuntimeGraphFilePath(); Assert.True(cni.Execute(), FormatBuildMessages(errors)); newProjectDir.Delete(true); @@ -193,13 +203,11 @@ public void Tasks_EndToEnd_With_EnvironmentVariable_Validation() cni.BaseImageTag = pcp.ParsedContainerTag; cni.Repository = pcp.NewContainerRepository; cni.OutputRegistry = pcp.NewContainerRegistry; - cni.PublishDirectory = Path.Combine(newProjectDir.FullName, "bin", "release", EndToEndTests._oldFramework, "linux-x64"); + cni.PublishFiles = MakeItemsForPublishDir(Path.Combine(newProjectDir.FullName, "bin", "release", EndToEndTests._oldFramework, "linux-x64")); cni.WorkingDirectory = "/app"; cni.Entrypoint = new TaskItem[] { new($"/app/{newProjectDir.Name}") }; cni.ImageTags = pcp.NewContainerTags; cni.ContainerEnvironmentVariables = pcp.NewContainerEnvironmentVariables; - cni.ContainerRuntimeIdentifier = "linux-x64"; - cni.RuntimeIdentifierGraphPath = ToolsetUtils.GetRuntimeGraphFilePath(); cni.LocalRegistry = DockerAvailableFactAttribute.LocalRegistry; Assert.True(cni.Execute(), FormatBuildMessages(errors)); @@ -276,13 +284,11 @@ public async System.Threading.Tasks.Task CreateNewImage_RootlessBaseImage() task.BaseImageTag = "latest"; task.OutputRegistry = "localhost:5010"; - task.PublishDirectory = Path.Combine(newProjectDir.FullName, "bin", "Release", ToolsetInfo.CurrentTargetFramework, "linux-x64", "publish"); + task.PublishFiles = MakeItemsForPublishDir(Path.Combine(newProjectDir.FullName, "bin", "Release", ToolsetInfo.CurrentTargetFramework, "linux-x64", "publish")); task.Repository = AppImage; task.ImageTags = new[] { "latest" }; task.WorkingDirectory = "app/"; - task.ContainerRuntimeIdentifier = "linux-x64"; task.Entrypoint = new TaskItem[] { new("dotnet"), new("build") }; - task.RuntimeIdentifierGraphPath = ToolsetUtils.GetRuntimeGraphFilePath(); Assert.True(task.Execute()); newProjectDir.Delete(true); @@ -348,12 +354,10 @@ public void CanOverrideContainerImageFormat() cni.BaseImageTag = pcp.ParsedContainerTag; cni.Repository = pcp.NewContainerRepository; cni.OutputRegistry = "localhost:5010"; - cni.PublishDirectory = Path.Combine(newProjectDir.FullName, "bin", "release", ToolsetInfo.CurrentTargetFramework); + cni.PublishFiles = MakeItemsForPublishDir(Path.Combine(newProjectDir.FullName, "bin", "release", ToolsetInfo.CurrentTargetFramework)); cni.WorkingDirectory = "app/"; cni.Entrypoint = new TaskItem[] { new(newProjectDir.Name) }; cni.ImageTags = pcp.NewContainerTags; - cni.ContainerRuntimeIdentifier = "linux-x64"; - cni.RuntimeIdentifierGraphPath = ToolsetUtils.GetRuntimeGraphFilePath(); cni.ImageFormat = KnownImageFormats.OCI.ToString(); diff --git a/test/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs b/test/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs index 7aefe1ca7791..bc6d658a9787 100644 --- a/test/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs +++ b/test/Microsoft.NET.Build.Containers.IntegrationTests/EndToEndTests.cs @@ -17,11 +17,13 @@ public class EndToEndTests : IDisposable { private ITestOutputHelper _testOutput; private readonly TestLoggerFactory _loggerFactory; + private readonly ContentStore _store; public EndToEndTests(ITestOutputHelper testOutput) { _testOutput = testOutput; _loggerFactory = new TestLoggerFactory(testOutput); + _store = new ContentStore(new(Path.GetTempPath())); } public static string NewImageName([CallerMemberName] string callerMemberName = "") @@ -53,6 +55,17 @@ internal static void ChangeTargetFrameworkAfterAppCreation(string path) File.WriteAllText(Path.Combine(path, csprojFilename), text); } + private static (string absolutefilePath, string relativePath)[] MakeItemsForPublishDir(string publishDir) + { + var files = Directory.GetFiles(publishDir, "*", new EnumerationOptions() + { + RecurseSubdirectories = true + }); + + return files.Select(f => new(f, Path.GetRelativePath(publishDir, f))).ToArray(); + } + + [DockerAvailableFact] public async Task ApiEndToEndWithRegistryPushAndPull() { @@ -72,7 +85,9 @@ public async Task ApiEndToEndWithRegistryPushAndPull() Assert.NotNull(imageBuilder); - Layer l = Layer.FromDirectory(publishDirectory, "/app", false, imageBuilder.ManifestMediaType); + var layerFilePath = new FileInfo(_store.GetTempFile()); + + Layer l = await Layer.FromFiles(MakeItemsForPublishDir(publishDirectory), "/app", false, imageBuilder.ManifestMediaType, _store, layerFilePath, CancellationToken.None); imageBuilder.AddLayer(l); @@ -118,7 +133,9 @@ public async Task ApiEndToEndWithLocalLoad() cancellationToken: default).ConfigureAwait(false); Assert.NotNull(imageBuilder); - Layer l = Layer.FromDirectory(publishDirectory, "/app", false, imageBuilder.ManifestMediaType); + var layerFilePath = new FileInfo(_store.GetTempFile()); + + Layer l = await Layer.FromFiles(MakeItemsForPublishDir(publishDirectory), "/app", false, imageBuilder.ManifestMediaType, _store, layerFilePath, CancellationToken.None); imageBuilder.AddLayer(l); @@ -158,8 +175,9 @@ public async Task ApiEndToEndWithArchiveWritingAndLoad() ToolsetUtils.RidGraphManifestPicker, cancellationToken: default).ConfigureAwait(false); Assert.NotNull(imageBuilder); + var layerFilePath = new FileInfo(_store.GetTempFile()); - Layer l = Layer.FromDirectory(publishDirectory, "/app", false, imageBuilder.ManifestMediaType); + Layer l = await Layer.FromFiles(MakeItemsForPublishDir(publishDirectory), "/app", false, imageBuilder.ManifestMediaType, _store, layerFilePath, CancellationToken.None); imageBuilder.AddLayer(l); @@ -245,11 +263,14 @@ private BuiltImage ConvertToOciImage(BuiltImage builtImage) var ociImage = new BuiltImage { Config = builtImage.Config, - ImageDigest = builtImage.ImageDigest, - ImageSha = builtImage.ImageSha, - Manifest = builtImage.Manifest, - ManifestDigest = builtImage.ManifestDigest, - ManifestMediaType = SchemaTypes.OciManifestV1, + Manifest = new() + { + Config = builtImage.Manifest.Config, + Layers = builtImage.Manifest.Layers, + SchemaVersion = builtImage.Manifest.SchemaVersion, + MediaType = SchemaTypes.OciManifestV1, + KnownDigest = builtImage.Manifest.KnownDigest, + }, Layers = builtImage.Layers }; @@ -1395,8 +1416,9 @@ public async Task CanPackageForAllSupportedContainerRIDs(string dockerPlatform, ToolsetUtils.RidGraphManifestPicker, cancellationToken: default).ConfigureAwait(false); Assert.NotNull(imageBuilder); + var layerFilePath = new FileInfo(_store.GetTempFile()); - Layer l = Layer.FromDirectory(publishDirectory, isWin ? "C:\\app" : "/app", isWin, imageBuilder.ManifestMediaType); + Layer l = await Layer.FromFiles(MakeItemsForPublishDir(publishDirectory), isWin ? "C:\\app" : "/app", isWin, imageBuilder.ManifestMediaType, _store, layerFilePath, CancellationToken.None); imageBuilder.AddLayer(l); imageBuilder.SetWorkingDirectory(workingDir); @@ -1446,8 +1468,9 @@ public async Task CheckErrorMessageWhenSourceRepositoryThrows() ToolsetUtils.RidGraphManifestPicker, cancellationToken: default).ConfigureAwait(false); Assert.NotNull(imageBuilder); + var layerFilePath = new FileInfo(_store.GetTempFile()); - Layer l = Layer.FromDirectory(publishDirectory, "C:\\app", true, imageBuilder.ManifestMediaType); + Layer l = await Layer.FromFiles(MakeItemsForPublishDir(publishDirectory), "C:\\app", true, imageBuilder.ManifestMediaType, _store, layerFilePath, CancellationToken.None); imageBuilder.AddLayer(l); imageBuilder.SetWorkingDirectory("C:\\app"); diff --git a/test/Microsoft.NET.Build.Containers.IntegrationTests/FullFramework/CreateNewImageToolTaskTests.cs b/test/Microsoft.NET.Build.Containers.IntegrationTests/FullFramework/CreateNewImageToolTaskTests.cs index cfffd3a8ca47..9ea5733a5827 100644 --- a/test/Microsoft.NET.Build.Containers.IntegrationTests/FullFramework/CreateNewImageToolTaskTests.cs +++ b/test/Microsoft.NET.Build.Containers.IntegrationTests/FullFramework/CreateNewImageToolTaskTests.cs @@ -1,601 +1,601 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. +// // Licensed to the .NET Foundation under one or more agreements. +// // The .NET Foundation licenses this file to you under the MIT license. -using FakeItEasy; -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using Microsoft.NET.Build.Containers.Tasks; - -namespace Microsoft.NET.Build.Containers.IntegrationTests.FullFramework; - -public class CreateNewImageToolTaskTests -{ - private ITestOutputHelper _testOutput; - - public CreateNewImageToolTaskTests(ITestOutputHelper testOutput) - { - _testOutput = testOutput; - } - - [Fact] - public void GenerateCommandLineCommands_ThrowsWhenRequiredPropertiesNotSet() - { - CreateNewImage task = new(); - - Exception e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); - Assert.Equal("CONTAINER4001: Required property 'PublishDirectory' was not set or empty.", e.Message); - - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - - task.PublishDirectory = publishDir.FullName; - - e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); - Assert.Equal("CONTAINER4001: Required property 'BaseRegistry' was not set or empty.", e.Message); - - task.BaseRegistry = "MyBaseRegistry"; - - e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); - Assert.Equal("CONTAINER4001: Required property 'BaseImageName' was not set or empty.", e.Message); - - task.BaseImageName = "MyBaseImageName"; - - e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); - Assert.Equal("CONTAINER4001: Required property 'Repository' was not set or empty.", e.Message); - - task.Repository = "MyImageName"; - - e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); - Assert.Equal("CONTAINER4001: Required property 'WorkingDirectory' was not set or empty.", e.Message); - - task.WorkingDirectory = "MyWorkingDirectory"; - - string args = task.GenerateCommandLineCommandsInt(); - string workDir = GetPathToContainerize(); - - new DotnetCommand(_testOutput, args) - .WithRawArguments() - .WithWorkingDirectory(workDir) - .Execute().Should().Fail() - .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error - - } - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - [InlineData("ValidTag", true)] - public void GenerateCommandLineCommands_BaseImageTag(string? value, bool optionExpected = false) - { - CreateNewImage task = new(); - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - if (value != null) - { - task.BaseImageTag = value; - } - - string args = task.GenerateCommandLineCommandsInt(); - - if (optionExpected) - { - Assert.Contains($"--baseimagetag {value}", args); - } - else - { - Assert.DoesNotContain("--baseimagetag", args); - } - } - - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - [InlineData("Valid", true)] - public void GenerateCommandLineCommands_OutputRegistry(string? value, bool optionExpected = false) - { - CreateNewImage task = new(); - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - if (value != null) - { - task.OutputRegistry = value; - } - - string args = task.GenerateCommandLineCommandsInt(); - - if (optionExpected) - { - Assert.Contains($"--outputregistry {value}", args); - } - else - { - Assert.DoesNotContain("--outputregistry", args); - } - } - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - [InlineData("Valid", true)] - public void GenerateCommandLineCommands_ContainerRuntimeIdentifier(string? value, bool optionExpected = false) - { - CreateNewImage task = new(); - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - if (value != null) - { - task.ContainerRuntimeIdentifier = value; - } - - string args = task.GenerateCommandLineCommandsInt(); - if (optionExpected) - { - Assert.Contains($"--rid {value}", args); - } - else - { - Assert.DoesNotContain("--rid", args); - } - } - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - [InlineData("Valid", true)] - public void GenerateCommandLineCommands_RuntimeIdentifierGraphPath(string? value, bool optionExpected = false) - { - CreateNewImage task = new(); - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - if (value != null) - { - task.RuntimeIdentifierGraphPath = value; - } - - string args = task.GenerateCommandLineCommandsInt(); - - if (optionExpected) - { - Assert.Contains($"--ridgraphpath {value}", args); - } - else - { - Assert.DoesNotContain("--ridgraphpath", args); - } - } - - [Fact] - public void GenerateCommandLineCommands_Labels() - { - CreateNewImage task = new(); - - List warnings = new(); - IBuildEngine buildEngine = A.Fake(); - A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); - - task.BuildEngine = buildEngine; - - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - task.Labels = new[] - { - new TaskItem("NoValue"), - new TaskItem(" "), - new TaskItem("Valid1", new Dictionary() {{ "Value", "Val1" }}), - new TaskItem("Valid12", new Dictionary() {{ "Value", "Val2" }}), - new TaskItem("Valid12", new Dictionary() {{ "Value", "" }}), - new TaskItem("Valid3", new Dictionary() {{ "Value", "has space" }}), - new TaskItem("Valid4", new Dictionary() {{ "Value", "has\"quotes\"" }}) - }; - - string args = task.GenerateCommandLineCommandsInt(); - - Assert.Contains(""" - --labels NoValue= Valid1=Val1 Valid12=Val2 Valid12= "Valid3=has space" "Valid4=has\"quotes\"" - """, args); - Assert.Equal("Items 'Labels' contain empty item(s) which will be ignored.", Assert.Single(warnings)); - - string workDir = GetPathToContainerize(); - - new DotnetCommand(_testOutput, args) - .WithRawArguments() - .WithWorkingDirectory(workDir) - .Execute().Should().Fail() - .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error - } - - [Fact] - public void GenerateCommandLineCommands_ContainerEnvironmentVariables() - { - CreateNewImage task = new(); - - List warnings = new(); - IBuildEngine buildEngine = A.Fake(); - A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); - - task.BuildEngine = buildEngine; - - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - task.ContainerEnvironmentVariables = new[] - { - new TaskItem("NoValue"), - new TaskItem(" "), - new TaskItem("Valid1", new Dictionary() {{ "Value", "Val1" }}), - new TaskItem("Valid12", new Dictionary() {{ "Value", "Val2" }}), - new TaskItem("Valid12", new Dictionary() {{ "Value", "" }}), - new TaskItem("Valid3", new Dictionary() {{ "Value", "has space" }}), - new TaskItem("Valid4", new Dictionary() {{ "Value", "has\"quotes\"" }}) - }; - - string args = task.GenerateCommandLineCommandsInt(); - - Assert.Contains(""" - --environmentvariables NoValue= Valid1=Val1 Valid12=Val2 Valid12= "Valid3=has space" "Valid4=has\"quotes\"" - """, args); - Assert.Equal("Items 'ContainerEnvironmentVariables' contain empty item(s) which will be ignored.", Assert.Single(warnings)); - - string workDir = GetPathToContainerize(); - - new DotnetCommand(_testOutput, args) - .WithRawArguments() - .WithWorkingDirectory(workDir) - .Execute().Should().Fail() - .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error - } - - [InlineData(nameof(CreateNewImage.Entrypoint), "entrypoint")] - [InlineData(nameof(CreateNewImage.EntrypointArgs), "entrypointargs", true)] - [InlineData(nameof(CreateNewImage.DefaultArgs), "defaultargs", true)] - [InlineData(nameof(CreateNewImage.AppCommand), "appcommand", true)] - [InlineData(nameof(CreateNewImage.AppCommandArgs), "appcommandargs", true)] - [Theory] - public void GenerateCommandLineCommands_EntryPointAndCommand(string propertyName, string commandArgName, bool warningExpected = false) - { - CreateNewImage task = new(); - - List warnings = new(); - IBuildEngine buildEngine = A.Fake(); - A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); - - task.BuildEngine = buildEngine; - - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - switch (propertyName) - { - case nameof(CreateNewImage.Entrypoint): - task.Entrypoint = new[] - { - new TaskItem("Valid1"), - new TaskItem("Valid2"), - new TaskItem("Quoted item") - }; - break; - case nameof(CreateNewImage.EntrypointArgs): - task.EntrypointArgs = new[] - { - new TaskItem(""), - new TaskItem(" "), - new TaskItem("Valid1"), - new TaskItem("Valid2"), - new TaskItem("Quoted item") - }; - break; - case nameof(CreateNewImage.DefaultArgs): - task.DefaultArgs = new[] - { - new TaskItem(""), - new TaskItem(" "), - new TaskItem("Valid1"), - new TaskItem("Valid2"), - new TaskItem("Quoted item") - }; - break; - case nameof(CreateNewImage.AppCommand): - task.AppCommand = new[] - { - new TaskItem(""), - new TaskItem(" "), - new TaskItem("Valid1"), - new TaskItem("Valid2"), - new TaskItem("Quoted item") - }; - break; - case nameof(CreateNewImage.AppCommandArgs): - task.AppCommandArgs = new[] - { - new TaskItem(""), - new TaskItem(" "), - new TaskItem("Valid1"), - new TaskItem("Valid2"), - new TaskItem("Quoted item") - }; - break; - } - - string args = task.GenerateCommandLineCommandsInt(); - - Assert.Contains($""" - --{commandArgName} Valid1 Valid2 "Quoted item" - """, args); - - if (warningExpected) - { - Assert.Equal($"Items '{propertyName}' contain empty item(s) which will be ignored.", Assert.Single(warnings)); - } - - string workDir = GetPathToContainerize(); - - new DotnetCommand(_testOutput, args) - .WithRawArguments() - .WithWorkingDirectory(workDir) - .Execute().Should().Fail() - .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error - } - - [InlineData("")] - [InlineData(" ")] - [Theory] - public void GenerateCommandLineCommands_EntryPointCanHaveEmptyItems(string itemValue) - { - CreateNewImage task = new(); - IBuildEngine buildEngine = A.Fake(); - - task.BuildEngine = buildEngine; - - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem(itemValue) }; - - task.GenerateCommandLineCommandsInt(); - } - - [Theory] - [InlineData(null)] - [InlineData("")] - [InlineData(" ")] - [InlineData("Valid", true)] - public void GenerateCommandLineCommands_AppCommandInstruction(string? value, bool optionExpected = false) - { - CreateNewImage task = new(); - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - - if (value != null) - { - task.AppCommandInstruction = value; - } - - string args = task.GenerateCommandLineCommandsInt(); - - if (optionExpected) - { - Assert.Contains($"--appcommandinstruction {value}", args); - } - else - { - Assert.DoesNotContain("--appcommandinstruction", args); - } - } - - [Fact] - public void GenerateCommandLineCommands_ImageTags() - { - CreateNewImage task = new(); - - List warnings = new(); - IBuildEngine buildEngine = A.Fake(); - A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); - - task.BuildEngine = buildEngine; - - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - task.ImageTags = new[] { "", " ", "Valid1", "To be quoted" }; - - string args = task.GenerateCommandLineCommandsInt(); - - Assert.Contains(""" - --imagetags Valid1 "To be quoted" - """, actualString: args); - Assert.Equal("Property 'ImageTags' is empty or contains whitespace and will be ignored.", Assert.Single(warnings)); - - string workDir = GetPathToContainerize(); - - new DotnetCommand(_testOutput, args) - .WithRawArguments() - .WithWorkingDirectory(workDir) - .Execute().Should().Fail() - .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error - } - - [Fact] - public void GenerateCommandLineCommands_ExposedPorts() - { - CreateNewImage task = new(); - - List warnings = new(); - IBuildEngine buildEngine = A.Fake(); - A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); - - task.BuildEngine = buildEngine; - - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - task.ExposedPorts = new[] - { - new TaskItem("1500"), - new TaskItem(" "), - new TaskItem("1501", new Dictionary() {{ "Type", "udp" }}), - new TaskItem("1501", new Dictionary() {{ "Type", "tcp" }}), - new TaskItem("1502", new Dictionary() {{ "Type", "tcp" }}), - new TaskItem("1503", new Dictionary() {{ "Type", "" }}) - }; - - string args = task.GenerateCommandLineCommandsInt(); - - Assert.Contains(""" - --ports 1500 1501/udp 1501/tcp 1502/tcp 1503 - """, args); - Assert.Equal("Items 'ExposedPorts' contain empty item(s) which will be ignored.", Assert.Single(warnings)); - - string workDir = GetPathToContainerize(); - - new DotnetCommand(_testOutput, args) - .WithRawArguments() - .WithWorkingDirectory(workDir) - .Execute().Should().Fail() - .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error - } - - [Fact] - public void Logging_CanEnableTraceLogging() - { - CreateNewImage task = new(); - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("") }; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - string args = task.GenerateCommandLineCommandsInt(); - string workDir = GetPathToContainerize(); - - new DotnetCommand(_testOutput, args) - .WithRawArguments() - .WithWorkingDirectory(workDir) - .WithEnvironmentVariable("CONTAINERIZE_TRACE_LOGGING_ENABLED", "1") - .Execute().Should().Fail() - .And.NotHaveStdOutContaining("Description:") //standard help output for parse error - .And.HaveStdOutContaining("Trace logging: enabled."); - } - - [Fact] - public void Logging_TraceLoggingIsDisabledByDefault() - { - CreateNewImage task = new(); - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("") }; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - - string args = task.GenerateCommandLineCommandsInt(); - string workDir = GetPathToContainerize(); - - new DotnetCommand(_testOutput, args) - .WithRawArguments() - .WithWorkingDirectory(workDir) - .Execute().Should().Fail() - .And.NotHaveStdOutContaining("Description:") //standard help output for parse error - .And.NotHaveStdOutContaining("Trace logging: enabled."); - } - - [Fact] - public void GenerateCommandLineCommands_LabelGeneration() - { - CreateNewImage task = new(); - - List warnings = new(); - IBuildEngine buildEngine = A.Fake(); - - task.BuildEngine = buildEngine; - - DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); - task.PublishDirectory = publishDir.FullName; - task.BaseRegistry = "MyBaseRegistry"; - task.BaseImageName = "MyBaseImageName"; - task.Repository = "MyImageName"; - task.WorkingDirectory = "MyWorkingDirectory"; - task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; - task.GenerateLabels = true; - task.GenerateDigestLabel = true; - - string args = task.GenerateCommandLineCommandsInt(); - - Assert.Contains("--generate-labels", args); - Assert.Contains("--generate-digest-label", args); - } - - - - private static string GetPathToContainerize() - { - return Path.Combine(TestContext.Current.TestExecutionDirectory, "Container", "containerize"); - } -} +// using FakeItEasy; +// using Microsoft.Build.Framework; +// using Microsoft.Build.Utilities; +// using Microsoft.NET.Build.Containers.Tasks; + +// namespace Microsoft.NET.Build.Containers.IntegrationTests.FullFramework; + +// public class CreateNewImageToolTaskTests +// { +// private ITestOutputHelper _testOutput; + +// public CreateNewImageToolTaskTests(ITestOutputHelper testOutput) +// { +// _testOutput = testOutput; +// } + +// [Fact] +// public void GenerateCommandLineCommands_ThrowsWhenRequiredPropertiesNotSet() +// { +// CreateNewImage task = new(); + +// Exception e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); +// Assert.Equal("CONTAINER4001: Required property 'PublishDirectory' was not set or empty.", e.Message); + +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); + +// task.PublishDirectory = publishDir.FullName; + +// e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); +// Assert.Equal("CONTAINER4001: Required property 'BaseRegistry' was not set or empty.", e.Message); + +// task.BaseRegistry = "MyBaseRegistry"; + +// e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); +// Assert.Equal("CONTAINER4001: Required property 'BaseImageName' was not set or empty.", e.Message); + +// task.BaseImageName = "MyBaseImageName"; + +// e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); +// Assert.Equal("CONTAINER4001: Required property 'Repository' was not set or empty.", e.Message); + +// task.Repository = "MyImageName"; + +// e = Assert.Throws(() => task.GenerateCommandLineCommandsInt()); +// Assert.Equal("CONTAINER4001: Required property 'WorkingDirectory' was not set or empty.", e.Message); + +// task.WorkingDirectory = "MyWorkingDirectory"; + +// string args = task.GenerateCommandLineCommandsInt(); +// string workDir = GetPathToContainerize(); + +// new DotnetCommand(_testOutput, args) +// .WithRawArguments() +// .WithWorkingDirectory(workDir) +// .Execute().Should().Fail() +// .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error + +// } + +// [Theory] +// [InlineData(null)] +// [InlineData("")] +// [InlineData(" ")] +// [InlineData("ValidTag", true)] +// public void GenerateCommandLineCommands_BaseImageTag(string? value, bool optionExpected = false) +// { +// CreateNewImage task = new(); +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// if (value != null) +// { +// task.BaseImageTag = value; +// } + +// string args = task.GenerateCommandLineCommandsInt(); + +// if (optionExpected) +// { +// Assert.Contains($"--baseimagetag {value}", args); +// } +// else +// { +// Assert.DoesNotContain("--baseimagetag", args); +// } +// } + + +// [Theory] +// [InlineData(null)] +// [InlineData("")] +// [InlineData(" ")] +// [InlineData("Valid", true)] +// public void GenerateCommandLineCommands_OutputRegistry(string? value, bool optionExpected = false) +// { +// CreateNewImage task = new(); +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// if (value != null) +// { +// task.OutputRegistry = value; +// } + +// string args = task.GenerateCommandLineCommandsInt(); + +// if (optionExpected) +// { +// Assert.Contains($"--outputregistry {value}", args); +// } +// else +// { +// Assert.DoesNotContain("--outputregistry", args); +// } +// } + +// [Theory] +// [InlineData(null)] +// [InlineData("")] +// [InlineData(" ")] +// [InlineData("Valid", true)] +// public void GenerateCommandLineCommands_ContainerRuntimeIdentifier(string? value, bool optionExpected = false) +// { +// CreateNewImage task = new(); +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// if (value != null) +// { +// task.ContainerRuntimeIdentifier = value; +// } + +// string args = task.GenerateCommandLineCommandsInt(); +// if (optionExpected) +// { +// Assert.Contains($"--rid {value}", args); +// } +// else +// { +// Assert.DoesNotContain("--rid", args); +// } +// } + +// [Theory] +// [InlineData(null)] +// [InlineData("")] +// [InlineData(" ")] +// [InlineData("Valid", true)] +// public void GenerateCommandLineCommands_RuntimeIdentifierGraphPath(string? value, bool optionExpected = false) +// { +// CreateNewImage task = new(); +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// if (value != null) +// { +// task.RuntimeIdentifierGraphPath = value; +// } + +// string args = task.GenerateCommandLineCommandsInt(); + +// if (optionExpected) +// { +// Assert.Contains($"--ridgraphpath {value}", args); +// } +// else +// { +// Assert.DoesNotContain("--ridgraphpath", args); +// } +// } + +// [Fact] +// public void GenerateCommandLineCommands_Labels() +// { +// CreateNewImage task = new(); + +// List warnings = new(); +// IBuildEngine buildEngine = A.Fake(); +// A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); + +// task.BuildEngine = buildEngine; + +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// task.Labels = new[] +// { +// new TaskItem("NoValue"), +// new TaskItem(" "), +// new TaskItem("Valid1", new Dictionary() {{ "Value", "Val1" }}), +// new TaskItem("Valid12", new Dictionary() {{ "Value", "Val2" }}), +// new TaskItem("Valid12", new Dictionary() {{ "Value", "" }}), +// new TaskItem("Valid3", new Dictionary() {{ "Value", "has space" }}), +// new TaskItem("Valid4", new Dictionary() {{ "Value", "has\"quotes\"" }}) +// }; + +// string args = task.GenerateCommandLineCommandsInt(); + +// Assert.Contains(""" +// --labels NoValue= Valid1=Val1 Valid12=Val2 Valid12= "Valid3=has space" "Valid4=has\"quotes\"" +// """, args); +// Assert.Equal("Items 'Labels' contain empty item(s) which will be ignored.", Assert.Single(warnings)); + +// string workDir = GetPathToContainerize(); + +// new DotnetCommand(_testOutput, args) +// .WithRawArguments() +// .WithWorkingDirectory(workDir) +// .Execute().Should().Fail() +// .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error +// } + +// [Fact] +// public void GenerateCommandLineCommands_ContainerEnvironmentVariables() +// { +// CreateNewImage task = new(); + +// List warnings = new(); +// IBuildEngine buildEngine = A.Fake(); +// A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); + +// task.BuildEngine = buildEngine; + +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// task.ContainerEnvironmentVariables = new[] +// { +// new TaskItem("NoValue"), +// new TaskItem(" "), +// new TaskItem("Valid1", new Dictionary() {{ "Value", "Val1" }}), +// new TaskItem("Valid12", new Dictionary() {{ "Value", "Val2" }}), +// new TaskItem("Valid12", new Dictionary() {{ "Value", "" }}), +// new TaskItem("Valid3", new Dictionary() {{ "Value", "has space" }}), +// new TaskItem("Valid4", new Dictionary() {{ "Value", "has\"quotes\"" }}) +// }; + +// string args = task.GenerateCommandLineCommandsInt(); + +// Assert.Contains(""" +// --environmentvariables NoValue= Valid1=Val1 Valid12=Val2 Valid12= "Valid3=has space" "Valid4=has\"quotes\"" +// """, args); +// Assert.Equal("Items 'ContainerEnvironmentVariables' contain empty item(s) which will be ignored.", Assert.Single(warnings)); + +// string workDir = GetPathToContainerize(); + +// new DotnetCommand(_testOutput, args) +// .WithRawArguments() +// .WithWorkingDirectory(workDir) +// .Execute().Should().Fail() +// .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error +// } + +// [InlineData(nameof(CreateNewImage.Entrypoint), "entrypoint")] +// [InlineData(nameof(CreateNewImage.EntrypointArgs), "entrypointargs", true)] +// [InlineData(nameof(CreateNewImage.DefaultArgs), "defaultargs", true)] +// [InlineData(nameof(CreateNewImage.AppCommand), "appcommand", true)] +// [InlineData(nameof(CreateNewImage.AppCommandArgs), "appcommandargs", true)] +// [Theory] +// public void GenerateCommandLineCommands_EntryPointAndCommand(string propertyName, string commandArgName, bool warningExpected = false) +// { +// CreateNewImage task = new(); + +// List warnings = new(); +// IBuildEngine buildEngine = A.Fake(); +// A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); + +// task.BuildEngine = buildEngine; + +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// switch (propertyName) +// { +// case nameof(CreateNewImage.Entrypoint): +// task.Entrypoint = new[] +// { +// new TaskItem("Valid1"), +// new TaskItem("Valid2"), +// new TaskItem("Quoted item") +// }; +// break; +// case nameof(CreateNewImage.EntrypointArgs): +// task.EntrypointArgs = new[] +// { +// new TaskItem(""), +// new TaskItem(" "), +// new TaskItem("Valid1"), +// new TaskItem("Valid2"), +// new TaskItem("Quoted item") +// }; +// break; +// case nameof(CreateNewImage.DefaultArgs): +// task.DefaultArgs = new[] +// { +// new TaskItem(""), +// new TaskItem(" "), +// new TaskItem("Valid1"), +// new TaskItem("Valid2"), +// new TaskItem("Quoted item") +// }; +// break; +// case nameof(CreateNewImage.AppCommand): +// task.AppCommand = new[] +// { +// new TaskItem(""), +// new TaskItem(" "), +// new TaskItem("Valid1"), +// new TaskItem("Valid2"), +// new TaskItem("Quoted item") +// }; +// break; +// case nameof(CreateNewImage.AppCommandArgs): +// task.AppCommandArgs = new[] +// { +// new TaskItem(""), +// new TaskItem(" "), +// new TaskItem("Valid1"), +// new TaskItem("Valid2"), +// new TaskItem("Quoted item") +// }; +// break; +// } + +// string args = task.GenerateCommandLineCommandsInt(); + +// Assert.Contains($""" +// --{commandArgName} Valid1 Valid2 "Quoted item" +// """, args); + +// if (warningExpected) +// { +// Assert.Equal($"Items '{propertyName}' contain empty item(s) which will be ignored.", Assert.Single(warnings)); +// } + +// string workDir = GetPathToContainerize(); + +// new DotnetCommand(_testOutput, args) +// .WithRawArguments() +// .WithWorkingDirectory(workDir) +// .Execute().Should().Fail() +// .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error +// } + +// [InlineData("")] +// [InlineData(" ")] +// [Theory] +// public void GenerateCommandLineCommands_EntryPointCanHaveEmptyItems(string itemValue) +// { +// CreateNewImage task = new(); +// IBuildEngine buildEngine = A.Fake(); + +// task.BuildEngine = buildEngine; + +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem(itemValue) }; + +// task.GenerateCommandLineCommandsInt(); +// } + +// [Theory] +// [InlineData(null)] +// [InlineData("")] +// [InlineData(" ")] +// [InlineData("Valid", true)] +// public void GenerateCommandLineCommands_AppCommandInstruction(string? value, bool optionExpected = false) +// { +// CreateNewImage task = new(); +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; + +// if (value != null) +// { +// task.AppCommandInstruction = value; +// } + +// string args = task.GenerateCommandLineCommandsInt(); + +// if (optionExpected) +// { +// Assert.Contains($"--appcommandinstruction {value}", args); +// } +// else +// { +// Assert.DoesNotContain("--appcommandinstruction", args); +// } +// } + +// [Fact] +// public void GenerateCommandLineCommands_ImageTags() +// { +// CreateNewImage task = new(); + +// List warnings = new(); +// IBuildEngine buildEngine = A.Fake(); +// A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); + +// task.BuildEngine = buildEngine; + +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// task.ImageTags = new[] { "", " ", "Valid1", "To be quoted" }; + +// string args = task.GenerateCommandLineCommandsInt(); + +// Assert.Contains(""" +// --imagetags Valid1 "To be quoted" +// """, actualString: args); +// Assert.Equal("Property 'ImageTags' is empty or contains whitespace and will be ignored.", Assert.Single(warnings)); + +// string workDir = GetPathToContainerize(); + +// new DotnetCommand(_testOutput, args) +// .WithRawArguments() +// .WithWorkingDirectory(workDir) +// .Execute().Should().Fail() +// .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error +// } + +// [Fact] +// public void GenerateCommandLineCommands_ExposedPorts() +// { +// CreateNewImage task = new(); + +// List warnings = new(); +// IBuildEngine buildEngine = A.Fake(); +// A.CallTo(() => buildEngine.LogWarningEvent(A.Ignored)).Invokes((BuildWarningEventArgs e) => warnings.Add(e.Message)); + +// task.BuildEngine = buildEngine; + +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// task.ExposedPorts = new[] +// { +// new TaskItem("1500"), +// new TaskItem(" "), +// new TaskItem("1501", new Dictionary() {{ "Type", "udp" }}), +// new TaskItem("1501", new Dictionary() {{ "Type", "tcp" }}), +// new TaskItem("1502", new Dictionary() {{ "Type", "tcp" }}), +// new TaskItem("1503", new Dictionary() {{ "Type", "" }}) +// }; + +// string args = task.GenerateCommandLineCommandsInt(); + +// Assert.Contains(""" +// --ports 1500 1501/udp 1501/tcp 1502/tcp 1503 +// """, args); +// Assert.Equal("Items 'ExposedPorts' contain empty item(s) which will be ignored.", Assert.Single(warnings)); + +// string workDir = GetPathToContainerize(); + +// new DotnetCommand(_testOutput, args) +// .WithRawArguments() +// .WithWorkingDirectory(workDir) +// .Execute().Should().Fail() +// .And.NotHaveStdOutContaining("Description:"); //standard help output for parse error +// } + +// [Fact] +// public void Logging_CanEnableTraceLogging() +// { +// CreateNewImage task = new(); +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); + +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("") }; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// string args = task.GenerateCommandLineCommandsInt(); +// string workDir = GetPathToContainerize(); + +// new DotnetCommand(_testOutput, args) +// .WithRawArguments() +// .WithWorkingDirectory(workDir) +// .WithEnvironmentVariable("CONTAINERIZE_TRACE_LOGGING_ENABLED", "1") +// .Execute().Should().Fail() +// .And.NotHaveStdOutContaining("Description:") //standard help output for parse error +// .And.HaveStdOutContaining("Trace logging: enabled."); +// } + +// [Fact] +// public void Logging_TraceLoggingIsDisabledByDefault() +// { +// CreateNewImage task = new(); +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); + +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("") }; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; + +// string args = task.GenerateCommandLineCommandsInt(); +// string workDir = GetPathToContainerize(); + +// new DotnetCommand(_testOutput, args) +// .WithRawArguments() +// .WithWorkingDirectory(workDir) +// .Execute().Should().Fail() +// .And.NotHaveStdOutContaining("Description:") //standard help output for parse error +// .And.NotHaveStdOutContaining("Trace logging: enabled."); +// } + +// [Fact] +// public void GenerateCommandLineCommands_LabelGeneration() +// { +// CreateNewImage task = new(); + +// List warnings = new(); +// IBuildEngine buildEngine = A.Fake(); + +// task.BuildEngine = buildEngine; + +// DirectoryInfo publishDir = Directory.CreateDirectory(Path.Combine(Path.GetTempPath(), DateTime.Now.ToString("yyyyMMddHHmmssfff"))); +// task.PublishDirectory = publishDir.FullName; +// task.BaseRegistry = "MyBaseRegistry"; +// task.BaseImageName = "MyBaseImageName"; +// task.Repository = "MyImageName"; +// task.WorkingDirectory = "MyWorkingDirectory"; +// task.Entrypoint = new[] { new TaskItem("MyEntryPoint") }; +// task.GenerateLabels = true; +// task.GenerateDigestLabel = true; + +// string args = task.GenerateCommandLineCommandsInt(); + +// Assert.Contains("--generate-labels", args); +// Assert.Contains("--generate-digest-label", args); +// } + + + +// private static string GetPathToContainerize() +// { +// return Path.Combine(TestContext.Current.TestExecutionDirectory, "Container", "containerize"); +// } +// } diff --git a/test/Microsoft.NET.Build.Containers.IntegrationTests/LayerEndToEndTests.cs b/test/Microsoft.NET.Build.Containers.IntegrationTests/LayerEndToEndTests.cs index b0a23aabf122..6dff3d9c0118 100644 --- a/test/Microsoft.NET.Build.Containers.IntegrationTests/LayerEndToEndTests.cs +++ b/test/Microsoft.NET.Build.Containers.IntegrationTests/LayerEndToEndTests.cs @@ -4,6 +4,7 @@ using System.Formats.Tar; using System.IO.Compression; using System.Security.Cryptography; +using System.Threading.Tasks; namespace Microsoft.NET.Build.Containers.IntegrationTests; @@ -15,12 +16,11 @@ public LayerEndToEndTests(ITestOutputHelper testOutput) { _testOutput = testOutput; testSpecificArtifactRoot = new(); - priorArtifactRoot = ContentStore.ArtifactRoot; - ContentStore.ArtifactRoot = testSpecificArtifactRoot.Path; + _store = new ContentStore(new(testSpecificArtifactRoot.Path)); } [Fact] - public void SingleFileInFolder() + public async Task SingleFileInFolder() { using TransientTestFolder folder = new(); @@ -29,7 +29,9 @@ public void SingleFileInFolder() File.WriteAllText(testFilePath, testString); - Layer l = Layer.FromDirectory(directory: folder.Path, containerPath: "/app", false, SchemaTypes.DockerManifestV2); + var layerFilePath = new FileInfo(_store.GetTempFile()); + + Layer l = await Layer.FromFiles([(Path.GetFullPath(testFilePath), "TestFile.txt")], containerPath: "/app", false, SchemaTypes.DockerManifestV2, _store, layerFilePath, CancellationToken.None); Console.WriteLine(l.Descriptor); @@ -45,7 +47,7 @@ public void SingleFileInFolder() } [Fact] - public void SingleFileInFolderWindows() + public async Task SingleFileInFolderWindows() { using TransientTestFolder folder = new(); @@ -53,8 +55,9 @@ public void SingleFileInFolderWindows() string testString = $"Test content for {nameof(SingleFileInFolder)}"; File.WriteAllText(testFilePath, testString); + var layerFilePath = new FileInfo(_store.GetTempFile()); - Layer l = Layer.FromDirectory(directory: folder.Path, containerPath: "C:\\app", true, SchemaTypes.DockerManifestV2); + Layer l = await Layer.FromFiles([(Path.GetFullPath(testFilePath), "TestFile.txt")], containerPath: "C:\\app", true, SchemaTypes.DockerManifestV2, _store, layerFilePath, CancellationToken.None); var allEntries = LoadAllTarEntries(l.BackingFile); Assert.True(allEntries.TryGetValue("Files", out var filesEntry) && filesEntry.EntryType == TarEntryType.Directory, "Missing Files directory entry"); @@ -73,7 +76,7 @@ public void SingleFileInFolderWindows() } [Fact] // https://github.com/dotnet/sdk/issues/40511 - public void SingleFileInHiddenFolder() + public async Task SingleFileInHiddenFolder() { using TransientTestFolder folder = new(); @@ -87,8 +90,9 @@ public void SingleFileInHiddenFolder() string testString = $"Test content for {nameof(SingleFileInHiddenFolder)}"; File.WriteAllText(testFilePath, testString); + var layerFilePath = new FileInfo(_store.GetTempFile()); - Layer l = Layer.FromDirectory(directory: folder.Path, containerPath: "/app", false, SchemaTypes.DockerManifestV2); + Layer l = await Layer.FromFiles([(Path.GetFullPath(testFilePath), ".well-known/wwwroot")], containerPath: "/app", false, SchemaTypes.DockerManifestV2, _store, layerFilePath, CancellationToken.None); VerifyDescriptorInfo(l); @@ -101,12 +105,12 @@ public void SingleFileInHiddenFolder() private static void VerifyDescriptorInfo(Layer l) { - Assert.Equal(l.Descriptor.Size, new FileInfo(l.BackingFile).Length); + Assert.Equal(l.Descriptor.Size, l.BackingFile.Length); byte[] hashBytes; byte[] uncompressedHashBytes; - using (FileStream fs = File.OpenRead(l.BackingFile)) + using (FileStream fs = l.BackingFile.OpenRead()) { hashBytes = SHA256.HashData(fs); @@ -123,21 +127,16 @@ private static void VerifyDescriptorInfo(Layer l) } TransientTestFolder? testSpecificArtifactRoot; - string? priorArtifactRoot; + private readonly ContentStore _store; public void Dispose() { testSpecificArtifactRoot?.Dispose(); - if (priorArtifactRoot is not null) - { - ContentStore.ArtifactRoot = priorArtifactRoot; - } } - - private static Dictionary LoadAllTarEntries(string file) + private static Dictionary LoadAllTarEntries(FileInfo file) { - using var gzip = new GZipStream(File.OpenRead(file), CompressionMode.Decompress); + using var gzip = new GZipStream(file.OpenRead(), CompressionMode.Decompress); using var tar = new TarReader(gzip); var entries = new Dictionary(); diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs index 6a359c970480..d6b134b47f64 100644 --- a/test/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs +++ b/test/Microsoft.NET.Build.Containers.UnitTests/ImageBuilderTests.cs @@ -75,10 +75,7 @@ public void CanAddLabelsToImage() baseConfig.AddLabel("testLabel1", "v1"); baseConfig.AddLabel("testLabel2", "v2"); - string readyImage = baseConfig.BuildConfig(); - - JsonNode? result = JsonNode.Parse(readyImage); - + var result = baseConfig.BuildConfig(); var resultLabels = result?["config"]?["Labels"] as JsonObject; Assert.NotNull(resultLabels); @@ -146,9 +143,7 @@ public void CanPreserveExistingLabels() baseConfig.AddLabel("testLabel1", "v1"); baseConfig.AddLabel("existing2", "v2"); - string readyImage = baseConfig.BuildConfig(); - - JsonNode? result = JsonNode.Parse(readyImage); + var result = baseConfig.BuildConfig(); var resultLabels = result?["config"]?["Labels"] as JsonObject; Assert.NotNull(resultLabels); @@ -214,9 +209,7 @@ public void CanAddPortsToImage() baseConfig.ExposePort(6000, PortType.tcp); baseConfig.ExposePort(6010, PortType.udp); - string readyImage = baseConfig.BuildConfig(); - - JsonNode? result = JsonNode.Parse(readyImage); + var result = baseConfig.BuildConfig(); var resultPorts = result?["config"]?["ExposedPorts"] as JsonObject; Assert.NotNull(resultPorts); @@ -288,9 +281,7 @@ public void CanPreserveExistingPorts() baseConfig.ExposePort(6100, PortType.udp); baseConfig.ExposePort(6200, PortType.tcp); - string readyImage = baseConfig.BuildConfig(); - - JsonNode? result = JsonNode.Parse(readyImage); + var result = baseConfig.BuildConfig(); var resultPorts = result?["config"]?["ExposedPorts"] as JsonObject; Assert.NotNull(resultPorts); @@ -372,9 +363,9 @@ public void HistoryEntriesMatchNonEmptyLayers() ImageConfig baseConfig = new(node); - string readyImage = baseConfig.BuildConfig(); + var result = baseConfig.BuildConfig(); - JsonNode? result = JsonNode.Parse(readyImage); + var historyNode = result?["history"]; Assert.NotNull(historyNode); @@ -425,7 +416,7 @@ public void CanSetUserFromAppUIDEnvVarFromBaseImage() var builtImage = builder.Build(); - JsonNode? result = JsonNode.Parse(builtImage.Config); + JsonNode? result = builtImage.Config; Assert.NotNull(result); var assignedUid = result["config"]?["User"]?.GetValue(); Assert.Equal(assignedUid, expectedUid); @@ -467,7 +458,7 @@ public void CanSetUserFromAppUIDEnvVarFromUser() builder.AddEnvironmentVariable(ImageBuilder.EnvironmentVariables.APP_UID, "12345"); var builtImage = builder.Build(); - JsonNode? result = JsonNode.Parse(builtImage.Config); + JsonNode? result = builtImage.Config; Assert.NotNull(result); var assignedUser = result["config"]?["User"]?.GetValue(); Assert.Equal(assignedUser, expectedUid); @@ -511,7 +502,7 @@ public void CanSetPortFromEnvVarFromBaseImage(string envVar, string envValue, pa var builtImage = builder.Build(); - JsonNode? result = JsonNode.Parse(builtImage.Config); + JsonNode? result = builtImage.Config; Assert.NotNull(result); var portsObject = result["config"]?["ExposedPorts"]?.AsObject(); var assignedPorts = portsObject?.AsEnumerable().Select(portString => int.Parse(portString.Key.Split('/')[0])).ToArray(); @@ -557,7 +548,7 @@ public void CanSetPortFromEnvVarFromUser(string envVar, string envValue, params var builtImage = builder.Build(); - JsonNode? result = JsonNode.Parse(builtImage.Config); + JsonNode? result = builtImage.Config; Assert.NotNull(result); var portsObject = result["config"]?["ExposedPorts"]?.AsObject(); var assignedPorts = portsObject?.AsEnumerable().Select(portString => int.Parse(portString.Key.Split('/')[0])).ToArray(); @@ -600,7 +591,7 @@ public void CanSetContainerUserAndOverrideAppUID() """); baseConfigBuilder.SetUser(userId); - var config = JsonNode.Parse(baseConfigBuilder.Build().Config); + var config = baseConfigBuilder.Build().Config; config!["config"]?["User"]?.GetValue().Should().Be(expected: userId, because: "The precedence of SetUser should override inferred user ids"); } @@ -640,7 +631,7 @@ public void WhenMultipleUrlSourcesAreSetOnlyAspnetcoreUrlsIsUsed() builder.AddEnvironmentVariable(ImageBuilder.EnvironmentVariables.ASPNETCORE_URLS, "https://*:12345"); builder.AddEnvironmentVariable(ImageBuilder.EnvironmentVariables.ASPNETCORE_HTTPS_PORTS, "456"); var builtImage = builder.Build(); - JsonNode? result = JsonNode.Parse(builtImage.Config); + JsonNode? result = builtImage.Config; Assert.NotNull(result); var portsObject = result["config"]?["ExposedPorts"]?.AsObject(); var assignedPorts = portsObject?.AsEnumerable().Select(portString => int.Parse(portString.Key.Split('/')[0])).ToArray(); @@ -681,7 +672,7 @@ public void CanSetBaseImageDigestLabel() builder.AddBaseImageDigestLabel(); var builtImage = builder.Build(); - JsonNode? result = JsonNode.Parse(builtImage.Config); + JsonNode? result = builtImage.Config; Assert.NotNull(result); var labels = result["config"]?["Labels"]?.AsObject(); var digest = labels?.AsEnumerable().First(label => label.Key == "org.opencontainers.image.base.digest").Value!; diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs index 25d3cafe2268..1e9dc1864272 100644 --- a/test/Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs +++ b/test/Microsoft.NET.Build.Containers.UnitTests/ImageConfigTests.cs @@ -50,7 +50,7 @@ public class ImageConfigTests public void PassesThroughPropertyEvenThoughPropertyIsntExplicitlyHandled(string property) { ImageConfig c = new(SampleImageConfig); - JsonNode after = JsonNode.Parse(c.BuildConfig())!; + JsonNode after = c.BuildConfig(); JsonNode? prop = after["config"]?[property]; Assert.NotNull(prop); } diff --git a/test/Microsoft.NET.Build.Containers.UnitTests/ImageIndexGeneratorTests.cs b/test/Microsoft.NET.Build.Containers.UnitTests/ImageIndexGeneratorTests.cs index 1acc846dcdcd..57979747782b 100644 --- a/test/Microsoft.NET.Build.Containers.UnitTests/ImageIndexGeneratorTests.cs +++ b/test/Microsoft.NET.Build.Containers.UnitTests/ImageIndexGeneratorTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Text.Json; using Microsoft.NET.Build.Containers.Resources; namespace Microsoft.NET.Build.Containers.UnitTests; @@ -19,22 +20,29 @@ public void ImagesCannotBeEmpty() public void ImagesCannotBeEmpty_SpecifiedMediaType() { BuiltImage[] images = Array.Empty(); - var ex = Assert.Throws(() => ImageIndexGenerator.GenerateImageIndex(images, "manifestMediaType", "imageIndexMediaType")); + var ex = Assert.Throws(() => ImageIndexGenerator.GenerateDockerManifestList(images, "manifestMediaType", "imageIndexMediaType")); Assert.Equal(Strings.ImagesEmpty, ex.Message); } + static BuiltImage EmptyWithMediaType(string mediaType) => + new() + { + Config = new(), + Manifest = new() + { + MediaType = mediaType, + Layers = [], + SchemaVersion = 2, + Config = new(), + }, + }; + [Fact] public void UnsupportedMediaTypeThrows() { BuiltImage[] images = [ - new BuiltImage - { - Config = "", - Manifest = "", - ManifestDigest = "", - ManifestMediaType = "unsupported" - } + EmptyWithMediaType("unsupported"), ]; var ex = Assert.Throws(() => ImageIndexGenerator.GenerateImageIndex(images)); @@ -48,20 +56,8 @@ public void ImagesWithMixedMediaTypes(string supportedMediaType) { BuiltImage[] images = [ - new BuiltImage - { - Config = "", - Manifest = "", - ManifestDigest = "", - ManifestMediaType = supportedMediaType, - }, - new BuiltImage - { - Config = "", - Manifest = "", - ManifestDigest = "", - ManifestMediaType = "anotherMediaType" - } + EmptyWithMediaType(supportedMediaType), + EmptyWithMediaType("unsupported"), ]; var ex = Assert.Throws(() => ImageIndexGenerator.GenerateImageIndex(images)); @@ -75,27 +71,38 @@ public void GenerateDockerManifestList() [ new BuiltImage { - Config = "", - Manifest = "123", - ManifestDigest = "sha256:digest1", - ManifestMediaType = SchemaTypes.DockerManifestV2, + Config = new(){ + + }, + Manifest = new(){ + KnownDigest = "sha256:digest1", + MediaType = SchemaTypes.DockerManifestV2, + SchemaVersion = 2, + Config = new(), + Layers = [], + }, Architecture = "arch1", OS = "os1" }, new BuiltImage { - Config = "", - Manifest = "123", - ManifestDigest = "sha256:digest2", - ManifestMediaType = SchemaTypes.DockerManifestV2, + Config = new(), + Manifest = new(){ + KnownDigest = "sha256:digest2", + MediaType = SchemaTypes.DockerManifestV2, + SchemaVersion = 2, + Config = new(), + Layers = [], + }, Architecture = "arch2", OS = "os2" } ]; - var (imageIndex, mediaType) = ImageIndexGenerator.GenerateImageIndex(images); - Assert.Equal("{\"schemaVersion\":2,\"mediaType\":\"application/vnd.docker.distribution.manifest.list.v2+json\",\"manifests\":[{\"mediaType\":\"application/vnd.docker.distribution.manifest.v2+json\",\"size\":3,\"digest\":\"sha256:digest1\",\"platform\":{\"architecture\":\"arch1\",\"os\":\"os1\"}},{\"mediaType\":\"application/vnd.docker.distribution.manifest.v2+json\",\"size\":3,\"digest\":\"sha256:digest2\",\"platform\":{\"architecture\":\"arch2\",\"os\":\"os2\"}}]}", imageIndex); - Assert.Equal(SchemaTypes.DockerManifestListV2, mediaType); + var imageIndex = ImageIndexGenerator.GenerateImageIndex(images); + var imageIndexJson = JsonSerializer.Serialize(imageIndex); + Assert.Equal("{\"schemaVersion\":2,\"mediaType\":\"application/vnd.docker.distribution.manifest.list.v2+json\",\"manifests\":[{\"mediaType\":\"application/vnd.docker.distribution.manifest.v2+json\",\"size\":3,\"digest\":\"sha256:digest1\",\"platform\":{\"architecture\":\"arch1\",\"os\":\"os1\"}},{\"mediaType\":\"application/vnd.docker.distribution.manifest.v2+json\",\"size\":3,\"digest\":\"sha256:digest2\",\"platform\":{\"architecture\":\"arch2\",\"os\":\"os2\"}}]}", imageIndexJson); + Assert.Equal(SchemaTypes.DockerManifestListV2, imageIndex.MediaType); } [Fact] @@ -105,33 +112,45 @@ public void GenerateOciImageIndex() [ new BuiltImage { - Config = "", - Manifest = "123", - ManifestDigest = "sha256:digest1", - ManifestMediaType = SchemaTypes.OciManifestV1, + Config = new(){ + + }, + Manifest = new(){ + KnownDigest = "sha256:digest1", + MediaType = SchemaTypes.OciManifestV1, + SchemaVersion = 2, + Config = new(), + Layers = [], + }, Architecture = "arch1", OS = "os1" }, new BuiltImage { - Config = "", - Manifest = "123", - ManifestDigest = "sha256:digest2", - ManifestMediaType = SchemaTypes.OciManifestV1, + Config = new(), + Manifest = new(){ + KnownDigest = "sha256:digest2", + MediaType = SchemaTypes.OciManifestV1, + SchemaVersion = 2, + Config = new(), + Layers = [], + }, Architecture = "arch2", OS = "os2" } ]; - var (imageIndex, mediaType) = ImageIndexGenerator.GenerateImageIndex(images); - Assert.Equal("{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.index.v1+json\",\"manifests\":[{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"size\":3,\"digest\":\"sha256:digest1\",\"platform\":{\"architecture\":\"arch1\",\"os\":\"os1\"}},{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"size\":3,\"digest\":\"sha256:digest2\",\"platform\":{\"architecture\":\"arch2\",\"os\":\"os2\"}}]}", imageIndex); - Assert.Equal(SchemaTypes.OciImageIndexV1, mediaType); + var imageIndex = ImageIndexGenerator.GenerateImageIndex(images); + var imageIndexJson = JsonSerializer.Serialize(imageIndex); + Assert.Equal("{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.index.v1+json\",\"manifests\":[{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"size\":3,\"digest\":\"sha256:digest1\",\"platform\":{\"architecture\":\"arch1\",\"os\":\"os1\"}},{\"mediaType\":\"application/vnd.oci.image.manifest.v1+json\",\"size\":3,\"digest\":\"sha256:digest2\",\"platform\":{\"architecture\":\"arch2\",\"os\":\"os2\"}}]}", imageIndexJson); + Assert.Equal(SchemaTypes.OciImageIndexV1, imageIndex.MediaType); } [Fact] public void GenerateImageIndexWithAnnotations() { - string imageIndex = ImageIndexGenerator.GenerateImageIndexWithAnnotations("mediaType", "sha256:digest", 3, "repository", ["1.0", "2.0"]); - Assert.Equal("{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.index.v1+json\",\"manifests\":[{\"mediaType\":\"mediaType\",\"size\":3,\"digest\":\"sha256:digest\",\"platform\":{},\"annotations\":{\"io.containerd.image.name\":\"docker.io/library/repository:1.0\",\"org.opencontainers.image.ref.name\":\"1.0\"}},{\"mediaType\":\"mediaType\",\"size\":3,\"digest\":\"sha256:digest\",\"platform\":{},\"annotations\":{\"io.containerd.image.name\":\"docker.io/library/repository:2.0\",\"org.opencontainers.image.ref.name\":\"2.0\"}}]}", imageIndex); + var imageIndex = ImageIndexGenerator.GenerateImageIndexWithAnnotations("mediaType", "sha256:digest", 3, "repository", ["1.0", "2.0"]); + var imageIndexJson = JsonSerializer.Serialize(imageIndex); + Assert.Equal("{\"schemaVersion\":2,\"mediaType\":\"application/vnd.oci.image.index.v1+json\",\"manifests\":[{\"mediaType\":\"mediaType\",\"size\":3,\"digest\":\"sha256:digest\",\"platform\":{},\"annotations\":{\"io.containerd.image.name\":\"docker.io/library/repository:1.0\",\"org.opencontainers.image.ref.name\":\"1.0\"}},{\"mediaType\":\"mediaType\",\"size\":3,\"digest\":\"sha256:digest\",\"platform\":{},\"annotations\":{\"io.containerd.image.name\":\"docker.io/library/repository:2.0\",\"org.opencontainers.image.ref.name\":\"2.0\"}}]}", imageIndexJson); } }