Improve source-built cache conflict warning (#878)

Try to find where the conflicting package came from and include the path in the message.

For example, this identifies that Microsoft.NETCore.Targets in the tarball build is being used from prebuilt/source-built. Before this change, it would incorrectly suggest the package was restored from the internet.

Add source-build infrastructure MSBuild binlogging for better diagnosis.

Author: dagood

build.sh

@@ -26,5 +26,5 @@ SDKPATH="$CLIPATH/sdk/$SDK_VERSION"
 
 set -x
 
-$CLIPATH/dotnet $SDKPATH/MSBuild.dll $SCRIPT_ROOT/build.proj /flp:v=diag /clp:v=m "$@"
+$CLIPATH/dotnet $SDKPATH/MSBuild.dll $SCRIPT_ROOT/build.proj /bl /flp:v=diag /clp:v=m "$@"
 

repos/dir.targets

@@ -189,8 +189,15 @@
       <Output TaskParameter="PackageInfoItems" ItemName="_PreviouslySourceBuiltPackageInfos" />
     </ReadNuGetPackageInfos>
 
+    <ItemGroup>
+      <_KnownOriginPackagePaths Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
+      <_KnownOriginPackagePaths Include="$(PrebuiltPackagesPath)*.nupkg" />
+      <_KnownOriginPackagePaths Include="$(ReferencePackagesDir)*.nupkg" />
+    </ItemGroup>
+
     <GetSourceBuiltNupkgCacheConflicts SourceBuiltPackageInfos="@(_PreviouslySourceBuiltPackageInfos)"
-                                       PackageCacheDir="$(PackagesDir)">
+                                       PackageCacheDir="$(PackagesDir)"
+                                       KnownOriginPackagePaths="@(_KnownOriginPackagePaths)">
       <Output TaskParameter="ConflictingPackageInfos" ItemName="ConflictingPackageInfos" />
     </GetSourceBuiltNupkgCacheConflicts>
   </Target>
@@ -213,7 +220,8 @@
     <WriteUsageReports DataFile="$(_ReportDataFile)"
                        OutputDirectory="$(_ReportDir)" />
 
-    <Warning Text="Detected package id/version(s) in the cache that were source-built, but contents don't match. They were probably downloaded. See $(_ReportDir) for usage details. @(ConflictingPackageInfos->'%(PackageId) %(PackageVersion)', ', ')" />
+    <Warning Text="Detected packages in the cache that should be source-built, but contents don't match. See $(_ReportDir) for usage details:" />
+    <Warning Text="%(ConflictingPackageInfos.PackageId)/%(ConflictingPackageInfos.PackageVersion) : %(ConflictingPackageInfos.WarningMessage)" />
   </Target>
 
   <Target Name="CreateRestoreSourceProps"

support/tarball/build.sh

@@ -14,7 +14,7 @@ export NUGET_PACKAGES="$SCRIPT_ROOT/packages/"
 MSBUILD_ARGUMENTS=("/p:OfflineBuild=true" "/flp:v=detailed")
 
 echo "Rebuild reference assemblies"
-$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/tools-local/init-build.proj /t:BuildReferenceAssemblies ${MSBUILD_ARGUMENTS[@]} "$@"
+$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll /bl:initBuildReferenceAssemblies.binlog $SCRIPT_ROOT/tools-local/init-build.proj /t:BuildReferenceAssemblies ${MSBUILD_ARGUMENTS[@]} "$@"
 
 echo "Expanding BuildTools dependencies into packages directory..."
 # init-tools tries to copy from its script directory to Tools, which in this case is a copy to
@@ -38,5 +38,5 @@ REF_PACKAGE_SOURCE="$SCRIPT_ROOT/reference-packages/packages"
     exit 1
 )
 
-$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/tools-local/init-build.proj /t:WriteDynamicPropsToStaticPropsFiles /p:GeneratingStaticPropertiesFile=true ${MSBUILD_ARGUMENTS[@]} "$@"
-$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@"
+$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll /bl:initWriteDynamicPropsToStaticPropsFiles.binlog $SCRIPT_ROOT/tools-local/init-build.proj /t:WriteDynamicPropsToStaticPropsFiles /p:GeneratingStaticPropertiesFile=true ${MSBUILD_ARGUMENTS[@]} "$@"
+$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll /bl:build.binlog $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@"

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/GetSourceBuiltNupkgCacheConflicts.cs

@@ -4,6 +4,8 @@
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
+using NuGet.Packaging.Core;
+using NuGet.Versioning;
 using System;
 using System.IO;
 using System.Linq;
@@ -40,13 +42,33 @@ public class GetSourceBuiltNupkgCacheConflicts : Task
         [Required]
         public string PackageCacheDir { get; set; }
 
+        /// <summary>
+        /// Paths to packages to compare against when conflicts are detected. Knowing where the
+        /// package in the cache came from can help diagnose a conflict. For example, is it from
+        /// prebuilt/source-built? Or does the build not have the nupkg anywhere else, and
+        /// therefore it most likely came from the internet?
+        /// </summary>
+        public string[] KnownOriginPackagePaths { get; set; }
+
         [Output]
         public ITaskItem[] ConflictingPackageInfos { get; set; }
 
         public override bool Execute()
         {
             DateTime startTime = DateTime.Now;
 
+            var knownNupkgs = new Lazy<ILookup<PackageIdentity, string>>(() =>
+            {
+                Log.LogMessage(
+                    MessageImportance.Low,
+                    $"Reading all {nameof(KnownOriginPackagePaths)} package identities to search " +
+                        "for conflicting package origin...");
+
+                return KnownOriginPackagePaths.NullAsEmpty().ToLookup(
+                    ReadNuGetPackageInfos.ReadIdentity,
+                    path => path);
+            });
+
             ConflictingPackageInfos = SourceBuiltPackageInfos
                 .Where(item =>
                 {
@@ -72,22 +94,50 @@ public override bool Execute()
                         MessageImportance.Low,
                         $"Package id/version found in package cache, verifying: {id} {version}");
 
-                    bool identical = File.ReadAllBytes(sourceBuiltPath)
-                        .SequenceEqual(File.ReadAllBytes(packageCachePath));
+                    byte[] packageCacheBytes = File.ReadAllBytes(packageCachePath);
 
-                    if (!identical)
+                    if (packageCacheBytes.SequenceEqual(File.ReadAllBytes(sourceBuiltPath)))
                     {
                         Log.LogMessage(
                             MessageImportance.Low,
-                            "BAD: Source-built nupkg is not byte-for-byte identical " +
-                                $"to nupkg in cache: {id} {version}");
-                        return true;
+                            $"OK: Package in cache is identical to source-built: {id} {version}");
+                        return false;
                     }
 
                     Log.LogMessage(
                         MessageImportance.Low,
-                        $"OK: Package in cache is identical to source-built: {id} {version}");
-                    return false;
+                        "BAD: Source-built nupkg is not byte-for-byte identical " +
+                            $"to nupkg in cache: {id} {version}");
+
+                    var ident = new PackageIdentity(id, NuGetVersion.Parse(version));
+
+                    string message = null;
+
+                    foreach (string knownNupkg in knownNupkgs.Value[ident])
+                    {
+                        if (packageCacheBytes.SequenceEqual(File.ReadAllBytes(knownNupkg)))
+                        {
+                            Log.LogMessage(
+                                MessageImportance.Low,
+                                $"Found identity match with identical contents: {knownNupkg}");
+
+                            message = (message ?? "Nupkg found at") + $" '{knownNupkg}'";
+                        }
+                        else
+                        {
+                            Log.LogMessage(
+                                MessageImportance.Low,
+                                $"Package identity match, but contents differ: {knownNupkg}");
+                        }
+                    }
+
+                    item.SetMetadata(
+                        "WarningMessage",
+                        message ??
+                            "Origin nupkg not found in build directory. It may have been " +
+                            "downloaded by NuGet restore.");
+
+                    return true;
                 })
                 .ToArray();