Merge pull request #877 from dotnet-maestro-bot/merge/release/2.1-to-release/2.2

dagood · web-flow · commit 3926ef1686ba · 2018-11-19T20:07:51.000-06:00
[automated] Merge branch 'release/2.1' =&gt; 'release/2.2'
diff --git a/.vsts.pipelines/builds/ci.yml b/.vsts.pipelines/builds/ci.yml
@@ -10,6 +10,7 @@ jobs:
   parameters:
     job: centos71
     imageName: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
+    reportPrebuiltLeaks: true
     matrix:
       Production: {}
       Online: { type: Online }
diff --git a/.vsts.pipelines/jobs/ci-linux.yml b/.vsts.pipelines/jobs/ci-linux.yml
@@ -5,6 +5,7 @@ parameters:
   pool:
     name: Hosted Ubuntu 1604
   imageName: null
+  reportPrebuiltLeaks: false
 
 jobs:
 - job: ${{ parameters.job }}
@@ -28,6 +29,7 @@ jobs:
     docker.tb.work: -w /tb
     dropDirectory: $(stagingDirectory)/drop
     imageName: ${{ parameters.imageName }}
+    reportPrebuiltLeaks: ${{ parameters.reportPrebuiltLeaks }}
     rootDirectory: $(Build.SourcesDirectory)/..
     stagingDirectory: $(rootDirectory)/sb/staging
     tarballName: tarball_$(Build.BuildId)
@@ -68,7 +70,7 @@ jobs:
     displayName: Build source-build
     timeoutInMinutes: 120
 
-  # Run smoke tests.
+  # Run smoke tests. This is needed even in tarball legs to create the smoke-test-prereqs archive.
   - script: |
       set -x
       df -h
@@ -82,9 +84,13 @@ jobs:
   - script: |
       set -x
       df -h
+      args="--skip-build"
+      if [ "$(reportPrebuiltLeaks)" = "true" ]; then
+        args="$args --enable-leak-detection"
+      fi
       $(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(imageName) ./build-source-tarball.sh \
         "/tb/$(tarballName)" \
-        --skip-build
+        $args
     displayName: Create tarball
     condition: and(succeeded(), eq(variables['sb.tarball'], true))
 
@@ -114,10 +120,15 @@ jobs:
       if [ "$(sb.tarballOffline)" = "true" ]; then
         networkArg="--network=none"
       fi
+      poisonArg=
+      if [ "$(sb.tarball)" != "true" ]; then
+        poisonArg="/p:EnablePoison=true"
+      fi
       $(docker.run) $(docker.tb.map) $(docker.tb.work) $networkArg $(imageName) "$(tarballName)/build.sh" \
         /p:Configuration=$(sb.configuration) \
         /p:PortableBuild=$(sb.portable) \
-        /p:FailOnPrebuiltBaselineError=true
+        /p:FailOnPrebuiltBaselineError=true \
+        $poisonArg
     displayName: Build tarball
     timeoutInMinutes: 120
     condition: and(succeeded(), eq(variables['sb.tarball'], true))
@@ -195,6 +206,15 @@ jobs:
       ArtifactName: Tarball $(artifactName)
       ArtifactType: Container
 
+  # Publish prebuilt report data to blob storage.
+  - template: ../steps/publish-prebuilt-data-sh.yml
+    parameters:
+      relativeBlobPath: prebuilt-data/$(Build.SourceBranchName)/latest/$(artifactName)
+  # Also publish per-build data to a unique location for future use.
+  - template: ../steps/publish-prebuilt-data-sh.yml
+    parameters:
+      relativeBlobPath: prebuilt-data/$(Build.SourceBranchName)/rolling/$(Build.SourceVersion)/$(Build.BuildNumber)/$(Build.DefinitionName)/$(artifactName)
+
   # Clean up (very large) working directory. root owner makes it difficult for others to remove.
   - script: $(docker.run) $(docker.root.map) $(imageName) bash -c 'rm -rf /root/sb'
     displayName: Clean sb directory
diff --git a/.vsts.pipelines/jobs/ci-osx.yml b/.vsts.pipelines/jobs/ci-osx.yml
@@ -13,7 +13,7 @@ jobs:
   timeoutInMinutes: 240
   variables:
     # Prefix to distinguish artifacts from different legs.
-    artifactName: ${{ format('$(type) {0}', parameters.job) }}
+    artifactName: ${{ format('{0} $(type)', parameters.job) }}
     logsDirectory: $(Build.ArtifactStagingDirectory)/logs
     SOURCE_BUILD_SKIP_SUBMODULE_CHECK: true
     # Default type, can be overridden by matrix legs.
diff --git a/.vsts.pipelines/steps/publish-prebuilt-data-sh.yml b/.vsts.pipelines/steps/publish-prebuilt-data-sh.yml
@@ -0,0 +1,19 @@
+parameters:
+  relativeBlobPath: ''
+
+steps:
+  - script: |
+      set -x
+      # Publish tarball's prebuilt data. Use ./build.sh from production build.
+      $(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
+        /t:PublishPrebuiltReportData \
+        /p:ProjectDir=/tb/$(tarballName)/ \
+        "/p:RelativeBlobPath=$relativeBlobPath" \
+        /p:ContainerName=$(publish.blobStorage.container) \
+        /p:AzureAccountName=$(publish.blobStorage.account) \
+        /p:AzureAccessToken=$(publish.blobStorage.accessToken) \
+        /clp:v=D
+    displayName: ${{ format('Publish prebuilt report data ({0})', parameters.relativeBlobPath) }}
+    env:
+      relativeBlobPath: ${{ parameters.relativeBlobPath }}
+    condition: and(succeeded(), ne(variables['publish.blobStorage.accessToken'], ''), eq(variables['sb.tarball'], true))
diff --git a/build.proj b/build.proj
@@ -137,6 +137,30 @@
           EnvironmentVariables="prodConBlobFeedUrl=$(ProdConBlobFeedUrl)" />
   </Target>
 
+  <UsingTask AssemblyFile="$(BuildToolsTaskDir)Microsoft.DotNet.Build.CloudTestTasks.dll" TaskName="UploadToAzure" />
+
+  <Target Name="PublishPrebuiltReportData">
+    <Error Text="RelativeBlobPath must be set to a non-empty string." Condition="'$(RelativeBlobPath)' == ''" />
+    <Error Text="ContainerName must be set to a non-empty string." Condition="'$(ContainerName)' == ''" />
+    <Error Text="AzureAccountName must be set to a non-empty string." Condition="'$(AzureAccountName)' == ''" />
+    <Error Text="AzureAccessToken must be set to a non-empty string." Condition="'$(AzureAccessToken)' == ''" />
+
+    <ItemGroup>
+      <ItemsToPublish Include="$(PackageReportDir)*.xml" />
+      <ItemsToPublish>
+        <RelativeBlobPath>$(RelativeBlobPath)/%(Filename)%(Extension)</RelativeBlobPath>
+      </ItemsToPublish>
+    </ItemGroup>
+
+    <Message Text="Uploading files to '$(AzureAccountName)' blob storage at $(ContainerName)/$(RelativeBlobPath)" />
+
+    <UploadToAzure AccountName="$(AzureAccountName)"
+                   AccountKey="$(AzureAccessToken)"
+                   ContainerName="$(ContainerName)"
+                   Items="@(ItemsToPublish)"
+                   Overwrite="true" />
+  </Target>
+
   <Import Project="$(ProjectDir)dependencies.targets" />
 
   <Import Project="$(ToolsDir)VersionTools.targets" />
diff --git a/build.sh b/build.sh
@@ -26,5 +26,5 @@ SDKPATH="$CLIPATH/sdk/$SDK_VERSION"
 
 set -x
 
-$CLIPATH/dotnet $SDKPATH/MSBuild.dll $SCRIPT_ROOT/build.proj /flp:v=diag /clp:v=m "$@"
+$CLIPATH/dotnet $SDKPATH/MSBuild.dll $SCRIPT_ROOT/build.proj /bl /flp:v=diag /clp:v=m "$@"
 
diff --git a/repos/application-insights.proj b/repos/application-insights.proj
@@ -2,7 +2,7 @@
 <Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))/dir.props" />
   <PropertyGroup>
-    <PackagesOutput>$(SubmoduleDirectory)/bin/$(Configuration)</PackagesOutput>
+    <PackagesOutput>$(ProjectDirectory)/bin/$(Configuration)</PackagesOutput>
     <RepoApiImplemented>false</RepoApiImplemented>
     <OrchestratedManifestBuildName>N/A</OrchestratedManifestBuildName>
   </PropertyGroup>
@@ -13,6 +13,9 @@
     <PropertyGroup>
       <BuildCommandArgs>$(ProjectDirectory)/Microsoft.ApplicationInsights.csproj</BuildCommandArgs>
       <BuildCommandArgs>$(BuildCommandArgs) /p:Configuration=$(Configuration)</BuildCommandArgs>
+      <!-- Re-assign EnlistmentRoot property so output directories end up under src/application-insights -->
+      <BuildCommandArgs>$(BuildCommandArgs) /p:EnlistmentRoot=$(ProjectDirectory)/src</BuildCommandArgs>
+      <BuildCommandArgs>$(BuildCommandArgs) /p:RelativeOutputPathBase=</BuildCommandArgs>
       <BuildCommandArgs>$(BuildCommandArgs) $(RedirectRepoOutputToLog)</BuildCommandArgs>
     </PropertyGroup>
 
diff --git a/repos/dir.targets b/repos/dir.targets
@@ -189,8 +189,15 @@
       <Output TaskParameter="PackageInfoItems" ItemName="_PreviouslySourceBuiltPackageInfos" />
     </ReadNuGetPackageInfos>
 
+    <ItemGroup>
+      <_KnownOriginPackagePaths Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" />
+      <_KnownOriginPackagePaths Include="$(PrebuiltPackagesPath)*.nupkg" />
+      <_KnownOriginPackagePaths Include="$(ReferencePackagesDir)*.nupkg" />
+    </ItemGroup>
+
     <GetSourceBuiltNupkgCacheConflicts SourceBuiltPackageInfos="@(_PreviouslySourceBuiltPackageInfos)"
-                                       PackageCacheDir="$(PackagesDir)">
+                                       PackageCacheDir="$(PackagesDir)"
+                                       KnownOriginPackagePaths="@(_KnownOriginPackagePaths)">
       <Output TaskParameter="ConflictingPackageInfos" ItemName="ConflictingPackageInfos" />
     </GetSourceBuiltNupkgCacheConflicts>
   </Target>
@@ -213,7 +220,8 @@
     <WriteUsageReports DataFile="$(_ReportDataFile)"
                        OutputDirectory="$(_ReportDir)" />
 
-    <Warning Text="Detected package id/version(s) in the cache that were source-built, but contents don't match. They were probably downloaded. See $(_ReportDir) for usage details. @(ConflictingPackageInfos->'%(PackageId) %(PackageVersion)', ', ')" />
+    <Warning Text="Detected packages in the cache that should be source-built, but contents don't match. See $(_ReportDir) for usage details:" />
+    <Warning Text="%(ConflictingPackageInfos.PackageId)/%(ConflictingPackageInfos.PackageVersion) : %(ConflictingPackageInfos.WarningMessage)" />
   </Target>
 
   <Target Name="CreateRestoreSourceProps"
diff --git a/support/tarball/build.sh b/support/tarball/build.sh
@@ -14,7 +14,7 @@ export NUGET_PACKAGES="$SCRIPT_ROOT/packages/"
 MSBUILD_ARGUMENTS=("/p:OfflineBuild=true" "/flp:v=detailed")
 
 echo "Rebuild reference assemblies"
-$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/tools-local/init-build.proj /t:BuildReferenceAssemblies ${MSBUILD_ARGUMENTS[@]} "$@"
+$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll /bl:initBuildReferenceAssemblies.binlog $SCRIPT_ROOT/tools-local/init-build.proj /t:BuildReferenceAssemblies ${MSBUILD_ARGUMENTS[@]} "$@"
 
 echo "Expanding BuildTools dependencies into packages directory..."
 # init-tools tries to copy from its script directory to Tools, which in this case is a copy to
@@ -38,5 +38,5 @@ REF_PACKAGE_SOURCE="$SCRIPT_ROOT/reference-packages/packages"
     exit 1
 )
 
-$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/tools-local/init-build.proj /t:WriteDynamicPropsToStaticPropsFiles /p:GeneratingStaticPropertiesFile=true ${MSBUILD_ARGUMENTS[@]} "$@"
-$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@"
+$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll /bl:initWriteDynamicPropsToStaticPropsFiles.binlog $SCRIPT_ROOT/tools-local/init-build.proj /t:WriteDynamicPropsToStaticPropsFiles /p:GeneratingStaticPropertiesFile=true ${MSBUILD_ARGUMENTS[@]} "$@"
+$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll /bl:build.binlog $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@"
diff --git a/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/GetSourceBuiltNupkgCacheConflicts.cs b/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/GetSourceBuiltNupkgCacheConflicts.cs
@@ -4,6 +4,8 @@
 
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
+using NuGet.Packaging.Core;
+using NuGet.Versioning;
 using System;
 using System.IO;
 using System.Linq;
@@ -40,13 +42,33 @@ public class GetSourceBuiltNupkgCacheConflicts : Task
         [Required]
         public string PackageCacheDir { get; set; }
 
+        /// <summary>
+        /// Paths to packages to compare against when conflicts are detected. Knowing where the
+        /// package in the cache came from can help diagnose a conflict. For example, is it from
+        /// prebuilt/source-built? Or does the build not have the nupkg anywhere else, and
+        /// therefore it most likely came from the internet?
+        /// </summary>
+        public string[] KnownOriginPackagePaths { get; set; }
+
         [Output]
         public ITaskItem[] ConflictingPackageInfos { get; set; }
 
         public override bool Execute()
         {
             DateTime startTime = DateTime.Now;
 
+            var knownNupkgs = new Lazy<ILookup<PackageIdentity, string>>(() =>
+            {
+                Log.LogMessage(
+                    MessageImportance.Low,
+                    $"Reading all {nameof(KnownOriginPackagePaths)} package identities to search " +
+                        "for conflicting package origin...");
+
+                return KnownOriginPackagePaths.NullAsEmpty().ToLookup(
+                    ReadNuGetPackageInfos.ReadIdentity,
+                    path => path);
+            });
+
             ConflictingPackageInfos = SourceBuiltPackageInfos
                 .Where(item =>
                 {
@@ -72,22 +94,50 @@ public override bool Execute()
                         MessageImportance.Low,
                         $"Package id/version found in package cache, verifying: {id} {version}");
 
-                    bool identical = File.ReadAllBytes(sourceBuiltPath)
-                        .SequenceEqual(File.ReadAllBytes(packageCachePath));
+                    byte[] packageCacheBytes = File.ReadAllBytes(packageCachePath);
 
-                    if (!identical)
+                    if (packageCacheBytes.SequenceEqual(File.ReadAllBytes(sourceBuiltPath)))
                     {
                         Log.LogMessage(
                             MessageImportance.Low,
-                            "BAD: Source-built nupkg is not byte-for-byte identical " +
-                                $"to nupkg in cache: {id} {version}");
-                        return true;
+                            $"OK: Package in cache is identical to source-built: {id} {version}");
+                        return false;
                     }
 
                     Log.LogMessage(
                         MessageImportance.Low,
-                        $"OK: Package in cache is identical to source-built: {id} {version}");
-                    return false;
+                        "BAD: Source-built nupkg is not byte-for-byte identical " +
+                            $"to nupkg in cache: {id} {version}");
+
+                    var ident = new PackageIdentity(id, NuGetVersion.Parse(version));
+
+                    string message = null;
+
+                    foreach (string knownNupkg in knownNupkgs.Value[ident])
+                    {
+                        if (packageCacheBytes.SequenceEqual(File.ReadAllBytes(knownNupkg)))
+                        {
+                            Log.LogMessage(
+                                MessageImportance.Low,
+                                $"Found identity match with identical contents: {knownNupkg}");
+
+                            message = (message ?? "Nupkg found at") + $" '{knownNupkg}'";
+                        }
+                        else
+                        {
+                            Log.LogMessage(
+                                MessageImportance.Low,
+                                $"Package identity match, but contents differ: {knownNupkg}");
+                        }
+                    }
+
+                    item.SetMetadata(
+                        "WarningMessage",
+                        message ??
+                            "Origin nupkg not found in build directory. It may have been " +
+                            "downloaded by NuGet restore.");
+
+                    return true;
                 })
                 .ToArray();
 

Original file line number	Diff line number	Diff line change
`@@ -26,5 +26,5 @@ SDKPATH="$CLIPATH/sdk/$SDK_VERSION"`
`26`	`26`
`27`	`27`	`set -x`
`28`	`28`
`29`		`-$CLIPATH/dotnet $SDKPATH/MSBuild.dll $SCRIPT_ROOT/build.proj /flp:v=diag /clp:v=m "$@"`
	`29`	`+$CLIPATH/dotnet $SDKPATH/MSBuild.dll $SCRIPT_ROOT/build.proj /bl /flp:v=diag /clp:v=m "$@"`
`30`	`30`