Merge pull request #803 from dotnet-maestro-bot/merge/release/2.1-to-release/2.2

[automated] Merge branch 'release/2.1' => 'release/2.2'

Author: crummel

build-source-tarball.sh

@@ -3,7 +3,7 @@ set -euo pipefail
 IFS=$'\n\t'
 
 usage() {
-    echo "usage: $0 <path-to-tarball-root> [--skip-build]"
+    echo "usage: $0 <path-to-tarball-root> [--skip-build] [--enable-leak-detection]"
     echo ""
 }
 
@@ -16,6 +16,8 @@ TARBALL_ROOT=$1
 shift
 
 SKIP_BUILD=0
+INCLUDE_LEAK_DETECTION=0
+export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 
 while :; do
     if [ $# -le 0 ]; then
@@ -31,6 +33,9 @@ while :; do
         --skip-build)
             SKIP_BUILD=1
             ;;
+        --enable-leak-detection)
+            INCLUDE_LEAK_DETECTION=1
+            ;;
         *)
             echo "Unrecognized argument '$1'"
             usage
@@ -158,6 +163,12 @@ SOURCE_BUILT_SDK_TOOLS_DIR="$TARBALL_ROOT/Tools/source-built/$ROSLYN_TOOLS_PACKA
 cp "$REPO_TOOLSET_PACKAGE_DIR/tools/"*.props "$SOURCE_BUILT_SDK_TOOLS_DIR"
 cp "$REPO_TOOLSET_PACKAGE_DIR/tools/"*.targets "$SOURCE_BUILT_SDK_TOOLS_DIR"
 
+if [ $INCLUDE_LEAK_DETECTION -eq 1 ]; then
+  echo 'Building leak detection MSBuild tasks...'
+  ./Tools/dotnetcli/dotnet restore $SCRIPT_ROOT/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj --source $FULL_TARBALL_ROOT/prebuilt/source-built --source $FULL_TARBALL_ROOT/prebuilt/nuget-packages
+  ./Tools/dotnetcli/dotnet publish -o $FULL_TARBALL_ROOT/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection $SCRIPT_ROOT/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj
+fi
+
 echo 'Recording commits for the source-build repo and all submodules, to aid in reproducibility...'
 
 cat >$TARBALL_ROOT/source-build-info.txt << EOF

build.proj

@@ -2,6 +2,8 @@
 <Project ToolsVersion="15.0" InitialTargets="PrepareOutput" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="dir.props" />
 
+  <UsingTask AssemblyFile="$(LeakDetectionTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll" TaskName="CheckForPoison" />
+
   <Target Name="Build" DependsOnTargets="PrepareOutput;InitBuild">
     <Message Text="Build Environment: $(Platform) $(Configuration) $(TargetOS) $(TargetRid)" />
 
@@ -45,6 +47,18 @@
     <MSBuild Projects="repos\$(RootRepo).proj" Targets="ReportPrebuiltUsage" />
   </Target>
 
+  <Target Name="ReportPoisonUsage"
+          AfterTargets="Build"
+          Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'">
+    <ItemGroup>
+      <FinalCliTarball Include="$(SourceBuiltTarBallPath)**/*$(TarBallExtension)" />
+    </ItemGroup>
+    <CheckForPoison FilesToCheck="@(FinalCliTarball)"
+                    HashCatalogFilePath="$(PoisonReportDataFile)"
+                    MarkerFileName="$(PoisonMarkerFile)"
+                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
+  </Target>
+
   <Target Name="RunSmokeTest" DependsOnTargets="GetProdConBlobFeedUrl">
     <!--
       Pass prodConBlobFeedUrl via EnvironmentVariables because it has '//' in it, which is

dir.props

@@ -47,6 +47,7 @@
     <ToolsLocalDir>$(ProjectDir)tools-local/</ToolsLocalDir>
     <TaskDirectory>$(ToolsLocalDir)tasks/</TaskDirectory>
     <TasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks/bin/Debug/netstandard1.5/</TasksBinDir>
+    <LeakDetectionTasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/</LeakDetectionTasksBinDir>
     <BaseIntermediatePath>$(BaseOutputPath)obj/</BaseIntermediatePath>
     <OutputPath>$(BaseOutputPath)$(Platform)/$(Configuration)/</OutputPath>
     <IntermediatePath>$(BaseIntermediatePath)$(Platform)/$(Configuration)/</IntermediatePath>
@@ -70,6 +71,11 @@
     <GitInfoOfflineDir>$(ProjectDir)git-info/</GitInfoOfflineDir>
     <PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
     <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
+    <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
+    <PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
+    <PoisonMarkerFile>.prebuilt.xml</PoisonMarkerFile>
+    <SourceBuiltPoisonReportDataFile>$(PackageReportDir)poison-source-built-catalog.xml</SourceBuiltPoisonReportDataFile>
+    <SourceBuiltPoisonMarkerFile>.source-built.xml</SourceBuiltPoisonMarkerFile>
     <ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
     <ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
     <PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>

netci.groovy

@@ -145,21 +145,29 @@ def addPushJob(String project, String branch, String os, String configuration, b
 
         def shortJobName = "${os}_Tarball_${configuration}";
         def contextString = "${os} Tarball ${configuration}";
+        def poisonProductionBuildArgument = ""
+        def poisonConsumptionBuildArugment = ""
 
         if (portable) {
           shortJobName += "_Portable"
           contextString += " Portable"
         }
 
+        // only run poisoning on one platform for now
+        if (os == "CentOS7.1") {
+          poisonProductionBuildArgument = "--enable-leak-detection"
+          poisonConsumptionBuildArugment = "/p:EnablePoison=true"
+        }
+
         def triggerPhrase = "(?i).*test\\W+${contextString}.*";
 
         def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
           steps{
               shell("cd ./source-build;git submodule update --init --recursive");
               shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} ${loggingOptions}");
-              shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");
+              shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build ${poisonProductionBuildArgument}");
 
-              shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
+              shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${poisonConsumptionBuildArugment} ${loggingOptions}")
               shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
           }
         }
@@ -200,12 +208,20 @@ def addPushJob(String project, String branch, String os, String configuration, b
 
         def shortJobName = "${os}_Unshared_${configuration}";
         def contextString = "${os} Unshared ${configuration}";
+        def poisonProductionBuildArgument = ""
+        def poisonConsumptionBuildArugment = ""
 
         if (portable) {
           shortJobName += "_Portable"
           contextString += " Portable"
         }
 
+        // only run poisoning on one platform for now
+        if (os == "CentOS7.1") {
+          poisonProductionBuildArgument = "--enable-leak-detection"
+          poisonConsumptionBuildArugment = "/p:EnablePoison=true"
+        }
+
         def triggerPhrase = "(?i).*test\\W+${contextString}.*";
         def imageName = getDockerImageForOs(os);
 
@@ -217,9 +233,9 @@ def addPushJob(String project, String branch, String os, String configuration, b
               // Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
               shell("mkdir tarball-output");
               // now build the tarball
-              shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
+              shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build ${poisonProductionBuildArgument}");
               // now build from the tarball offline and without access to the regular non-tarball build
-              shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
+              shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${poisonConsumptionBuildArugment} ${loggingOptions}");
               // finally, run a smoke-test on the result
               shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
           }

tools-local/init-build.proj

@@ -7,6 +7,7 @@
   <Import Project="..\dir.props" />
 
   <UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="GetHostInformation" />
+  <UsingTask AssemblyFile="$(LeakDetectionTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll" TaskName="MarkAndCatalogPackages" />
 
   <PropertyGroup>
     <BuildCompetedSuccessSemaphore>$(BaseIntermediatePath)/init-build-proj.complete</BuildCompetedSuccessSemaphore>
@@ -17,7 +18,7 @@
     Inputs="$(TargetInfoProps)"
     Outputs="$(BuildCompetedSuccessSemaphore)"
     >
-    <CallTarget Targets="BuildTasks;InstallSourceBuildSdkResolver;WriteDynamicPropsToStaticPropsFiles;GenerateRootFs;CreateAllGitInfoProps;ApplyPatches" />
+    <CallTarget Targets="BuildTasks;InstallSourceBuildSdkResolver;WriteDynamicPropsToStaticPropsFiles;GenerateRootFs;CreateAllGitInfoProps;ApplyPatches;PoisonPrebuiltPackages" />
     <Touch Files="$(BuildCompetedSuccessSemaphore)" AlwaysCreate="true" />
   </Target>
 
@@ -93,4 +94,13 @@
     <WriteLinesToFile File="$(TargetInfoProps)" Lines="$(TargetInfoPropsContent)" Overwrite="True" />
   </Target>
 
+  <Target Name="PoisonPrebuiltPackages" Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'">
+    <ItemGroup>
+      <PrebuiltPackages Include="$(PrebuiltPackagesPath)**/*.nupkg" />
+      <PrebuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)**/*.nupkg" />
+    </ItemGroup>
+    <MarkAndCatalogPackages PackagesToMark="@(PrebuiltPackages)" CatalogOutputFilePath="$(PoisonReportDataFile)" MarkerFileName="$(PoisonMarkerFile)" />
+    <MarkAndCatalogPackages PackagesToMark="@(PrebuiltSourceBuiltPackages)" CatalogOutputFilePath="$(SourceBuiltPoisonReportDataFile)" MarkerFileName="$(SourceBuiltPoisonMarkerFile)" />
+  </Target>
+
 </Project>

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs

@@ -0,0 +1,26 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Xml.Linq;
+
+namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
+{
+    internal class CatalogFileEntry
+    {
+        const string ElementName = "File";
+
+        internal string Path { get; set; }
+        internal byte[] OriginalHash { get; set; }
+        internal byte[] PoisonedHash { get; set; }
+
+        public XElement ToXml() => new XElement(ElementName,
+            new XAttribute(nameof(Path), Path),
+            new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()),
+            PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString())
+        );
+    }
+}

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs

@@ -0,0 +1,38 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Xml.Linq;
+
+namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
+{
+    internal class CatalogPackageEntry
+    {
+        const string ElementName = "Package";
+
+        internal string Path { get; set; }
+        internal string Id { get; set; }
+        internal string Version { get; set; }
+        internal byte[] OriginalHash { get; set; }
+        internal byte[] PoisonedHash { get; set; }
+        internal List<CatalogFileEntry> Files { get; }
+
+        public CatalogPackageEntry()
+        {
+            this.Files = new List<CatalogFileEntry>();
+        }
+
+        public XElement ToXml() => new XElement(ElementName,
+            new XAttribute(nameof(Path), Path),
+            new XAttribute(nameof(Id), Id),
+            new XAttribute(nameof(Version), Version),
+            new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()),
+            PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString()),
+            Files.Select(f => f.ToXml())
+        );
+    }
+}