Add tasks for poisoning and checking for poison in packages. (#797)

* Add tasks for poisoning and checking for poison in packages.

* Fix false positives in archives.

* Address code review comments and remove debug code.

Author: crummel

build-source-tarball.sh

@@ -3,7 +3,7 @@ set -euo pipefail
 IFS=$'\n\t'
 
 usage() {
-    echo "usage: $0 <path-to-tarball-root> [--skip-build]"
+    echo "usage: $0 <path-to-tarball-root> [--skip-build] [--enable-leak-detection]"
     echo ""
 }
 
@@ -16,6 +16,8 @@ TARBALL_ROOT=$1
 shift
 
 SKIP_BUILD=0
+INCLUDE_LEAK_DETECTION=0
+export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 
 while :; do
     if [ $# -le 0 ]; then
@@ -31,6 +33,9 @@ while :; do
         --skip-build)
             SKIP_BUILD=1
             ;;
+        --enable-leak-detection)
+            INCLUDE_LEAK_DETECTION=1
+            ;;
         *)
             echo "Unrecognized argument '$1'"
             usage
@@ -158,6 +163,12 @@ SOURCE_BUILT_SDK_TOOLS_DIR="$TARBALL_ROOT/Tools/source-built/$ROSLYN_TOOLS_PACKA
 cp "$REPO_TOOLSET_PACKAGE_DIR/tools/"*.props "$SOURCE_BUILT_SDK_TOOLS_DIR"
 cp "$REPO_TOOLSET_PACKAGE_DIR/tools/"*.targets "$SOURCE_BUILT_SDK_TOOLS_DIR"
 
+if [ $INCLUDE_LEAK_DETECTION -eq 1 ]; then
+  echo 'Building leak detection MSBuild tasks...'
+  ./Tools/dotnetcli/dotnet restore $SCRIPT_ROOT/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj --source $FULL_TARBALL_ROOT/prebuilt/source-built --source $FULL_TARBALL_ROOT/prebuilt/nuget-packages
+  ./Tools/dotnetcli/dotnet publish -o $FULL_TARBALL_ROOT/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection $SCRIPT_ROOT/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj
+fi
+
 echo 'Recording commits for the source-build repo and all submodules, to aid in reproducibility...'
 
 cat >$TARBALL_ROOT/source-build-info.txt << EOF

build.proj

@@ -2,6 +2,8 @@
 <Project ToolsVersion="15.0" InitialTargets="PrepareOutput" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <Import Project="dir.props" />
 
+  <UsingTask AssemblyFile="$(LeakDetectionTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll" TaskName="CheckForPoison" />
+
   <Target Name="Build" DependsOnTargets="PrepareOutput;InitBuild">
     <Message Text="Build Environment: $(Platform) $(Configuration) $(TargetOS) $(TargetRid)" />
 
@@ -45,6 +47,18 @@
     <MSBuild Projects="repos\$(RootRepo).proj" Targets="ReportPrebuiltUsage" />
   </Target>
 
+  <Target Name="ReportPoisonUsage"
+          AfterTargets="Build"
+          Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'">
+    <ItemGroup>
+      <FinalCliTarball Include="$(SourceBuiltTarBallPath)**/*$(TarBallExtension)" />
+    </ItemGroup>
+    <CheckForPoison FilesToCheck="@(FinalCliTarball)"
+                    HashCatalogFilePath="$(PoisonReportDataFile)"
+                    MarkerFileName="$(PoisonMarkerFile)"
+                    PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
+  </Target>
+
   <Target Name="RunSmokeTest" DependsOnTargets="GetProdConBlobFeedUrl">
     <!--
       Pass prodConBlobFeedUrl via EnvironmentVariables because it has '//' in it, which is

dir.props

@@ -47,6 +47,7 @@
     <ToolsLocalDir>$(ProjectDir)tools-local/</ToolsLocalDir>
     <TaskDirectory>$(ToolsLocalDir)tasks/</TaskDirectory>
     <TasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks/bin/Debug/netstandard1.5/</TasksBinDir>
+    <LeakDetectionTasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/</LeakDetectionTasksBinDir>
     <BaseIntermediatePath>$(BaseOutputPath)obj/</BaseIntermediatePath>
     <OutputPath>$(BaseOutputPath)$(Platform)/$(Configuration)/</OutputPath>
     <IntermediatePath>$(BaseIntermediatePath)$(Platform)/$(Configuration)/</IntermediatePath>
@@ -70,6 +71,11 @@
     <GitInfoOfflineDir>$(ProjectDir)git-info/</GitInfoOfflineDir>
     <PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
     <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
+    <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
+    <PoisonReportDataFile>$(PackageReportDir)poison-catalog.xml</PoisonReportDataFile>
+    <PoisonMarkerFile>.prebuilt.xml</PoisonMarkerFile>
+    <SourceBuiltPoisonReportDataFile>$(PackageReportDir)poison-source-built-catalog.xml</SourceBuiltPoisonReportDataFile>
+    <SourceBuiltPoisonMarkerFile>.source-built.xml</SourceBuiltPoisonMarkerFile>
     <ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
     <ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
     <PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>

tools-local/init-build.proj

@@ -7,6 +7,7 @@
   <Import Project="..\dir.props" />
 
   <UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="GetHostInformation" />
+  <UsingTask AssemblyFile="$(LeakDetectionTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll" TaskName="MarkAndCatalogPackages" />
 
   <PropertyGroup>
     <BuildCompetedSuccessSemaphore>$(BaseIntermediatePath)/init-build-proj.complete</BuildCompetedSuccessSemaphore>
@@ -17,7 +18,7 @@
     Inputs="$(TargetInfoProps)"
     Outputs="$(BuildCompetedSuccessSemaphore)"
     >
-    <CallTarget Targets="BuildTasks;InstallSourceBuildSdkResolver;WriteDynamicPropsToStaticPropsFiles;GenerateRootFs;CreateAllGitInfoProps;ApplyPatches" />
+    <CallTarget Targets="BuildTasks;InstallSourceBuildSdkResolver;WriteDynamicPropsToStaticPropsFiles;GenerateRootFs;CreateAllGitInfoProps;ApplyPatches;PoisonPrebuiltPackages" />
     <Touch Files="$(BuildCompetedSuccessSemaphore)" AlwaysCreate="true" />
   </Target>
 
@@ -93,4 +94,13 @@
     <WriteLinesToFile File="$(TargetInfoProps)" Lines="$(TargetInfoPropsContent)" Overwrite="True" />
   </Target>
 
+  <Target Name="PoisonPrebuiltPackages" Condition="'$(EnablePoison)' == 'true' and '$(OfflineBuild)' == 'true'">
+    <ItemGroup>
+      <PrebuiltPackages Include="$(PrebuiltPackagesPath)**/*.nupkg" />
+      <PrebuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)**/*.nupkg" />
+    </ItemGroup>
+    <MarkAndCatalogPackages PackagesToMark="@(PrebuiltPackages)" CatalogOutputFilePath="$(PoisonReportDataFile)" MarkerFileName="$(PoisonMarkerFile)" />
+    <MarkAndCatalogPackages PackagesToMark="@(PrebuiltSourceBuiltPackages)" CatalogOutputFilePath="$(SourceBuiltPoisonReportDataFile)" MarkerFileName="$(SourceBuiltPoisonMarkerFile)" />
+  </Target>
+
 </Project>

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs

@@ -0,0 +1,26 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Xml.Linq;
+
+namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
+{
+    internal class CatalogFileEntry
+    {
+        const string ElementName = "File";
+
+        internal string Path { get; set; }
+        internal byte[] OriginalHash { get; set; }
+        internal byte[] PoisonedHash { get; set; }
+
+        public XElement ToXml() => new XElement(ElementName,
+            new XAttribute(nameof(Path), Path),
+            new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()),
+            PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString())
+        );
+    }
+}

tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs

@@ -0,0 +1,38 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Xml.Linq;
+
+namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection
+{
+    internal class CatalogPackageEntry
+    {
+        const string ElementName = "Package";
+
+        internal string Path { get; set; }
+        internal string Id { get; set; }
+        internal string Version { get; set; }
+        internal byte[] OriginalHash { get; set; }
+        internal byte[] PoisonedHash { get; set; }
+        internal List<CatalogFileEntry> Files { get; }
+
+        public CatalogPackageEntry()
+        {
+            this.Files = new List<CatalogFileEntry>();
+        }
+
+        public XElement ToXml() => new XElement(ElementName,
+            new XAttribute(nameof(Path), Path),
+            new XAttribute(nameof(Id), Id),
+            new XAttribute(nameof(Version), Version),
+            new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()),
+            PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString()),
+            Files.Select(f => f.ToXml())
+        );
+    }
+}