Merge pull request #779 from dotnet-maestro-bot/merge/release/2.1-to-release/2.2

[automated] Merge branch 'release/2.1' => 'release/2.2'

Author: dagood

.gitmodules

@@ -63,3 +63,6 @@
 [submodule "src/roslyn-tools"]
 	path = src/roslyn-tools
 	url = https://github.com/dotnet/roslyn-tools
+[submodule "src/linker"]
+	path = src/linker
+	url = https://github.com/mono/linker.git

build.proj

@@ -17,6 +17,7 @@
     <MakeDir Directories="$(SourceBuiltPackagesPath)" />
     <MakeDir Directories="$(SourceBuiltAssetsDir)" />
     <MakeDir Directories="$(LocalBlobStorageRoot)" />
+    <MakeDir Directories="$(MSBuildDebugPathTargetDir)" />
   </Target>
 
   <Target Name="InitBuild">

dependencies.props

@@ -17,12 +17,6 @@
     <CurrentRefXmlPath>$(MSBuildThisFileFullPath)</CurrentRefXmlPath>
   </PropertyGroup>
 
-  <!-- ILLink.Tasks package version -->
-  <PropertyGroup>
-    <ILLinkTasksPackage>ILLink.Tasks</ILLinkTasksPackage>
-    <ILLinkTasksPackageVersion>0.1.5-preview-1461378</ILLinkTasksPackageVersion>
-  </PropertyGroup>
-
   <!--
     Packages built by ProdCon, but not source-build. These are included as prebuilts, or embedded in
     the product as version strings to be used by the SDK to fetch extra content.

dir.props

@@ -62,12 +62,14 @@
     <RestoreSourcePropsPath>$(IntermediatePath)RestoreSources.props</RestoreSourcePropsPath>
     <PackageVersionPropsPath>$(IntermediatePath)PackageVersions.props</PackageVersionPropsPath>
     <LoggingDir>$(BaseOutputPath)logs/</LoggingDir>
+    <MSBuildDebugPathTargetDir>$(BaseOutputPath)msbuild-debug/</MSBuildDebugPathTargetDir>
     <!-- Dir where git info is generated during online builds. -->
     <GitInfoOutputDir>$(BaseOutputPath)git-info/</GitInfoOutputDir>
     <!-- Dir where git info is placed inside the tarball. -->
     <GitInfoOfflineDir>$(ProjectDir)git-info/</GitInfoOfflineDir>
     <PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
     <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
+    <ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
     <ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
     <PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
     <ConflictingPackageReportDir>$(BaseOutputPath)conflict-report/</ConflictingPackageReportDir>
@@ -83,11 +85,6 @@
 
   <Import Project="$(TargetInfoProps)" Condition="$(GeneratingStaticPropertiesFile) != 'true' AND Exists('$(TargetInfoProps)')" />
 
-  <PropertyGroup>
-    <!-- Always build portable runtime on OSX to match the CLI repo's filename expectations. https://github.com/dotnet/source-build/issues/438 -->
-    <PortableBuild Condition="'$(TargetOS)' == 'OSX'">true</PortableBuild>
-  </PropertyGroup>
-
   <Import Project="$(ProjectDir)dependencies.props" />
 
   <PropertyGroup>

init-tools.msbuild

@@ -11,6 +11,5 @@
                       Condition="'$(BuildToolsPackageVersion)' != ''"
                       Version="$(BuildToolsPackageVersion)" />
     <PackageReference Include="Microsoft.DotNet.BuildTools.Coreclr" Version="1.0.4-prerelease" />
-    <PackageReference Include="$(ILLinkTasksPackage)" Version="$(ILLinkTasksPackageVersion)" />
   </ItemGroup>
 </Project>

netci.groovy

@@ -13,7 +13,9 @@ def addArchival(def job) {
   // Grab these logs from all of those locations.
   [ "", "source-build/", "tarball-output/"].each { logRoot ->
     archivalSettings.addFiles("${logRoot}bin/logs/*")
-    archivalSettings.addFiles("${logRoot}bin/prebuilt-report/*")
+    archivalSettings.addFiles("${logRoot}bin/prebuilt-report/**/*")
+    archivalSettings.addFiles("${logRoot}bin/conflict-report/**/*")
+    archivalSettings.addFiles("${logRoot}bin/msbuild-debug/**/*")
     archivalSettings.addFiles("${logRoot}src/**/*.binlog")
     archivalSettings.addFiles("${logRoot}src/**/*.log")
     archivalSettings.addFiles("${logRoot}init-tools.log")
@@ -53,16 +55,16 @@ def getDockerImageForOs(os) {
   return imageMap.get(os)
 }
 
-def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration) {
+def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration, boolean portable) {
   job.with {
     steps {
       if (os == "Windows_NT") {
         batchFile("git submodule update --init --recursive");
-        batchFile(".\\build.cmd /p:Configuration=${configuration} ${loggingOptions}")
+        batchFile(".\\build.cmd /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
       }
       else {
         shell("git submodule update --init --recursive");
-        shell("./build.sh /p:Configuration=${configuration} ${loggingOptions}");
+        shell("./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
         smokeTestExcludes = "";
         if (os == "Fedora24" || os == "OSX10.12") {
           // Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560
@@ -76,90 +78,116 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration)
   setMachineAffinity(job, os);
 }
 
-def addPullRequestJob(String project, String branch, String os, String configuration, boolean runByDefault)
+def addPullRequestJob(String project, String branch, String os, String configuration, boolean portable, boolean runByDefault)
 {
-  def newJobName = Utilities.getFullJobName(project, "${os}_${configuration}", true);
+  def config = configuration;
+  if (portable) {
+    config += "_Portable";
+  }
+  def newJobName = Utilities.getFullJobName(project, "${os}_${config}", true);
   def contextString = "${os} ${configuration}";
+  if (portable) {
+    contextString += " Portable";
+  }
   def triggerPhrase = "(?i).*test\\W+${contextString}.*";
 
   def newJob = job(newJobName);
 
-  addBuildStepsAndSetMachineAffinity(newJob, os, configuration);
+  addBuildStepsAndSetMachineAffinity(newJob, os, configuration, portable);
   addArchival(newJob);
   Utilities.standardJobSetup(newJob, project, true, "*/${branch}");
   Utilities.setJobTimeout(newJob, 180);
   Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase, !runByDefault);
 }
 
-def addPushJob(String project, String branch, String os, String configuration)
+def addPushJob(String project, String branch, String os, String configuration, boolean portable)
 {
     def shortJobName = "${os}_${configuration}";
+    if (portable) {
+      shortJobName += "_Portable";
+    }
 
     def newJobName = Utilities.getFullJobName(project, shortJobName, false);
     def newJob = job(newJobName);
 
-    addBuildStepsAndSetMachineAffinity(newJob, os, configuration);
+    addBuildStepsAndSetMachineAffinity(newJob, os, configuration, portable);
     addArchival(newJob);
     Utilities.standardJobSetup(newJob, project, false, "*/${branch}");
     Utilities.setJobTimeout(newJob, 180);
     Utilities.addGithubPushTrigger(newJob);
 }
 
 ["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "CentOS7.1", "OSX10.12"].each { os ->
-  addPullRequestJob(project, branch, os, "Release", true);
-  addPullRequestJob(project, branch, os, "Debug", false);
+  // Release non-portable run by default
+  addPullRequestJob(project, branch, os, "Release", false, true);
+  // Debug non-portable can be triggered
+  addPullRequestJob(project, branch, os, "Debug", false, false);
+  // Release portable can be triggered
+  addPullRequestJob(project, branch, os, "Release", true, false);
+  // Debug portable can be triggered
+  addPullRequestJob(project, branch, os, "Debug", true, false);
 };
 
 // Per push, run all the jobs
 ["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "Windows_NT", "CentOS7.1", "OSX10.12"].each { os ->
   ["Release", "Debug"].each { configuration ->
-    addPushJob(project, branch, os, configuration);
+    [true, false].each { portable ->
+      addPushJob(project, branch, os, configuration, portable);
+    };
   };
 };
 
 // Tarball builds that are not enforced to be offline
 [true, false].each { isPR ->
   ["RHEL7.2", "CentOS7.1"].each { os ->
     ["Release", "Debug"].each { configuration ->
+      [true, false].each { portable ->
 
-      def shortJobName = "${os}_Tarball_${configuration}";
-      def contextString = "${os} Tarball ${configuration}";
-      def triggerPhrase = "(?i).*test\\W+${contextString}.*";
+        def shortJobName = "${os}_Tarball_${configuration}";
+        def contextString = "${os} Tarball ${configuration}";
 
-      def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
-        steps{
-            shell("cd ./source-build;git submodule update --init --recursive");
-            shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
-            shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");
+        if (portable) {
+          shortJobName += "_Portable"
+          contextString += " Portable"
+        }
 
-            shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} ${loggingOptions}")
-            shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
+        def triggerPhrase = "(?i).*test\\W+${contextString}.*";
+
+        def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
+          steps{
+              shell("cd ./source-build;git submodule update --init --recursive");
+              shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} ${loggingOptions}");
+              shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");
+
+              shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
+              shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
+          }
         }
-      }
 
-      setMachineAffinity(newJob, os);
+        setMachineAffinity(newJob, os);
 
-      Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
+        Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
 
-      // Increase timeout. The tarball builds can take longer than the 2 hour default.
-      Utilities.setJobTimeout(newJob, 240);
+        // Increase timeout. The tarball builds can take longer than the 2 hour default.
+        Utilities.setJobTimeout(newJob, 240);
 
-      // Clone into the source-build directory
-      Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
+        // Clone into the source-build directory
+        Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
 
-      addArchival(newJob);
-      if(isPR){
-        if(configuration == "Release"){
-          Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
+        addArchival(newJob);
+        if(isPR){
+          if(configuration == "Release"){
+            Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
+          }
+          else{
+            Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
+          }
         }
         else{
-          Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
+          Utilities.addGithubPushTrigger(newJob);
         }
-      }
-      else{
-        Utilities.addGithubPushTrigger(newJob);
-      }
 
+      }
     }
   }
 }
@@ -168,52 +196,60 @@ def addPushJob(String project, String branch, String os, String configuration)
 [true, false].each { isPR ->
   ["RHEL7.2", "CentOS7.1"].each { os->
     ["Release", "Debug"].each { configuration ->
+      [true, false].each { portable ->
+
+        def shortJobName = "${os}_Unshared_${configuration}";
+        def contextString = "${os} Unshared ${configuration}";
 
-      def shortJobName = "${os}_Unshared_${configuration}";
-      def contextString = "${os} Unshared ${configuration}";
-      def triggerPhrase = "(?i).*test\\W+${contextString}.*";
-      def imageName = getDockerImageForOs(os);
-
-      def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
-        steps{
-            shell("cd ./source-build;git submodule update --init --recursive");
-            // First build the product itself
-            shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}");
-            // Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
-            shell("mkdir tarball-output");
-            // now build the tarball
-            shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
-            // now build from the tarball offline and without access to the regular non-tarball build
-            shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} ${loggingOptions}");
-            // finally, run a smoke-test on the result
-            shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
+        if (portable) {
+          shortJobName += "_Portable"
+          contextString += " Portable"
         }
-      }
 
-      // Only Ubuntu Jenkins machines have Docker
-      setMachineAffinity(newJob, "Ubuntu16.04");
+        def triggerPhrase = "(?i).*test\\W+${contextString}.*";
+        def imageName = getDockerImageForOs(os);
+
+        def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
+          steps{
+              shell("cd ./source-build;git submodule update --init --recursive");
+              // First build the product itself
+              shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}");
+              // Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
+              shell("mkdir tarball-output");
+              // now build the tarball
+              shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
+              // now build from the tarball offline and without access to the regular non-tarball build
+              shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
+              // finally, run a smoke-test on the result
+              shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
+          }
+        }
 
-      Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
+        // Only Ubuntu Jenkins machines have Docker
+        setMachineAffinity(newJob, "Ubuntu16.04");
 
-      // Increase timeout. The offline build in Docker takes more than 2 hours.
-      Utilities.setJobTimeout(newJob, 240);
+        Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
 
-      // Clone into the source-build directory
-      Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
+        // Increase timeout. The offline build in Docker takes more than 2 hours.
+        Utilities.setJobTimeout(newJob, 240);
 
-      addArchival(newJob);
-      if(isPR){
-        if(configuration == "Release"){
-          Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
+        // Clone into the source-build directory
+        Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
+
+        addArchival(newJob);
+        if(isPR){
+          if(configuration == "Release"){
+            Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
+          }
+          else{
+            Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
+          }
         }
         else{
-          Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
+          Utilities.addGithubPushTrigger(newJob);
         }
-      }
-      else{
-        Utilities.addGithubPushTrigger(newJob);
-      }
 
+      }
     }
   }
 }