Merge pull request #1663 from dagood/3.1.6

Author: dagood

.vsts.pipelines/jobs/ci-linux.yml

@@ -41,6 +41,12 @@ jobs:
     # Default type, can be overridden by matrix legs.
     type: Production
 
+    ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}:
+      SetInternalPackageFeedPatDockerArg: ''
+    ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      SetInternalPackageFeedPatDockerArg: >-
+        -e internalPackageFeedPat=$(dn-bot-dnceng-artifact-feeds-rw)
+
   steps:
   - template: ../steps/cleanup-unneeded-files.yml
   - template: ../steps/docker-cleanup-linux.yml
@@ -64,7 +70,7 @@ jobs:
       if [ "$(sb.tarball)" != "true" ]; then
         failOnBaselineError=true
       fi
-      $(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
+      $(docker.run) $(docker.src.map) $(docker.src.work) $(SetInternalPackageFeedPatDockerArg) $(imageName) ./build.sh \
         /p:Configuration=$(sb.configuration) \
         /p:PortableBuild=$(sb.portable) \
         /p:UseSystemLibunwind=$(systemLibunwind) \
@@ -90,7 +96,7 @@ jobs:
   - script: |
       set -ex
       df -h
-      $(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
+      $(docker.run) $(docker.src.map) $(docker.src.work) $(SetInternalPackageFeedPatDockerArg) $(imageName) ./build.sh \
         --run-smoke-test \
         /p:Configuration=$(sb.configuration) \
         /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
@@ -119,7 +125,7 @@ jobs:
       if [ "$(reportPrebuiltLeaks)" = "true" ]; then
         args="$args --enable-leak-detection"
       fi
-      $(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(imageName) ./build-source-tarball.sh \
+      $(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(SetInternalPackageFeedPatDockerArg) $(imageName) ./build-source-tarball.sh \
         "/tb/$(tarballName)" \
         $args
       du -h $(rootDirectory) | sort -h | tail -n 50

.vsts.pipelines/jobs/ci-local.yml

@@ -35,6 +35,11 @@ jobs:
     # Default type, can be overridden by matrix legs.
     type: Production
 
+    ${{ if not(and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'))) }}:
+      InternalPackageFeedPatEnv: ''
+    ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+      InternalPackageFeedPatEnv: $(dn-bot-dnceng-artifact-feeds-rw)
+
   steps:
   - checkout: self
     clean: true
@@ -64,6 +69,8 @@ jobs:
 
   # Build source-build.
   - script: ${{ format('{0}build{1} $(args.build)', parameters.scriptPrefix, parameters.scriptSuffix) }}
+    env:
+      internalPackageFeedPat: $(InternalPackageFeedPatEnv)
     displayName: Build source-build
     timeoutInMinutes: 150
 
@@ -72,6 +79,8 @@ jobs:
   # Run smoke tests.
   - ${{ if ne(parameters.skipSmokeTest, true) }}:
     - bash: ${{ format('{0}build{1} $(args.smokeTest)', parameters.scriptPrefix, parameters.scriptSuffix) }}
+      env:
+        internalPackageFeedPat: $(InternalPackageFeedPatEnv)
       displayName: Run smoke-test
 
   # Gather artifacts. Uses git bash on Windows.

Directory.Build.targets

@@ -25,7 +25,6 @@
       <IgnoredRepos>$(IgnoredRepos);https://dev.azure.com/devdiv/DevDiv/_git/DotNet-Trusted</IgnoredRepos>
       <IgnoredRepos>$(IgnoredRepos);https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Trusted</IgnoredRepos>
       <IgnoredRepos>$(IgnoredRepos);https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-optimization</IgnoredRepos>
-      <IgnoredRepos>$(IgnoredRepos);https://dev.azure.com/dnceng/internal/_git/dotnet-core-setup</IgnoredRepos>
       <IgnoredRepos>$(IgnoredRepos);https://github.com/dotnet/source-build-reference-packages</IgnoredRepos>
 
       <DarcCloneReposFolder Condition="'$(DarcCloneReposFolder)' == ''">$(ClonedSubmoduleDirectory)</DarcCloneReposFolder>

NuGet.config

@@ -3,10 +3,6 @@
   <packageSources>
     <clear />
     <!--Begin: Package sources managed by Dependency Flow automation. Do not edit the sources below.-->
-    <add key="darc-pub-dotnet-templating-4609143" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-templating-46091435/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-extensions-7d9baaf" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-extensions-7d9baaf6/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-aspnetcore-tooling-5ecfad7" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-aspnetcore-tooling-5ecfad7e/nuget/v3/index.json" />
-    <add key="darc-pub-dotnet-aspnetcore-e81033e" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/darc-pub-dotnet-aspnetcore-e81033e0/nuget/v3/index.json" />
     <!--End: Package sources managed by Dependency Flow automation. Do not edit the sources above.-->
     <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="nuget-build" value="https://dotnet.myget.org/F/nuget-build/api/v3/index.json" />

build.proj

@@ -163,7 +163,7 @@
           AfterTargets="Build"
           Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'">
     <PropertyGroup Condition="'$(DownloadSourceBuiltArtifactsTimeoutSeconds)' == ''">
-      <DownloadSourceBuiltArtifactsTimeoutSeconds>600</DownloadSourceBuiltArtifactsTimeoutSeconds>
+      <DownloadSourceBuiltArtifactsTimeoutSeconds>1500</DownloadSourceBuiltArtifactsTimeoutSeconds>
     </PropertyGroup>
 
     <DownloadFileSB 

build.sh

@@ -102,6 +102,21 @@ export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 export DOTNET_MULTILEVEL_LOOKUP=0
 export NUGET_PACKAGES="$scriptroot/packages/restored/"
 
+if [ "${internalPackageFeedPat:-}" ]; then
+  echo "Setting up NuGet credential provider using PAT from env var 'internalPackageFeedPat'..."
+  . "$scriptroot/eng/install-nuget-credprovider.sh"
+  # TODO: Read these from nuget.config
+  # The internal transport isn't added by Darc, though, so it will still need special-casing.
+  export VSS_NUGET_EXTERNAL_FEED_ENDPOINTS='{"endpointCredentials": [
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/darc-int-dotnet-templating-3a2d6426/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/darc-int-dotnet-extensions-be18161f/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/darc-int-dotnet-core-setup-3acd9b0c/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/darc-int-dotnet-aspnetcore-tooling-d63ec4a1/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/darc-int-dotnet-aspnetcore-62c098bc/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"},
+    {"endpoint":"https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v3/index.json", "username":"optional", "password":"'$internalPackageFeedPat'"}
+  ]}'
+fi
+
 set -x
 scriptroot="$( cd -P "$( dirname "$source" )" && pwd )"
 

eng/SourceBuild.Tarball.KnownExtraPrebuilts.targets

@@ -9,13 +9,13 @@
           DependsOnTargets="GetTarballDirProps">
     <ItemGroup>
       <KnownExtraPrebuiltFile Include="
-        $(TarballPrebuiltPackageDir)dotnet-dev-certs.3.0.0-rc2.19465.2.nupkg;
-        $(TarballPrebuiltPackageDir)dotnet-user-secrets.3.0.0-rc2.19465.2.nupkg;
-        $(TarballPrebuiltPackageDir)dotnet-watch.3.0.0-rc2.19465.2.nupkg;
-        $(TarballPrebuiltPackageDir)microsoft.aspnetcore.analyzers.3.0.0-rc2.19465.2.nupkg;
-        $(TarballPrebuiltPackageDir)microsoft.aspnetcore.developercertificates.xplat.3.0.0-rc2.19465.2.nupkg;
-        $(TarballPrebuiltPackageDir)microsoft.aspnetcore.mvc.analyzers.3.0.0-rc2.19465.2.nupkg;
-        $(TarballPrebuiltPackageDir)microsoft.aspnetcore.mvc.api.analyzers.3.0.0-rc2.19465.2.nupkg;
+        $(TarballPrebuiltPackageDir)dotnet-dev-certs.3.1.6-servicing.20326.1.nupkg;
+        $(TarballPrebuiltPackageDir)dotnet-user-secrets.3.1.6-servicing.20326.1.nupkg;
+        $(TarballPrebuiltPackageDir)dotnet-watch.3.1.6-servicing.20326.1.nupkg;
+        $(TarballPrebuiltPackageDir)microsoft.aspnetcore.analyzers.3.1.6-servicing.20326.1.nupkg;
+        $(TarballPrebuiltPackageDir)microsoft.aspnetcore.developercertificates.xplat.3.1.6-servicing.20326.1.nupkg;
+        $(TarballPrebuiltPackageDir)microsoft.aspnetcore.mvc.analyzers.3.1.6-servicing.20326.1.nupkg;
+        $(TarballPrebuiltPackageDir)microsoft.aspnetcore.mvc.api.analyzers.3.1.6-servicing.20326.1.nupkg;
         $(TarballPrebuiltPackageDir)microsoft.codeanalysis.analyzers.2.6.1.nupkg;
         $(TarballPrebuiltPackageDir)microsoft.codeanalysis.common.2.9.0.nupkg;
         $(TarballPrebuiltPackageDir)microsoft.codeanalysis.csharp.2.9.0.nupkg;
@@ -55,6 +55,17 @@
         $(TarballPrebuiltPackageDir)system.xml.xpath.xdocument.4.3.0.nupkg;
         $(TarballPrebuiltPackageDir)vswhere.2.6.7.nupkg;
         " />
+      <!--
+        Remove packages that have had a new release during servicing, and we want to use the
+        source-built version while building the tarball.
+
+        For example, a dependency on A/1.0.0 is checked into the repos, but now we build A/1.0.1 and
+        we have A/1.0.1 as previously-source-built. During the tarball build, we can use A/1.0.1
+        fine, but need to remove A/1.0.0 manually from the production build.
+      -->
+      <KnownExtraPrebuiltFile Include="
+        $(TarballPrebuiltPackageDir)microsoft.bcl.asyncinterfaces.1.1.0.nupkg;
+        " />
       <!--
         Remove source-link packages to throw away tooling that Arcade imports unconditionally in the
         production build.

eng/SourceBuild.Tarball.targets

@@ -151,6 +151,18 @@
         Include="$(ProjectDir)support/tarball/build.sh"
         RelativeDestination="build.sh" />
 
+      <!--
+        When building an internal build, the main smoke-test nuget config has authenticated sources
+        in it to generate a full set of smoke-test-prereqs. This isn't necessary in the tarball
+        because we have those smoke-test-prereqs, but we hit authentication errors nonetheless. Copy
+        a trimmed down nuget.config in this case.
+        TODO: Automatically trim down the main nuget.config.
+      -->
+      <TarballCopyFile
+        Condition="Exists('$(ProjectDir)support\tarball\smoke-testNuGet.Config')"
+        Include="$(ProjectDir)support\tarball\smoke-testNuGet.Config"
+        RelativeDestination="smoke-testNuGet.Config" />
+
       <!-- Setup package version props to include both source-built and running PackageVersions.props -->
       <TarballCopyFile
         Include="$(ProjectDir)support\tarball\PackageVersions.props"
@@ -161,7 +173,7 @@
   <Target Name="AddTarballPackages">
     <!-- Find reference-packages to exclude from tarball prebuilts...-->
     <ItemGroup Condition="'$(CustomRefPackagesDir)' != ''">
-      <SourceBuildReferencePackageFile Include="$(CustomRefPackagesDir)**\*.nupkg" />
+      <SourceBuildReferencePackageFile Include="$([MSBuild]::NormalizeDirectory('$(CustomRefPackagesDir)'))**\*.nupkg" />
     </ItemGroup>
 
     <Exec
@@ -178,7 +190,7 @@
 
     <!-- Find previously source-built packages to exclude from tarball prebuilts... -->
     <ItemGroup Condition="'$(CustomPreviouslySourceBuiltPackagesDir)' != ''">
-      <PreviouslySourceBuiltPackageFile Include="$(CustomPreviouslySourceBuiltPackagesDir)**\*.nupkg" />
+      <PreviouslySourceBuiltPackageFile Include="$([MSBuild]::NormalizeDirectory('$(CustomPreviouslySourceBuiltPackagesDir)'))**\*.nupkg" />
     </ItemGroup>
 
     <Exec
@@ -212,18 +224,26 @@
         RelativeDestination="packages\prebuilt\%(Filename)%(Extension)" />
 
       <SmokeTestPackageFile
-        Include="$(ProjectDir)testing-smoke\smoke-test-packages\**"
+        Include="$(ProjectDir)testing-smoke\smoke-test-packages\**\*.nupkg"
         RelativeRoot="packages\smoke-test-packages\" />
 
-      <!-- Set identity to the filename to allow for Remove, but keep old identity to swap back. -->
+      <!--
+        Set up filename-based items, in order to filter out some packages that aren't necessary as
+        prebuilts, or are generated/acquired another way.
+      -->
+      <IncludedPrebuiltPackageCandidate Include="@(PrebuiltPackageFile)" />
+
+      <!--
+        For internal builds, all smoke-test packages should be kept. For external builds, they can
+        be restored during smoke-test execution from public sources, so can be pruned.
+      -->
+      <IncludedPrebuiltPackageFile Include="@(SmokeTestPackageFile)" Condition="'$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' != ''" />
+      <IncludedPrebuiltPackageCandidate Include="@(SmokeTestPackageFile)" Condition="'$(VSS_NUGET_EXTERNAL_FEED_ENDPOINTS)' == ''" />
+
       <IncludedPrebuiltPackageFilename
-        Include="
-          @(PrebuiltPackageFile -> '%(Filename)');
-          @(SmokeTestPackageFile -> '%(Filename)');
-          "
+        Include="@(IncludedPrebuiltPackageCandidate -> '%(Filename)')"
         OriginalIdentity="%(Identity)" />
 
-      <!-- Exclude prebuilt nupkgs supplied a different way or not actually needed as prebuilts. -->
       <IncludedPrebuiltPackageFilename Remove="
         @(SourceBuiltPackageFilename);
         @(SourceBuildReferencePackageFilename);