Update bootstrap scripts to use params for source-build and s-b-r-p build.sh (#1476)

* Update bootstrap scripts to use params for source-build build.sh and
source-build-reference-packages build.sh

* Update CI to work with updated bootstrap script

* Fix cleanup of reference packages

* Enable smoke-test-prereqs in bootstrapping final SDK smoke tests.

Author: dseefeld

.vsts.pipelines/jobs/ci-linux_bootstrap.yml

@@ -42,7 +42,7 @@ jobs:
     displayName: Download CentOS artifacts
     inputs:
       buildType: 'current'
-      itemPattern: 'Tarball centos71 Offline/$(tarballName).tar.gz'
+      itemPattern: 'Tarball centos71 Offline/$(tarballName)*.tar.gz'
       artifactName: 'Tarball centos71 Offline'
       downloadPath: '$(System.ArtifactsDirectory)/boostrap_root'
 

.vsts.pipelines/steps/run-bootstrap.yml

@@ -7,7 +7,7 @@ steps:
       $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
       mkdir -p /bst/sb/tarball
       tar -xf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz" -C /bst/sb/tarball --no-same-owner --strip-components=1
-      rm -rf "/bst/Tarball centos71 Offline"'
+      rm -rf "/bst/Tarball centos71 Offline/$(tarballName).tar.gz"'
       df -h
   displayName: Unpack Tarball
   condition: always()
@@ -84,16 +84,18 @@ steps:
   displayName: Build stage1 source-build-reference-packages
   condition: succeeded()
 
-# Copy ref-packages tarball to drop directory.
+# Create and copy ref-packages tarball to drop directory.
 - script: |
     set -ex
     df -h
     $(docker.run) $(docker.bst.map) $(docker.drop.map) $(docker.bst.work) $(imageName) /bin/bash -c "
       mkdir -p /drop/tarball/reference-packages
       pushd /bst/bootstrap_dir/reference-packages
+      tar --numeric-owner -czf Private.SourceBuild.ReferencePackages.bootstrap.tar.gz *.nupkg
       find . \( \
         -iname 'Private.SourceBuild.ReferencePackages.bootstrap.tar.gz' \) \
         -exec cp {} /drop/tarball/reference-packages \;
+      rm -rf Private.SourceBuild.ReferencePackages.bootstrap.tar.gz
       popd"
     du -h $(rootDirectory) | sort -h | tail -n 50
   displayName: Copy ref-packages tarball 
@@ -108,8 +110,8 @@ steps:
       rm -rf /bst/source-build-reference-packages/*
       rm -rf /bst/source-build-reference-packages/.git
       pushd /bst/bootstrap_dir/reference-packages
-      echo "Removing everything in reference-packages except artifacts"
-      find . -type f -not -name '*SourceBuild.ReferencePackages.bootstrap.tar.gz' -delete
+      echo "Removing everything in reference-packages except nupkg artifacts"
+      find . -type f -not -name '*.nupkg' -delete
       popd'
   displayName: Clean all except tarball in source-build-reference-packages
   condition: always()
@@ -177,6 +179,17 @@ steps:
   condition: always()
   continueOnError: true
 
+- script: |
+      set -ex
+      mount
+      df -h
+      $(docker.run) --network=none $(docker.bst.map) $(docker.bst.work) $(imageName) bash -c '
+      tar -xf "/bst/Tarball centos71 Offline/$(tarballName)-smoke-test-prereqs.tar.gz" -C /bst/bootstrap_dir/final-sdk --no-same-owner --strip-components=1
+      rm -rf "/bst/Tarball centos71 Offline/$(tarballName)-smoke-test-prereqs.tar.gz"'
+      df -h
+  displayName: Unpack smoke test prereqs
+  condition: always()
+
 - script: |
       set -ex 
       df -h 

smoke-test.sh

@@ -30,7 +30,7 @@ restoredPackagesDir="$testingDir/packages"
 testingHome="$testingDir/home"
 archiveRestoredPackages=false
 archivedPackagesDir="$testingDir/smoke-test-packages"
-smokeTestPrebuilts="$SCRIPT_ROOT/prebuilt/smoke-test-packages"
+smokeTestPrebuilts="$SCRIPT_ROOT/packages/smoke-test-packages"
 runningOnline=false
 runningHttps=false
 

support/tarball/tool-bootstrapping/bootstrap.sh

@@ -80,32 +80,16 @@ function BuildRefPkgs {
     LogMessage "Copy $refPkgSourceDir to $refPkgsDir"
     cp -r $refPkgSourceDir $refPkgsDir
 
-    LogMessage "Copy coreclr-tools to $refPkgsDir/source-built"
-    cp -r $stage1Dir/Tools/source-built/coreclr-tools/ $refPkgsDir/source-built
-
-    LogMessage "Un-tar private source-built artifacts into $refPkgsDir/source-built/"
-    tar -xzf $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts.*.tar.gz --directory $refPkgsDir/source-built/
-
-    LogMessage "Un-tar sdk into $refPkgsDir/.dotnet/"
-    if [ -d "$refPkgsDir/.dotnet/" ]; then
-        rm -rf $refPkgsDir/.dotnet/
+    LogMessage "Un-tar sdk into $stage1SdkInstallDir"
+    if [ -d "$stage1SdkInstallDir" ]; then
+        rm -rf $stage1SdkInstallDir
     fi
-    mkdir $refPkgsDir/.dotnet/
-    tar -xzf $stage1Dir/bin/x64/Release/dotnet-sdk-3*.tar.gz --directory $refPkgsDir/.dotnet/
-
-    LogMessage "Update SDK version in global.json file"
-    sdkVer=`ls $refPkgsDir/.dotnet/sdk`
-    LogMessage "    Sdk version = [$sdkVer]"
-    sed -i "s|\"dotnet\": \".*|\"dotnet\": \"$sdkVer\"|g" $refPkgsDir/global.json
+    mkdir $stage1SdkInstallDir
+    tar -xzf $stage1Dir/bin/x64/Release/dotnet-sdk-3*.tar.gz --directory $stage1SdkInstallDir
 
     LogMessage "Build stage1 ref-pkgs"
     pushd $refPkgsDir
-    ./build.sh
-    popd
-
-    LogMessage "Package built reference packages into new tarball"
-    pushd $refPkgsDir/artifacts/reference-packages
-    tar --numeric-owner -czf $refPkgsDir/artifacts/Private.SourceBuild.ReferencePackages.bootstrap.tar.gz *.nupkg
+    ./build.sh --with-packages $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts.*.tar.gz --with-sdk $stage1SdkInstallDir
     popd
 
     LogMessage "Completing Step2 - Build Reference Packages" 
@@ -139,45 +123,22 @@ function buildFinalSdk {
     fi
 
     if [ "${1-default}" != "test" ]; then
-    LogMessage "Copy $tarballSourceDir to $finalSdkDir"
-    cp -r $tarballSourceDir $finalSdkDir
+        LogMessage "Copy $tarballSourceDir to $finalSdkDir"
+        cp -r $tarballSourceDir $finalSdkDir
     fi
 
     LogMessage "Remove archive packages, source-built packages and SDK from final-sdk dir"
-    rm -rf $finalSdkDir/packages/archive/*
+    rm -rf $finalSdkDir/packages/archive
     rm -rf $finalSdkDir/packages/source-built/*
     rm -rf $finalSdkDir/.dotnet
 
-    LogMessage "Copy stage1 coreclr-tools to $finalSdkDir/packages/source-built"
-    cp -r $stage1Dir/Tools/source-built/coreclr-tools $finalSdkDir/packages/source-built
-
-    LogMessage "Copy source-built packages archive from $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts*.tar.gz to $finalSdkDir/packages/archive"
-    cp $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts*.tar.gz $finalSdkDir/packages/archive
-
-    LogMessage "Extract .NET Core SDK from $stage1Dir/bin/x64/Release/dotnet-sdk-3*.tar.gz to $finalSdkDir/.dotnet/"
-    if [ -d "$finalSdkDir/.dotnet/" ]; then
-        rm -rf $finalSdkDir/.dotnet/
-    fi
-    mkdir $finalSdkDir/.dotnet/
-    tar -xzf $stage1Dir/bin/x64/Release/dotnet-sdk-3*.tar.gz --directory $finalSdkDir/.dotnet/
-
-    LogMessage "Copy reference packages from $refPkgsDir/artifacts/Private.SourceBuild.ReferencePackages.bootstrap.tar.gz to $finalSdkDir/packages/archive"
-    cp $refPkgsDir/artifacts/Private.SourceBuild.ReferencePackages.bootstrap.tar.gz $finalSdkDir/packages/archive
-
-    LogMessage "Update version of SDK and arcade in global.json"
-    arcadeName=$(tar -tf $finalSdkDir/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz Microsoft.DotNet.Arcade.Sdk.*.nupkg)
-    arcadeVer=${arcadeName//Microsoft.DotNet.Arcade.Sdk.}
-    arcadeVer=${arcadeVer//.nupkg}
-    LogMessage "    Arcade version = [$arcadeVer]"
-    sed -i "s|\(\"Microsoft.DotNet.Arcade.Sdk\": \"\)[^\"]*\"|\1$arcadeVer\"|g" $finalSdkDir/global.json
-
-    sdkVer=$(ls $finalSdkDir/.dotnet/sdk)
-    LogMessage "    Sdk version = [$sdkVer]"
-    sed -i "s|\"dotnet\": \".*|\"dotnet\": \"$sdkVer\"|g" $finalSdkDir/global.json
+    LogMessage "Install previously source-built archive from $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts*.tar.gz to $stage1SourceBuiltArtifactsInstallDir"
+    mkdir -p $stage1SourceBuiltArtifactsInstallDir
+    tar -xzf $stage1Dir/bin/x64/Release/Private.SourceBuilt.Artifacts*.tar.gz --directory $stage1SourceBuiltArtifactsInstallDir
 
     LogMessage "Build final sdk"
     pushd $finalSdkDir
-    ./build.sh
+    ./build.sh --with-ref-packages $refPkgsDir/artifacts/reference-packages --with-packages $stage1SourceBuiltArtifactsInstallDir --with-sdk $stage1SdkInstallDir
     popd
 
     LogMessage "Completing Step3 - Build final sdk"
@@ -205,6 +166,8 @@ refPkgSourceDir="$(abspath $3)"
 startStep="${4:-}"
 
 stage1Dir="$bootstrapDir/stage1-sdk"
+stage1SdkInstallDir="$bootstrapDir/stage1-sdk-install"
+stage1SourceBuiltArtifactsInstallDir="$bootstrapDir/stage1-source-built-artifacts-install"
 refPkgsDir="$bootstrapDir/reference-packages"
 finalSdkDir="$bootstrapDir/final-sdk"