Merged PR 130846: Add archiving of packages restored during smoke-test

Add archiving of packages restored during smoke-test and include them in the tarball so they can be used by the tarball smoke-test.

Author: dseefeld

.vsts-ci.yml

@@ -16,7 +16,7 @@ phases:
   queue:
     name: DotNet-Build
     demands: agent.os -equals linux
-    timeoutInMinutes: 180
+    timeoutInMinutes: 240
     parallel: 2
     matrix:
       centos71:
@@ -69,7 +69,6 @@ phases:
     # run smoke tests
     - script: $(docker.runInSourceBuild) $(imageName) ./build.sh /t:RunSmokeTest /p:Configuration=$(buildConfiguration) /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
       displayName: Run smoke-test
-      condition: and(succeeded(), eq(variables['buildOfflineTarball'], false))
 
     # copy smoke test logs to working directory
     - script: $(docker.runInSourceBuild) -v $(dropDirectory)/logs:/logs $(imageName)  /bin/bash -c "mkdir -p /logs/source-build/smoke-test; find ./testing-smoke -name '*.log' -exec cp {} /logs/source-build/smoke-test \;"
@@ -94,10 +93,7 @@ phases:
       condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
 
     # run smoke tests
-    # "-e SKIP_INIT_BUILD=true": init-build was already done by the build. Now, skip it, because the
-    # support build.sh tries to pass all args to init-build.proj where there is no "RunSmokeTest".
-    - script: $(docker.runInTarball) -e SKIP_INIT_BUILD=true $(imageName) ./build.sh /t:RunSmokeTest /p:Configuration=$(buildConfiguration) /p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
-      displayName: Run smoke-test
+    - script: $(docker.runInTarball) $(imageName) ./smoke-test.sh --minimal --projectOutput --configuration $(buildConfiguration)
       condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
 
     # copy all tarball logs to working directory

build-source-tarball.sh

@@ -83,13 +83,20 @@ mkdir -p $TARBALL_ROOT/prebuilt/nuget-packages
 find $SCRIPT_ROOT/packages -name '*.nupkg' -exec cp {} $TARBALL_ROOT/prebuilt/nuget-packages/ \;
 find $SCRIPT_ROOT/bin/obj/x64/Release/nuget-packages -name '*.nupkg' -exec cp {} $TARBALL_ROOT/prebuilt/nuget-packages/ \;
 
+if [ -e $SCRIPT_ROOT/testing-smoke/smoke-test-packages ]; then
+    cp -rf $SCRIPT_ROOT/testing-smoke/smoke-test-packages $TARBALL_ROOT/prebuilt
+fi
+
 echo 'Removing source-built packages from tarball prebuilts...'
 
 for built_package in $(find $SCRIPT_ROOT/bin/obj/x64/Release/blob-feed/packages/ -name '*.nupkg' | tr '[:upper:]' '[:lower:]')
 do
     if [ -e $TARBALL_ROOT/prebuilt/nuget-packages/$(basename $built_package) ]; then
         rm $TARBALL_ROOT/prebuilt/nuget-packages/$(basename $built_package)
     fi
+    if [ -e $TARBALL_ROOT/prebuilt/smoke-test-packages/$(basename $built_package) ]; then
+        rm $TARBALL_ROOT/prebuilt/smoke-test-packages/$(basename $built_package)
+    fi
 done
 
 if [ -z "${SOURCE_BUILD_SKIP_PREBUILT_REPORT:-}" ]; then

build.proj

@@ -53,7 +53,7 @@
       translated into '/' if it's passed in the Command arg.
       This is also a problem when passing CLI feeds: https://github.com/dotnet/source-build/issues/561
     -->
-    <Exec Command="./smoke-test.sh --minimal --projectOutput --configuration $(Configuration)"
+    <Exec Command="./smoke-test.sh --minimal --projectOutput --configuration $(Configuration) --archiveRestoredPackages"
           EnvironmentVariables="prodConBlobFeedUrl=$(ProdConBlobFeedUrl)" />
   </Target>
 

smoke-test.sh

@@ -23,6 +23,9 @@ logFile="$testingDir/smoke-test.log"
 restoredPackagesDir="$testingDir/packages"
 testingHome="$testingDir/home"
 prodConBlobFeedUrl=${prodConBlobFeedUrl-}
+archiveRestoredPackages=false
+archivedPackagesDir="$testingDir/smoke-test-packages"
+smokeTestPrebuilts="$SCRIPT_ROOT/prebuilt/smoke-test-packages"
 
 function usage() {
     echo ""
@@ -40,6 +43,7 @@ function usage() {
     echo "  --excludeOnlineTests           exclude test that use online sources for nuget packages"
     echo "  --devCertsVersion <version>    use dotnet-dev-certs <version> instead of default $DEV_CERTS_VERSION_DEFAULT"
     echo "  --prodConBlobFeedUrl <url>     override the prodcon blob feed specified in ProdConFeed.txt"
+    echo "  --archiveRestoredPackages      capture all restored packages to $archivedPackagesDir"
     echo "environment:"
     echo "  prodConBlobFeedUrl    override the prodcon blob feed specified in ProdConFeed.txt"
     echo ""
@@ -99,6 +103,9 @@ while :; do
             shift
             prodConBlobFeedUrl="$1"
             ;;
+        --archiverestoredpackages)
+            archiveRestoredPackages=true
+            ;;
         *)
             usage
             exit 1
@@ -263,6 +270,22 @@ function setupProdConFeed() {
     sed -i.bakProdCon "s|PRODUCT_CONTRUCTION_PACKAGES|$prodConBlobFeedUrl|g" "$testingDir/NuGet.Config"
 }
 
+function setupSmokeTestFeed() {
+    # Setup smoke-test-packages if they exist
+    if [ -e "$smokeTestPrebuilts" ]; then
+        sed -i.bakSmokeTestFeed "s|SMOKE_TEST_PACKAGE_FEED|$smokeTestPrebuilts|g" "$testingDir/NuGet.Config"
+    else
+        sed -i.bakSmokeTestFeed "/SMOKE_TEST_PACKAGE_FEED/d" "$testingDir/NuGet.Config"
+    fi
+}
+
+function copyRestoredPackages() {
+    if [ "$archiveRestoredPackages" == "true" ]; then
+        mkdir -p "$archivedPackagesDir"
+        cp -rf "$restoredPackagesDir"/* "$archivedPackagesDir"
+    fi
+}
+
 # Clean up and create directory
 if [ -e "$testingDir"  ]; then
     read -p "testing-smoke directory exists, remove it? [Y]es / [n]o" -n 1 -r
@@ -302,11 +325,13 @@ if [ "$excludeLocalTests" == "false" ]; then
         cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config"
         sed -i.bak "s|SOURCE_BUILT_PACKAGES|$SOURCE_BUILT_PKGS_PATH|g" "$testingDir/NuGet.Config"
         setupProdConFeed
+        setupSmokeTestFeed
         echo "$testingDir/NuGet.Config Contents:"
         cat "$testingDir/NuGet.Config"
     fi
     echo "RUN ALL TESTS - LOCAL RESTORE SOURCE"
     runAllTests
+    copyRestoredPackages
     echo "LOCAL RESTORE SOURCE - ALL TESTS PASSED!"
 fi
 
@@ -317,11 +342,13 @@ if [ "$excludeOnlineTests" == "false" ]; then
         cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config"
         sed -i.bak "/SOURCE_BUILT_PACKAGES/d" "$testingDir/NuGet.Config"
         setupProdConFeed
+        setupSmokeTestFeed
         echo "$testingDir/NuGet.Config Contents:"
         cat "$testingDir/NuGet.Config"
     fi
     echo "RUN ALL TESTS - ONLINE RESTORE SOURCE"
     runAllTests
+    copyRestoredPackages
     echo "ONLINE RESTORE SOURCE - ALL TESTS PASSED!"
 fi
 

smoke-testNuGet.Config

@@ -7,5 +7,6 @@
     <add key="api.nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="dotnet.myget.org/feed/dotnet-core" value="https://dotnet.myget.org/F/dotnet-core/api/v3/index.json" />
     <add key="ProdCon" value="PRODUCT_CONTRUCTION_PACKAGES" />
+    <add key="smoke-test feed" value="SMOKE_TEST_PACKAGE_FEED" />
   </packageSources>
 </configuration>

support/tarball/build.sh

@@ -34,7 +34,5 @@ PREBUILT_PACKAGE_SOURCE="$SCRIPT_ROOT/prebuilt/nuget-packages"
     exit 1
 )
 
-if [ -z "${SKIP_INIT_BUILD:-}" ]; then
-    $CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/tools-local/init-build.proj /t:WriteDynamicPropsToStaticPropsFiles /p:GeneratingStaticPropertiesFile=true ${MSBUILD_ARGUMENTS[@]} "$@"
-fi
+$CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/tools-local/init-build.proj /t:WriteDynamicPropsToStaticPropsFiles /p:GeneratingStaticPropertiesFile=true ${MSBUILD_ARGUMENTS[@]} "$@"
 $CLI_ROOT/dotnet $CLI_ROOT/sdk/$CLI_VERSION/MSBuild.dll $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@"