Skip to content

Commit 5dcc817

Browse files
dseefelddseefeld
authored
Build base tarball (#2170)
* Add Artifacts text file to packages output * Add build of bare tarball plush prep.sh * Update CI to build base tarball * Build BaseTarball as part of centos71 Offline * Build base tarball with centos71 online * Add missing single quote * Change permissions on prep.sh file * Fixes based on PR comments
1 parent 8a18c55 commit 5dcc817

File tree

9 files changed

+190
-3
lines changed

9 files changed

+190
-3
lines changed

‎.vsts.pipelines/builds/matrix.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ stages:
2121
# Temporarily disable unit tests until entire product builds.
2222
# See https://github.com/dotnet/source-build/issues/1549
2323
Production: { generatePrebuiltBurndown: true, runUnitTests: false }
24-
Online: { type: Online }
24+
Online: { type: Online BaseTarball }
2525
Offline: { type: Offline }
2626
Offline Portable: { type: Offline Portable }
2727

‎.vsts.pipelines/jobs/ci-linux.yml

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ jobs:
3232
stagingDirectory: $(rootDirectory)/sb/staging
3333
systemLibunwind: ${{ parameters.systemLibunwind }}
3434
tarballName: tarball_$(Build.BuildId)
35+
baseTarballName: base_tarball_$(Build.BuildId)
3536
tarballDownloadArgs: >-
3637
/p:DownloadSourceBuildReferencePackagesTimeoutSeconds=600
3738
/p:DownloadSourceBuiltArtifactsTimeoutSeconds=1500
@@ -121,6 +122,21 @@ jobs:
121122
timeoutInMinutes: 45
122123
continueOnError: true
123124
125+
# Create base tarball.
126+
- script: |
127+
set -ex
128+
df -h
129+
args="--skip-build --base-tarball"
130+
if [ "$(reportPrebuiltLeaks)" = "true" ]; then
131+
args="$args --enable-leak-detection"
132+
fi
133+
$(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(SetInternalPackageFeedPatDockerArg) $(imageName) ./build-source-tarball.sh \
134+
"/tb/$(baseTarballName)" \
135+
$args
136+
du -h $(rootDirectory) | sort -h | tail -n 50
137+
displayName: Create base tarball
138+
condition: and(succeeded(), eq(variables['sb.baseTarball'], true))
139+
124140
# Create tarball.
125141
- script: |
126142
set -ex
@@ -206,6 +222,19 @@ jobs:
206222
displayName: Copy tarball to output
207223
condition: and(succeeded(), eq(variables['sb.tarball'], true))
208224
225+
# tar the base tarball directory into the drop directory.
226+
- script: |
227+
set -ex
228+
df -h
229+
$(docker.run) $(docker.tb.map) $(docker.drop.map) $(docker.tb.work) $(imageName) /bin/bash -c '
230+
mkdir -p /drop/tarball/
231+
# Make one .tar.gz for build, another for extras necessary to smoke test:
232+
tar --numeric-owner -zcf "/drop/tarball/$(baseTarballName).tar.gz" "$(baseTarballName)"'
233+
du -h $(rootDirectory) | sort -h | tail -n 50
234+
displayName: Copy base tarball to output
235+
condition: and(succeeded(), eq(variables['sb.baseTarball'], true))
236+
237+
209238
# Build tarball.
210239
- script: |
211240
set -ex

‎.vsts.pipelines/steps/calculate-config-flags.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,18 @@ steps:
99
1010
setvariable sb.configuration Release
1111
setvariable sb.portable false
12+
setvariable sb.baseTarball false
1213
setvariable sb.tarball false
1314
setvariable sb.tarballOffline false
1415
1516
echo "Setting up configuration variables for CI type '$TYPE'"
1617
1718
for x in $TYPE; do
1819
case $x in
20+
BaseTarball)
21+
setvariable sb.baseTarball true
22+
;;
23+
1924
Debug|Release)
2025
setvariable sb.configuration $x
2126
;;

‎Directory.Build.props

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,8 @@
171171
<SourceBuiltArtifactsTarballUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/</SourceBuiltArtifactsTarballUrl>
172172
<ReferencePackagesTarballName>Private.SourceBuild.ReferencePackages</ReferencePackagesTarballName>
173173
<ReferencePackagesTarballUrl>https://dotnetfeed.blob.core.windows.net/dotnet-core/assets/</ReferencePackagesTarballUrl>
174+
<ArchiveArtifactsTextFileName>archiveArtifacts.txt</ArchiveArtifactsTextFileName>
175+
<ArchiveArtifactsTextFile>$(ExternalTarballsDir)$(ArchiveArtifactsTextFileName)</ArchiveArtifactsTextFile>
174176
<BaselineDataFile>$(ToolsLocalDir)prebuilt-baseline-</BaselineDataFile>
175177
<OfflineBaselineDataFile>$(BaselineDataFile)offline.xml</OfflineBaselineDataFile>
176178
<OnlineBaselineDataFile>$(BaselineDataFile)online.xml</OnlineBaselineDataFile>

‎build-source-tarball.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ usage() {
1111
echo " --with-ref-packages <dir> use the specified directory of available reference packages to determine what prebuilts to delete, instead of downloading the most recent version"
1212
echo " --with-packages <dir> use the specified directory of available previously-built packages to determine what prebuilts to delete, instead of downloading the most recent version"
1313
echo " --with-sdk <dir> use the specified SDK to check out source code. do not copy it to the tarball. an external SDK will be required to build from the tarball."
14+
echo " --base-tarball creates a tarball without sdk, previously-build packages and reference packages"
1415
echo "use -- to send the remaining arguments to build.sh"
1516
}
1617

@@ -23,6 +24,7 @@ TARBALL_ROOT=$1
2324
shift
2425

2526
SKIP_BUILD=0
27+
BASE_TARBALL=0
2628
CUSTOM_SDK_DIR=''
2729
MINIMIZE_DISK_USAGE=0
2830
CUSTOM_REF_PACKAGES_DIR=''
@@ -79,6 +81,10 @@ while :; do
7981
MAIN_BUILD_ARGS+=( "/p:CustomPreviouslySourceBuiltPackagesDir=$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" )
8082
shift
8183
;;
84+
--base-tarball)
85+
BASE_TARBALL=1
86+
MAIN_BUILD_ARGS+=( "/p:PackBaseTarball=true" )
87+
;;
8288
--)
8389
shift
8490
echo "Detected '--': passing remaining parameters '$@' as build.sh arguments."
@@ -98,6 +104,11 @@ while :; do
98104
shift
99105
done
100106

107+
if [[ $BASE_TARBALL -eq 1 && ( "$CUSTOM_SDK_DIR" != "" || "$CUSTOM_REF_PACKAGES_DIR" != "" || "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" != "" ) ]]; then
108+
echo "ERROR: Cannot specify --base-tarball with other --with* parameters (--with-sdk --with-ref-packages --with-packages)"
109+
exit 1
110+
fi
111+
101112
if [ $MINIMIZE_DISK_USAGE -eq 1 ]; then
102113
echo "WARNING"
103114
echo "WARNING"

‎build.proj

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,21 @@
5151
<RemoveDir Directories="$(BaseOutputPath)" />
5252
</Target>
5353

54+
<Target Name="OutputArchiveArtifactsTextFile"
55+
AfterTargets="Build"
56+
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT'" >
57+
58+
<ItemGroup>
59+
<ArtifactUrls Include="$(ReferencePackagesTarballUrl)$(ReferencePackagesTarballName).$(PrivateSourceBuildReferencePackagesPackageVersion).tar.gz" />
60+
<ArtifactUrls Include="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz" />
61+
</ItemGroup>
62+
<WriteLinesToFile
63+
File="$(ArchiveArtifactsTextFile)"
64+
Lines="@(ArtifactUrls)"
65+
Overwrite="true" />
66+
</Target>
67+
68+
5469
<Target Name="DownloadSourceBuildReferencePackages"
5570
AfterTargets="Build"
5671
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'"

‎eng/SourceBuild.Tarball.targets

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,7 +136,7 @@
136136
$(DotNetCliToolDir)shared\2.1.0\**;
137137
$(DotNetCliToolDir)tools\**;
138138
"
139-
Condition="'$(CustomDotNetSdkDir)' == ''"/>
139+
Condition=" '$(CustomDotNetSdkDir)' == '' AND '$(PackBaseTarball)' != 'true' "/>
140140

141141
<TarballCopyFile
142142
Include="@(TarballRootToolFile)"
@@ -155,6 +155,11 @@
155155
<TarballCopyFile
156156
Include="$(ProjectDir)support/tarball/build.sh"
157157
RelativeDestination="build.sh" />
158+
159+
<TarballCopyFile
160+
Condition=" '$(PackBaseTarball)' == 'true' "
161+
Include="$(ProjectDir)support/tarball/prep.sh"
162+
RelativeDestination="prep.sh" />
158163

159164
<!--
160165
When building an internal build, the main smoke-test nuget config has authenticated sources
@@ -280,15 +285,21 @@
280285
DependsOnTargets="AddTarballPackages">
281286
<ItemGroup>
282287
<ExternalTarballFile
288+
Condition=" '$(PackBaseTarball)' != 'true' "
283289
Include="$(ExternalTarballsDir)*.tar.gz"
284290
RelativeRoot="packages\archive\" />
285291

286292
<!-- If not using previously source built artifacts tar.gz, use Production build output. -->
287293
<ExternalTarballFile
288-
Condition="'$(PrivateSourceBuiltArtifactsPackageVersion)' == ''"
294+
Condition=" '$(PrivateSourceBuiltArtifactsPackageVersion)' == '' AND '$(PackBaseTarball)' != 'true' "
289295
Include="@(PrivateSourceBuiltArtifactsTarGzFile)"
290296
RelativeRoot="packages\archive\" />
291297

298+
<ExternalTarballFile
299+
Condition=" '$(PackBaseTarball)' == 'true' "
300+
Include="$(ArchiveArtifactsTextFile)"
301+
RelativeRoot="packages\archive\" />
302+
292303
<TarballCopyFile Include="@(ExternalTarballFile);" />
293304
</ItemGroup>
294305
</Target>

‎support/tarball/build.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,31 @@ while :; do
7979
shift
8080
done
8181

82+
if [ -f "$SCRIPT_ROOT/packages/archive/archiveArtifacts.txt" ]; then
83+
ARCHIVE_ERROR=0
84+
if [ ! -d "$SCRIPT_ROOT/.dotnet" ] && [ "$CUSTOM_SDK_DIR" == "" ]; then
85+
echo "ERROR: SDK not found at $SCRIPT_ROOT/.dotnet"
86+
ARCHIVE_ERROR=1
87+
fi
88+
if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" == "" ]; then
89+
echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-packages parameter"
90+
ARCHIVE_ERROR=1
91+
fi
92+
if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuild.ReferencePackages*.tar.gz ] && [ "$CUSTOM_REF_PACKAGES_DIR" == "" ]; then
93+
echo "ERROR: Private.SourceBuild.ReferencePackages artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-ref-packages parameter"
94+
ARCHIVE_ERROR=1
95+
fi
96+
if [ $ARCHIVE_ERROR == 1 ]; then
97+
echo ""
98+
echo " Errors detected in tarball. To prep the tarball, run prep.sh while online to install an SDK,"
99+
echo " a Private.SourceBuilt.Artifacts tarball and a Private.SourceBuild.ReferencePackages tarball."
100+
echo " After prepping the tarball, the tarball can be built offline. As an alternative to prepping"
101+
echo " the tarball, these assets can be provided using the --with-sdk, --with-packages and"
102+
echo " --with-ref-packages parameters."
103+
exit 1
104+
fi
105+
fi
106+
82107
if [ -d "$CUSTOM_SDK_DIR" ]; then
83108
export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version`
84109
export CLI_ROOT="$CUSTOM_SDK_DIR"

‎support/tarball/prep.sh

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/usr/bin/env bash
2+
set -euo pipefail
3+
IFS=$'\n\t'
4+
5+
SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)"
6+
7+
usage() {
8+
echo "usage: $0"
9+
echo ""
10+
echo " Prepares a tarball to be built by downloading Private.SourceBuild.ReferencePackages.*.tar.gz and"
11+
echo " Private.SourceBuilt.Artifacts.*.tar.gz and installing the version of dotnet referenced in global.json"
12+
echo ""
13+
}
14+
15+
positional_args=()
16+
while :; do
17+
if [ $# -le 0 ]; then
18+
break
19+
fi
20+
lowerI="$(echo "$1" | awk '{print tolower($0)}')"
21+
case $lowerI in
22+
"-?"|-h|--help)
23+
usage
24+
exit 0
25+
;;
26+
*)
27+
positional_args+=("$1")
28+
;;
29+
esac
30+
31+
shift
32+
done
33+
34+
# Check for the archive text file which describes the location of the archive files to download
35+
if [ ! -f $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt ]; then
36+
echo " ERROR: $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt does not exist. Cannot determine which archives to download. Exiting..."
37+
exit -1
38+
fi
39+
40+
downloadRefPkgs=true
41+
downloadArtifacts=true
42+
installDotnet=true
43+
44+
# Check to make sure curl exists to download the archive files
45+
if ! command -v curl &> /dev/null
46+
then
47+
echo " ERROR: curl not found. Exiting..."
48+
exit -1
49+
fi
50+
51+
# Check if the reference packages archive exists
52+
if [ -f $SCRIPT_ROOT/packages/archive/Private.SourceBuild.ReferencePackages.*.tar.gz ]; then
53+
echo " Private.SourceBuild.ReferencePackages.*.tar.gz exists...it will not be downloaded"
54+
downloadRefPkgs=false
55+
fi
56+
57+
# Check if Private.SourceBuilt artifacts archive exists
58+
if [ -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts.*.tar.gz ]; then
59+
echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded"
60+
downloadArtifacts=false
61+
fi
62+
63+
# Check if dotnet is installed
64+
if [ -d $SCRIPT_ROOT/.dotnet ]; then
65+
echo " ./.dotnet SDK directory exists...it will not be installed"
66+
installDotnet=false;
67+
fi
68+
69+
# Read the archive text file to get the archives to download and download them
70+
while read -r line; do
71+
if [[ $line == *"Private.SourceBuild.ReferencePackages"* ]]; then
72+
if [ "$downloadRefPkgs" == "true" ]; then
73+
echo " Downloading ref pkgs..."
74+
(cd $SCRIPT_ROOT/packages/archive/ && curl -O $line)
75+
fi
76+
fi
77+
if [[ $line == *"Private.SourceBuilt.Artifacts"* ]]; then
78+
if [ "$downloadArtifacts" == "true" ]; then
79+
echo " Downloading source-built artifacts..."
80+
(cd $SCRIPT_ROOT/packages/archive/ && curl -O $line)
81+
fi
82+
fi
83+
done < $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt
84+
85+
# Check for the version of dotnet to install
86+
if [ "$installDotnet" == "true" ]; then
87+
echo " Installing dotnet..."
88+
(source ./eng/common/tools.sh && InitializeDotNetCli true)
89+
fi

0 commit comments

Comments
 (0)