Skip to content

Commit 0d4efc8

Browse files
dagooddagood
authored
Add RHEL7.5 internal CI capability, refactor YAML (#688)
* Add RHEL7.5 CI capability, refactor YAML * Fix comment indentation * Make -w explicit, remove unused cases * Separate tarball and logs for separate download * Only publish tarball artifact if tarball built (cherry picked from commit 1a04f52)
1 parent dffa908 commit 0d4efc8

File tree

4 files changed

+237
-131
lines changed

4 files changed

+237
-131
lines changed

‎.vsts-ci.yml

Lines changed: 0 additions & 131 deletions
This file was deleted.

‎.vsts.pipelines/builds/ci.yml

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
phases:
2+
- template: ../phases/ci-linux.yml
3+
parameters:
4+
name: ci_Docker
5+
matrix:
6+
centos71-unshared:
7+
artifactName: ci_Docker_centos71-unshared
8+
buildOfflineTarball: true
9+
imageName: microsoft/dotnet-buildtools-prereqs:centos711503_prereqs_2
10+
- template: ../phases/ci-linux.yml
11+
parameters:
12+
name: ci_Docker_RHEL_host
13+
dockerRegistryPassword: $(BotAccount-dotnet-docker-acr-bot-password)
14+
dockerRegistryServer: $(acr.server)
15+
dockerRegistryUserName: $(acr.userName)
16+
queueName: DotNetCore-Infra
17+
queueDemands: VSTS_OS -equals Linux_RHEL_7.4
18+
matrix:
19+
rhel7.5:
20+
artifactName: ci_Docker_RHEL_host_rhel7.5
21+
imageName: dotnetdocker.azurecr.io/dotnet-buildtools-prereqs:rhel-7.5-b4560f6-20180725153037

‎.vsts.pipelines/phases/ci-linux.yml

Lines changed: 211 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,211 @@
1+
parameters:
2+
dockerRegistryPassword: ''
3+
dockerRegistryServer: ''
4+
dockerRegistryUserName: ''
5+
matrix: null
6+
name: ci
7+
queueDemands: Agent.OS -equals Linux
8+
queueName: DotNet-Build
9+
10+
phases:
11+
- phase: ${{ parameters.name }}
12+
variables:
13+
# Prefix to distinguish artifacts from different legs. No documented variable for this exists.
14+
artifactName: no_artifact_name
15+
buildLoggingOptions: ''
16+
buildConfiguration: Release
17+
buildOfflineTarball: false
18+
# Use ":z" to set selinux flag for sharing in build-owned root dir. https://docs.docker.com/storage/bind-mounts/#configure-the-selinux-label
19+
docker.agentSrc.map: -v $(Build.SourcesDirectory):/agentSrc:z
20+
docker.agentSrc.work: -w /agentSrc
21+
docker.drop.map: -v $(dropDirectory):/drop:z
22+
docker.logs.map: -v $(dropDirectory)/logs:/logs:z
23+
docker.root.map: -v $(rootDirectory):/root:z
24+
docker.run: docker run --rm
25+
docker.src.map: -v $(rootDirectory)/sb/source-build:/src:z
26+
docker.src.work: -w /src
27+
docker.tb.map: -v $(rootDirectory)/sb/tarball:/tb:z
28+
docker.tb.work: -w /tb
29+
dockerRegistry.password: ${{ parameters.dockerRegistryPassword }}
30+
dockerRegistry.server: ${{ parameters.dockerRegistryServer }}
31+
dockerRegistry.userName: ${{ parameters.dockerRegistryUserName }}
32+
dropDirectory: $(stagingDirectory)/drop
33+
rootDirectory: $(Build.SourcesDirectory)/..
34+
stagingDirectory: $(rootDirectory)/sb/staging
35+
tarballName: tarball_$(Build.BuildId)
36+
queue:
37+
name: ${{ parameters.queueName }}
38+
demands: ${{ parameters.queueDemands }}
39+
timeoutInMinutes: 240
40+
parallel: 2
41+
matrix: ${{ parameters.matrix }}
42+
steps:
43+
- template: ../steps/docker-cleanup-linux.yml
44+
45+
# Docker registry login and pull, if one is defined.
46+
- script: |
47+
docker login -u $(dockerRegistry.userName) -p $(dockerRegistry.password) $(dockerRegistry.server)
48+
docker pull $(imageName)
49+
displayName: Docker login and image pull
50+
condition: and(succeeded(), ne(variables['dockerRegistry.server'], ''))
51+
52+
# Docker registry logout, if one is defined. Do this immediately: avoid unnecessary login time.
53+
- script: docker logout $(dockerRegistry.server)
54+
displayName: Docker logout
55+
condition: ne(variables['dockerRegistry.server'], '')
56+
57+
# Create working directory and copy source into it.
58+
- script: |
59+
set -x
60+
$(docker.run) $(docker.root.map) $(docker.agentSrc.map) $(docker.agentSrc.work) $(imageName) bash -c '
61+
rm -rf /root/sb/
62+
mkdir -p /root/sb/tarball
63+
cp -r . /root/sb/source-build'
64+
displayName: Clean sb directory and copy source from cloned directory
65+
66+
# Fetch vsts commits if building internally.
67+
- script: |
68+
set -x
69+
# Ignore failure for the first command. It will intentionally fail if the commit is only
70+
# available in VSTS. "submodule update --init" is the simplest way to set up the submodule
71+
# directory. ("submodule init" only sets up .git/config, not the e.g. src/coreclr/.git and
72+
# .git/modules/src/coreclr/ directories.)
73+
$(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) git submodule update --init --recursive
74+
$(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./fetch-vsts-commits.sh $(user.PAT)
75+
displayName: Fetch internal vsts commits
76+
condition: and(succeeded(), ne(variables['user.PAT'], ''))
77+
78+
# Initialize submodules.
79+
- script: $(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) git submodule update --init --recursive
80+
displayName: Initialize submodules
81+
82+
# Build source-build.
83+
- script: |
84+
$(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
85+
/p:ArchiveDownloadedPackages=true \
86+
/p:Configuration=$(buildConfiguration) \
87+
/p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix) \
88+
$(buildLoggingOptions)
89+
displayName: Build source-build
90+
timeoutInMinutes: 90
91+
92+
# Copy logs to working directory.
93+
- script: |
94+
set -x
95+
$(docker.run) $(docker.logs.map) $(docker.src.map) $(docker.src.work) $(imageName) /bin/bash -c "
96+
mkdir -p /logs/source-build/logs
97+
find . \( \
98+
-iname '*.binlog' -o \
99+
-iname '*.log' \) \
100+
-exec cp {} --parents /logs/source-build/logs \;"
101+
displayName: Copy source-build logs
102+
condition: always()
103+
continueOnError: true
104+
105+
# Run smoke tests.
106+
- script: |
107+
$(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
108+
/t:RunSmokeTest \
109+
/p:Configuration=$(buildConfiguration) \
110+
/p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix)
111+
displayName: Run smoke-test
112+
113+
# Copy smoke test logs to working directory.
114+
- script: |
115+
$(docker.run) $(docker.logs.map) $(docker.src.map) $(docker.src.work) $(imageName) /bin/bash -c "
116+
mkdir -p /logs/source-build/smoke-test
117+
find ./testing-smoke -name '*.log' -exec cp {} /logs/source-build/smoke-test \;"
118+
displayName: Copy smoke-test logs
119+
condition: always()
120+
continueOnError: true
121+
122+
# Create tarball.
123+
- script: |
124+
$(docker.run) $(docker.tb.map) $(docker.src.map) $(docker.src.work) $(imageName) ./build-source-tarball.sh \
125+
"/tb/$(tarballName)" \
126+
--skip-build
127+
displayName: Create tarball
128+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
129+
130+
# tar the tarball directory into the drop directory.
131+
- script: |
132+
$(docker.run) $(docker.tb.map) $(docker.drop.map) $(imageName) /bin/bash -c '
133+
mkdir -p /drop/tarball/
134+
tar -zcf "/drop/tarball/$(tarballName).tar.gz" "/tb/$(tarballName)"'
135+
displayName: Copy tarball to output
136+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
137+
138+
# Build tarball.
139+
- script: |
140+
$(docker.run) $(docker.tb.map) $(docker.tb.work) --network='none' $(imageName) "$(tarballName)/build.sh" \
141+
/p:Configuration=$(buildConfiguration) \
142+
$(buildLoggingOptions)
143+
displayName: Build tarball
144+
timeoutInMinutes: 90
145+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
146+
147+
# Run smoke tests.
148+
- script: |
149+
$(docker.run) $(docker.tb.map) $(docker.tb.work) $(imageName) "$(tarballName)/smoke-test.sh" \
150+
--minimal \
151+
--projectOutput \
152+
--configuration $(buildConfiguration) \
153+
--prodConBlobFeedUrl ''
154+
displayName: Run smoke-test in tarball
155+
condition: and(succeeded(), eq(variables['buildOfflineTarball'], true))
156+
157+
# Copy all tarball logs to working directory.
158+
- script: |
159+
set -x
160+
$(docker.run) $(docker.logs.map) $(docker.tb.map) $(docker.tb.work) $(imageName) /bin/bash -c "
161+
mkdir -p /logs/tarball/logs
162+
cd \"$(tarballName)\"
163+
find . \( \
164+
-iname '*.binlog' -o \
165+
-iname '*.log' \) \
166+
-exec cp {} --parents /logs/tarball/logs \;
167+
# Copy offline prebuilt report
168+
mkdir -p /logs/prebuilt-report/offline
169+
cp -r ./bin/prebuilt-report/* /logs/prebuilt-report/offline"
170+
$(docker.run) $(docker.logs.map) $(docker.src.map) $(docker.src.work) $(imageName) /bin/bash -c "
171+
# Copy online prebuilt report
172+
mkdir -p /logs/prebuilt-report/online
173+
cp -r ./bin/prebuilt-report/* /logs/prebuilt-report/online"
174+
displayName: Copy tarball logs
175+
condition: eq(variables['buildOfflineTarball'], true)
176+
continueOnError: true
177+
178+
# Copy artifacts to staging - Copy to VSTS owned folder is done outside of docker so copied files
179+
# have correct ownership so VSTS can clean them up later.
180+
- task: CopyFiles@2
181+
condition: always()
182+
continueOnError: true
183+
inputs:
184+
sourceFolder: $(stagingDirectory)
185+
targetFolder: $(Build.ArtifactStagingDirectory)
186+
187+
# Publish artifacts.
188+
- task: PublishBuildArtifacts@1
189+
displayName: Publish Logs artifact
190+
condition: always()
191+
continueOnError: true
192+
inputs:
193+
PathtoPublish: $(dropDirectory)/logs
194+
ArtifactName: Logs $(artifactName)
195+
ArtifactType: Container
196+
- task: PublishBuildArtifacts@1
197+
displayName: Publish Tarball artifact
198+
condition: eq(variables['buildOfflineTarball'], true)
199+
continueOnError: true
200+
inputs:
201+
PathtoPublish: $(dropDirectory)/tarball
202+
ArtifactName: Tarball $(artifactName)
203+
ArtifactType: Container
204+
205+
# Clean up (very large) working directory. root owner makes it difficult for others to remove.
206+
- script: $(docker.run) $(docker.root.map) $(imageName) bash -c 'rm -rf /root/sb'
207+
displayName: Clean sb directory
208+
condition: always()
209+
continueOnError: true
210+
211+
- template: ../steps/docker-cleanup-linux.yml

‎.vsts.pipelines/steps/docker-cleanup-linux.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
steps:
2+
# TODO: Remove this redundant step once agent supports "--volumes".
3+
- script: docker system prune -a -f
4+
displayName: Cleanup Docker (without --volumes, for agent compatibility)
5+
condition: always()
6+
continueOnError: true
27
- script: docker system prune -a -f --volumes
38
displayName: Cleanup Docker
49
condition: always()

0 commit comments

Comments
 (0)