Skip to content

Commit 5d303ed

Browse files
crummelcrummel
authored
[release/2.1] Add portable builds to CI. (#718)
* Add portable builds to CI. * Address code review feedback. * Change to only do offline and tarball builds for portable since they require the online build anyway. * Fix tarball and offline build names. * Add PR jobs that can be triggered for portable testing.
1 parent 76df50f commit 5d303ed

File tree

1 file changed

+106
-72
lines changed

1 file changed

+106
-72
lines changed

‎netci.groovy

Lines changed: 106 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -55,16 +55,16 @@ def getDockerImageForOs(os) {
5555
return imageMap.get(os)
5656
}
5757

58-
def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration) {
58+
def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration, boolean portable) {
5959
job.with {
6060
steps {
6161
if (os == "Windows_NT") {
6262
batchFile("git submodule update --init --recursive");
63-
batchFile(".\\build.cmd /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
63+
batchFile(".\\build.cmd /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
6464
}
6565
else {
6666
shell("git submodule update --init --recursive");
67-
shell("./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
67+
shell("./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
6868
smokeTestExcludes = "";
6969
if (os == "Fedora24" || os == "OSX10.12") {
7070
// Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560
@@ -78,90 +78,116 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration)
7878
setMachineAffinity(job, os);
7979
}
8080

81-
def addPullRequestJob(String project, String branch, String os, String configuration, boolean runByDefault)
81+
def addPullRequestJob(String project, String branch, String os, String configuration, boolean portable, boolean runByDefault)
8282
{
83-
def newJobName = Utilities.getFullJobName(project, "${os}_${configuration}", true);
83+
def config = configuration;
84+
if (portable) {
85+
config += "_Portable";
86+
}
87+
def newJobName = Utilities.getFullJobName(project, "${os}_${config}", true);
8488
def contextString = "${os} ${configuration}";
89+
if (portable) {
90+
contextString += " Portable";
91+
}
8592
def triggerPhrase = "(?i).*test\\W+${contextString}.*";
8693

8794
def newJob = job(newJobName);
8895

89-
addBuildStepsAndSetMachineAffinity(newJob, os, configuration);
96+
addBuildStepsAndSetMachineAffinity(newJob, os, configuration, portable);
9097
addArchival(newJob);
9198
Utilities.standardJobSetup(newJob, project, true, "*/${branch}");
9299
Utilities.setJobTimeout(newJob, 180);
93100
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase, !runByDefault);
94101
}
95102

96-
def addPushJob(String project, String branch, String os, String configuration)
103+
def addPushJob(String project, String branch, String os, String configuration, boolean portable)
97104
{
98105
def shortJobName = "${os}_${configuration}";
106+
if (portable) {
107+
shortJobName += "_Portable";
108+
}
99109

100110
def newJobName = Utilities.getFullJobName(project, shortJobName, false);
101111
def newJob = job(newJobName);
102112

103-
addBuildStepsAndSetMachineAffinity(newJob, os, configuration);
113+
addBuildStepsAndSetMachineAffinity(newJob, os, configuration, portable);
104114
addArchival(newJob);
105115
Utilities.standardJobSetup(newJob, project, false, "*/${branch}");
106116
Utilities.setJobTimeout(newJob, 180);
107117
Utilities.addGithubPushTrigger(newJob);
108118
}
109119

110120
["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "CentOS7.1", "OSX10.12"].each { os ->
111-
addPullRequestJob(project, branch, os, "Release", true);
112-
addPullRequestJob(project, branch, os, "Debug", false);
121+
// Release non-portable run by default
122+
addPullRequestJob(project, branch, os, "Release", false, true);
123+
// Debug non-portable can be triggered
124+
addPullRequestJob(project, branch, os, "Debug", false, false);
125+
// Release portable can be triggered
126+
addPullRequestJob(project, branch, os, "Release", true, false);
127+
// Debug portable can be triggered
128+
addPullRequestJob(project, branch, os, "Debug", true, false);
113129
};
114130

115131
// Per push, run all the jobs
116132
["Ubuntu16.04", "Fedora24", "Debian8.4", "RHEL7.2", "Windows_NT", "CentOS7.1", "OSX10.12"].each { os ->
117133
["Release", "Debug"].each { configuration ->
118-
addPushJob(project, branch, os, configuration);
134+
[true, false].each { portable ->
135+
addPushJob(project, branch, os, configuration, portable);
136+
};
119137
};
120138
};
121139

122140
// Tarball builds that are not enforced to be offline
123141
[true, false].each { isPR ->
124142
["RHEL7.2", "CentOS7.1"].each { os ->
125143
["Release", "Debug"].each { configuration ->
144+
[true, false].each { portable ->
126145

127-
def shortJobName = "${os}_Tarball_${configuration}";
128-
def contextString = "${os} Tarball ${configuration}";
129-
def triggerPhrase = "(?i).*test\\W+${contextString}.*";
146+
def shortJobName = "${os}_Tarball_${configuration}";
147+
def contextString = "${os} Tarball ${configuration}";
130148

131-
def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
132-
steps{
133-
shell("cd ./source-build;git submodule update --init --recursive");
134-
shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
135-
shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");
149+
if (portable) {
150+
shortJobName += "_Portable"
151+
contextString += " Portable"
152+
}
136153

137-
shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
138-
shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
154+
def triggerPhrase = "(?i).*test\\W+${contextString}.*";
155+
156+
def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
157+
steps{
158+
shell("cd ./source-build;git submodule update --init --recursive");
159+
shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} ${loggingOptions}");
160+
shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");
161+
162+
shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
163+
shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
164+
}
139165
}
140-
}
141166

142-
setMachineAffinity(newJob, os);
167+
setMachineAffinity(newJob, os);
143168

144-
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
169+
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
145170

146-
// Increase timeout. The tarball builds can take longer than the 2 hour default.
147-
Utilities.setJobTimeout(newJob, 240);
171+
// Increase timeout. The tarball builds can take longer than the 2 hour default.
172+
Utilities.setJobTimeout(newJob, 240);
148173

149-
// Clone into the source-build directory
150-
Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
174+
// Clone into the source-build directory
175+
Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
151176

152-
addArchival(newJob);
153-
if(isPR){
154-
if(configuration == "Release"){
155-
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
177+
addArchival(newJob);
178+
if(isPR){
179+
if(configuration == "Release"){
180+
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
181+
}
182+
else{
183+
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
184+
}
156185
}
157186
else{
158-
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
187+
Utilities.addGithubPushTrigger(newJob);
159188
}
160-
}
161-
else{
162-
Utilities.addGithubPushTrigger(newJob);
163-
}
164189

190+
}
165191
}
166192
}
167193
}
@@ -170,52 +196,60 @@ def addPushJob(String project, String branch, String os, String configuration)
170196
[true, false].each { isPR ->
171197
["RHEL7.2", "CentOS7.1"].each { os->
172198
["Release", "Debug"].each { configuration ->
199+
[true, false].each { portable ->
200+
201+
def shortJobName = "${os}_Unshared_${configuration}";
202+
def contextString = "${os} Unshared ${configuration}";
173203

174-
def shortJobName = "${os}_Unshared_${configuration}";
175-
def contextString = "${os} Unshared ${configuration}";
176-
def triggerPhrase = "(?i).*test\\W+${contextString}.*";
177-
def imageName = getDockerImageForOs(os);
178-
179-
def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
180-
steps{
181-
shell("cd ./source-build;git submodule update --init --recursive");
182-
// First build the product itself
183-
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
184-
// Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
185-
shell("mkdir tarball-output");
186-
// now build the tarball
187-
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
188-
// now build from the tarball offline and without access to the regular non-tarball build
189-
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
190-
// finally, run a smoke-test on the result
191-
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
204+
if (portable) {
205+
shortJobName += "_Portable"
206+
contextString += " Portable"
192207
}
193-
}
194208

195-
// Only Ubuntu Jenkins machines have Docker
196-
setMachineAffinity(newJob, "Ubuntu16.04");
209+
def triggerPhrase = "(?i).*test\\W+${contextString}.*";
210+
def imageName = getDockerImageForOs(os);
211+
212+
def newJob = job(Utilities.getFullJobName(project, shortJobName, isPR)){
213+
steps{
214+
shell("cd ./source-build;git submodule update --init --recursive");
215+
// First build the product itself
216+
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}");
217+
// Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
218+
shell("mkdir tarball-output");
219+
// now build the tarball
220+
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
221+
// now build from the tarball offline and without access to the regular non-tarball build
222+
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:PortableBuild=${portable} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
223+
// finally, run a smoke-test on the result
224+
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
225+
}
226+
}
197227

198-
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
228+
// Only Ubuntu Jenkins machines have Docker
229+
setMachineAffinity(newJob, "Ubuntu16.04");
199230

200-
// Increase timeout. The offline build in Docker takes more than 2 hours.
201-
Utilities.setJobTimeout(newJob, 240);
231+
Utilities.standardJobSetup(newJob, project, isPR, "*/${branch}");
202232

203-
// Clone into the source-build directory
204-
Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
233+
// Increase timeout. The offline build in Docker takes more than 2 hours.
234+
Utilities.setJobTimeout(newJob, 240);
205235

206-
addArchival(newJob);
207-
if(isPR){
208-
if(configuration == "Release"){
209-
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
236+
// Clone into the source-build directory
237+
Utilities.addScmInSubDirectory(newJob, project, isPR, 'source-build');
238+
239+
addArchival(newJob);
240+
if(isPR){
241+
if(configuration == "Release"){
242+
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString);
243+
}
244+
else{
245+
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
246+
}
210247
}
211248
else{
212-
Utilities.addGithubPRTriggerForBranch(newJob, branch, contextString, triggerPhrase);
249+
Utilities.addGithubPushTrigger(newJob);
213250
}
214-
}
215-
else{
216-
Utilities.addGithubPushTrigger(newJob);
217-
}
218251

252+
}
219253
}
220254
}
221255
}

0 commit comments

Comments
 (0)