Skip to content

Commit f61faa7

Browse files
dagooddagood
authored
Improve prebuilt baselining: warn on error by default, save all project.assets.json (#776)
* By default, continue on prebuilt baseline error * Save all potential reports for more diagnosability * Create zip with all project.assets.json files * Set MSBUILDDEBUGPATH env var for all builds Save contents in CI in case the outputs help troubleshooting.
1 parent 63d84cf commit f61faa7

File tree

6 files changed

+61
-9
lines changed

6 files changed

+61
-9
lines changed

‎build.proj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
<MakeDir Directories="$(SourceBuiltPackagesPath)" />
1818
<MakeDir Directories="$(SourceBuiltAssetsDir)" />
1919
<MakeDir Directories="$(LocalBlobStorageRoot)" />
20+
<MakeDir Directories="$(MSBuildDebugPathTargetDir)" />
2021
</Target>
2122

2223
<Target Name="InitBuild">

‎dir.props

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,12 +62,14 @@
6262
<RestoreSourcePropsPath>$(IntermediatePath)RestoreSources.props</RestoreSourcePropsPath>
6363
<PackageVersionPropsPath>$(IntermediatePath)PackageVersions.props</PackageVersionPropsPath>
6464
<LoggingDir>$(BaseOutputPath)logs/</LoggingDir>
65+
<MSBuildDebugPathTargetDir>$(BaseOutputPath)msbuild-debug/</MSBuildDebugPathTargetDir>
6566
<!-- Dir where git info is generated during online builds. -->
6667
<GitInfoOutputDir>$(BaseOutputPath)git-info/</GitInfoOutputDir>
6768
<!-- Dir where git info is placed inside the tarball. -->
6869
<GitInfoOfflineDir>$(ProjectDir)git-info/</GitInfoOfflineDir>
6970
<PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
7071
<PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
72+
<ProjectAssetsJsonArchiveFile>$(PackageReportDir)all-project-assets-json-files.zip</ProjectAssetsJsonArchiveFile>
7173
<ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
7274
<PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
7375
<ConflictingPackageReportDir>$(BaseOutputPath)conflict-report/</ConflictingPackageReportDir>

‎netci.groovy

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,9 @@ def addArchival(def job) {
1313
// Grab these logs from all of those locations.
1414
[ "", "source-build/", "tarball-output/"].each { logRoot ->
1515
archivalSettings.addFiles("${logRoot}bin/logs/*")
16-
archivalSettings.addFiles("${logRoot}bin/prebuilt-report/*")
16+
archivalSettings.addFiles("${logRoot}bin/prebuilt-report/**/*")
17+
archivalSettings.addFiles("${logRoot}bin/conflict-report/**/*")
18+
archivalSettings.addFiles("${logRoot}bin/msbuild-debug/**/*")
1719
archivalSettings.addFiles("${logRoot}src/**/*.binlog")
1820
archivalSettings.addFiles("${logRoot}src/**/*.log")
1921
archivalSettings.addFiles("${logRoot}init-tools.log")
@@ -58,11 +60,11 @@ def addBuildStepsAndSetMachineAffinity(def job, String os, String configuration)
5860
steps {
5961
if (os == "Windows_NT") {
6062
batchFile("git submodule update --init --recursive");
61-
batchFile(".\\build.cmd /p:Configuration=${configuration} ${loggingOptions}")
63+
batchFile(".\\build.cmd /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
6264
}
6365
else {
6466
shell("git submodule update --init --recursive");
65-
shell("./build.sh /p:Configuration=${configuration} ${loggingOptions}");
67+
shell("./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
6668
smokeTestExcludes = "";
6769
if (os == "Fedora24" || os == "OSX10.12") {
6870
// Dev certs doesn't seem to work in these platforms. https://github.com/dotnet/source-build/issues/560
@@ -132,7 +134,7 @@ def addPushJob(String project, String branch, String os, String configuration)
132134
shell("cd ./source-build;./build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
133135
shell("cd ./source-build;./build-source-tarball.sh ../tarball-output --skip-build");
134136

135-
shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} ${loggingOptions}")
137+
shell("cd ./tarball-output;./build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}")
136138
shell("cd ./tarball-output;./smoke-test.sh --minimal --configuration ${configuration}")
137139
}
138140
}
@@ -178,13 +180,13 @@ def addPushJob(String project, String branch, String os, String configuration)
178180
steps{
179181
shell("cd ./source-build;git submodule update --init --recursive");
180182
// First build the product itself
181-
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}");
183+
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
182184
// Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
183185
shell("mkdir tarball-output");
184186
// now build the tarball
185187
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home --network none -v \$(pwd)/source-build:/opt/code -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/code ${imageName} /opt/code/build-source-tarball.sh /opt/tarball --skip-build");
186188
// now build from the tarball offline and without access to the regular non-tarball build
187-
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} ${loggingOptions}");
189+
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home --network none -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/build.sh /p:Configuration=${configuration} /p:FailOnPrebuiltBaselineError=true ${loggingOptions}");
188190
// finally, run a smoke-test on the result
189191
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/tarball/home -v \$(pwd)/tarball-output:/opt/tarball --rm -w /opt/tarball ${imageName} /opt/tarball/smoke-test.sh --minimal --configuration ${configuration}");
190192
}

‎repos/dir.props

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@
5656
<EnvironmentVariables Include="PB_IsStable=$(UseStableVersions)" Condition="'$(UseStableVersions)' != ''" />
5757
<EnvironmentVariables Include="PackageVersionStamp=$(VersionStamp)" Condition="'$(VersionStamp)' != ''" />
5858

59+
<!-- If MSBuild exits early, make sure debug output like 'MSBuild_*.failure.txt' ends up in a place we can see it. -->
60+
<EnvironmentVariables Include="MSBUILDDEBUGPATH=$(MSBuildDebugPathTargetDir)" />
61+
5962
<!-- Turn off node reuse for source build because repos use conflicting versions
6063
of compilers which cause assembly load errors.
6164
See https://github.com/dotnet/source-build/issues/541 -->

‎repos/dir.targets

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -192,11 +192,13 @@
192192
<PropertyGroup>
193193
<_ReportDir>$(ConflictingPackageReportDir)before-$(RepositoryName)/</_ReportDir>
194194
<_ReportDataFile>$(_ReportDir)usage.xml</_ReportDataFile>
195+
<_ProjectAssetsJsonArchiveFile>$(_ReportDir)all-project-assets-json-files.zip</_ProjectAssetsJsonArchiveFile>
195196
</PropertyGroup>
196197

197198
<WritePackageUsageData NuGetPackageInfos="@(ConflictingPackageInfos)"
198199
RootDir="$(ProjectDir)"
199-
DataFile="$(_ReportDataFile)" />
200+
DataFile="$(_ReportDataFile)"
201+
ProjectAssetsJsonArchiveFile="$(_ProjectAssetsJsonArchiveFile)" />
200202

201203
<WriteUsageReports DataFile="$(_ReportDataFile)"
202204
OutputDirectory="$(_ReportDir)" />
@@ -385,7 +387,8 @@
385387
TargetRid="$(TargetRid)"
386388
ProjectDirectories="@(ProjectDirectories)"
387389
RootDir="$(ProjectDir)"
388-
DataFile="$(PackageReportDataFile)" />
390+
DataFile="$(PackageReportDataFile)"
391+
ProjectAssetsJsonArchiveFile="$(ProjectAssetsJsonArchiveFile)" />
389392
</Target>
390393

391394
<Target Name="GetAllProjectDirectories">
@@ -407,7 +410,12 @@
407410
<BaselineDataFile Condition="'$(OfflineBuild)' == 'true'">$(BaselineDataFile)offline.xml</BaselineDataFile>
408411
<BaselineDataFile Condition="'$(OfflineBuild)' != 'true'">$(BaselineDataFile)online.xml</BaselineDataFile>
409412

410-
<ContinueOnPrebuiltBaselineError Condition="'$(ContinueOnPrebuiltBaselineError)' == ''">false</ContinueOnPrebuiltBaselineError>
413+
<FailOnPrebuiltBaselineError Condition="'$(FailOnPrebuiltBaselineError)' == ''">false</FailOnPrebuiltBaselineError>
414+
</PropertyGroup>
415+
416+
<PropertyGroup Condition="'$(ContinueOnPrebuiltBaselineError)' == ''">
417+
<ContinueOnPrebuiltBaselineError>false</ContinueOnPrebuiltBaselineError>
418+
<ContinueOnPrebuiltBaselineError Condition="'$(FailOnPrebuiltBaselineError)' != 'true'">true</ContinueOnPrebuiltBaselineError>
411419
</PropertyGroup>
412420

413421
<ValidateUsageAgainstBaseline

‎tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/UsageReport/WritePackageUsageData.cs

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
using System.Collections.Concurrent;
1212
using System.Collections.Generic;
1313
using System.IO;
14+
using System.IO.Compression;
1415
using System.Linq;
1516
using System.Threading.Tasks;
1617
using Task = Microsoft.Build.Utilities.Task;
@@ -68,6 +69,12 @@ public class WritePackageUsageData : Task
6869
[Required]
6970
public string DataFile { get; set; }
7071

72+
/// <summary>
73+
/// If passed, the path of the archive file to generate that includes a copy of all
74+
/// project.asset.json files found.
75+
/// </summary>
76+
public string ProjectAssetsJsonArchiveFile { get; set; }
77+
7178
public override bool Execute()
7279
{
7380
DateTime startTime = DateTime.Now;
@@ -124,6 +131,35 @@ public override bool Execute()
124131
"project.assets.json",
125132
SearchOption.AllDirectories);
126133

134+
if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile))
135+
{
136+
Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files...");
137+
138+
Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile));
139+
140+
using (var projectAssetArchive = new ZipArchive(
141+
File.Open(
142+
ProjectAssetsJsonArchiveFile,
143+
FileMode.Create,
144+
FileAccess.ReadWrite),
145+
ZipArchiveMode.Create))
146+
{
147+
// Only one entry can be open at a time, so don't do this during the Parallel
148+
// ForEach later.
149+
foreach (var file in assetFiles)
150+
{
151+
string relativePath = file.Substring(RootDir.Length);
152+
using (var stream = File.OpenRead(file))
153+
using (Stream entryWriter = projectAssetArchive
154+
.CreateEntry(relativePath, CompressionLevel.Optimal)
155+
.Open())
156+
{
157+
stream.CopyTo(entryWriter);
158+
}
159+
}
160+
}
161+
}
162+
127163
Log.LogMessage(MessageImportance.Low, "Reading usage info...");
128164

129165
var usages = new ConcurrentBag<Usage>();

0 commit comments

Comments
 (0)