Skip to content

Commit 8ff7eb6

Browse files
dagooddagood
authored
Merge pull request #768 from dotnet-maestro-bot/merge/release/2.2-to-master
[automated] Merge branch 'release/2.2' => 'master'
2 parents 13f8cdf + 9780779 commit 8ff7eb6

22 files changed

+2075
-458
lines changed

‎.vsts.pipelines/phases/ci-linux.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ phases:
8484
$(docker.run) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
8585
/p:ArchiveDownloadedPackages=true \
8686
/p:Configuration=$(buildConfiguration) \
87+
/p:ContinueOnPrebuiltBaselineError=$(buildOfflineTarball) \
8788
/p:ProdConBlobFeedUrlPrefix=$(prodConBlobFeedUrlPrefix) \
8889
$(buildLoggingOptions)
8990
displayName: Build source-build

‎build-source-tarball.sh

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -129,15 +129,6 @@ do
129129
fi
130130
done
131131

132-
if [ -z "${SOURCE_BUILD_SKIP_PREBUILT_REPORT:-}" ]; then
133-
echo 'Creating prebuilt package usage report...'
134-
(
135-
# Don't clean up (or ask to clean up).
136-
export SOURCE_BUILD_SKIP_SUBMODULE_CHECK=1
137-
"$SCRIPT_ROOT/build.sh" /nologo /t:ReportTarballPrebuiltUsage /p:TarballPrebuiltPackagesPath="$FULL_TARBALL_ROOT/prebuilt/nuget-packages/"
138-
)
139-
fi
140-
141132
echo 'WORKAROUND: Overwriting the source-built roslyn-tools MSBuild files with prebuilt so that roslyn-tools can successfully build in the tarball... (https://github.com/dotnet/source-build/issues/654)'
142133

143134
ROSLYN_TOOLS_PACKAGE='RoslynTools.RepoToolset'

‎build.proj

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -31,23 +31,20 @@
3131
<RemoveDir Directories="$(BaseOutputPath)" />
3232
</Target>
3333

34-
<!-- After building offline, create a prebuilt usage report. -->
35-
<Target Name="ReportPrebuiltUsageAfterOfflineBuild"
34+
<!-- After building, generate a prebuilt usage report. -->
35+
<Target Name="ReportPrebuiltUsage"
3636
AfterTargets="Build"
37-
Condition="'$(OfflineBuild)' == 'true' and '$(SkipReportPrebuiltUsageAfterOfflineBuild)' != 'true'">
38-
<MSBuild Projects="repos\$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
39-
</Target>
40-
41-
<!-- After generating a tarball, check why/where the online build downloaded prebuilts. -->
42-
<Target Name="ReportTarballPrebuiltUsage">
37+
Condition="'$(SkipReportPrebuiltUsage)' != 'true'">
4338
<MSBuild Projects="repos\$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
4439
</Target>
4540

4641
<!--
47-
Dev scenario: regenerate a prebuilt-report. This makes it easy to add data to an existing
48-
prebuilt report without performing another full build.
42+
Dev scenario: rewrite a prebuilt-report. This makes it easy to add data to an existing
43+
prebuilt report without performing another full build. This doesn't reevalutate which packages
44+
are prebuilts or search the projects: it uses the existing usage.xml file to generate report
45+
XMLs based on the info put in bin/.
4946
-->
50-
<Target Name="CreatePrebuiltUsageReport">
47+
<Target Name="RewritePrebuiltUsageReport">
5148
<MSBuild Projects="repos\$(RootRepo).proj" Targets="ReportPrebuiltUsage" />
5249
</Target>
5350

‎dir.props

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -44,8 +44,9 @@
4444
<BaseOutputPath>$(ProjectDir)bin/</BaseOutputPath>
4545
<ToolsDir>$(ProjectDir)Tools/</ToolsDir>
4646
<ToolPackageExtractDir>$(ToolsDir)source-built/</ToolPackageExtractDir>
47-
<TaskDirectory>$(ProjectDir)tools-local/tasks/</TaskDirectory>
48-
<TasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks/bin/Debug/netstandard1.5/</TasksBinDir>
47+
<ToolsLocalDir>$(ProjectDir)tools-local/</ToolsLocalDir>
48+
<TaskDirectory>$(ToolsLocalDir)tasks/</TaskDirectory>
49+
<TasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks/bin/Debug/netstandard2.0/</TasksBinDir>
4950
<BaseIntermediatePath>$(BaseOutputPath)obj/</BaseIntermediatePath>
5051
<OutputPath>$(BaseOutputPath)$(Platform)/$(Configuration)/</OutputPath>
5152
<IntermediatePath>$(BaseIntermediatePath)$(Platform)/$(Configuration)/</IntermediatePath>
@@ -68,7 +69,7 @@
6869
<!-- Dir where git info is placed inside the tarball. -->
6970
<GitInfoOfflineDir>$(ProjectDir)git-info/</GitInfoOfflineDir>
7071
<PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
71-
<PackageReportDataFile>$(PackageReportDir)prebuilt-usage.json</PackageReportDataFile>
72+
<PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
7273
<ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
7374
<PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
7475
<ConflictingPackageReportDir>$(BaseOutputPath)conflict-report/</ConflictingPackageReportDir>

‎netci.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -205,7 +205,7 @@ def addPushJob(String project, String branch, String os, String configuration, b
205205
steps{
206206
shell("cd ./source-build;git submodule update --init --recursive");
207207
// First build the product itself
208-
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} ${loggingOptions}");
208+
shell("docker run -u=\"\$(id -u):\$(id -g)\" -t --sig-proxy=true -e HOME=/opt/code/home -v \$(pwd)/source-build:/opt/code --rm -w /opt/code ${imageName} /opt/code/build.sh /p:ArchiveDownloadedPackages=true /p:Configuration=${configuration} /p:ContinueOnPrebuiltBaselineError=true ${loggingOptions}");
209209
// Have to make this directory before volume-sharing it unlike non-docker build - existing directory is really only a warning in build-source-tarball.sh
210210
shell("mkdir tarball-output");
211211
// now build the tarball

‎repos/dir.targets

Lines changed: 50 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="ReadNuGetPackageInfos" />
1212
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="RemoveInternetSourcesFromNuGetConfig" />
1313
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="UpdateJson" />
14+
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="ValidateUsageAgainstBaseline" />
1415
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="WriteBuildOutputProps" />
1516
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="WritePackageUsageData" />
1617
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="WriteRestoreSourceProps" />
@@ -166,6 +167,7 @@
166167
OutputPath="$(PackageVersionPropsPath)" />
167168

168169
<WriteBuildOutputProps NuGetPackages="@(_PreviouslySourceBuiltPackages)"
170+
IncludeCreationTimeProperty="true"
169171
OutputPath="$(PackageVersionPropsPath).pre.$(RepositoryName).xml" />
170172

171173
<ReadLinesFromFile File="$(PackageVersionPropsPath)">
@@ -191,11 +193,11 @@
191193
Condition="'@(ConflictingPackageInfos)' != ''">
192194
<PropertyGroup>
193195
<_ReportDir>$(ConflictingPackageReportDir)before-$(RepositoryName)/</_ReportDir>
194-
<_ReportDataFile>$(_ReportDir)usage.json</_ReportDataFile>
196+
<_ReportDataFile>$(_ReportDir)usage.xml</_ReportDataFile>
195197
</PropertyGroup>
196198

197199
<WritePackageUsageData NuGetPackageInfos="@(ConflictingPackageInfos)"
198-
ProjectDirectories="@(ProjectDirectories)"
200+
RootDir="$(ProjectDir)"
199201
DataFile="$(_ReportDataFile)" />
200202

201203
<WriteUsageReports DataFile="$(_ReportDataFile)"
@@ -352,28 +354,40 @@
352354

353355
<Target Name="WritePrebuiltUsageData"
354356
DependsOnTargets="GetAllProjectDirectories">
357+
<!-- Save the PVP snapshot of each build step to be evaluated while building the report. -->
355358
<ItemGroup>
356-
<PrebuiltPackages Include="$(PrebuiltPackagesPath)*.nupkg" />
357-
<PrebuiltPackages Include="$(TarballPrebuiltPackagesPath)*.nupkg" Condition="'$(TarballPrebuiltPackagesPath)' != ''"/>
359+
<PackageVersionPropsSnapshotFiles Include="$(IntermediatePath)PackageVersions.props.pre.*.xml" />
360+
</ItemGroup>
361+
<Copy SourceFiles="@(PackageVersionPropsSnapshotFiles)" DestinationFolder="$(PackageReportDir)snapshots/" />
358362

359-
<PackageVersionPropsSnapshotSourceFiles Include="$(IntermediatePath)PackageVersions.props.pre.*.xml" />
363+
<ItemGroup>
364+
<AllRestoredPackageFiles Include="$(LocalNuGetPackagesRoot)**/*.nupkg" />
365+
<AllRestoredPackageFiles Include="$(PackagesDir)**/*.nupkg" />
360366

361-
<ProjectDirectories Include="$(ToolsDir);$(TaskDirectory);$(BaseIntermediatePath)" />
362-
<!-- Scan the entire project, in case project.assets.json ends up in an unexpected place. -->
363-
<ProjectDirectories Include="$(ProjectDir)" />
364-
</ItemGroup>
367+
<!-- Only contains packages when building a tarball. -->
368+
<TarballPrebuiltPackageFiles Include="$(PrebuiltPackagesPath)*.nupkg" />
365369

366-
<Copy SourceFiles="@(PackageVersionPropsSnapshotSourceFiles)" DestinationFolder="$(PackageReportDir)" />
370+
<SourceBuiltPackageFiles Include="$(SourceBuiltBlobFeedDir)**/*.nupkg" />
367371

368-
<Message Importance="High" Text="Reading prebuilt nupkg identities..." />
372+
<!-- Check all RIDs from all restored Microsoft.NETCore.Platforms packages. -->
373+
<PlatformsRuntimeJsonFiles Include="$(LocalNuGetPackagesRoot)*/microsoft.netcore.platforms/*/runtime.json" />
374+
<PlatformsRuntimeJsonFiles Include="$(PackagesDir)microsoft.netcore.platforms/*/runtime.json" />
369375

370-
<ReadNuGetPackageInfos PackagePaths="@(PrebuiltPackages)">
371-
<Output TaskParameter="PackageInfoItems" ItemName="PrebuiltPackageInfoItems" />
372-
</ReadNuGetPackageInfos>
376+
<!-- Add some other potential top-level project directories for a more specific report. -->
377+
<ProjectDirectories Include="$(ToolsDir);$(TaskDirectory);$(BaseIntermediatePath)" />
378+
<!-- Finally, scan entire source-build, in case project.assets.json ends up in an unexpected place. -->
379+
<ProjectDirectories Include="$(ProjectDir)" />
380+
</ItemGroup>
373381

374-
<WritePackageUsageData NuGetPackageInfos="@(PrebuiltPackageInfoItems)"
375-
ProjectDirectories="@(ProjectDirectories)"
376-
DataFile="$(PackageReportDataFile)" />
382+
<WritePackageUsageData
383+
RestoredPackageFiles="@(AllRestoredPackageFiles)"
384+
TarballPrebuiltPackageFiles="@(TarballPrebuiltPackageFiles)"
385+
SourceBuiltPackageFiles="@(SourceBuiltPackageFiles)"
386+
PlatformsRuntimeJsonFiles="@(PlatformsRuntimeJsonFiles)"
387+
TargetRid="$(TargetRid)"
388+
ProjectDirectories="@(ProjectDirectories)"
389+
RootDir="$(ProjectDir)"
390+
DataFile="$(PackageReportDataFile)" />
377391
</Target>
378392

379393
<Target Name="GetAllProjectDirectories">
@@ -389,21 +403,33 @@
389403
</MSBuild>
390404
</Target>
391405

392-
<Target Name="ReportPrebuiltUsage"
393-
DependsOnTargets="GetPreviousReleasePrebuiltPackageInfos">
406+
<Target Name="ReportPrebuiltUsage">
407+
<PropertyGroup>
408+
<BaselineDataFile>$(ToolsLocalDir)prebuilt-baseline-</BaselineDataFile>
409+
<BaselineDataFile Condition="'$(OfflineBuild)' == 'true'">$(BaselineDataFile)offline.xml</BaselineDataFile>
410+
<BaselineDataFile Condition="'$(OfflineBuild)' != 'true'">$(BaselineDataFile)online.xml</BaselineDataFile>
411+
412+
<ContinueOnPrebuiltBaselineError Condition="'$(ContinueOnPrebuiltBaselineError)' == ''">false</ContinueOnPrebuiltBaselineError>
413+
</PropertyGroup>
414+
415+
<ValidateUsageAgainstBaseline
416+
DataFile="$(PackageReportDataFile)"
417+
BaselineDataFile="$(BaselineDataFile)"
418+
OutputBaselineFile="$(PackageReportDir)generated-new-baseline.xml"
419+
OutputReportFile="$(PackageReportDir)baseline-comparison.xml"
420+
ContinueOnError="$(ContinueOnPrebuiltBaselineError)" />
421+
394422
<ItemGroup>
395-
<PackageVersionPropsSnapshotFiles Include="$(PackageReportDir)PackageVersions.props.pre.*.xml" />
423+
<PackageVersionPropsSavedSnapshotFiles Include="$(PackageReportDir)snapshots/PackageVersions.props.pre.*.xml" />
396424
</ItemGroup>
425+
397426
<WriteUsageReports DataFile="$(PackageReportDataFile)"
398-
PackageVersionPropsSnapshots="@(PackageVersionPropsSnapshotFiles)"
427+
PackageVersionPropsSnapshots="@(PackageVersionPropsSavedSnapshotFiles)"
399428
ProdConBuildManifestFile="$(ProdConManifestFile)"
400429
PoisonedReportFile="$(PoisonedReportFile)"
401-
PreviousReleasePrebuiltPackageInfos="@(PreviousReleasePrebuiltPackageInfos)"
402430
OutputDirectory="$(PackageReportDir)" />
403431
</Target>
404432

405-
<Import Project="$(ProjectDir)tools-local/earlier-prebuilts.targets" />
406-
407433
<Target Name="GetProjectDirectory" Outputs="$(ProjectDirectory)" />
408434
<Target Name="GetOrchestratedManifestBuildName" Outputs="$(OrchestratedManifestBuildName)" />
409435
<Target Name="GetOfficialBuildId" Outputs="$(OfficialBuildId)" />

0 commit comments

Comments
 (0)