Skip to content

Commit 5b090ae

Browse files
f-alizadaf-alizada
authored
Merge branch 'main' into dev/f-alizada/add-automation-for-respect-custom-culture-property
2 parents 70ea017 + 79c270e commit 5b090ae

File tree

11 files changed

+658
-287
lines changed

11 files changed

+658
-287
lines changed

‎eng/Version.Details.xml

Lines changed: 184 additions & 184 deletions
Large diffs are not rendered by default.

‎eng/Versions.props

Lines changed: 77 additions & 77 deletions
Large diffs are not rendered by default.

‎eng/pipelines/templates/jobs/vmr-build.yml

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,7 @@ jobs:
314314
315315
if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
316316
customBuildArgs="$customBuildArgs --source-only"
317+
extraBuildProperties="$extraBuildProperties /p:ReportSbrpUsage=true"
317318
fi
318319
319320
if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then
@@ -433,8 +434,7 @@ jobs:
433434
434435
cd "$(sourcesPath)"
435436
436-
CopyWithRelativeFolders "artifacts/log/" $targetFolder "*.binlog"
437-
CopyWithRelativeFolders "artifacts/log/" $targetFolder "*.log"
437+
CopyWithRelativeFolders "artifacts/log/" $targetFolder "*"
438438
CopyWithRelativeFolders "src/" $targetFolder "*.binlog"
439439
CopyWithRelativeFolders "src/" $targetFolder "*.log"
440440
@@ -468,8 +468,7 @@ jobs:
468468
mkdir -p ${targetFolder}
469469
470470
cd "$(sourcesPath)"
471-
find artifacts/log/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
472-
find artifacts/log/ -type f -name "*.log" -exec rsync -R {} -t ${targetFolder} \;
471+
find artifacts/log/ -exec rsync -R {} -t ${targetFolder} \;
473472
if [ -d "artifacts/scenario-tests/" ]; then
474473
find artifacts/scenario-tests/ -type f -name "*.binlog" -exec rsync -R {} -t ${targetFolder} \;
475474
echo "##vso[task.setvariable variable=hasScenarioTestResults]true"

‎src/Microsoft.Net.Sdk.Compilers.Toolset/Microsoft.Net.Sdk.Compilers.Toolset.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
<TargetFramework>$(SdkTargetFramework)</TargetFramework>
55
<Description>Transport package for Microsoft.Net.Compilers.Toolset.Framework assemblies. For internal use only.</Description>
66
<IsPackable>true</IsPackable>
7+
<IsShippingPackage>true</IsShippingPackage>
78
<IncludeBuildOutput>false</IncludeBuildOutput>
89
<NoPackageAnalysis>true</NoPackageAnalysis>
910
</PropertyGroup>

‎src/SourceBuild/content/Directory.Build.props

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@
172172
<PackageReportDir>$([MSBuild]::NormalizeDirectory('$(ArtifactsDir)', 'prebuilt-report'))</PackageReportDir>
173173
<ResultingPrebuiltPackagesDir>$([MSBuild]::NormalizeDirectory('$(PackageReportDir)', 'prebuilt-packages'))</ResultingPrebuiltPackagesDir>
174174

175+
<SbrpRepoSrcDir>$([MSBuild]::NormalizeDirectory('$(SrcDir)', 'source-build-reference-packages', 'src'))</SbrpRepoSrcDir>
175176
<ReferencePackagesDir>$([MSBuild]::NormalizeDirectory('$(PrereqsPackagesDir)', 'reference'))</ReferencePackagesDir>
176177
<SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
177178
<SourceBuiltPrebuiltsTarballName>Private.SourceBuilt.Prebuilts</SourceBuiltPrebuiltsTarballName>

‎src/SourceBuild/content/eng/finish-source-only.proj

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,15 @@
2121
<MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
2222
</Target>
2323

24+
<UsingTask TaskName="Microsoft.DotNet.UnifiedBuild.Tasks.WriteSBRPUsageReport" AssemblyFile="$(MicrosoftDotNetUnifiedBuildTasksAssembly)" TaskFactory="TaskHostFactory" />
25+
<Target Name="ReportSbrpUsage"
26+
AfterTargets="Build"
27+
Condition="'$(ReportSbrpUsage)' == 'true'">
28+
<WriteSbrpUsageReport SbrpRepoSrcPath="$(SbrpRepoSrcDir)"
29+
SrcPath="$(SrcDir)"
30+
OutputPath="$(ArtifactsLogDir)" />
31+
</Target>
32+
2433
<!--
2534
Determine symbols tarball names and discover all intermediate symbols,
2635
to be used as inputs and outputs of symbols repackaging targets.

‎src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.UnifiedBuild.Tasks/WriteSbrpUsageReport.cs

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
#nullable enable
6+
7+
using System.Collections.Generic;
8+
using System.IO;
9+
using System.Linq;
10+
using System.Text.Json;
11+
using System.Xml.Linq;
12+
using Microsoft.Build.Framework;
13+
using Microsoft.Build.Utilities;
14+
using NuGet.ProjectModel;
15+
16+
namespace Microsoft.DotNet.UnifiedBuild.Tasks;
17+
18+
/// <summary>
19+
/// Reports the usage of the source-build-reference-packages:
20+
/// 1. SBRP references
21+
/// 2. Unreferenced packages
22+
/// </summary>
23+
public class WriteSbrpUsageReport : Task
24+
{
25+
private const string SbrpRepoName = "source-build-reference-packages";
26+
27+
private readonly Dictionary<string, PackageInfo> _sbrpPackages = [];
28+
29+
/// <summary>
30+
/// Path to the SBRP src directory.
31+
/// </summary>
32+
[Required]
33+
public required string SbrpRepoSrcPath { get; set; }
34+
35+
/// <summary>
36+
/// Path to the VMR src directory.
37+
/// </summary>
38+
[Required]
39+
public required string SrcPath { get; set; }
40+
41+
/// <summary>
42+
/// Path to the usage report to.
43+
/// </summary>
44+
[Required]
45+
public required string OutputPath { get; set; }
46+
47+
public override bool Execute()
48+
{
49+
Log.LogMessage($"Scanning for SBRP Package Usage...");
50+
51+
ReadSbrpPackages("referencePackages", trackTfms: true);
52+
ReadSbrpPackages("textOnlyPackages", trackTfms: false);
53+
54+
ScanProjectReferences();
55+
56+
GenerateUsageReport();
57+
58+
return !Log.HasLoggedErrors;
59+
}
60+
61+
private void GenerateUsageReport()
62+
{
63+
PackageInfo[] existingSbrps = [.. _sbrpPackages.Values.OrderBy(pkg => pkg.Id)];
64+
PurgeNonReferencedReferences();
65+
IEnumerable<string> unreferencedSbrps = GetUnreferencedSbrps().Select(pkg => pkg.Id).OrderBy(id => id);
66+
Report report = new(existingSbrps, unreferencedSbrps);
67+
68+
string reportFilePath = Path.Combine(OutputPath, "sbrpPackageUsage.json");
69+
#pragma warning disable CA1869 // Cache and reuse 'JsonSerializerOptions' instances
70+
string jsonContent = JsonSerializer.Serialize(report, new JsonSerializerOptions { WriteIndented = true });
71+
#pragma warning restore CA1869 // Cache and reuse 'JsonSerializerOptions' instances
72+
File.WriteAllText(reportFilePath, jsonContent);
73+
}
74+
75+
/// <summary>
76+
/// Removes all references from unreferenced SBRP packages. This is necessary to determine the
77+
/// complete set of unreferenced SBRP packages.
78+
/// </summary>
79+
private void PurgeNonReferencedReferences()
80+
{
81+
bool hasPurged;
82+
do
83+
{
84+
hasPurged = false;
85+
PackageInfo[] unrefPkgs = GetUnreferencedSbrps().ToArray();
86+
87+
foreach (PackageInfo sbrpPkg in _sbrpPackages.Values)
88+
{
89+
foreach (PackageInfo unrefPkg in unrefPkgs)
90+
{
91+
var unref = sbrpPkg.References.Keys
92+
.SingleOrDefault(path => path.Contains(SbrpRepoName) && path.Contains($"{unrefPkg.Name}.{unrefPkg.Version}"));
93+
if (unref != null)
94+
{
95+
Log.LogMessage($"Removing {unrefPkg.Id} from {sbrpPkg.Id}'s references.");
96+
sbrpPkg.References.Remove(unref);
97+
hasPurged = true;
98+
}
99+
}
100+
}
101+
} while (hasPurged);
102+
}
103+
104+
private IEnumerable<PackageInfo> GetUnreferencedSbrps() =>
105+
_sbrpPackages.Values.Where(pkg => pkg.References.Count == 0);
106+
107+
private string GetSbrpPackagesPath(string packageType) => Path.Combine(SbrpRepoSrcPath, packageType, "src");
108+
109+
private void ReadSbrpPackages(string packageType, bool trackTfms)
110+
{
111+
foreach (string projectPath in Directory.GetFiles(GetSbrpPackagesPath(packageType), "*.csproj", SearchOption.AllDirectories))
112+
{
113+
DirectoryInfo? directory = Directory.GetParent(projectPath);
114+
string version = directory!.Name;
115+
string projectName = Path.GetFileNameWithoutExtension(projectPath);
116+
HashSet<string>? tfms = null;
117+
118+
if (trackTfms)
119+
{
120+
XDocument xmlDoc = XDocument.Load(projectPath);
121+
// Reference packages are generated using the TargetFrameworks property
122+
// so there is no need to handle the TargetFramework property.
123+
tfms = xmlDoc.Element("Project")?
124+
.Elements("PropertyGroup")
125+
.Elements("TargetFrameworks")
126+
.FirstOrDefault()?.Value?
127+
.Split(';')
128+
.ToHashSet();
129+
130+
if (tfms == null || tfms.Count == 0)
131+
{
132+
Log.LogError($"No TargetFrameworks were detected in {projectPath}.");
133+
continue;
134+
}
135+
}
136+
137+
PackageInfo info = new(projectName[..(projectName.Length - 1 - version.Length)],
138+
version,
139+
directory.FullName,
140+
tfms);
141+
142+
_sbrpPackages.Add(info.Id, info);
143+
Log.LogMessage($"Detected package: {info.Id}");
144+
}
145+
}
146+
147+
private void ScanProjectReferences()
148+
{
149+
foreach (string projectJsonFile in Directory.GetFiles(SrcPath, "project.assets.json", SearchOption.AllDirectories))
150+
{
151+
LockFile lockFile = new LockFileFormat().Read(projectJsonFile);
152+
foreach (LockFileTargetLibrary lib in lockFile.Targets.SelectMany(t => t.Libraries))
153+
{
154+
if (!_sbrpPackages.TryGetValue($"{lib.Name}/{lib.Version}", out PackageInfo? info))
155+
{
156+
continue;
157+
}
158+
159+
if (!info.References.TryGetValue(lockFile.Path, out HashSet<string>? referencedTfms))
160+
{
161+
referencedTfms = [];
162+
info.References.Add(lockFile.Path, referencedTfms);
163+
}
164+
165+
IEnumerable<string> tfms = lib.CompileTimeAssemblies
166+
.Where(asm => asm.Path.StartsWith("lib") || asm.Path.StartsWith("ref"))
167+
.Select(asm => asm.Path.Split('/')[1]);
168+
foreach (string tfm in tfms)
169+
{
170+
referencedTfms.Add(tfm);
171+
}
172+
}
173+
}
174+
}
175+
176+
private record PackageInfo(string Name, string Version, string Path, HashSet<string>? Tfms = default)
177+
{
178+
public string Id => $"{Name}/{Version}";
179+
180+
// Dictionary of projects referencing the SBRP and the TFMs referenced by each project
181+
public Dictionary<string, HashSet<string>> References { get; } = [];
182+
}
183+
184+
private record Report(IEnumerable<PackageInfo> Sbrps, IEnumerable<string> UnreferencedSbrps);
185+
}

‎src/SourceBuild/content/repo-projects/Directory.Build.targets

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,6 @@
103103
<PropertyGroup>
104104
<!-- Dev innerloop opt-in feed: /p:ExtraRestoreSourcePath=... -->
105105
<ExtraSourcesNuGetSourceName>ExtraSources</ExtraSourcesNuGetSourceName>
106-
<SbrpRepoSrcPath>$([MSBuild]::NormalizeDirectory('$(RepoRoot)', 'src', 'source-build-reference-packages', 'src'))</SbrpRepoSrcPath>
107106
</PropertyGroup>
108107

109108
<PropertyGroup Condition="'$(DotNetBuildSourceOnly)' == 'true'">
@@ -187,7 +186,7 @@
187186
NuGetConfigFile="$(NuGetConfigFile)"
188187
BuildWithOnlineFeeds="$(DotNetBuildWithOnlineFeeds)"
189188
SourceBuildSources="@(_BuildSources)"
190-
SbrpRepoSrcPath="$(SbrpRepoSrcPath)"
189+
SbrpRepoSrcPath="$(SbrpRepoSrcDir)"
191190
SbrpCacheSourceName="$(SbrpCacheNuGetSourceName)"
192191
ReferencePackagesSourceName="$(ReferencePackagesNuGetSourceName)"
193192
PreviouslySourceBuiltSourceName="$(PreviouslySourceBuiltNuGetSourceName)"

‎src/SourceBuild/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/ArtifactsSizeTests/centos.9-x64.txt

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,6 @@ Microsoft.NET.Sdk.WorkloadMSBuildSdkResolver.x.y.z.nupkg: 14414
208208
Microsoft.NET.Sdk.x.y.z.nupkg: 994666
209209
Microsoft.NET.StringTools.x.y.z.nupkg: 74038
210210
Microsoft.NET.WebAssembly.Threading.x.y.z.nupkg: 44064
211-
Microsoft.NET.WebAssembly.Webcil.x.y.z.nupkg: 31426
212211
Microsoft.NET.Workload.Emscripten.Current.Manifest-x.y.z-1.nupkg: 6044
213212
Microsoft.NET.Workload.Emscripten.Current.Manifest-x.y.z.nupkg: 6036
214213
Microsoft.NET.Workload.Emscripten.net6.Manifest-x.y.z.nupkg: 5613
@@ -1642,8 +1641,8 @@ packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runt
16421641
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Numerics.pdb: 62392
16431642
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.pdb: 1152
16441643
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Serialization.dll: 6656
1645-
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Serialization.Formatters.dll: 126464
1646-
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Serialization.Formatters.pdb: 53552
1644+
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Serialization.Formatters.dll: 55808
1645+
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Serialization.Formatters.pdb: 20816
16471646
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Serialization.Json.dll: 5632
16481647
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Serialization.Json.pdb: 1164
16491648
packs/runtime.banana-rid.Microsoft.DotNet.ILCompiler/x.y.z/framework/System.Runtime.Serialization.pdb: 2576
@@ -3349,7 +3348,7 @@ sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/targets/Sdk.Razor.CurrentVersion.props: 4
33493348
sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/targets/Sdk.Razor.CurrentVersion.targets: 42907
33503349
sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tasks/netx.y/Microsoft.Extensions.FileSystemGlobbing.dll: 80896
33513350
sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tasks/netx.y/Microsoft.NET.Sdk.Razor.Tasks.dll: 124928
3352-
sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tools/Microsoft.AspNetCore.Razor.Utilities.Shared.dll: 150016
3351+
sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tools/Microsoft.AspNetCore.Razor.Utilities.Shared.dll: 193024
33533352
sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tools/Microsoft.CodeAnalysis.Razor.Compiler.dll: 3578368
33543353
sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tools/Microsoft.Css.Parser.dll: 329728
33553354
sdk/x.y.z/Sdks/Microsoft.NET.Sdk.Razor/tools/Microsoft.Extensions.ObjectPool.dll: 25600
@@ -4783,7 +4782,7 @@ shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Intrinsics.dll: 6656
47834782
shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Loader.dll: 5632
47844783
shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Numerics.dll: 306688
47854784
shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Serialization.dll: 6656
4786-
shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Serialization.Formatters.dll: 294400
4785+
shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Serialization.Formatters.dll: 107520
47874786
shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Serialization.Json.dll: 5632
47884787
shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Serialization.Primitives.dll: 18432
47894788
shared/Microsoft.NETCore.App/x.y.z/System.Runtime.Serialization.Xml.dll: 6656

‎src/StaticWebAssetsSdk/Tasks/GenerateStaticWebAssetsDevelopmentManifest.cs

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,9 +151,7 @@ private StaticWebAssetsDevelopmentManifest CreateManifest(
151151
}
152152
var matchingAsset = new StaticWebAssetMatch
153153
{
154-
SubPath = asset.Identity.StartsWith(asset.ContentRoot) ?
155-
StaticWebAsset.Normalize(asset.Identity.Substring(asset.ContentRoot.Length)) :
156-
asset.RelativePath,
154+
SubPath = ResolveSubPath(asset),
157155
ContentRootIndex = index
158156
};
159157
currentNode.Children ??= new Dictionary<string, StaticWebAssetNode>(StringComparer.Ordinal);
@@ -271,6 +269,33 @@ private StaticWebAssetsDevelopmentManifest CreateManifest(
271269
ContentRoots = contentRootIndex.OrderBy(kvp => kvp.Value).Select(kvp => kvp.Key).ToArray(),
272270
Root = root
273271
};
272+
273+
static string ResolveSubPath(StaticWebAsset asset)
274+
{
275+
if (File.Exists(asset.Identity))
276+
{
277+
if (asset.Identity.StartsWith(asset.ContentRoot, OSPath.PathComparison))
278+
{
279+
// We need an extra check that the file exist to avoid pointing out to a non-existing file. This can happen
280+
// when the asset is defined with an identity that doesn't exist yet, but that will be materialized later
281+
// when the asset is copied to the wwwroot folder.
282+
return StaticWebAsset.Normalize(asset.Identity.Substring(asset.ContentRoot.Length));
283+
}
284+
else
285+
{
286+
// This is a content root that we don't know about, so we can't resolve the subpath based on the identity, and
287+
// we need to rely on the assumption that the file will be available at contentRoot + relativePath.
288+
return asset.ReplaceTokens(asset.RelativePath, StaticWebAssetTokenResolver.Instance);
289+
}
290+
}
291+
else
292+
{
293+
// In any other case where the file doesn't exist, we expect the file to end up at the correct final location
294+
// which is defined by contentRoot + relativePath, and since the file will be copied there, the tokens will be
295+
// replaced as needed so that the file can be found.
296+
return asset.ReplaceTokens(asset.RelativePath, StaticWebAssetTokenResolver.Instance);
297+
}
298+
}
274299
}
275300

276301
public class StaticWebAssetsDevelopmentManifest

0 commit comments

Comments
 (0)