Skip to content

Commit 4a909b6

Browse files
dseefelddseefeld
authored
Add task to generate burndown data from the prebuilt report (#922)
* Add task to generate burndown data from the prebuilt report data checked in to git. The burndown data counts the number of prebuilt packages and versions and number of prebuilt packages alone.
1 parent e40b0e7 commit 4a909b6

File tree

8 files changed

+164
-11
lines changed

8 files changed

+164
-11
lines changed

‎.vsts.pipelines/builds/matrix.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ jobs:
1515
imageName: microsoft/dotnet-buildtools-prereqs:centos-7-b46d863-20180719033416
1616
reportPrebuiltLeaks: true
1717
matrix:
18-
Production: {}
18+
Production: { generatePrebuiltBurndown: true }
1919
Online: { type: Online }
2020
Offline: { type: Offline }
2121
Offline Portable: { type: Offline Portable }

‎.vsts.pipelines/jobs/ci-linux.yml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ parameters:
66
name: Hosted Ubuntu 1604
77
imageName: null
88
reportPrebuiltLeaks: false
9+
generatePrebuiltBurndown: false
910

1011
jobs:
1112
- job: ${{ parameters.job }}
@@ -30,9 +31,11 @@ jobs:
3031
dropDirectory: $(stagingDirectory)/drop
3132
imageName: ${{ parameters.imageName }}
3233
reportPrebuiltLeaks: ${{ parameters.reportPrebuiltLeaks }}
34+
generatePrebuiltBurndown: ${{ parameters.generatePrebuiltBurndown }}
3335
rootDirectory: $(Build.SourcesDirectory)/..
3436
stagingDirectory: $(rootDirectory)/sb/staging
3537
tarballName: tarball_$(Build.BuildId)
38+
SOURCE_BUILD_SKIP_SUBMODULE_CHECK: true
3639
# Default type, can be overridden by matrix legs.
3740
type: Production
3841

@@ -70,6 +73,16 @@ jobs:
7073
displayName: Build source-build
7174
timeoutInMinutes: 120
7275
76+
# Generate prebuilt burndown data
77+
- script: |
78+
set -x
79+
df -h
80+
$(docker.run) $(docker.src.map) $(docker.src.work) -e SOURCE_BUILD_SKIP_SUBMODULE_CHECK $(imageName) ./build.sh \
81+
/t:GeneratePrebuiltBurndownData
82+
displayName: Generate prebuilt burndown data
83+
condition: and(succeeded(), eq(variables['generatePrebuiltBurndown'], true))
84+
continueOnError: true
85+
7386
# Run smoke tests. This is needed even in tarball legs to create the smoke-test-prereqs archive.
7487
- script: |
7588
set -x

‎build.proj

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
<UsingTask AssemblyFile="$(LeakDetectionTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll" TaskName="CheckForPoison" />
66
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="CopyReferenceOnlyPackages" />
7+
<UsingTask AssemblyFile="$(TasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.dll" TaskName="WriteUsageBurndownData" />
78

89
<Target Name="Build" DependsOnTargets="PrepareOutput;InitBuild">
910
<Message Text="Build Environment: $(Platform) $(Configuration) $(TargetOS) $(TargetRid)" />
@@ -131,6 +132,15 @@
131132
PoisonReportOutputFilePath="$(PoisonUsageReportFile)" />
132133
</Target>
133134

135+
<Target Name="GeneratePrebuiltBurndownData">
136+
<WriteUsageBurndownData RootDirectory="$(ProjectDir)"
137+
PrebuiltBaselineFile="$(OnlineBaselineDataFile)"
138+
OutputFilePath="$(OnlinePrebuiltBurndownDataFile)" />
139+
<WriteUsageBurndownData RootDirectory="$(ProjectDir)"
140+
PrebuiltBaselineFile="$(OfflineBaselineDataFile)"
141+
OutputFilePath="$(OfflinePrebuiltBurndownDataFile)" />
142+
</Target>
143+
134144
<Target Name="RunSmokeTest" DependsOnTargets="GetProdConBlobFeedUrl">
135145
<PropertyGroup>
136146
<SmokeTestCommand>./smoke-test.sh</SmokeTestCommand>

‎dir.props

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
<IldasmPath>$(ToolsDir)ilasm/ildasm</IldasmPath>
4848
<ToolsLocalDir>$(ProjectDir)tools-local/</ToolsLocalDir>
4949
<TaskDirectory>$(ToolsLocalDir)tasks/</TaskDirectory>
50-
<TasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks/bin/Debug/netstandard1.5/</TasksBinDir>
50+
<TasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks/bin/Debug/netstandard2.0/</TasksBinDir>
5151
<LeakDetectionTasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/</LeakDetectionTasksBinDir>
5252
<BaseIntermediatePath>$(BaseOutputPath)obj/</BaseIntermediatePath>
5353
<OutputPath>$(BaseOutputPath)$(Platform)/$(Configuration)/</OutputPath>
@@ -83,6 +83,8 @@
8383
<ProdConManifestFile>$(PackageReportDir)prodcon-build.xml</ProdConManifestFile>
8484
<PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
8585
<ConflictingPackageReportDir>$(BaseOutputPath)conflict-report/</ConflictingPackageReportDir>
86+
<OfflinePrebuiltBurndownDataFile>$(PackageReportDir)PrebuiltBurndownData-offline.csv</OfflinePrebuiltBurndownDataFile>
87+
<OnlinePrebuiltBurndownDataFile>$(PackageReportDir)PrebuiltBurndownData-online.csv</OnlinePrebuiltBurndownDataFile>
8688
<ReferencePackagesBaseDir>$(IntermediatePath)reference-packages/</ReferencePackagesBaseDir>
8789
<!--
8890
Change ReferencePackagesBaseDir conditionally in offline build.
@@ -94,6 +96,11 @@
9496
<ReferencePackagesSourceDir>$(ReferencePackagesBaseDir)source/</ReferencePackagesSourceDir>
9597
<ReferencePackagesDir>$(ReferencePackagesBaseDir)packages/</ReferencePackagesDir>
9698
<ReferencePackagesToDeleteDir>$(ReferencePackagesBaseDir)packages-to-delete/</ReferencePackagesToDeleteDir>
99+
<BaselineDataFile>$(ToolsLocalDir)prebuilt-baseline-</BaselineDataFile>
100+
<OfflineBaselineDataFile>$(BaselineDataFile)offline.xml</OfflineBaselineDataFile>
101+
<OnlineBaselineDataFile>$(BaselineDataFile)online.xml</OnlineBaselineDataFile>
102+
<BaselineDataFile Condition="'$(OfflineBuild)' == 'true'">$(OfflineBaselineDataFile)</BaselineDataFile>
103+
<BaselineDataFile Condition="'$(OfflineBuild)' != 'true'">$(OnlineBaselineDataFile)</BaselineDataFile>
97104
</PropertyGroup>
98105

99106
<!-- Import Build tools common props file where repo-independent properties are found -->

‎repos/dir.targets

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -467,10 +467,6 @@
467467

468468
<Target Name="ReportPrebuiltUsage">
469469
<PropertyGroup>
470-
<BaselineDataFile>$(ToolsLocalDir)prebuilt-baseline-</BaselineDataFile>
471-
<BaselineDataFile Condition="'$(OfflineBuild)' == 'true'">$(BaselineDataFile)offline.xml</BaselineDataFile>
472-
<BaselineDataFile Condition="'$(OfflineBuild)' != 'true'">$(BaselineDataFile)online.xml</BaselineDataFile>
473-
474470
<FailOnPrebuiltBaselineError Condition="'$(FailOnPrebuiltBaselineError)' == ''">false</FailOnPrebuiltBaselineError>
475471
</PropertyGroup>
476472

‎tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
</PropertyGroup>
66
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))/dir.props" />
77
<PropertyGroup>
8-
<TargetFramework>netstandard1.5</TargetFramework>
8+
<TargetFramework>netstandard2.0</TargetFramework>
99
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
1010
<CoreSetupPlatformAbstractionsPath>$(SubmoduleDirectory)core-setup/src/managed/Microsoft.DotNet.PlatformAbstractions/</CoreSetupPlatformAbstractionsPath>
1111
<OutputPath>$(MSBuildThisFileDirectory)bin/$(Configuration)</OutputPath>

‎tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/Microsoft.DotNet.SourceBuild.Tasks.csproj

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
</PropertyGroup>
66
<Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), dir.props))/dir.props" />
77
<PropertyGroup>
8-
<TargetFramework>netstandard1.5</TargetFramework>
8+
<TargetFramework>netstandard2.0</TargetFramework>
99
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
1010
<CoreSetupPlatformAbstractionsPath>$(SubmoduleDirectory)core-setup/src/managed/Microsoft.DotNet.PlatformAbstractions/</CoreSetupPlatformAbstractionsPath>
1111
<OutputPath>$(MSBuildThisFileDirectory)bin/$(Configuration)</OutputPath>
@@ -27,9 +27,6 @@
2727
<PackageReference Include="Microsoft.Build.Utilities.Core">
2828
<Version>15.6.82</Version>
2929
</PackageReference>
30-
<PackageReference Include="System.ValueTuple">
31-
<Version>4.4.0</Version>
32-
</PackageReference>
3330
</ItemGroup>
3431

3532
<!--

‎tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks/UsageReport/WriteUsageBurndownData.cs

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
using Microsoft.Build.Framework;
6+
using Newtonsoft.Json;
7+
using Newtonsoft.Json.Linq;
8+
using NuGet.Packaging.Core;
9+
using NuGet.Versioning;
10+
using System;
11+
using System.Collections.Concurrent;
12+
using System.Collections.Generic;
13+
using System.Diagnostics;
14+
using System.IO;
15+
using System.IO.Compression;
16+
using System.Linq;
17+
using System.Net;
18+
using System.Threading.Tasks;
19+
using System.Xml.Linq;
20+
using Task = Microsoft.Build.Utilities.Task;
21+
22+
namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport
23+
{
24+
public class WriteUsageBurndownData : Task
25+
{
26+
/// <summary>
27+
/// Specifies the root directory for git.
28+
/// Note: Requires a trailing "/" when specifying the directory.
29+
/// </summary>
30+
[Required]
31+
public string RootDirectory { get; set; }
32+
33+
/// <summary>
34+
/// Specifies the path to the prebuilt baseline file
35+
/// to be used to generate the burndown.
36+
/// </summary>
37+
[Required]
38+
public string PrebuiltBaselineFile { get; set; }
39+
40+
/// <summary>
41+
/// Output data CSV file.
42+
/// </summary>
43+
[Required]
44+
public string OutputFilePath { get; set; }
45+
46+
public override bool Execute()
47+
{
48+
string baselineRelativeFileName = PrebuiltBaselineFile.Replace(RootDirectory, "");
49+
string gitLogCommand = $"log --first-parent --pretty=format:%H,%f,%ci -- {PrebuiltBaselineFile}";
50+
51+
DateTime startTime = DateTime.Now;
52+
Log.LogMessage(MessageImportance.High, "Generating summary usage burndown data...");
53+
54+
ParallelQuery<string> data = ExecuteGitCommand(RootDirectory, gitLogCommand).AsParallel().Select(commitLine =>
55+
{
56+
var splitLine = commitLine.Split(',');
57+
var commit = new Commit()
58+
{
59+
Sha = splitLine[0],
60+
Title = splitLine[1],
61+
CommitDate = DateTime.Parse(splitLine[2])
62+
};
63+
string fileContents = GetFileContents(baselineRelativeFileName, commit.Sha);
64+
Usage[] usages = UsageData.Parse(XElement.Parse(fileContents)).Usages.NullAsEmpty().ToArray();
65+
commit.PackageVersionCount = usages.Count();
66+
commit.PackageCount = usages.GroupBy(i => i.PackageIdentity.Id).Select(grp => grp.First()).Count();
67+
return commit;
68+
})
69+
.Select(c => c.ToString());
70+
71+
Directory.CreateDirectory(Path.GetDirectoryName(OutputFilePath));
72+
73+
File.WriteAllLines(OutputFilePath, data);
74+
75+
Log.LogMessage(
76+
MessageImportance.High,
77+
$"Generating summary usage burndown data at {OutputFilePath} done. Took {DateTime.Now - startTime}");
78+
79+
return !Log.HasLoggedErrors;
80+
}
81+
82+
/// <summary>
83+
/// Get the contents of a git file based on the commit sha.
84+
/// </summary>
85+
/// <param name="relativeFilePath">The relative path (from the git root) to the file.</param>
86+
/// <param name="commitSha">The commit sha for the version of the file to get.</param>
87+
/// <returns>The contents of the specified file.</returns>
88+
private string GetFileContents(string relativeFilePath, string commitSha)
89+
{
90+
WebClient client = new WebClient();
91+
var xmlString = client.DownloadString($"https://raw.githubusercontent.com/dotnet/source-build/{commitSha}/{relativeFilePath.Replace('\\', '/')}");
92+
return xmlString;
93+
}
94+
95+
/// <summary>
96+
/// Executes a git command and returns the result.
97+
/// </summary>
98+
/// <param name="workingDirectory">The working directory for the git command.</param>
99+
/// <param name="command">The git command to execute.</param>
100+
/// <returns>An array of the output lines of the git command.</returns>
101+
private string[] ExecuteGitCommand(string workingDirectory, string command)
102+
{
103+
string[] returnData;
104+
Process _process = new Process();
105+
_process.StartInfo.FileName = "git";
106+
_process.StartInfo.Arguments = command;
107+
_process.StartInfo.WorkingDirectory = workingDirectory;
108+
_process.StartInfo.RedirectStandardOutput = true;
109+
_process.StartInfo.UseShellExecute = false;
110+
_process.Start();
111+
returnData = _process.StandardOutput.ReadToEnd().Split('\n');
112+
_process.WaitForExit();
113+
return returnData;
114+
}
115+
116+
private class Commit
117+
{
118+
public string Sha { get; set; }
119+
public string Title { get; set; }
120+
public DateTime CommitDate { get; set; }
121+
public int PackageVersionCount { get; set; }
122+
public int PackageCount { get; set; }
123+
124+
public override string ToString()
125+
{
126+
return $"{Sha}, {Title}, {CommitDate}, {PackageVersionCount}, {PackageCount}";
127+
}
128+
}
129+
}
130+
}

0 commit comments

Comments
 (0)