Skip to content

Commit 5d575d4

Browse files
adaggarwaldagood
adaggarwal
and
dagood
authored
uptake downloadFile task locally to fix timeouts (#1531)
* add download file as a local source-build task * Use sourcebuild BuildTasks * Update tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/DownloadFileSB.cs Co-authored-by: Davis Goodin <dagood@users.noreply.github.com>
1 parent a6c33a1 commit 5d575d4

File tree

2 files changed

+318
-2
lines changed

2 files changed

+318
-2
lines changed

‎build.proj

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="CopyReferenceOnlyPackages" />
88
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="WriteUsageBurndownData" />
99
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="ReplaceTextInFile" />
10+
<UsingTask AssemblyFile="$(XPlatSourceBuildTasksAssembly)" TaskName="DownloadFileSB" />
1011

1112
<Target Name="Build" DependsOnTargets="PrepareOutput;InitBuild">
1213
<Message Text="Build Environment: $(Platform) $(Configuration) $(TargetOS) $(TargetRid)" />
@@ -148,15 +149,15 @@
148149
<Target Name="DownloadSourceBuildReferencePackages"
149150
AfterTargets="Build"
150151
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingReferencePackages)' != 'true'">
151-
<DownloadFile
152+
<DownloadFileSB
152153
SourceUrl="$(ReferencePackagesTarballUrl)$(ReferencePackagesTarballName).$(PrivateSourceBuildReferencePackagesPackageVersion).tar.gz"
153154
DestinationFolder="$(ExternalTarballsDir)" />
154155
</Target>
155156

156157
<Target Name="DownloadSourceBuiltArtifacts"
157158
AfterTargets="Build"
158159
Condition="'$(OfflineBuild)' != 'true' and '$(OS)' != 'Windows_NT' and '$(SkipDownloadingPreviouslySourceBuiltPackages)' != 'true'">
159-
<DownloadFile
160+
<DownloadFileSB
160161
SourceUrl="$(SourceBuiltArtifactsTarballUrl)$(SourceBuiltArtifactsTarballName).$(PrivateSourceBuiltArtifactsPackageVersion).tar.gz"
161162
DestinationFolder="$(ExternalTarballsDir)" />
162163
</Target>

‎tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/DownloadFileSB.cs

Lines changed: 315 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,315 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
/// This task is sourced from https://github.com/microsoft/msbuild/blob/04e508c36f9c1fe826264aef7c26ffb8f16e9bdc/src/Tasks/DownloadFile.cs
5+
/// It alleviates the problem of time outs on DownloadFile Task. We are not the version of msbuild that has this fix, hence we have to locally
6+
/// build it to get rid of the issue.
7+
8+
using Microsoft.Build.Framework;
9+
using Microsoft.Build.Utilities;
10+
using Microsoft.DotNet.Build.Tasks;
11+
using System;
12+
using System.IO;
13+
using System.Net;
14+
using System.Net.Http;
15+
using System.Threading;
16+
using System.Threading.Tasks;
17+
using Task = System.Threading.Tasks.Task;
18+
19+
namespace Microsoft.Build.Tasks
20+
{
21+
/// <summary>
22+
/// Represents a task that can download a file.
23+
/// </summary>
24+
public sealed class DownloadFileSB : BuildTask, ICancelableTask
25+
{
26+
private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
27+
28+
/// <summary>
29+
/// Gets or sets an optional filename for the destination file. By default, the filename is derived from the <see cref="SourceUrl"/> if possible.
30+
/// </summary>
31+
public ITaskItem DestinationFileName { get; set; }
32+
33+
/// <summary>
34+
/// Gets or sets a <see cref="ITaskItem"/> that specifies the destination folder to download the file to.
35+
/// </summary>
36+
[Required]
37+
public ITaskItem DestinationFolder { get; set; }
38+
39+
/// <summary>
40+
/// Gets or sets a <see cref="ITaskItem"/> that contains details about the downloaded file.
41+
/// </summary>
42+
[Output]
43+
public ITaskItem DownloadedFile { get; set; }
44+
45+
/// <summary>
46+
/// Gets or sets an optional number of times to retry if possible.
47+
/// </summary>
48+
public int Retries { get; set; }
49+
50+
/// <summary>
51+
/// Gets or sets the number of milliseconds to wait before retrying.
52+
/// </summary>
53+
public int RetryDelayMilliseconds { get; set; } = 5 * 1000;
54+
55+
/// <summary>
56+
/// Gets or sets an optional value indicating whether or not the download should be skipped if the file is up-to-date.
57+
/// </summary>
58+
public bool SkipUnchangedFiles { get; set; } = true;
59+
60+
/// <summary>
61+
/// Gets or sets the URL to download.
62+
/// </summary>
63+
[Required]
64+
public string SourceUrl { get; set; }
65+
66+
/// <summary>
67+
/// Gets or sets a <see cref="HttpMessageHandler"/> to use. This is used by unit tests to mock a connection to a remote server.
68+
/// </summary>
69+
internal HttpMessageHandler HttpMessageHandler { get; set; }
70+
71+
/// <inheritdoc cref="ICancelableTask.Cancel"/>
72+
public void Cancel()
73+
{
74+
_cancellationTokenSource.Cancel();
75+
}
76+
77+
public override bool Execute()
78+
{
79+
return ExecuteAsync().GetAwaiter().GetResult();
80+
}
81+
82+
private async Task<bool> ExecuteAsync()
83+
{
84+
if (!Uri.TryCreate(SourceUrl, UriKind.Absolute, out Uri uri))
85+
{
86+
Log.LogError($"DownloadFileSB.ErrorInvalidUrl {SourceUrl}");
87+
return false;
88+
}
89+
90+
int retryAttemptCount = 0;
91+
92+
CancellationToken cancellationToken = _cancellationTokenSource.Token;
93+
94+
while(true)
95+
{
96+
try
97+
{
98+
await DownloadAsync(uri, cancellationToken);
99+
break;
100+
}
101+
catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken)
102+
{
103+
// This task is being cancelled. Exit the loop.
104+
break;
105+
}
106+
catch (Exception e)
107+
{
108+
bool canRetry = IsRetriable(e, out Exception actualException) && retryAttemptCount++ < Retries;
109+
110+
if (canRetry)
111+
{
112+
Log.LogWarning($"DownloadFileSB.Retrying {SourceUrl} {retryAttemptCount + 1} {RetryDelayMilliseconds} {actualException}");
113+
114+
try
115+
{
116+
await Task.Delay(RetryDelayMilliseconds, cancellationToken).ConfigureAwait(false);
117+
}
118+
catch (OperationCanceledException delayException) when (delayException.CancellationToken == cancellationToken)
119+
{
120+
// This task is being cancelled, exit the loop
121+
break;
122+
}
123+
}
124+
else
125+
{
126+
Log.LogError($"DownloadFileSB.ErrorDownloading {SourceUrl} {actualException}");
127+
break;
128+
}
129+
}
130+
}
131+
132+
return !_cancellationTokenSource.IsCancellationRequested && !Log.HasLoggedErrors;
133+
}
134+
135+
/// <summary>
136+
/// Attempts to download the file.
137+
/// </summary>
138+
/// <param name="uri">The parsed <see cref="Uri"/> of the request.</param>
139+
private async Task DownloadAsync(Uri uri, CancellationToken cancellationToken)
140+
{
141+
// The main reason to use HttpClient vs WebClient is because we can pass a message handler for unit tests to mock
142+
using (var client = new HttpClient(HttpMessageHandler ?? new HttpClientHandler(), disposeHandler: true))
143+
{
144+
// Only get the response without downloading the file so we can determine if the file is already up-to-date
145+
using (HttpResponseMessage response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false))
146+
{
147+
try
148+
{
149+
response.EnsureSuccessStatusCode();
150+
}
151+
catch (HttpRequestException e)
152+
{
153+
// HttpRequestException does not have the status code so its wrapped and thrown here so that later on we can determine
154+
// if a retry is possible based on the status code
155+
throw new CustomHttpRequestException(e.Message, e.InnerException, response.StatusCode);
156+
}
157+
158+
if (!TryGetFileName(response, out string filename))
159+
{
160+
Log.LogError($"DownloadFileSB.ErrorUnknownFileName {SourceUrl} {nameof(DestinationFileName)}");
161+
return;
162+
}
163+
164+
DirectoryInfo destinationDirectory = Directory.CreateDirectory(DestinationFolder.ItemSpec);
165+
166+
var destinationFile = new FileInfo(Path.Combine(destinationDirectory.FullName, filename));
167+
168+
// The file is considered up-to-date if its the same length. This could be inaccurate, we can consider alternatives in the future
169+
if (ShouldSkip(response, destinationFile))
170+
{
171+
Log.LogMessage(MessageImportance.Normal, $"DownloadFileSB.DidNotDownloadBecauseOfFileMatch {SourceUrl}", destinationFile.FullName, nameof(SkipUnchangedFiles), "true");
172+
173+
DownloadedFile = new TaskItem(destinationFile.FullName);
174+
175+
return;
176+
}
177+
178+
try
179+
{
180+
cancellationToken.ThrowIfCancellationRequested();
181+
182+
using (var target = new FileStream(destinationFile.FullName, FileMode.Create, FileAccess.Write, FileShare.None))
183+
{
184+
Log.LogMessage(MessageImportance.High, $"DownloadFileSB.Downloading {SourceUrl}", destinationFile.FullName, response.Content.Headers.ContentLength);
185+
186+
using (Stream responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false))
187+
{
188+
await responseStream.CopyToAsync(target, 1024, cancellationToken).ConfigureAwait(false);
189+
}
190+
191+
DownloadedFile = new TaskItem(destinationFile.FullName);
192+
}
193+
}
194+
finally
195+
{
196+
if (DownloadedFile == null)
197+
{
198+
// Delete the file if anything goes wrong during download. This could be destructive but we don't want to leave
199+
// partially downloaded files on disk either. Alternatively we could download to a temporary location and copy
200+
// on success but we are concerned about the added I/O
201+
destinationFile.Delete();
202+
}
203+
}
204+
}
205+
}
206+
}
207+
208+
/// <summary>
209+
/// Determines if the specified exception is considered retriable.
210+
/// </summary>
211+
/// <param name="exception">The originally thrown exception.</param>
212+
/// <param name="actualException">The actual exception to be used for logging errors.</param>
213+
/// <returns><code>true</code> if the exception is retriable, otherwise <code>false</code>.</returns>
214+
private static bool IsRetriable(Exception exception, out Exception actualException)
215+
{
216+
actualException = exception;
217+
218+
// Get aggregate inner exception
219+
if (actualException is AggregateException aggregateException && aggregateException.InnerException != null)
220+
{
221+
actualException = aggregateException.InnerException;
222+
}
223+
224+
// Some HttpRequestException have an inner exception that has the real error
225+
if (actualException is HttpRequestException httpRequestException && httpRequestException.InnerException != null)
226+
{
227+
actualException = httpRequestException.InnerException;
228+
229+
// An IOException inside of a HttpRequestException means that something went wrong while downloading
230+
if (actualException is IOException)
231+
{
232+
return true;
233+
}
234+
}
235+
236+
if (actualException is CustomHttpRequestException customHttpRequestException)
237+
{
238+
// A wrapped CustomHttpRequestException has the status code from the error
239+
switch (customHttpRequestException.StatusCode)
240+
{
241+
case HttpStatusCode.InternalServerError:
242+
case HttpStatusCode.RequestTimeout:
243+
return true;
244+
}
245+
}
246+
247+
if (actualException is WebException webException)
248+
{
249+
// WebException is thrown when accessing the Content of the response
250+
switch (webException.Status)
251+
{
252+
// Don't retry on anything that cannot be compensated for
253+
case WebExceptionStatus.TrustFailure:
254+
case WebExceptionStatus.MessageLengthLimitExceeded:
255+
case WebExceptionStatus.RequestProhibitedByCachePolicy:
256+
case WebExceptionStatus.RequestProhibitedByProxy:
257+
return false;
258+
259+
default:
260+
// Retry on all other WebExceptions
261+
return true;
262+
}
263+
}
264+
265+
return false;
266+
}
267+
268+
/// <summary>
269+
/// Attempts to get the file name to use when downloading the file.
270+
/// </summary>
271+
/// <param name="response">The <see cref="HttpResponseMessage"/> with information about the response.</param>
272+
/// <param name="filename">Receives the name of the file.</param>
273+
/// <returns><code>true</code> if a file name could be determined, otherwise <code>false</code>.</returns>
274+
private bool TryGetFileName(HttpResponseMessage response, out string filename)
275+
{
276+
if (response == null)
277+
{
278+
throw new ArgumentNullException(nameof(response));
279+
}
280+
281+
// Not all URIs contain a file name so users will have to specify one
282+
// Example: http://www.download.com/file/1/
283+
284+
filename = !String.IsNullOrWhiteSpace(DestinationFileName?.ItemSpec)
285+
? DestinationFileName.ItemSpec // Get the file name from what the user specified
286+
: response.Content?.Headers?.ContentDisposition?.FileName // Attempt to get the file name from the content-disposition header value
287+
?? Path.GetFileName(response.RequestMessage.RequestUri.LocalPath); // Otherwise attempt to get a file name from the URI
288+
289+
return !String.IsNullOrWhiteSpace(filename);
290+
}
291+
292+
/// <summary>
293+
/// Represents a wrapper around the <see cref="HttpRequestException"/> that also contains the <see cref="HttpStatusCode"/>.
294+
/// </summary>
295+
private sealed class CustomHttpRequestException : HttpRequestException
296+
{
297+
public CustomHttpRequestException(string message, Exception inner, HttpStatusCode statusCode)
298+
: base(message, inner)
299+
{
300+
StatusCode = statusCode;
301+
}
302+
303+
public HttpStatusCode StatusCode { get; }
304+
}
305+
306+
private bool ShouldSkip(HttpResponseMessage response, FileInfo destinationFile)
307+
{
308+
return SkipUnchangedFiles
309+
&& destinationFile.Exists
310+
&& destinationFile.Length == response.Content.Headers.ContentLength
311+
&& response.Content.Headers.LastModified.HasValue
312+
&& destinationFile.LastWriteTimeUtc > response.Content.Headers.LastModified.Value.UtcDateTime;
313+
}
314+
}
315+
}

0 commit comments

Comments
 (0)