Skip to content

Commit e23a791

Browse files
nikita-petkonikita-petko
authored
Fix for #96 (#102)
Introduces cases for timeout and internal server error.
1 parent e150225 commit e23a791

File tree

5 files changed

+120
-90
lines changed

5 files changed

+120
-90
lines changed

‎Assemblies/MFDLabs.Logging/Implementation/Logger.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public class Logger : ILogger, IDisposable
4646
static Logger()
4747
{
4848
var stdout = Console.OpenStandardOutput();
49-
var stdoutStream = new System.IO.StreamWriter(stdout, System.Text.Encoding.ASCII)
49+
var stdoutStream = new StreamWriter(stdout, System.Text.Encoding.ASCII)
5050
{
5151
AutoFlush = true
5252
};

‎MFDLabs.Grid.AutoDeployer/MFDLabs.Grid.AutoDeployer.csproj

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@
8686
<ItemGroup>
8787
<None Update="runservice.bat">
8888
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
89+
<DependentUpon>RunService.ps1</DependentUpon>
8990
</None>
9091
<None Update="RunService.ps1">
9192
<CopyToOutputDirectory>Always</CopyToOutputDirectory>

‎MFDLabs.Grid.AutoDeployer/Program.cs

Lines changed: 34 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.IO;
33
using System.Linq;
4+
using System.Diagnostics;
45
using System.CommandLine;
56
using System.Threading.Tasks;
67
using System.CommandLine.Parsing;
@@ -12,19 +13,26 @@
1213
using MFDLabs.FileSystem;
1314
using MFDLabs.Diagnostics;
1415
using MFDLabs.Text.Extensions;
16+
using MFDLabs.Configuration.Logging;
1517
using MFDLabs.Grid.AutoDeployer.Service;
1618

1719
namespace MFDLabs.Grid.AutoDeployer;
1820

1921
internal static class Program
2022
{
21-
private static readonly Option<bool> PurgeOption = new (new[] { "-purge", "/purge", "--purge" }, "Purge all known deployer info.") { IsRequired = false };
23+
private static readonly Option<bool> PurgeOption = new(new[] { "-purge", "/purge", "--purge" }, "Purge all known deployer info.") { IsRequired = false };
2224

2325
public static async Task Main(params string[] args)
2426
{
2527
Logger.Singleton.LogLevel = global::MFDLabs.Grid.AutoDeployer.Properties.Settings.Default.EnvironmentLogLevel;
2628
EventLogLogger.Singleton.LogLevel = global::MFDLabs.Grid.AutoDeployer.Properties.Settings.Default.EnvironmentLogLevel;
2729

30+
ConfigurationLogging.OverrideDefaultConfigurationLogging(
31+
EventLogLogger.Singleton.Error,
32+
Logger.Singleton.Warning,
33+
Logger.NoopSingleton.Info
34+
);
35+
2836
// If args has -purge or --purge etc.
2937
// purge all known deployer info.
3038
var rootCommand = new RootCommand(
@@ -137,14 +145,37 @@ private static void Run(InvocationContext context)
137145
return;
138146
}
139147

148+
TryCreateEventLog(typeof(Program).Namespace, typeof(Program).Namespace);
149+
140150
var app = new ServiceHostApp();
141-
app.EventLog.Source = "MFDLabs.Grid.AutoDeployer";
142-
app.EventLog.Log = "MFDLabs.Grid.AutoDeployer";
151+
app.EventLog.Source = typeof(Program).Namespace;
152+
app.EventLog.Log = typeof(Program).Namespace;
143153

144154
Console.CancelKeyPress += (_, _) => app.Stop();
145155

146156
app.HostOpening += AutoDeployerService.Start;
147157
app.HostClosing += AutoDeployerService.Stop;
148158
app.Process(args.ToArray());
149159
}
160+
161+
private static void TryCreateEventLog(string source, string log)
162+
{
163+
#if NETFRAMEWORK
164+
try
165+
{
166+
if (EventLog.Exists(log) && EventLog.SourceExists(source)) return;
167+
168+
EventLog.CreateEventSource(source, log);
169+
170+
var eventLog = new EventLog(source, ".", log);
171+
172+
if (eventLog.OverflowAction != OverflowAction.OverwriteAsNeeded)
173+
{
174+
eventLog.ModifyOverflowPolicy(OverflowAction.OverwriteAsNeeded, 5);
175+
eventLog.MaximumKilobytes = 16000;
176+
}
177+
}
178+
catch { }
179+
#endif
180+
}
150181
}

‎MFDLabs.Grid.AutoDeployer/Service/AutoDeployerService.cs

Lines changed: 84 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Net;
66
using System.Linq;
77
using System.Text;
8+
using System.Net.Http;
89
using System.Threading;
910
using System.Diagnostics;
1011
using System.Net.Sockets;
@@ -21,6 +22,7 @@
2122
using MFDLabs.Threading.Extensions;
2223
using MFDLabs.Reflection.Extensions;
2324
using MFDLabs.Diagnostics.Extensions;
25+
using MFDLabs.ErrorHandling.Extensions;
2426

2527
// ReSharper disable InconsistentNaming
2628
// ReSharper disable EmptyGeneralCatchClause
@@ -29,6 +31,7 @@
2931
/** TODO: Report to Infrastructure that node deployed this App **/
3032
/** TODO: Expose some API to force Rollbacks and Force Deployments? **/
3133
/** TODO: Rate limit detection for not just the REST API but also the GraphQL API **/
34+
/** TODO: Store some stuff in the registry like skipped version. **/
3235

3336
namespace MFDLabs.Grid.AutoDeployer.Service
3437
{
@@ -145,6 +148,12 @@ private static bool CachedVersionExistsRemotely(out bool wasRatelimited, out Tim
145148
return false;
146149

147150
}
151+
/* Last cases for this, we assume it is either timeout or 500. */
152+
catch (Exception ex) when (ex is HttpRequestException or ApiException)
153+
{
154+
EventLogLogger.Singleton.Warning(ex.ToDetailedString());
155+
return true;
156+
}
148157
catch (Exception ex)
149158
{
150159
Logger.Singleton.Error(ex);
@@ -238,7 +247,7 @@ private static void Work()
238247
{
239248
var uri = new Uri(gheUrl);
240249

241-
_gitHubClient = new(ghApiPhv, new Uri(gheUrl));
250+
_gitHubClient = new(ghApiPhv, uri);
242251
_gitHubQLClient = new(ghQlPhv, new($"{uri.Scheme}://{uri.Host}/graphql"), _githubToken);
243252
}
244253

@@ -275,87 +284,95 @@ private static void SkippedVersionInvalidationWork()
275284

276285
private static void Run()
277286
{
278-
SetupRegistry();
279-
SetupDeploymentPath();
280-
_markerLock = new FileLock(Path.Combine(_deploymentPath, DeploymentMarkerFilename));
281-
_markerLock.Lock();
282-
PurgeCachedVersionIfNoLongerExists(out _, out _);
283-
DetermineLatestVersion();
284-
LaunchIfNotRunning();
285-
286-
while (_isRunning)
287+
try
287288
{
288-
if (DetermineIfNewReleaseAvailable(out var r))
289+
SetupRegistry();
290+
SetupDeploymentPath();
291+
_markerLock = new FileLock(Path.Combine(_deploymentPath, DeploymentMarkerFilename));
292+
_markerLock.Lock();
293+
PurgeCachedVersionIfNoLongerExists(out _, out _);
294+
DetermineLatestVersion();
295+
LaunchIfNotRunning();
296+
297+
while (_isRunning)
289298
{
290-
// We have the release, it should have the following files:
291-
// A .mfdlabs-config-archive file -> This file contains .config files
292-
// A .Unpacker.ps1 file -> This file runs a 7Zip command that unpacks the contents to the archives into the directory it's in.
293-
// A .Unpacker.bat file -> This file is a wrapper to the .ps1 file.
294-
// A .mfdlabs-archive file -> Contains everything else other than the .config files.
295-
// There should only be 4 assets in the release, and each of these should match the deployment id regex and have any of those extensions at the end.
296-
if (r.Assets.Count != 4)
299+
if (DetermineIfNewReleaseAvailable(out var r))
297300
{
298-
SkipVersion(r.TagName);
299-
EventLogLogger.Singleton.Warning("Skipping '{0}' because it had more or less than 4 assets.", r.TagName);
300-
goto SLEEP;
301-
}
302-
303-
var regex = $@"{r.TagName}_?(((?i)(net(?i)(standard|coreapp)?)(([0-9]{{1,3}}\.?){{1,2}}))-(?i)(release|debug)(?i)(config(?i)(uration)?)?)?(?i)(\.mfdlabs-(config-)?archive|Unpacker\.(ps1|bat))?";
304-
305-
foreach (var a in r.Assets)
306-
{
307-
if (!a.Name.IsMatch(regex))
301+
// We have the release, it should have the following files:
302+
// A .mfdlabs-config-archive file -> This file contains .config files
303+
// A .Unpacker.ps1 file -> This file runs a 7Zip command that unpacks the contents to the archives into the directory it's in.
304+
// A .Unpacker.bat file -> This file is a wrapper to the .ps1 file.
305+
// A .mfdlabs-archive file -> Contains everything else other than the .config files.
306+
// There should only be 4 assets in the release, and each of these should match the deployment id regex and have any of those extensions at the end.
307+
if (r.Assets.Count != 4)
308308
{
309309
SkipVersion(r.TagName);
310-
EventLogLogger.Singleton.Warning("Skipping '{0}' because the asset '{1}' didn't match '{2}'.", r.TagName, a.Name, regex);
310+
EventLogLogger.Singleton.Warning("Skipping '{0}' because it had more or less than 4 assets.", r.TagName);
311311
goto SLEEP;
312312
}
313-
}
313+
314+
var regex = $@"{r.TagName}_?(((?i)(net(?i)(standard|coreapp)?)(([0-9]{{1,3}}\.?){{1,2}}))-(?i)(release|debug)(?i)(config(?i)(uration)?)?)?(?i)(\.mfdlabs-(config-)?archive|Unpacker\.(ps1|bat))?";
315+
316+
foreach (var a in r.Assets)
317+
{
318+
if (!a.Name.IsMatch(regex))
319+
{
320+
SkipVersion(r.TagName);
321+
EventLogLogger.Singleton.Warning("Skipping '{0}' because the asset '{1}' didn't match '{2}'.", r.TagName, a.Name, regex);
322+
goto SLEEP;
323+
}
324+
}
314325

315326
#if GRID_BOT
316-
if (ProcessHelper.GetProcessByName(_primaryDeploymentExecutable.ToLower().Replace(".exe", ""), out _))
317-
{
318-
InvokeMaintenanceCommandOnPreviousExe(r.TagName);
319-
EventLogLogger.Singleton.Warning("Invoked upgrade message onto bot, sleep for 15 seconds to ensure it receives it.");
320-
Thread.Sleep(TimeSpan.FromSeconds(15));
321-
}
327+
if (ProcessHelper.GetProcessByName(_primaryDeploymentExecutable.ToLower().Replace(".exe", ""), out _))
328+
{
329+
InvokeMaintenanceCommandOnPreviousExe(r.TagName);
330+
EventLogLogger.Singleton.Warning("Invoked upgrade message onto bot, sleep for 15 seconds to ensure it receives it.");
331+
Thread.Sleep(TimeSpan.FromSeconds(15));
332+
}
322333
#endif
323334

324-
foreach (var a in r.Assets)
325-
{
326-
var fqp = Path.Combine(_deploymentPath, a.Name);
327-
DownloadArtifact(a.Name, a.Url, fqp);
328-
}
335+
foreach (var a in r.Assets)
336+
{
337+
var fqp = Path.Combine(_deploymentPath, a.Name);
338+
DownloadArtifact(a.Name, a.Url, fqp);
339+
}
329340

330-
if (!RunUnpacker(r, out var versionDeploymentPath))
331-
{
341+
if (!RunUnpacker(r, out var versionDeploymentPath))
342+
{
343+
CleanupArtifacts(r.Assets.ToArray());
344+
try { Directory.Delete(versionDeploymentPath, true); } catch { }
345+
try { Directory.Delete(Path.Combine(_deploymentPath, r.TagName), true); } catch { }
346+
SkipVersion(r.TagName);
347+
goto SLEEP;
348+
}
332349
CleanupArtifacts(r.Assets.ToArray());
333-
try { Directory.Delete(versionDeploymentPath, true); } catch { }
334-
try { Directory.Delete(Path.Combine(_deploymentPath, r.TagName), true); } catch { }
335-
SkipVersion(r.TagName);
336-
goto SLEEP;
337-
}
338-
CleanupArtifacts(r.Assets.ToArray());
339350

340-
var primaryExe = Path.Combine(versionDeploymentPath, _primaryDeploymentExecutable);
351+
var primaryExe = Path.Combine(versionDeploymentPath, _primaryDeploymentExecutable);
341352

342-
if (!File.Exists(primaryExe))
343-
{
344-
SkipVersion(r.TagName);
345-
versionDeploymentPath.PollDeletionBlocking();
346-
EventLogLogger.Singleton.Error("Unable to deploy version '{0}': The file '{1}' was not found.", r.TagName, primaryExe);
347-
goto SLEEP;
348-
}
353+
if (!File.Exists(primaryExe))
354+
{
355+
SkipVersion(r.TagName);
356+
versionDeploymentPath.PollDeletionBlocking();
357+
EventLogLogger.Singleton.Error("Unable to deploy version '{0}': The file '{1}' was not found.", r.TagName, primaryExe);
358+
goto SLEEP;
359+
}
349360

350-
KillAllProcessByNameSafe(_primaryDeploymentExecutable);
351-
StartNewProcess(primaryExe, versionDeploymentPath);
361+
KillAllProcessByNameSafe(_primaryDeploymentExecutable);
362+
StartNewProcess(primaryExe, versionDeploymentPath);
363+
364+
_cachedVersion = r.TagName;
365+
WriteVersionToRegistry();
366+
}
352367

353-
_cachedVersion = r.TagName;
354-
WriteVersionToRegistry();
368+
SLEEP:
369+
Thread.Sleep(_pollingInterval);
355370
}
356-
357-
SLEEP:
358-
Thread.Sleep(_pollingInterval);
371+
}
372+
catch (Exception ex)
373+
{
374+
EventLogLogger.Singleton.Error(ex);
375+
Stop(null, null);
359376
}
360377
}
361378

@@ -561,8 +578,10 @@ private static void SkipVersion(string version)
561578
_skippedVersions.Add(version);
562579
}
563580

564-
private static bool DetermineIfNewReleaseAvailable(out MinimalRelease latestRelease, string nextPageCursor = null)
581+
private static bool DetermineIfNewReleaseAvailable(out MinimalRelease latestRelease)
565582
{
583+
string nextPageCursor = null;
584+
566585
while (true)
567586
{
568587
latestRelease = null;

‎Services.sln

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -55,10 +55,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Metrics", "Metrics", "{6A9E
5555
EndProject
5656
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MFDLabs.Instrumentation", "Assemblies\MFDLabs.Instrumentation\MFDLabs.Instrumentation.csproj", "{5D59D8C8-6B3C-45F9-BB2A-3F09F1F3326E}"
5757
EndProject
58-
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{711A73BD-9A06-412B-9CA8-C67E265ECFD3}"
59-
EndProject
60-
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "MFDLabs.Grid.AutoDeployer.Unit.Tests", "Unit Tests\MFDLabs.Grid.AutoDeployer.Unit.Tests\MFDLabs.Grid.AutoDeployer.Unit.Tests.csproj", "{CD35574A-0146-49E7-B4F7-624A61AC9D3C}"
61-
EndProject
6258
Global
6359
GlobalSection(SolutionConfigurationPlatforms) = preSolution
6460
Debug|Any CPU = Debug|Any CPU
@@ -327,22 +323,6 @@ Global
327323
{5D59D8C8-6B3C-45F9-BB2A-3F09F1F3326E}.ReleaseGrid|Any CPU.Build.0 = Release|Any CPU
328324
{5D59D8C8-6B3C-45F9-BB2A-3F09F1F3326E}.ReleaseGridDeploy|Any CPU.ActiveCfg = Release|Any CPU
329325
{5D59D8C8-6B3C-45F9-BB2A-3F09F1F3326E}.ReleaseGridDeploy|Any CPU.Build.0 = Release|Any CPU
330-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
331-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.Debug|Any CPU.Build.0 = Debug|Any CPU
332-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.DebugDeploy|Any CPU.ActiveCfg = Debug|Any CPU
333-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.DebugDeploy|Any CPU.Build.0 = Debug|Any CPU
334-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.DebugGrid|Any CPU.ActiveCfg = Debug|Any CPU
335-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.DebugGrid|Any CPU.Build.0 = Debug|Any CPU
336-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.DebugGridDeploy|Any CPU.ActiveCfg = Debug|Any CPU
337-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.DebugGridDeploy|Any CPU.Build.0 = Debug|Any CPU
338-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.Release|Any CPU.ActiveCfg = Release|Any CPU
339-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.Release|Any CPU.Build.0 = Release|Any CPU
340-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.ReleaseDeploy|Any CPU.ActiveCfg = Release|Any CPU
341-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.ReleaseDeploy|Any CPU.Build.0 = Release|Any CPU
342-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.ReleaseGrid|Any CPU.ActiveCfg = Release|Any CPU
343-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.ReleaseGrid|Any CPU.Build.0 = Release|Any CPU
344-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.ReleaseGridDeploy|Any CPU.ActiveCfg = Release|Any CPU
345-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C}.ReleaseGridDeploy|Any CPU.Build.0 = Release|Any CPU
346326
EndGlobalSection
347327
GlobalSection(SolutionProperties) = preSolution
348328
HideSolutionNode = FALSE
@@ -371,7 +351,6 @@ Global
371351
{6131788B-81D1-410C-9CA1-6BDE5822DB28} = {437C251F-C5CA-4D9D-BDCF-B9DD2FFB936F}
372352
{6A9E7A46-52AF-4F98-B8C8-90FC1974757D} = {72541B93-52D9-47CE-ACCD-9279015ABBC8}
373353
{5D59D8C8-6B3C-45F9-BB2A-3F09F1F3326E} = {6A9E7A46-52AF-4F98-B8C8-90FC1974757D}
374-
{CD35574A-0146-49E7-B4F7-624A61AC9D3C} = {711A73BD-9A06-412B-9CA8-C67E265ECFD3}
375354
EndGlobalSection
376355
GlobalSection(ExtensibilityGlobals) = postSolution
377356
SolutionGuid = {1738DAC0-DF8A-4C95-8B03-CF31A9A1966E}

0 commit comments

Comments
 (0)