Skip to content

Commit 1b7379e

Browse files
AlexCSDevAlexCSDev
authored
Merge pull request #157 from AlexCSDev/release_23
Release 23
2 parents 01f5e70 + 35e889a commit 1b7379e

32 files changed

+71
-1110
lines changed

‎PatreonDownloader.App/PatreonDownloader.App.csproj

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@
1414

1515
<ItemGroup>
1616
<ProjectReference Include="..\PatreonDownloader.Implementation\PatreonDownloader.Implementation.csproj" />
17-
<ProjectReference Include="..\PatreonDownloader.PuppeteerEngine\PatreonDownloader.PuppeteerEngine.csproj" />
1817
<ProjectReference Include="..\submodules\UniversalDownloaderPlatform\UniversalDownloaderPlatform.Engine\UniversalDownloaderPlatform.Engine.csproj" />
1918
</ItemGroup>
2019

‎PatreonDownloader.App/Program.cs

Lines changed: 2 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ class Program
1818
{
1919
private static readonly Logger _logger = LogManager.GetCurrentClassLogger();
2020
private static UniversalDownloader _universalDownloader;
21-
private static PuppeteerEngine.PuppeteerCookieRetriever _cookieRetriever;
2221
private static IConfiguration _configuration;
2322
private static int _filesDownloaded;
2423

@@ -103,20 +102,6 @@ private static void Cleanup()
103102
_logger.Fatal($"Error during patreon downloader disposal! Exception: {ex}");
104103
}
105104
}
106-
107-
if (_cookieRetriever != null)
108-
{
109-
_logger.Debug("Disposing cookie retriever...");
110-
try
111-
{
112-
_cookieRetriever.Dispose();
113-
_cookieRetriever = null;
114-
}
115-
catch (Exception ex)
116-
{
117-
_logger.Fatal($"Error during cookie retriever disposal! Exception: {ex}");
118-
}
119-
}
120105
}
121106

122107
private static async Task RunPatreonDownloader(CommandLineOptions commandLineOptions)
@@ -160,34 +145,11 @@ private static async Task<PatreonDownloaderSettings> InitializeSettings(CommandL
160145
throw new Exception($"Invalid proxy server address: {commandLineOptions.ProxyServerAddress}");
161146
}
162147

163-
_logger.Info("Retrieving cookies...");
164-
if (!string.IsNullOrWhiteSpace(commandLineOptions.RemoteBrowserAddress))
165-
_cookieRetriever =
166-
new PuppeteerEngine.PuppeteerCookieRetriever(new Uri(commandLineOptions.RemoteBrowserAddress));
167-
else
168-
_cookieRetriever = new PuppeteerEngine.PuppeteerCookieRetriever(true, commandLineOptions.ProxyServerAddress);
169-
CookieContainer cookieContainer = await _cookieRetriever.RetrieveCookies();
170-
if (cookieContainer == null)
171-
{
172-
throw new Exception("Unable to retrieve cookies");
173-
}
174-
175-
string userAgent = await _cookieRetriever.GetUserAgent();
176-
if (string.IsNullOrWhiteSpace(userAgent))
177-
{
178-
throw new Exception("Unable to retrieve user agent");
179-
}
180-
181-
_cookieRetriever.Dispose();
182-
_cookieRetriever = null;
183-
184-
await Task.Delay(1000); //wait for PuppeteerCookieRetriever to close the browser
185-
186148
PatreonDownloaderSettings settings = new PatreonDownloaderSettings
187149
{
188150
UrlBlackList = (_configuration["UrlBlackList"] ?? "").ToLowerInvariant().Split("|").ToList(),
189-
UserAgent = userAgent,
190-
CookieContainer = cookieContainer,
151+
UserAgent = null,
152+
CookieContainer = null,
191153
SaveAvatarAndCover = commandLineOptions.SaveAvatarAndCover,
192154
SaveDescriptions = commandLineOptions.SaveDescriptions,
193155
SaveEmbeds = commandLineOptions.SaveEmbeds,

‎PatreonDownloader.App/Properties/AssemblyInfo.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,5 @@
2929
// Build Number
3030
// Revision
3131
//
32-
[assembly: AssemblyVersion("22.0.0.0")]
33-
[assembly: AssemblyFileVersion("22.0.0.0")]
32+
[assembly: AssemblyVersion("23.0.0.0")]
33+
[assembly: AssemblyFileVersion("23.0.0.0")]

‎PatreonDownloader.Common/Interfaces/ICaptchaSolver.cs

Lines changed: 0 additions & 14 deletions
This file was deleted.

‎PatreonDownloader.Common/Interfaces/ICookieRetriever.cs

Lines changed: 0 additions & 15 deletions
This file was deleted.

‎PatreonDownloader.Common/PatreonDownloader.Common.csproj

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎PatreonDownloader.Common/Properties/AssemblyInfo.cs

Lines changed: 0 additions & 33 deletions
This file was deleted.

‎PatreonDownloader.Implementation/Helpers/PostSubdirectoryHelper.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ public static string CreateNameFromPattern(PatreonCrawledUrl crawledUrl, string
3131
.Replace("%postid%", crawledUrl.PostId);
3232

3333
if (retString.Length > lengthLimit)
34-
retString = retString.Substring(0, lengthLimit);
34+
retString = $"{retString.Substring(0, lengthLimit - 1).Trim()}~";
3535

3636
return PathSanitizer.SanitizePath(retString);
3737
}

‎PatreonDownloader.Implementation/Models/PatreonDownloaderSettings.cs

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,11 @@
55
using UniversalDownloaderPlatform.Common.Helpers;
66
using UniversalDownloaderPlatform.Common.Interfaces.Models;
77
using UniversalDownloaderPlatform.DefaultImplementations.Models;
8+
using UniversalDownloaderPlatform.PuppeteerEngine.Interfaces;
89

910
namespace PatreonDownloader.Implementation.Models
1011
{
11-
public record PatreonDownloaderSettings : UniversalDownloaderPlatformSettings
12+
public record PatreonDownloaderSettings : UniversalDownloaderPlatformSettings, IPuppeteerSettings
1213
{
1314
public bool SaveDescriptions { get; init; }
1415

@@ -48,6 +49,12 @@ public record PatreonDownloaderSettings : UniversalDownloaderPlatformSettings
4849
/// </summary>
4950
public bool IsUseLegacyFilenaming { get; init; }
5051

52+
public string LoginPageAddress { get { return "https://www.patreon.com/login"; } }
53+
public string LoginCheckAddress { get { return "https://www.patreon.com/api/current_user"; } }
54+
public string CaptchaCookieRetrievalAddress { get { return "https://www.patreon.com/home"; } }
55+
public Uri RemoteBrowserAddress { get; init; }
56+
public bool IsHeadlessBrowser { get; init; }
57+
5158
public PatreonDownloaderSettings()
5259
{
5360
SaveDescriptions = true;
@@ -60,6 +67,7 @@ public PatreonDownloaderSettings()
6067
MaxFilenameLength = 100;
6168
MaxSubdirectoryNameLength = 100;
6269
IsUseLegacyFilenaming = false;
70+
IsHeadlessBrowser = true;
6371
}
6472
}
6573
}

‎PatreonDownloader.Implementation/PatreonCrawledUrlProcessor.cs

Lines changed: 8 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -26,20 +26,15 @@ class PatreonCrawledUrlProcessor : ICrawledUrlProcessor
2626

2727
private ConcurrentDictionary<string, int> _fileCountDict; //file counter for duplicate check
2828
private PatreonDownloaderSettings _patreonDownloaderSettings;
29-
private static readonly Regex _googleDriveRegex;
30-
private static readonly Regex _fileIdRegex; //Regex used to retrieve file id from its url
29+
30+
private static readonly Regex _fileIdRegex = new Regex(
31+
"https:\\/\\/(.+)\\.patreonusercontent\\.com\\/(.+)\\/patreon-media\\/p\\/post\\/([0-9]+)\\/([a-z0-9]+)",
32+
RegexOptions.Compiled | RegexOptions.IgnoreCase); //Regex used to retrieve file id from its url
3133

3234
static PatreonCrawledUrlProcessor()
3335
{
3436
_invalidFilenameCharacters = new HashSet<char>(Path.GetInvalidFileNameChars());
3537
_invalidFilenameCharacters.Add(':');
36-
37-
_fileIdRegex =
38-
new Regex(
39-
"https:\\/\\/(.+)\\.patreonusercontent\\.com\\/(.+)\\/patreon-media\\/p\\/post\\/([0-9]+)\\/([a-z0-9]+)",
40-
RegexOptions.Compiled | RegexOptions.IgnoreCase);
41-
42-
_googleDriveRegex = new Regex("https:\\/\\/drive\\.google\\.com\\/(?:file\\/d\\/|open\\?id\\=|drive\\/folders\\/|folderview\\?id=|drive\\/u\\/[0-9]+\\/folders\\/)([A-Za-z0-9_-]+)");
4338
}
4439

4540
public PatreonCrawledUrlProcessor(IRemoteFilenameRetriever remoteFilenameRetriever)
@@ -61,30 +56,7 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl)
6156
{
6257
PatreonCrawledUrl crawledUrl = (PatreonCrawledUrl)udpCrawledUrl;
6358

64-
bool skipChecks = false; //skip sanitization, duplicate and other checks, do not pass filename to download path
65-
if (crawledUrl.Url.IndexOf("dropbox.com/", StringComparison.Ordinal) != -1)
66-
{
67-
if (!crawledUrl.Url.EndsWith("?dl=1"))
68-
{
69-
if (crawledUrl.Url.EndsWith("?dl=0"))
70-
crawledUrl.Url = crawledUrl.Url.Replace("?dl=0", "?dl=1");
71-
else
72-
crawledUrl.Url = $"{crawledUrl.Url}?dl=1";
73-
}
74-
75-
_logger.Debug($"[{crawledUrl.PostId}] This is a dropbox entry: {crawledUrl.Url}");
76-
}
77-
else if (crawledUrl.Url.StartsWith("https://mega.nz/"))
78-
{
79-
_logger.Debug($"[{crawledUrl.PostId}] mega found: {crawledUrl.Url}");
80-
skipChecks = true; //mega plugin expects to see only path to the folder where everything will be saved
81-
}
82-
else if (_googleDriveRegex.Match(crawledUrl.Url).Success)
83-
{
84-
_logger.Debug($"[{crawledUrl.PostId}] google drive found: {crawledUrl.Url}");
85-
skipChecks = true; //no need for checks if we use google drive plugin
86-
}
87-
else if (crawledUrl.Url.IndexOf("youtube.com/watch?v=", StringComparison.Ordinal) != -1 ||
59+
if (crawledUrl.Url.IndexOf("youtube.com/watch?v=", StringComparison.Ordinal) != -1 ||
8860
crawledUrl.Url.IndexOf("youtu.be/", StringComparison.Ordinal) != -1)
8961
{
9062
//TODO: YOUTUBE SUPPORT?
@@ -100,7 +72,7 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl)
10072

10173
string filename = crawledUrl.Filename;
10274

103-
if (!skipChecks)
75+
if (!crawledUrl.IsProcessedByPlugin)
10476
{
10577
if (!_patreonDownloaderSettings.IsUseSubDirectories)
10678
filename = $"{crawledUrl.PostId}_";
@@ -199,14 +171,14 @@ public async Task<bool> ProcessCrawledUrl(ICrawledUrl udpCrawledUrl)
199171
}
200172
}
201173

202-
string downloadDirectory = _patreonDownloaderSettings.DownloadDirectory;
174+
string downloadDirectory = "";
203175

204176
if (_patreonDownloaderSettings.IsUseSubDirectories &&
205177
crawledUrl.UrlType != PatreonCrawledUrlType.AvatarFile &&
206178
crawledUrl.UrlType != PatreonCrawledUrlType.CoverFile)
207179
downloadDirectory = Path.Combine(downloadDirectory, PostSubdirectoryHelper.CreateNameFromPattern(crawledUrl, _patreonDownloaderSettings.SubDirectoryPattern, _patreonDownloaderSettings.MaxSubdirectoryNameLength));
208180

209-
crawledUrl.DownloadPath = !skipChecks ? Path.Combine(downloadDirectory, filename) : downloadDirectory + Path.DirectorySeparatorChar;
181+
crawledUrl.DownloadPath = !crawledUrl.IsProcessedByPlugin ? Path.Combine(downloadDirectory, filename) : downloadDirectory + Path.DirectorySeparatorChar;
210182

211183
return true;
212184
}

‎PatreonDownloader.Implementation/PatreonDefaultPlugin.cs

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.IO;
4+
using System.Runtime;
45
using System.Text.RegularExpressions;
56
using System.Threading.Tasks;
67
using HtmlAgilityPack;
@@ -31,11 +32,18 @@ internal sealed class PatreonDefaultPlugin : IPlugin
3132
public string Author => "Aleksey Tsutsey";
3233
public string ContactInformation => "https://github.com/Megalan/PatreonDownloader";
3334

35+
private IUniversalDownloaderPlatformSettings _settings;
36+
3437
public PatreonDefaultPlugin(IWebDownloader webDownloader)
3538
{
3639
_webDownloader = webDownloader ?? throw new ArgumentNullException(nameof(webDownloader));
3740
}
3841

42+
public void OnLoad(IDependencyResolver dependencyResolver)
43+
{
44+
//do nothing
45+
}
46+
3947
public async Task<bool> IsSupportedUrl(string url)
4048
{
4149
if (string.IsNullOrEmpty(url))
@@ -49,12 +57,12 @@ public async Task Download(ICrawledUrl crawledUrl)
4957
if(crawledUrl == null)
5058
throw new ArgumentNullException(nameof(crawledUrl));
5159

52-
await _webDownloader.DownloadFile(crawledUrl.Url, crawledUrl.DownloadPath, null); //referer is set in PatreonWebDownloader
60+
await _webDownloader.DownloadFile(crawledUrl.Url, Path.Combine(_settings.DownloadDirectory, crawledUrl.DownloadPath), null); //referer is set in PatreonWebDownloader
5361
}
5462

5563
public Task BeforeStart(IUniversalDownloaderPlatformSettings settings)
5664
{
57-
//Do nothing
65+
_settings = settings;
5866
return Task.CompletedTask;
5967
}
6068

@@ -118,5 +126,23 @@ private bool IsAllowedUrl(string url)
118126

119127
return true;
120128
}
129+
130+
public Task<bool> ProcessCrawledUrl(ICrawledUrl crawledUrl)
131+
{
132+
if (crawledUrl.Url.IndexOf("dropbox.com/", StringComparison.Ordinal) != -1)
133+
{
134+
if (!crawledUrl.Url.EndsWith("?dl=1"))
135+
{
136+
if (crawledUrl.Url.EndsWith("?dl=0"))
137+
crawledUrl.Url = crawledUrl.Url.Replace("?dl=0", "?dl=1");
138+
else
139+
crawledUrl.Url = $"{crawledUrl.Url}?dl=1";
140+
}
141+
142+
_logger.Trace($"Dropbox url found: {crawledUrl.Url}");
143+
}
144+
145+
return Task.FromResult(false); //we still want full processing for all crawled urls passed here
146+
}
121147
}
122148
}

‎PatreonDownloader.Implementation/PatreonDownloader.Implementation.csproj

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,7 @@
1414
</ItemGroup>
1515

1616
<ItemGroup>
17-
<ProjectReference Include="..\PatreonDownloader.Common\PatreonDownloader.Common.csproj" />
18-
<ProjectReference Include="..\PatreonDownloader.PuppeteerEngine\PatreonDownloader.PuppeteerEngine.csproj" />
17+
<ProjectReference Include="..\..\XMADownloader\submodules\UniversalDownloaderPlatform\UniversalDownloaderPlatform.PuppeteerEngine\UniversalDownloaderPlatform.PuppeteerEngine.csproj" />
1918
<ProjectReference Include="..\submodules\UniversalDownloaderPlatform\UniversalDownloaderPlatform.DefaultImplementations\UniversalDownloaderPlatform.DefaultImplementations.csproj" />
2019
</ItemGroup>
2120

‎PatreonDownloader.Implementation/PatreonDownloaderModule.cs

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,23 @@
1-
using System;
2-
using System.Collections.Generic;
3-
using System.Text;
4-
using Ninject;
1+
using Ninject;
52
using Ninject.Modules;
63
using PatreonDownloader.Engine;
74
using PatreonDownloader.Implementation.Interfaces;
85
using PatreonDownloader.Implementation.Models;
9-
using PatreonDownloader.PuppeteerEngine;
106
using UniversalDownloaderPlatform.Common.Interfaces;
117
using UniversalDownloaderPlatform.Common.Interfaces.Models;
128
using UniversalDownloaderPlatform.Common.Interfaces.Plugins;
139
using UniversalDownloaderPlatform.DefaultImplementations;
1410
using UniversalDownloaderPlatform.DefaultImplementations.Interfaces;
15-
using UniversalDownloaderPlatform.DefaultImplementations.Models;
11+
using UniversalDownloaderPlatform.PuppeteerEngine;
1612

1713
namespace PatreonDownloader.Implementation
1814
{
1915
public class PatreonDownloaderModule : NinjectModule
2016
{
2117
public override void Load()
2218
{
19+
Kernel.Load(new PuppeteerEngineModule());
20+
2321
Bind<IRemoteFileSizeChecker>().To<RemoteFileSizeChecker>().InSingletonScope();
2422
Bind<IWebDownloader>().To<PatreonWebDownloader>().InSingletonScope();
2523
Bind<IRemoteFilenameRetriever>().To<PatreonRemoteFilenameRetriever>().InSingletonScope();
@@ -28,7 +26,7 @@ public override void Load()
2826
Bind<IPageCrawler>().To<PatreonPageCrawler>().InSingletonScope();
2927
Bind<IPlugin>().To<PatreonDefaultPlugin>().WhenInjectedInto<IPluginManager>();
3028
Bind<IUniversalDownloaderPlatformSettings>().To<PatreonDownloaderSettings>();
31-
Bind<ICookieValidator>().To<PatreonCookieValidator>().InSingletonScope();
29+
Rebind<ICookieValidator>().To<PatreonCookieValidator>().InSingletonScope();
3230
}
3331
}
3432
}

0 commit comments

Comments
 (0)