From 1da80956dd83217ba83131557c58b0b3f2bafd8f Mon Sep 17 00:00:00 2001 From: Tyrrrz Date: Sat, 24 Oct 2020 21:15:58 +0300 Subject: [PATCH] Refactor --- .../Commands/Base/ExportCommandBase.cs | 24 +++-- .../Base/ExportMultipleCommandBase.cs | 13 ++- .../Commands/Base/TokenCommandBase.cs | 15 ++- .../Commands/ExportAllCommand.cs | 5 +- .../Commands/ExportChannelCommand.cs | 6 +- .../Commands/ExportGuildCommand.cs | 3 +- .../Commands/GetChannelsCommand.cs | 3 +- .../Commands/GuideCommand.cs | 24 +++-- .../Discord/DiscordClient.cs | 43 +++------ .../Exporting/ExportContext.cs | 5 +- .../Exporting/MediaDownloader.cs | 91 +++++++++---------- .../Exporting/MessageExporter.cs | 17 ++-- .../Internal/Extensions/BinaryExtensions.cs | 19 ++++ DiscordChatExporter.Domain/Internal/Http.cs | 61 +++++++++++++ DiscordChatExporter.Domain/Internal/PathEx.cs | 22 ----- .../Internal/Singleton.cs | 23 ----- .../Utilities/AsyncExtensions.cs | 11 ++- DiscordChatExporter.Gui/App.xaml.cs | 12 ++- .../MultiSelectionListBoxBehavior.cs | 4 +- DiscordChatExporter.Gui/Bootstrapper.cs | 3 +- .../ExportFormatToStringConverter.cs | 6 +- DiscordChatExporter.Gui/Internal/ProcessEx.cs | 3 +- .../Services/SettingsService.cs | 2 +- .../Services/UpdateService.cs | 3 +- .../Dialogs/ExportSetupViewModel.cs | 25 ++++- .../ViewModels/Framework/DialogManager.cs | 11 +-- .../ViewModels/Framework/Extensions.cs | 19 ---- .../ViewModels/RootViewModel.cs | 23 +++-- .../Views/Dialogs/ExportSetupView.xaml | 18 ++-- .../Views/Dialogs/ExportSetupView.xaml.cs | 11 +-- .../Views/Dialogs/SettingsView.xaml | 8 +- .../Views/Dialogs/SettingsView.xaml.cs | 8 +- DiscordChatExporter.Gui/Views/RootView.xaml | 6 +- .../Views/RootView.xaml.cs | 11 +-- 34 files changed, 298 insertions(+), 260 deletions(-) create mode 100644 DiscordChatExporter.Domain/Internal/Extensions/BinaryExtensions.cs create mode 100644 DiscordChatExporter.Domain/Internal/Http.cs delete mode 100644 DiscordChatExporter.Domain/Internal/Singleton.cs delete mode 100644 DiscordChatExporter.Gui/ViewModels/Framework/Extensions.cs diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs index 4aa49b8..ebc1b56 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs @@ -11,28 +11,36 @@ namespace DiscordChatExporter.Cli.Commands.Base { public abstract class ExportCommandBase : TokenCommandBase { - [CommandOption("output", 'o', Description = "Output file or directory path.")] + [CommandOption("output", 'o', + Description = "Output file or directory path.")] public string OutputPath { get; set; } = Directory.GetCurrentDirectory(); - [CommandOption("format", 'f', Description = "Output file format.")] + [CommandOption("format", 'f', + Description = "Export format.")] public ExportFormat ExportFormat { get; set; } = ExportFormat.HtmlDark; - [CommandOption("after", Description = "Limit to messages sent after this date.")] + [CommandOption("after", + Description = "Only include messages sent after this date.")] public DateTimeOffset? After { get; set; } - [CommandOption("before", Description = "Limit to messages sent before this date.")] + [CommandOption("before", + Description = "Only include messages sent before this date.")] public DateTimeOffset? Before { get; set; } - [CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")] + [CommandOption("partition", 'p', + Description = "Split output into partitions limited to this number of messages.")] public int? PartitionLimit { get; set; } - [CommandOption("media", Description = "Download referenced media content.")] + [CommandOption("media", + Description = "Download referenced media content.")] public bool ShouldDownloadMedia { get; set; } - [CommandOption("reuse-media", Description = "If the media folder already exists, reuse media inside it to skip downloads.")] + [CommandOption("reuse-media", + Description = "Reuse already existing media content to skip redundant downloads.")] public bool ShouldReuseMedia { get; set; } - [CommandOption("dateformat", Description = "Date format used in output.")] + [CommandOption("dateformat", + Description = "Format used when writing dates.")] public string DateFormat { get; set; } = "dd-MMM-yy hh:mm tt"; protected ChannelExporter GetChannelExporter() => new ChannelExporter(GetDiscordClient()); diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs index 716ca22..7b28168 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportMultipleCommandBase.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -16,21 +17,23 @@ namespace DiscordChatExporter.Cli.Commands.Base { public abstract class ExportMultipleCommandBase : ExportCommandBase { - [CommandOption("parallel", Description = "Export this number of channels in parallel.")] + [CommandOption("parallel", + Description = "Limits how many channels can be exported in parallel.")] public int ParallelLimit { get; set; } = 1; protected async ValueTask ExportMultipleAsync(IConsole console, IReadOnlyList channels) { - // HACK: this uses a separate route from ExportCommandBase because the progress ticker is not thread-safe + // This uses a different route from ExportCommandBase.ExportAsync() because it runs + // in parallel and needs another way to report progress to console. console.Output.Write($"Exporting {channels.Count} channels... "); var progress = console.CreateProgressTicker(); var operations = progress.Wrap().CreateOperations(channels.Count); - var errors = new List(); - var successfulExportCount = 0; + var errors = new ConcurrentBag(); + await channels.Zip(operations).ParallelForEachAsync(async tuple => { var (channel, operation) = tuple; diff --git a/DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs index 715996f..e74c964 100644 --- a/DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs @@ -7,15 +7,22 @@ namespace DiscordChatExporter.Cli.Commands.Base { public abstract class TokenCommandBase : ICommand { - [CommandOption("token", 't', IsRequired = true, EnvironmentVariableName = "DISCORD_TOKEN", + [CommandOption("token", 't', IsRequired = true, + EnvironmentVariableName = "DISCORD_TOKEN", Description = "Authorization token.")] public string TokenValue { get; set; } = ""; - [CommandOption("bot", 'b', EnvironmentVariableName = "DISCORD_TOKEN_BOT", - Description = "Whether this authorization token belongs to a bot.")] + [CommandOption("bot", 'b', + EnvironmentVariableName = "DISCORD_TOKEN_BOT", + Description = "Authorize as a bot.")] public bool IsBotToken { get; set; } - protected AuthToken GetAuthToken() => new AuthToken(IsBotToken ? AuthTokenType.Bot : AuthTokenType.User, TokenValue); + protected AuthToken GetAuthToken() => new AuthToken( + IsBotToken + ? AuthTokenType.Bot + : AuthTokenType.User, + TokenValue + ); protected DiscordClient GetDiscordClient() => new DiscordClient(GetAuthToken()); diff --git a/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs b/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs index 153565b..a37880c 100644 --- a/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs @@ -10,7 +10,8 @@ namespace DiscordChatExporter.Cli.Commands [Command("exportall", Description = "Export all accessible channels.")] public class ExportAllCommand : ExportMultipleCommandBase { - [CommandOption("include-dm", Description = "Whether to also export direct message channels.")] + [CommandOption("include-dm", + Description = "Include direct message channels.")] public bool IncludeDirectMessages { get; set; } = true; public override async ValueTask ExecuteAsync(IConsole console) @@ -33,4 +34,4 @@ namespace DiscordChatExporter.Cli.Commands await ExportMultipleAsync(console, channels); } } -} +} \ No newline at end of file diff --git a/DiscordChatExporter.Cli/Commands/ExportChannelCommand.cs b/DiscordChatExporter.Cli/Commands/ExportChannelCommand.cs index 4811ddb..446d12a 100644 --- a/DiscordChatExporter.Cli/Commands/ExportChannelCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportChannelCommand.cs @@ -8,9 +8,11 @@ namespace DiscordChatExporter.Cli.Commands [Command("export", Description = "Export a channel.")] public class ExportChannelCommand : ExportCommandBase { - [CommandOption("channel", 'c', IsRequired = true, Description = "Channel ID.")] + [CommandOption("channel", 'c', IsRequired = true, + Description = "Channel ID.")] public string ChannelId { get; set; } = ""; - public override async ValueTask ExecuteAsync(IConsole console) => await ExportAsync(console, ChannelId); + public override async ValueTask ExecuteAsync(IConsole console) => + await ExportAsync(console, ChannelId); } } \ No newline at end of file diff --git a/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs b/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs index 9d2f228..8564b37 100644 --- a/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs @@ -9,7 +9,8 @@ namespace DiscordChatExporter.Cli.Commands [Command("exportguild", Description = "Export all channels within specified guild.")] public class ExportGuildCommand : ExportMultipleCommandBase { - [CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")] + [CommandOption("guild", 'g', IsRequired = true, + Description = "Guild ID.")] public string GuildId { get; set; } = ""; public override async ValueTask ExecuteAsync(IConsole console) diff --git a/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs b/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs index b73326f..5b1a0e5 100644 --- a/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs +++ b/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs @@ -10,7 +10,8 @@ namespace DiscordChatExporter.Cli.Commands [Command("channels", Description = "Get the list of channels in a guild.")] public class GetChannelsCommand : TokenCommandBase { - [CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")] + [CommandOption("guild", 'g', IsRequired = true, + Description = "Guild ID.")] public string GuildId { get; set; } = ""; public override async ValueTask ExecuteAsync(IConsole console) diff --git a/DiscordChatExporter.Cli/Commands/GuideCommand.cs b/DiscordChatExporter.Cli/Commands/GuideCommand.cs index adedccb..2996b69 100644 --- a/DiscordChatExporter.Cli/Commands/GuideCommand.cs +++ b/DiscordChatExporter.Cli/Commands/GuideCommand.cs @@ -10,7 +10,9 @@ namespace DiscordChatExporter.Cli.Commands { public ValueTask ExecuteAsync(IConsole console) { - console.WithForegroundColor(ConsoleColor.White, () => console.Output.WriteLine("To get user token:")); + console.WithForegroundColor(ConsoleColor.White, () => + console.Output.WriteLine("To get user token:") + ); console.Output.WriteLine(" 1. Open Discord"); console.Output.WriteLine(" 2. Press Ctrl+Shift+I to show developer tools"); console.Output.WriteLine(" 3. Navigate to the Application tab"); @@ -20,14 +22,18 @@ namespace DiscordChatExporter.Cli.Commands console.Output.WriteLine(" * Automating user accounts is technically against TOS, use at your own risk."); console.Output.WriteLine(); - console.WithForegroundColor(ConsoleColor.White, () => console.Output.WriteLine("To get bot token:")); + console.WithForegroundColor(ConsoleColor.White, () => + console.Output.WriteLine("To get bot token:") + ); console.Output.WriteLine(" 1. Go to Discord developer portal"); console.Output.WriteLine(" 2. Open your application's settings"); console.Output.WriteLine(" 3. Navigate to the Bot section on the left"); console.Output.WriteLine(" 4. Under Token click Copy"); console.Output.WriteLine(); - console.WithForegroundColor(ConsoleColor.White, () => console.Output.WriteLine("To get guild ID or guild channel ID:")); + console.WithForegroundColor(ConsoleColor.White, () => + console.Output.WriteLine("To get guild ID or guild channel ID:") + ); console.Output.WriteLine(" 1. Open Discord"); console.Output.WriteLine(" 2. Open Settings"); console.Output.WriteLine(" 3. Go to Appearance section"); @@ -35,7 +41,9 @@ namespace DiscordChatExporter.Cli.Commands console.Output.WriteLine(" 5. Right click on the desired guild or channel and click Copy ID"); console.Output.WriteLine(); - console.WithForegroundColor(ConsoleColor.White, () => console.Output.WriteLine("To get direct message channel ID:")); + console.WithForegroundColor(ConsoleColor.White, () => + console.Output.WriteLine("To get direct message channel ID:") + ); console.Output.WriteLine(" 1. Open Discord"); console.Output.WriteLine(" 2. Open the desired direct message channel"); console.Output.WriteLine(" 3. Press Ctrl+Shift+I to show developer tools"); @@ -44,8 +52,12 @@ namespace DiscordChatExporter.Cli.Commands console.Output.WriteLine(" 6. Copy the first long sequence of numbers inside the URL"); console.Output.WriteLine(); - console.Output.WriteLine("For more information, check out the wiki:"); - console.Output.WriteLine("https://github.com/Tyrrrz/DiscordChatExporter/wiki"); + console.WithForegroundColor(ConsoleColor.White, + () => console.Output.WriteLine("For more information, check out the wiki:") + ); + console.WithForegroundColor(ConsoleColor.Blue, + () => console.Output.WriteLine("https://github.com/Tyrrrz/DiscordChatExporter/wiki") + ); return default; } diff --git a/DiscordChatExporter.Domain/Discord/DiscordClient.cs b/DiscordChatExporter.Domain/Discord/DiscordClient.cs index 9014cb8..5484a6e 100644 --- a/DiscordChatExporter.Domain/Discord/DiscordClient.cs +++ b/DiscordChatExporter.Domain/Discord/DiscordClient.cs @@ -9,50 +9,33 @@ using DiscordChatExporter.Domain.Discord.Models; using DiscordChatExporter.Domain.Exceptions; using DiscordChatExporter.Domain.Internal; using DiscordChatExporter.Domain.Internal.Extensions; -using Polly; namespace DiscordChatExporter.Domain.Discord { public class DiscordClient { + private readonly HttpClient _httpClient; private readonly AuthToken _token; - private readonly HttpClient _httpClient = Singleton.HttpClient; - private readonly IAsyncPolicy _httpRequestPolicy; private readonly Uri _baseUri = new Uri("https://discord.com/api/v6/", UriKind.Absolute); - public DiscordClient(AuthToken token) + public DiscordClient(HttpClient httpClient, AuthToken token) { + _httpClient = httpClient; _token = token; - - // Discord seems to always respond with 429 on the first request with unreasonable wait time (10+ minutes). - // For that reason the policy will ignore such errors at first, then wait a constant amount of time, and - // finally wait the specified amount of time, based on how many requests have failed in a row. - _httpRequestPolicy = Policy - .HandleResult(m => m.StatusCode == HttpStatusCode.TooManyRequests) - .OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError) - .WaitAndRetryAsync(6, - (i, result, ctx) => - { - if (i <= 3) - return TimeSpan.FromSeconds(2 * i); - - if (i <= 5) - return TimeSpan.FromSeconds(5 * i); - - return result.Result.Headers.RetryAfter.Delta ?? TimeSpan.FromSeconds(10 * i); - }, - (response, timespan, retryCount, context) => Task.CompletedTask - ); } - private async ValueTask GetResponseAsync(string url) => await _httpRequestPolicy.ExecuteAsync(async () => - { - using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url)); - request.Headers.Authorization = _token.GetAuthorizationHeader(); + public DiscordClient(AuthToken token) + : this(Http.Client, token) {} + + private async ValueTask GetResponseAsync(string url) => + await Http.ResponsePolicy.ExecuteAsync(async () => + { + using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url)); + request.Headers.Authorization = _token.GetAuthorizationHeader(); - return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); - }); + return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); + }); private async ValueTask GetJsonResponseAsync(string url) { diff --git a/DiscordChatExporter.Domain/Exporting/ExportContext.cs b/DiscordChatExporter.Domain/Exporting/ExportContext.cs index c8e58a4..e7dea5e 100644 --- a/DiscordChatExporter.Domain/Exporting/ExportContext.cs +++ b/DiscordChatExporter.Domain/Exporting/ExportContext.cs @@ -41,7 +41,7 @@ namespace DiscordChatExporter.Domain.Exporting { "unix" => date.ToUnixTimeSeconds().ToString(), "unixms" => date.ToUnixTimeMilliseconds().ToString(), - var df => date.ToLocalString(df), + var dateFormat => date.ToLocalString(dateFormat) }; public Member? TryGetMember(string id) => @@ -77,7 +77,7 @@ namespace DiscordChatExporter.Domain.Exporting // We want relative path so that the output files can be copied around without breaking var relativeFilePath = Path.GetRelativePath(Request.OutputBaseDirPath, filePath); - // For HTML, we need to format the URL properly + // HACK: for HTML, we need to format the URL properly if (Request.Format == ExportFormat.HtmlDark || Request.Format == ExportFormat.HtmlLight) { // Need to escape each path segment while keeping the directory separators intact @@ -94,6 +94,7 @@ namespace DiscordChatExporter.Domain.Exporting // https://github.com/Tyrrrz/DiscordChatExporter/issues/372 catch (Exception ex) when (ex is HttpRequestException || ex is OperationCanceledException) { + // TODO: add logging so we can be more liberal with catching exceptions // We don't want this to crash the exporting process in case of failure return url; } diff --git a/DiscordChatExporter.Domain/Exporting/MediaDownloader.cs b/DiscordChatExporter.Domain/Exporting/MediaDownloader.cs index 3dff958..ad53d1c 100644 --- a/DiscordChatExporter.Domain/Exporting/MediaDownloader.cs +++ b/DiscordChatExporter.Domain/Exporting/MediaDownloader.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Net; using System.Net.Http; using System.Security.Cryptography; using System.Text; @@ -9,87 +8,79 @@ using System.Text.RegularExpressions; using System.Threading.Tasks; using DiscordChatExporter.Domain.Internal; using DiscordChatExporter.Domain.Internal.Extensions; -using Polly; -using Polly.Retry; namespace DiscordChatExporter.Domain.Exporting { internal partial class MediaDownloader { - private readonly HttpClient _httpClient = Singleton.HttpClient; + private readonly HttpClient _httpClient; private readonly string _workingDirPath; - private readonly bool _reuseMedia; - private readonly AsyncRetryPolicy _httpRequestPolicy; - private readonly Dictionary _pathMap = new Dictionary(); + // URL -> Local file path + private readonly Dictionary _pathCache = + new Dictionary(StringComparer.Ordinal); - public MediaDownloader(string workingDirPath, bool reuseMedia) + public MediaDownloader(HttpClient httpClient, string workingDirPath, bool reuseMedia) { + _httpClient = httpClient; _workingDirPath = workingDirPath; _reuseMedia = reuseMedia; - - _httpRequestPolicy = Policy - .Handle() - .WaitAndRetryAsync(8, i => TimeSpan.FromSeconds(0.5 * i)); } + public MediaDownloader(string workingDirPath, bool reuseMedia) + : this(Http.Client, workingDirPath, reuseMedia) {} + public async ValueTask DownloadAsync(string url) { - return await _httpRequestPolicy.ExecuteAsync(async () => - { - if (_pathMap.TryGetValue(url, out var cachedFilePath)) - return cachedFilePath; - - var fileName = GetFileNameFromUrl(url); - var filePath = Path.Combine(_workingDirPath, fileName); + if (_pathCache.TryGetValue(url, out var cachedFilePath)) + return cachedFilePath; - if (!_reuseMedia) - { - filePath = PathEx.MakeUniqueFilePath(filePath); - } + var fileName = GetFileNameFromUrl(url); + var filePath = Path.Combine(_workingDirPath, fileName); - if (!_reuseMedia || !File.Exists(filePath)) - { - Directory.CreateDirectory(_workingDirPath); - await _httpClient.DownloadAsync(url, filePath); - } + // Reuse existing files if we're allowed to + if (_reuseMedia && File.Exists(filePath)) + return _pathCache[url] = filePath; - return _pathMap[url] = filePath; + // Download it + Directory.CreateDirectory(_workingDirPath); + await Http.ExceptionPolicy.ExecuteAsync(async () => + { + // This catches IOExceptions which is dangerous as we're working also with files + await _httpClient.DownloadAsync(url, filePath); }); + + return _pathCache[url] = filePath; } } internal partial class MediaDownloader { - private static int URL_HASH_LENGTH = 5; - private static string HashUrl(string url) + private static string GetUrlHash(string url) { - using (var md5 = MD5.Create()) - { - var inputBytes = Encoding.UTF8.GetBytes(url); - var hashBytes = md5.ComputeHash(inputBytes); - - var hashBuilder = new StringBuilder(); - for (int i = 0; i < hashBytes.Length; i++) - { - hashBuilder.Append(hashBytes[i].ToString("X2")); - } - return hashBuilder.ToString().Truncate(URL_HASH_LENGTH); - } - } + using var hash = SHA256.Create(); - private static string GetRandomFileName() => Guid.NewGuid().ToString().Replace("-", "").Substring(0, 16); + var data = hash.ComputeHash(Encoding.UTF8.GetBytes(url)); + return data.ToHex().Truncate(5); // 5 chars ought to be enough for anybody + } private static string GetFileNameFromUrl(string url) { - var originalFileName = Regex.Match(url, @".+/([^?]*)").Groups[1].Value; + var urlHash = GetUrlHash(url); + + // Try to extract file name from URL + var fileName = Regex.Match(url, @".+/([^?]*)").Groups[1].Value; + + // If it's not there, just use the URL hash as the file name + if (string.IsNullOrWhiteSpace(fileName)) + return urlHash; - var fileName = !string.IsNullOrWhiteSpace(originalFileName) - ? $"{Path.GetFileNameWithoutExtension(originalFileName).Truncate(42)}-({HashUrl(url)}){Path.GetExtension(originalFileName)}" - : GetRandomFileName(); + // Otherwise, use the original file name but inject the hash in the middle + var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName); + var fileExtension = Path.GetExtension(fileName); - return PathEx.EscapePath(fileName); + return PathEx.EscapePath(fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension); } } } diff --git a/DiscordChatExporter.Domain/Exporting/MessageExporter.cs b/DiscordChatExporter.Domain/Exporting/MessageExporter.cs index ca3e53f..e632ec2 100644 --- a/DiscordChatExporter.Domain/Exporting/MessageExporter.cs +++ b/DiscordChatExporter.Domain/Exporting/MessageExporter.cs @@ -72,7 +72,9 @@ namespace DiscordChatExporter.Domain.Exporting internal partial class MessageExporter { - private static string GetPartitionFilePath(string baseFilePath, int partitionIndex) + private static string GetPartitionFilePath( + string baseFilePath, + int partitionIndex) { // First partition - don't change file name if (partitionIndex <= 0) @@ -82,16 +84,17 @@ namespace DiscordChatExporter.Domain.Exporting var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath); var fileExt = Path.GetExtension(baseFilePath); var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}"; - - // Generate new path var dirPath = Path.GetDirectoryName(baseFilePath); - if (!string.IsNullOrWhiteSpace(dirPath)) - return Path.Combine(dirPath, fileName); - return fileName; + return !string.IsNullOrWhiteSpace(dirPath) + ? Path.Combine(dirPath, fileName) + : fileName; } - private static MessageWriter CreateMessageWriter(string filePath, ExportFormat format, ExportContext context) + private static MessageWriter CreateMessageWriter( + string filePath, + ExportFormat format, + ExportContext context) { // Stream will be disposed by the underlying writer var stream = File.Create(filePath); diff --git a/DiscordChatExporter.Domain/Internal/Extensions/BinaryExtensions.cs b/DiscordChatExporter.Domain/Internal/Extensions/BinaryExtensions.cs new file mode 100644 index 0000000..e29c10f --- /dev/null +++ b/DiscordChatExporter.Domain/Internal/Extensions/BinaryExtensions.cs @@ -0,0 +1,19 @@ +using System.Text; + +namespace DiscordChatExporter.Domain.Internal.Extensions +{ + internal static class BinaryExtensions + { + public static string ToHex(this byte[] data) + { + var buffer = new StringBuilder(); + + foreach (var t in data) + { + buffer.Append(t.ToString("X2")); + } + + return buffer.ToString(); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Domain/Internal/Http.cs b/DiscordChatExporter.Domain/Internal/Http.cs new file mode 100644 index 0000000..3cd2e58 --- /dev/null +++ b/DiscordChatExporter.Domain/Internal/Http.cs @@ -0,0 +1,61 @@ +using System; +using System.Globalization; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using Polly; + +namespace DiscordChatExporter.Domain.Internal +{ + internal static class Http + { + public static HttpClient Client { get; } = new HttpClient(); + + public static IAsyncPolicy ResponsePolicy { get; } = + Policy + .Handle() + .Or() + .OrResult(m => m.StatusCode == HttpStatusCode.TooManyRequests) + .OrResult(m => m.StatusCode == HttpStatusCode.RequestTimeout) + .OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError) + .WaitAndRetryAsync(8, + (i, result, ctx) => + { + // If rate-limited, use retry-after as a guide + if (result.Result.StatusCode == HttpStatusCode.TooManyRequests) + { + // Only start respecting retry-after after a few attempts. + // The reason is that Discord often sends unreasonable (20+ minutes) retry-after + // on the very first request. + if (i > 3) + { + var retryAfterDelay = result.Result.Headers.RetryAfter.Delta; + if (retryAfterDelay != null) + return retryAfterDelay.Value + TimeSpan.FromSeconds(1); // margin just in case + } + } + + return TimeSpan.FromSeconds(Math.Pow(2, i) + 1); + }, + (response, timespan, retryCount, context) => Task.CompletedTask); + + private static HttpStatusCode? TryGetStatusCodeFromException(HttpRequestException ex) + { + // This is extremely frail, but there's no other way + var statusCodeRaw = Regex.Match(ex.Message, @": (\d+) \(").Groups[1].Value; + return !string.IsNullOrWhiteSpace(statusCodeRaw) + ? (HttpStatusCode) int.Parse(statusCodeRaw, CultureInfo.InvariantCulture) + : (HttpStatusCode?) null; + } + + public static IAsyncPolicy ExceptionPolicy { get; } = + Policy + .Handle() // dangerous + .Or(ex => TryGetStatusCodeFromException(ex) == HttpStatusCode.TooManyRequests) + .Or(ex => TryGetStatusCodeFromException(ex) == HttpStatusCode.RequestTimeout) + .Or(ex => TryGetStatusCodeFromException(ex) >= HttpStatusCode.InternalServerError) + .WaitAndRetryAsync(4, i => TimeSpan.FromSeconds(Math.Pow(2, i) + 1)); + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Domain/Internal/PathEx.cs b/DiscordChatExporter.Domain/Internal/PathEx.cs index 309c3c9..83b5949 100644 --- a/DiscordChatExporter.Domain/Internal/PathEx.cs +++ b/DiscordChatExporter.Domain/Internal/PathEx.cs @@ -14,27 +14,5 @@ namespace DiscordChatExporter.Domain.Internal } public static string EscapePath(string path) => EscapePath(new StringBuilder(path)).ToString(); - - public static string MakeUniqueFilePath(string baseFilePath, int maxAttempts = int.MaxValue) - { - if (!File.Exists(baseFilePath)) - return baseFilePath; - - var baseDirPath = Path.GetDirectoryName(baseFilePath); - var baseFileNameWithoutExtension = Path.GetFileNameWithoutExtension(baseFilePath); - var baseFileExtension = Path.GetExtension(baseFilePath); - - for (var i = 1; i <= maxAttempts; i++) - { - var filePath = $"{baseFileNameWithoutExtension} ({i}){baseFileExtension}"; - if (!string.IsNullOrWhiteSpace(baseDirPath)) - filePath = Path.Combine(baseDirPath, filePath); - - if (!File.Exists(filePath)) - return filePath; - } - - return baseFilePath; - } } } \ No newline at end of file diff --git a/DiscordChatExporter.Domain/Internal/Singleton.cs b/DiscordChatExporter.Domain/Internal/Singleton.cs deleted file mode 100644 index 594c988..0000000 --- a/DiscordChatExporter.Domain/Internal/Singleton.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.Net; -using System.Net.Http; - -namespace DiscordChatExporter.Domain.Internal -{ - internal static class Singleton - { - private static readonly Lazy LazyHttpClient = new Lazy(() => - { - var handler = new HttpClientHandler(); - - if (handler.SupportsAutomaticDecompression) - handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate; - - handler.UseCookies = false; - - return new HttpClient(handler, true); - }); - - public static HttpClient HttpClient { get; } = LazyHttpClient.Value; - } -} \ No newline at end of file diff --git a/DiscordChatExporter.Domain/Utilities/AsyncExtensions.cs b/DiscordChatExporter.Domain/Utilities/AsyncExtensions.cs index 43f16af..e6f0bd3 100644 --- a/DiscordChatExporter.Domain/Utilities/AsyncExtensions.cs +++ b/DiscordChatExporter.Domain/Utilities/AsyncExtensions.cs @@ -9,7 +9,8 @@ namespace DiscordChatExporter.Domain.Utilities { public static class AsyncExtensions { - private static async ValueTask> AggregateAsync(this IAsyncEnumerable asyncEnumerable) + private static async ValueTask> AggregateAsync( + this IAsyncEnumerable asyncEnumerable) { var list = new List(); @@ -19,10 +20,14 @@ namespace DiscordChatExporter.Domain.Utilities return list; } - public static ValueTaskAwaiter> GetAwaiter(this IAsyncEnumerable asyncEnumerable) => + public static ValueTaskAwaiter> GetAwaiter( + this IAsyncEnumerable asyncEnumerable) => asyncEnumerable.AggregateAsync().GetAwaiter(); - public static async ValueTask ParallelForEachAsync(this IEnumerable source, Func handleAsync, int degreeOfParallelism) + public static async ValueTask ParallelForEachAsync( + this IEnumerable source, + Func handleAsync, + int degreeOfParallelism) { using var semaphore = new SemaphoreSlim(degreeOfParallelism); diff --git a/DiscordChatExporter.Gui/App.xaml.cs b/DiscordChatExporter.Gui/App.xaml.cs index 2420703..b097bfc 100644 --- a/DiscordChatExporter.Gui/App.xaml.cs +++ b/DiscordChatExporter.Gui/App.xaml.cs @@ -7,13 +7,17 @@ namespace DiscordChatExporter.Gui { public partial class App { - private static readonly Assembly Assembly = typeof(App).Assembly; + private static Assembly Assembly { get; } = typeof(App).Assembly; - public static string Name => Assembly.GetName().Name!; + public static string Name { get; } = Assembly.GetName().Name!; - public static Version Version => Assembly.GetName().Version!; + public static Version Version { get; } = Assembly.GetName().Version!; - public static string VersionString => Version.ToString(3); + public static string VersionString { get; } = Version.ToString(3); + + public static string GitHubProjectUrl { get; } = "https://github.com/Tyrrrz/DiscordChatExporter"; + + public static string GitHubProjectWikiUrl { get; } = GitHubProjectUrl + "/wiki"; } public partial class App diff --git a/DiscordChatExporter.Gui/Behaviors/MultiSelectionListBoxBehavior.cs b/DiscordChatExporter.Gui/Behaviors/MultiSelectionListBoxBehavior.cs index 815be15..e3809c3 100644 --- a/DiscordChatExporter.Gui/Behaviors/MultiSelectionListBoxBehavior.cs +++ b/DiscordChatExporter.Gui/Behaviors/MultiSelectionListBoxBehavior.cs @@ -31,9 +31,9 @@ namespace DiscordChatExporter.Gui.Behaviors private bool _viewHandled; private bool _modelHandled; - public IList SelectedItems + public IList? SelectedItems { - get => (IList) GetValue(SelectedItemsProperty); + get => (IList?) GetValue(SelectedItemsProperty); set => SetValue(SelectedItemsProperty, value); } diff --git a/DiscordChatExporter.Gui/Bootstrapper.cs b/DiscordChatExporter.Gui/Bootstrapper.cs index a6f0460..5fb3533 100644 --- a/DiscordChatExporter.Gui/Bootstrapper.cs +++ b/DiscordChatExporter.Gui/Bootstrapper.cs @@ -17,7 +17,8 @@ namespace DiscordChatExporter.Gui { base.OnStart(); - // Light theme is the default + // Set default theme + // (preferred theme will be chosen later, once the settings are loaded) App.SetLightTheme(); } diff --git a/DiscordChatExporter.Gui/Converters/ExportFormatToStringConverter.cs b/DiscordChatExporter.Gui/Converters/ExportFormatToStringConverter.cs index 17bda21..4feadf1 100644 --- a/DiscordChatExporter.Gui/Converters/ExportFormatToStringConverter.cs +++ b/DiscordChatExporter.Gui/Converters/ExportFormatToStringConverter.cs @@ -18,9 +18,7 @@ namespace DiscordChatExporter.Gui.Converters return default(string); } - public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) - { - throw new NotImplementedException(); - } + public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) => + throw new NotSupportedException(); } } \ No newline at end of file diff --git a/DiscordChatExporter.Gui/Internal/ProcessEx.cs b/DiscordChatExporter.Gui/Internal/ProcessEx.cs index 7481b42..c9c8218 100644 --- a/DiscordChatExporter.Gui/Internal/ProcessEx.cs +++ b/DiscordChatExporter.Gui/Internal/ProcessEx.cs @@ -11,8 +11,7 @@ namespace DiscordChatExporter.Gui.Internal UseShellExecute = true }; - using (Process.Start(startInfo)) - { } + using (Process.Start(startInfo)) {} } } } \ No newline at end of file diff --git a/DiscordChatExporter.Gui/Services/SettingsService.cs b/DiscordChatExporter.Gui/Services/SettingsService.cs index 14afc67..b2b3b7d 100644 --- a/DiscordChatExporter.Gui/Services/SettingsService.cs +++ b/DiscordChatExporter.Gui/Services/SettingsService.cs @@ -16,7 +16,7 @@ namespace DiscordChatExporter.Gui.Services public int ParallelLimit { get; set; } = 1; - public bool ShouldReuseMedia { get; set; } = false; + public bool ShouldReuseMedia { get; set; } public AuthToken? LastToken { get; set; } diff --git a/DiscordChatExporter.Gui/Services/UpdateService.cs b/DiscordChatExporter.Gui/Services/UpdateService.cs index f33d4a7..39e5c96 100644 --- a/DiscordChatExporter.Gui/Services/UpdateService.cs +++ b/DiscordChatExporter.Gui/Services/UpdateService.cs @@ -10,7 +10,8 @@ namespace DiscordChatExporter.Gui.Services { private readonly IUpdateManager _updateManager = new UpdateManager( new GithubPackageResolver("Tyrrrz", "DiscordChatExporter", "DiscordChatExporter.zip"), - new ZipPackageExtractor()); + new ZipPackageExtractor() + ); private readonly SettingsService _settingsService; diff --git a/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs index 44bade9..37e3a60 100644 --- a/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs @@ -75,10 +75,16 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs _settingsService.LastShouldDownloadMedia = ShouldDownloadMedia; // If single channel - prompt file path - if (IsSingleChannel) + if (Channels != null && IsSingleChannel) { var channel = Channels.Single(); - var defaultFileName = ExportRequest.GetDefaultOutputFileName(Guild!, channel, SelectedFormat, After, Before); + var defaultFileName = ExportRequest.GetDefaultOutputFileName( + Guild!, + channel, + SelectedFormat, + After, + Before + ); // Filter var ext = SelectedFormat.GetFileExtension(); @@ -92,11 +98,24 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs OutputPath = _dialogManager.PromptDirectoryPath(); } - // If canceled - return if (string.IsNullOrWhiteSpace(OutputPath)) return; Close(true); } } + + public static class ExportSetupViewModelExtensions + { + public static ExportSetupViewModel CreateExportSetupViewModel(this IViewModelFactory factory, + Guild guild, IReadOnlyList channels) + { + var viewModel = factory.CreateExportSetupViewModel(); + + viewModel.Guild = guild; + viewModel.Channels = channels; + + return viewModel; + } + } } \ No newline at end of file diff --git a/DiscordChatExporter.Gui/ViewModels/Framework/DialogManager.cs b/DiscordChatExporter.Gui/ViewModels/Framework/DialogManager.cs index 37f8c0c..69f2abc 100644 --- a/DiscordChatExporter.Gui/ViewModels/Framework/DialogManager.cs +++ b/DiscordChatExporter.Gui/ViewModels/Framework/DialogManager.cs @@ -19,13 +19,10 @@ namespace DiscordChatExporter.Gui.ViewModels.Framework public async ValueTask ShowDialogAsync(DialogScreen dialogScreen) { - // Get the view that renders this viewmodel var view = _viewManager.CreateAndBindViewForModelIfNecessary(dialogScreen); - // Set up event routing that will close the view when called from viewmodel void OnDialogOpened(object? sender, DialogOpenedEventArgs openArgs) { - // Delegate to close the dialog and unregister event handler void OnScreenClosed(object? o, EventArgs closeArgs) { openArgs.Session.Close(); @@ -35,37 +32,31 @@ namespace DiscordChatExporter.Gui.ViewModels.Framework dialogScreen.Closed += OnScreenClosed; } - // Show view await DialogHost.Show(view, OnDialogOpened); - // Return the result return dialogScreen.DialogResult; } public string? PromptSaveFilePath(string filter = "All files|*.*", string defaultFilePath = "") { - // Create dialog var dialog = new SaveFileDialog { Filter = filter, AddExtension = true, FileName = defaultFilePath, - DefaultExt = Path.GetExtension(defaultFilePath) ?? "" + DefaultExt = Path.GetExtension(defaultFilePath) }; - // Show dialog and return result return dialog.ShowDialog() == true ? dialog.FileName : null; } public string? PromptDirectoryPath(string defaultDirPath = "") { - // Create dialog var dialog = new VistaFolderBrowserDialog { SelectedPath = defaultDirPath }; - // Show dialog and return result return dialog.ShowDialog() == true ? dialog.SelectedPath : null; } } diff --git a/DiscordChatExporter.Gui/ViewModels/Framework/Extensions.cs b/DiscordChatExporter.Gui/ViewModels/Framework/Extensions.cs deleted file mode 100644 index 5584e8d..0000000 --- a/DiscordChatExporter.Gui/ViewModels/Framework/Extensions.cs +++ /dev/null @@ -1,19 +0,0 @@ -using System.Collections.Generic; -using DiscordChatExporter.Domain.Discord.Models; -using DiscordChatExporter.Gui.ViewModels.Dialogs; - -namespace DiscordChatExporter.Gui.ViewModels.Framework -{ - public static class Extensions - { - public static ExportSetupViewModel CreateExportSetupViewModel(this IViewModelFactory factory, - Guild guild, IReadOnlyList channels) - { - var viewModel = factory.CreateExportSetupViewModel(); - viewModel.Guild = guild; - viewModel.Channels = channels; - - return viewModel; - } - } -} \ No newline at end of file diff --git a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs index 13721c8..754fc77 100644 --- a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs @@ -8,7 +8,9 @@ using DiscordChatExporter.Domain.Discord.Models; using DiscordChatExporter.Domain.Exceptions; using DiscordChatExporter.Domain.Exporting; using DiscordChatExporter.Domain.Utilities; +using DiscordChatExporter.Gui.Internal; using DiscordChatExporter.Gui.Services; +using DiscordChatExporter.Gui.ViewModels.Dialogs; using DiscordChatExporter.Gui.ViewModels.Framework; using Gress; using MaterialDesignThemes.Wpf; @@ -63,14 +65,21 @@ namespace DiscordChatExporter.Gui.ViewModels // Update busy state when progress manager changes ProgressManager.Bind(o => o.IsActive, - (sender, args) => IsBusy = ProgressManager.IsActive); + (sender, args) => IsBusy = ProgressManager.IsActive + ); + ProgressManager.Bind(o => o.IsActive, - (sender, args) => IsProgressIndeterminate = ProgressManager.IsActive && ProgressManager.Progress.IsEither(0, 1)); + (sender, args) => IsProgressIndeterminate = + ProgressManager.IsActive && ProgressManager.Progress.IsEither(0, 1) + ); + ProgressManager.Bind(o => o.Progress, - (sender, args) => IsProgressIndeterminate = ProgressManager.IsActive && ProgressManager.Progress.IsEither(0, 1)); + (sender, args) => IsProgressIndeterminate = + ProgressManager.IsActive && ProgressManager.Progress.IsEither(0, 1) + ); } - private async ValueTask HandleAutoUpdateAsync() + private async ValueTask CheckForUpdatesAsync() { try { @@ -117,7 +126,7 @@ namespace DiscordChatExporter.Gui.ViewModels App.SetLightTheme(); } - await HandleAutoUpdateAsync(); + await CheckForUpdatesAsync(); } protected override void OnClose() @@ -134,6 +143,8 @@ namespace DiscordChatExporter.Gui.ViewModels await _dialogManager.ShowDialogAsync(dialog); } + public void ShowHelp() => ProcessEx.StartShellExecute(App.GitHubProjectWikiUrl); + public bool CanPopulateGuildsAndChannels => !IsBusy && !string.IsNullOrWhiteSpace(TokenValue); @@ -187,8 +198,8 @@ namespace DiscordChatExporter.Gui.ViewModels var exporter = new ChannelExporter(token); var operations = ProgressManager.CreateOperations(dialog.Channels!.Count); - var successfulExportCount = 0; + await dialog.Channels.Zip(operations).ParallelForEachAsync(async tuple => { var (channel, operation) = tuple; diff --git a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml index 23aeaf0..91bd67b 100644 --- a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml +++ b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml @@ -95,7 +95,7 @@ materialDesign:HintAssist.IsFloating="True" DisplayDateEnd="{Binding BeforeDate, Converter={x:Static converters:DateTimeOffsetToDateTimeConverter.Instance}}" SelectedDate="{Binding AfterDate, Converter={x:Static converters:DateTimeOffsetToDateTimeConverter.Instance}}" - ToolTip="If this is set, only messages sent after this date will be exported" /> + ToolTip="Only include messages sent after this date" /> + ToolTip="Only include messages sent before this date" /> + ToolTip="Only include messages sent after this time" /> + ToolTip="Only include messages sent before this time" /> @@ -131,10 +131,10 @@ materialDesign:HintAssist.Hint="Messages per partition" materialDesign:HintAssist.IsFloating="True" Text="{Binding PartitionLimit, TargetNullValue=''}" - ToolTip="If this is set, the exported file will be split into multiple partitions, each containing no more than specified number of messages" /> + ToolTip="Split output into partitions limited to this number of messages" /> - + @@ -143,7 +143,7 @@ + Text="Download media" /> + ToolTip="Toggle advanced options" />