diff --git a/DiscordChatExporter.Cli/Commands/ExportCommandBase.cs b/DiscordChatExporter.Cli/Commands/ExportCommandBase.cs index b9b21ef..2edfff0 100644 --- a/DiscordChatExporter.Cli/Commands/ExportCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/ExportCommandBase.cs @@ -53,6 +53,7 @@ namespace DiscordChatExporter.Cli.Commands After, Before, progress); console.Output.WriteLine(); + console.Output.WriteLine("Done."); } protected async ValueTask ExportAsync(IConsole console, Channel channel) diff --git a/DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs b/DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs index 92e39b8..0e2f022 100644 --- a/DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs @@ -1,16 +1,13 @@ using System.Linq; -using System.Net; using System.Threading.Tasks; using CliFx; using CliFx.Attributes; -using DiscordChatExporter.Core.Models.Exceptions; using DiscordChatExporter.Core.Services; -using DiscordChatExporter.Core.Services.Exceptions; namespace DiscordChatExporter.Cli.Commands { [Command("exportdm", Description = "Export all direct message channels.")] - public class ExportDirectMessagesCommand : ExportCommandBase + public class ExportDirectMessagesCommand : ExportMultipleCommandBase { public ExportDirectMessagesCommand(SettingsService settingsService, DataService dataService, ExportService exportService) : base(settingsService, dataService, exportService) @@ -19,32 +16,10 @@ namespace DiscordChatExporter.Cli.Commands public override async ValueTask ExecuteAsync(IConsole console) { - // Get channels - var channels = await DataService.GetDirectMessageChannelsAsync(Token); + var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token); + var channels = directMessageChannels.OrderBy(c => c.Name).ToArray(); - // Order channels - channels = channels.OrderBy(c => c.Name).ToArray(); - - // Loop through channels - foreach (var channel in channels) - { - try - { - await ExportAsync(console, channel); - } - catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden) - { - console.Error.WriteLine("You don't have access to this channel."); - } - catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound) - { - console.Error.WriteLine("This channel doesn't exist."); - } - catch (DomainException ex) - { - console.Error.WriteLine(ex.Message); - } - } + await ExportMultipleAsync(console, channels); } } } \ No newline at end of file diff --git a/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs b/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs index c69941e..118fb6c 100644 --- a/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs @@ -1,17 +1,14 @@ using System.Linq; -using System.Net; using System.Threading.Tasks; using CliFx; using CliFx.Attributes; using DiscordChatExporter.Core.Models; -using DiscordChatExporter.Core.Models.Exceptions; using DiscordChatExporter.Core.Services; -using DiscordChatExporter.Core.Services.Exceptions; namespace DiscordChatExporter.Cli.Commands { [Command("exportguild", Description = "Export all channels within specified guild.")] - public class ExportGuildCommand : ExportCommandBase + public class ExportGuildCommand : ExportMultipleCommandBase { [CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")] public string GuildId { get; set; } = ""; @@ -23,32 +20,14 @@ namespace DiscordChatExporter.Cli.Commands public override async ValueTask ExecuteAsync(IConsole console) { - // Get channels - var channels = await DataService.GetGuildChannelsAsync(Token, GuildId); + var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId); - // Filter and order channels - channels = channels.Where(c => c.Type.IsExportable()).OrderBy(c => c.Name).ToArray(); + var channels = guildChannels + .Where(c => c.Type.IsExportable()) + .OrderBy(c => c.Name) + .ToArray(); - // Loop through channels - foreach (var channel in channels) - { - try - { - await ExportAsync(console, channel); - } - catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden) - { - console.Error.WriteLine("You don't have access to this channel."); - } - catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound) - { - console.Error.WriteLine("This channel doesn't exist."); - } - catch (DomainException ex) - { - console.Error.WriteLine(ex.Message); - } - } + await ExportMultipleAsync(console, channels); } } } \ No newline at end of file diff --git a/DiscordChatExporter.Cli/Commands/ExportMultipleCommandBase.cs b/DiscordChatExporter.Cli/Commands/ExportMultipleCommandBase.cs new file mode 100644 index 0000000..d6c86db --- /dev/null +++ b/DiscordChatExporter.Cli/Commands/ExportMultipleCommandBase.cs @@ -0,0 +1,92 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using CliFx; +using CliFx.Attributes; +using CliFx.Utilities; +using DiscordChatExporter.Core.Models; +using DiscordChatExporter.Core.Models.Exceptions; +using DiscordChatExporter.Core.Services; +using DiscordChatExporter.Core.Services.Exceptions; +using Gress; +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Cli.Commands +{ + public abstract class ExportMultipleCommandBase : ExportCommandBase + { + [CommandOption("parallel", Description = "Export this number of separate channels in parallel.")] + public int ParallelLimit { get; set; } = 1; + + protected ExportMultipleCommandBase(SettingsService settingsService, DataService dataService, ExportService exportService) + : base(settingsService, dataService, exportService) + { + } + + protected async ValueTask ExportMultipleAsync(IConsole console, IReadOnlyList channels) + { + // This uses a separate route from ExportCommandBase because the progress ticker is not thread-safe + // Ugly code ahead. Will need to refactor. + + if (!string.IsNullOrWhiteSpace(DateFormat)) + SettingsService.DateFormat = DateFormat; + + // Progress + console.Output.Write($"Exporting {channels.Count} channels... "); + var ticker = console.CreateProgressTicker(); + + // TODO: refactor this after improving Gress + var progressManager = new ProgressManager(); + progressManager.PropertyChanged += (sender, args) => ticker.Report(progressManager.Progress); + + var operations = progressManager.CreateOperations(channels.Count); + + // Export channels + using var semaphore = new SemaphoreSlim(ParallelLimit.ClampMin(1)); + + var errors = new List(); + + await Task.WhenAll(channels.Select(async (channel, i) => + { + var operation = operations[i]; + await semaphore.WaitAsync(); + + var guild = await DataService.GetGuildAsync(Token, channel.GuildId); + + try + { + await ExportService.ExportChatLogAsync(Token, guild, channel, + OutputPath, ExportFormat, PartitionLimit, + After, Before, operation); + } + catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden) + { + errors.Add("You don't have access to this channel."); + } + catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound) + { + errors.Add("This channel doesn't exist."); + } + catch (DomainException ex) + { + errors.Add(ex.Message); + } + finally + { + semaphore.Release(); + operation.Dispose(); + } + })); + + ticker.Report(1); + console.Output.WriteLine(); + + foreach (var error in errors) + console.Error.WriteLine(error); + + console.Output.WriteLine("Done."); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs b/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs index 4052dab..b08d437 100644 --- a/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs +++ b/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs @@ -20,13 +20,13 @@ namespace DiscordChatExporter.Cli.Commands public override async ValueTask ExecuteAsync(IConsole console) { - // Get channels - var channels = await DataService.GetGuildChannelsAsync(Token, GuildId); + var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId); - // Filter and order channels - channels = channels.Where(c => c.Type.IsExportable()).OrderBy(c => c.Name).ToArray(); + var channels = guildChannels + .Where(c => c.Type.IsExportable()) + .OrderBy(c => c.Name) + .ToArray(); - // Print result foreach (var channel in channels) console.Output.WriteLine($"{channel.Id} | {channel.Name}"); } diff --git a/DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs b/DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs index f4dcc2f..3c281d9 100644 --- a/DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs +++ b/DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs @@ -16,13 +16,9 @@ namespace DiscordChatExporter.Cli.Commands public override async ValueTask ExecuteAsync(IConsole console) { - // Get channels - var channels = await DataService.GetDirectMessageChannelsAsync(Token); + var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token); + var channels = directMessageChannels.OrderBy(c => c.Name).ToArray(); - // Order channels - channels = channels.OrderBy(c => c.Name).ToArray(); - - // Print result foreach (var channel in channels) console.Output.WriteLine($"{channel.Id} | {channel.Name}"); } diff --git a/DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs b/DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs index 5dd43e9..f90166b 100644 --- a/DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs +++ b/DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs @@ -16,14 +16,9 @@ namespace DiscordChatExporter.Cli.Commands public override async ValueTask ExecuteAsync(IConsole console) { - // Get guilds var guilds = await DataService.GetUserGuildsAsync(Token); - // Order guilds - guilds = guilds.OrderBy(g => g.Name).ToArray(); - - // Print result - foreach (var guild in guilds) + foreach (var guild in guilds.OrderBy(g => g.Name)) console.Output.WriteLine($"{guild.Id} | {guild.Name}"); } } diff --git a/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj b/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj index 539f641..59bd35c 100644 --- a/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj +++ b/DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj @@ -8,6 +8,7 @@ + diff --git a/DiscordChatExporter.Core.Services/SettingsService.cs b/DiscordChatExporter.Core.Services/SettingsService.cs index fa6f550..3ee3b6d 100644 --- a/DiscordChatExporter.Core.Services/SettingsService.cs +++ b/DiscordChatExporter.Core.Services/SettingsService.cs @@ -11,6 +11,8 @@ namespace DiscordChatExporter.Core.Services public bool IsTokenPersisted { get; set; } = true; + public int ParallelLimit { get; set; } = 1; + public AuthToken? LastToken { get; set; } public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark; diff --git a/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs index 7e87672..3acc2fa 100644 --- a/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs @@ -1,5 +1,6 @@ using DiscordChatExporter.Core.Services; using DiscordChatExporter.Gui.ViewModels.Framework; +using Tyrrrz.Extensions; namespace DiscordChatExporter.Gui.ViewModels.Dialogs { @@ -25,6 +26,12 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs set => _settingsService.IsTokenPersisted = value; } + public int ParallelLimit + { + get => _settingsService.ParallelLimit; + set => _settingsService.ParallelLimit = value.Clamp(1, 10); + } + public SettingsViewModel(SettingsService settingsService) { _settingsService = settingsService; diff --git a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs index f95e4fe..117425d 100644 --- a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using System.Net; +using System.Threading; using System.Threading.Tasks; using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models.Exceptions; @@ -260,10 +261,13 @@ namespace DiscordChatExporter.Gui.ViewModels // Export channels var successfulExportCount = 0; - for (var i = 0; i < dialog.Channels.Count; i++) + using var semaphore = new SemaphoreSlim(_settingsService.ParallelLimit.ClampMin(1)); + + await Task.WhenAll(dialog.Channels.Select(async (channel, i) => { var operation = operations[i]; - var channel = dialog.Channels[i]; + + await semaphore.WaitAsync(); try { @@ -288,8 +292,9 @@ namespace DiscordChatExporter.Gui.ViewModels finally { operation.Dispose(); + semaphore.Release(); } - } + })); // Notify of overall completion if (successfulExportCount > 0) diff --git a/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml b/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml index 1bd110f..1ce646f 100644 --- a/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml +++ b/DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml @@ -56,6 +56,21 @@ IsChecked="{Binding IsTokenPersisted}" /> + + + + + + + + +