open-source-mirrors
/
DiscordChatExporter
mirror of https://github.com/Tyrrrz/DiscordChatExporter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add parallel exporting

Browse Source 
Closes #264
pull/285/head
Alexey Golub 5 years ago
parent 70a1c9db8c
commit 9f4277ae84
12 changed files with 145 additions and 77 deletions
Show Stats Download Patch File Download Diff File

1
DiscordChatExporter.Cli/Commands/ExportCommandBase.cs
Unescape View File

@ -53,6 +53,7 @@ namespace DiscordChatExporter.Cli.Commands
After, Before, progress);
console.Output.WriteLine();
console.Output.WriteLine("Done.");
}
protected async ValueTask ExportAsync(IConsole console, Channel channel)

33
DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs
Unescape View File

@ -1,16 +1,13 @@
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions;
namespace DiscordChatExporter.Cli.Commands
{
[Command("exportdm", Description = "Export all direct message channels.")]
public class ExportDirectMessagesCommand : ExportCommandBase
public class ExportDirectMessagesCommand : ExportMultipleCommandBase
{
public ExportDirectMessagesCommand(SettingsService settingsService, DataService dataService, ExportService exportService)
: base(settingsService, dataService, exportService)
@ -19,32 +16,10 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console)
{
// Get channels
var channels = await DataService.GetDirectMessageChannelsAsync(Token);
var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token);
var channels = directMessageChannels.OrderBy(c => c.Name).ToArray();
// Order channels
channels = channels.OrderBy(c => c.Name).ToArray();
// Loop through channels
foreach (var channel in channels)
{
try
{
await ExportAsync(console, channel);
}
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
{
console.Error.WriteLine("You don't have access to this channel.");
}
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
{
console.Error.WriteLine("This channel doesn't exist.");
}
catch (DomainException ex)
{
console.Error.WriteLine(ex.Message);
}
}
await ExportMultipleAsync(console, channels);
}
}
}

35
DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs
Unescape View File

@ -1,17 +1,14 @@
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions;
namespace DiscordChatExporter.Cli.Commands
{
[Command("exportguild", Description = "Export all channels within specified guild.")]
public class ExportGuildCommand : ExportCommandBase
public class ExportGuildCommand : ExportMultipleCommandBase
{
[CommandOption("guild", 'g', IsRequired = true, Description = "Guild ID.")]
public string GuildId { get; set; } = "";
@ -23,32 +20,14 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console)
{
// Get channels
var channels = await DataService.GetGuildChannelsAsync(Token, GuildId);
var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId);
// Filter and order channels
channels = channels.Where(c => c.Type.IsExportable()).OrderBy(c => c.Name).ToArray();
var channels = guildChannels
.Where(c => c.Type.IsExportable())
.OrderBy(c => c.Name)
.ToArray();
// Loop through channels
foreach (var channel in channels)
{
try
{
await ExportAsync(console, channel);
}
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
{
console.Error.WriteLine("You don't have access to this channel.");
}
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
{
console.Error.WriteLine("This channel doesn't exist.");
}
catch (DomainException ex)
{
console.Error.WriteLine(ex.Message);
}
}
await ExportMultipleAsync(console, channels);
}
}
}

92
DiscordChatExporter.Cli/Commands/ExportMultipleCommandBase.cs
Unescape View File

@ -0,0 +1,92 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using CliFx.Utilities;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions;
using Gress;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Cli.Commands
{
public abstract class ExportMultipleCommandBase : ExportCommandBase
{
[CommandOption("parallel", Description = "Export this number of separate channels in parallel.")]
public int ParallelLimit { get; set; } = 1;
protected ExportMultipleCommandBase(SettingsService settingsService, DataService dataService, ExportService exportService)
: base(settingsService, dataService, exportService)
{
}
protected async ValueTask ExportMultipleAsync(IConsole console, IReadOnlyList<Channel> channels)
{
// This uses a separate route from ExportCommandBase because the progress ticker is not thread-safe
// Ugly code ahead. Will need to refactor.
if (!string.IsNullOrWhiteSpace(DateFormat))
SettingsService.DateFormat = DateFormat;
// Progress
console.Output.Write($"Exporting {channels.Count} channels... ");
var ticker = console.CreateProgressTicker();
// TODO: refactor this after improving Gress
var progressManager = new ProgressManager();
progressManager.PropertyChanged += (sender, args) => ticker.Report(progressManager.Progress);
var operations = progressManager.CreateOperations(channels.Count);
// Export channels
using var semaphore = new SemaphoreSlim(ParallelLimit.ClampMin(1));
var errors = new List<string>();
await Task.WhenAll(channels.Select(async (channel, i) =>
{
var operation = operations[i];
await semaphore.WaitAsync();
var guild = await DataService.GetGuildAsync(Token, channel.GuildId);
try
{
await ExportService.ExportChatLogAsync(Token, guild, channel,
OutputPath, ExportFormat, PartitionLimit,
After, Before, operation);
}
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
{
errors.Add("You don't have access to this channel.");
}
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.NotFound)
{
errors.Add("This channel doesn't exist.");
}
catch (DomainException ex)
{
errors.Add(ex.Message);
}
finally
{
semaphore.Release();
operation.Dispose();
}
}));
ticker.Report(1);
console.Output.WriteLine();
foreach (var error in errors)
console.Error.WriteLine(error);
console.Output.WriteLine("Done.");
}
}
}

10
DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs
Unescape View File

@ -20,13 +20,13 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console)
{
// Get channels
var channels = await DataService.GetGuildChannelsAsync(Token, GuildId);
var guildChannels = await DataService.GetGuildChannelsAsync(Token, GuildId);
// Filter and order channels
channels = channels.Where(c => c.Type.IsExportable()).OrderBy(c => c.Name).ToArray();
var channels = guildChannels
.Where(c => c.Type.IsExportable())
.OrderBy(c => c.Name)
.ToArray();
// Print result
foreach (var channel in channels)
console.Output.WriteLine($"{channel.Id} | {channel.Name}");
}

8
DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs
Unescape View File

@ -16,13 +16,9 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console)
{
// Get channels
var channels = await DataService.GetDirectMessageChannelsAsync(Token);
var directMessageChannels = await DataService.GetDirectMessageChannelsAsync(Token);
var channels = directMessageChannels.OrderBy(c => c.Name).ToArray();
// Order channels
channels = channels.OrderBy(c => c.Name).ToArray();
// Print result
foreach (var channel in channels)
console.Output.WriteLine($"{channel.Id} | {channel.Name}");
}

7
DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs
Unescape View File

@ -16,14 +16,9 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console)
{
// Get guilds
var guilds = await DataService.GetUserGuildsAsync(Token);
// Order guilds
guilds = guilds.OrderBy(g => g.Name).ToArray();
// Print result
foreach (var guild in guilds)
foreach (var guild in guilds.OrderBy(g => g.Name))
console.Output.WriteLine($"{guild.Id} | {guild.Name}");
}
}

1
DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj
Unescape View File

@ -8,6 +8,7 @@
<ItemGroup>
<PackageReference Include="CliFx" Version="1.0.0" />
<PackageReference Include="Gress" Version="1.1.1" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="3.1.1" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>

2
DiscordChatExporter.Core.Services/SettingsService.cs
Unescape View File

@ -11,6 +11,8 @@ namespace DiscordChatExporter.Core.Services
public bool IsTokenPersisted { get; set; } = true;
public int ParallelLimit { get; set; } = 1;
public AuthToken? LastToken { get; set; }
public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;

7
DiscordChatExporter.Gui/ViewModels/Dialogs/SettingsViewModel.cs
Unescape View File

@ -1,5 +1,6 @@
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Gui.ViewModels.Framework;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Gui.ViewModels.Dialogs
{
@ -25,6 +26,12 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
set => _settingsService.IsTokenPersisted = value;
}
public int ParallelLimit
{
get => _settingsService.ParallelLimit;
set => _settingsService.ParallelLimit = value.Clamp(1, 10);
}
public SettingsViewModel(SettingsService settingsService)
{
_settingsService = settingsService;

11
DiscordChatExporter.Gui/ViewModels/RootViewModel.cs
Unescape View File

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
@ -260,10 +261,13 @@ namespace DiscordChatExporter.Gui.ViewModels
// Export channels
var successfulExportCount = 0;
for (var i = 0; i < dialog.Channels.Count; i++)
using var semaphore = new SemaphoreSlim(_settingsService.ParallelLimit.ClampMin(1));
await Task.WhenAll(dialog.Channels.Select(async (channel, i) =>
{
var operation = operations[i];
var channel = dialog.Channels[i];
await semaphore.WaitAsync();
try
{
@ -288,8 +292,9 @@ namespace DiscordChatExporter.Gui.ViewModels
finally
{
operation.Dispose();
semaphore.Release();
}
}
}));
// Notify of overall completion
if (successfulExportCount > 0)

15
DiscordChatExporter.Gui/Views/Dialogs/SettingsView.xaml
Unescape View File

@ -56,6 +56,21 @@
IsChecked="{Binding IsTokenPersisted}" />
</DockPanel>
<!-- Parallel limit -->
<StackPanel Background="Transparent" ToolTip="How many channels can be exported at the same time">
<TextBlock Margin="16,8">
<Run Text="Parallel limit:" />
<Run Foreground="{DynamicResource PrimaryTextBrush}" Text="{Binding ParallelLimit, Mode=OneWay}" />
</TextBlock>
<Slider
Margin="16,8"
IsSnapToTickEnabled="True"
Maximum="10"
Minimum="1"
TickFrequency="1"
Value="{Binding ParallelLimit}" />
</StackPanel>
<!-- Save button -->
<Button
Margin="8"

Write Preview
Loading…
Cancel
Save