open-source-mirrors
/
DiscordChatExporter
mirror of https://github.com/Tyrrrz/DiscordChatExporter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add managed cancellation support

Browse Source 
Closes #716
pull/727/head
Tyrrrz 3 years ago
parent 2f3e165988
commit 21d89afa70
21 changed files with 274 additions and 147 deletions
Show Stats Download Patch File Download Diff File

12
DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
Unescape View File

@ -15,7 +15,6 @@ using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Filtering; using DiscordChatExporter.Core.Exporting.Filtering;
using DiscordChatExporter.Core.Exporting.Partitioning; using DiscordChatExporter.Core.Exporting.Partitioning;
using DiscordChatExporter.Core.Utils.Extensions; using DiscordChatExporter.Core.Utils.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Cli.Commands.Base namespace DiscordChatExporter.Cli.Commands.Base
{ {
@ -56,6 +55,8 @@ namespace DiscordChatExporter.Cli.Commands.Base
protected async ValueTask ExecuteAsync(IConsole console, IReadOnlyList<Channel> channels) protected async ValueTask ExecuteAsync(IConsole console, IReadOnlyList<Channel> channels)
{ {
var cancellationToken = console.RegisterCancellationHandler();
if (ShouldReuseMedia && !ShouldDownloadMedia) if (ShouldReuseMedia && !ShouldDownloadMedia)
{ {
throw new CommandException("Option --reuse-media cannot be used without --media."); throw new CommandException("Option --reuse-media cannot be used without --media.");
@ -73,7 +74,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
{ {
await progressContext.StartTaskAsync($"{channel.Category} / {channel.Name}", async progress => await progressContext.StartTaskAsync($"{channel.Category} / {channel.Name}", async progress =>
{ {
var guild = await Discord.GetGuildAsync(channel.GuildId); var guild = await Discord.GetGuildAsync(channel.GuildId, cancellationToken);
var request = new ExportRequest( var request = new ExportRequest(
guild, guild,
@ -89,14 +90,14 @@ namespace DiscordChatExporter.Cli.Commands.Base
DateFormat DateFormat
); );
await Exporter.ExportChannelAsync(request, progress); await Exporter.ExportChannelAsync(request, progress, cancellationToken);
}); });
} }
catch (DiscordChatExporterException ex) when (!ex.IsFatal) catch (DiscordChatExporterException ex) when (!ex.IsFatal)
{ {
errors[channel] = ex.Message; errors[channel] = ex.Message;
} }
}, ParallelLimit.ClampMin(1)); }, Math.Max(ParallelLimit, 1), cancellationToken);
}); });
// Print result // Print result
@ -140,11 +141,12 @@ namespace DiscordChatExporter.Cli.Commands.Base
protected async ValueTask ExecuteAsync(IConsole console, IReadOnlyList<Snowflake> channelIds) protected async ValueTask ExecuteAsync(IConsole console, IReadOnlyList<Snowflake> channelIds)
{ {
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>(); var channels = new List<Channel>();
foreach (var channelId in channelIds) foreach (var channelId in channelIds)
{ {
var channel = await Discord.GetChannelAsync(channelId); var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
channels.Add(channel); channels.Add(channel);
} }

5
DiscordChatExporter.Cli/Commands/ExportAllCommand.cs
Unescape View File

@ -15,16 +15,17 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console) public override async ValueTask ExecuteAsync(IConsole console)
{ {
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>(); var channels = new List<Channel>();
await console.Output.WriteLineAsync("Fetching channels..."); await console.Output.WriteLineAsync("Fetching channels...");
await foreach (var guild in Discord.GetUserGuildsAsync()) await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
{ {
// Skip DMs if instructed to // Skip DMs if instructed to
if (!IncludeDirectMessages && guild.Id == Guild.DirectMessages.Id) if (!IncludeDirectMessages && guild.Id == Guild.DirectMessages.Id)
continue; continue;
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id)) await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
{ {
// Skip non-text channels // Skip non-text channels
if (!channel.IsTextChannel) if (!channel.IsTextChannel)

4
DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs
Unescape View File

@ -13,8 +13,10 @@ namespace DiscordChatExporter.Cli.Commands
{ {
public override async ValueTask ExecuteAsync(IConsole console) public override async ValueTask ExecuteAsync(IConsole console)
{ {
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels..."); await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id); var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
var textChannels = channels.Where(c => c.IsTextChannel).ToArray(); var textChannels = channels.Where(c => c.IsTextChannel).ToArray();
await base.ExecuteAsync(console, textChannels); await base.ExecuteAsync(console, textChannels);

4
DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs
Unescape View File

@ -16,8 +16,10 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console) public override async ValueTask ExecuteAsync(IConsole console)
{ {
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels..."); await console.Output.WriteLineAsync("Fetching channels...");
var channels = await Discord.GetGuildChannelsAsync(GuildId); var channels = await Discord.GetGuildChannelsAsync(GuildId, cancellationToken);
var textChannels = channels.Where(c => c.IsTextChannel).ToArray(); var textChannels = channels.Where(c => c.IsTextChannel).ToArray();
await base.ExecuteAsync(console, textChannels); await base.ExecuteAsync(console, textChannels);

4
DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs
Unescape View File

@ -17,7 +17,9 @@ namespace DiscordChatExporter.Cli.Commands
public override async ValueTask ExecuteAsync(IConsole console) public override async ValueTask ExecuteAsync(IConsole console)
{ {
var channels = await Discord.GetGuildChannelsAsync(GuildId); var cancellationToken = console.RegisterCancellationHandler();
var channels = await Discord.GetGuildChannelsAsync(GuildId, cancellationToken);
var textChannels = channels var textChannels = channels
.Where(c => c.IsTextChannel) .Where(c => c.IsTextChannel)

4
DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs
Unescape View File

@ -14,7 +14,9 @@ namespace DiscordChatExporter.Cli.Commands
{ {
public override async ValueTask ExecuteAsync(IConsole console) public override async ValueTask ExecuteAsync(IConsole console)
{ {
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id); var cancellationToken = console.RegisterCancellationHandler();
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
var textChannels = channels var textChannels = channels
.Where(c => c.IsTextChannel) .Where(c => c.IsTextChannel)

4
DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs
Unescape View File

@ -13,7 +13,9 @@ namespace DiscordChatExporter.Cli.Commands
{ {
public override async ValueTask ExecuteAsync(IConsole console) public override async ValueTask ExecuteAsync(IConsole console)
{ {
var guilds = await Discord.GetUserGuildsAsync(); var cancellationToken = console.RegisterCancellationHandler();
var guilds = await Discord.GetUserGuildsAsync(cancellationToken);
foreach (var guild in guilds.OrderBy(g => g.Name)) foreach (var guild in guilds.OrderBy(g => g.Name))
{ {

94
DiscordChatExporter.Core/Discord/DiscordClient.cs
Unescape View File

@ -3,7 +3,9 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Net.Http; using System.Net.Http;
using System.Runtime.CompilerServices;
using System.Text.Json; using System.Text.Json;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions; using DiscordChatExporter.Core.Exceptions;
@ -21,18 +23,28 @@ namespace DiscordChatExporter.Core.Discord
public DiscordClient(AuthToken token) => _token = token; public DiscordClient(AuthToken token) => _token = token;
private async ValueTask<HttpResponseMessage> GetResponseAsync(string url) => private async ValueTask<HttpResponseMessage> GetResponseAsync(
await Http.ResponsePolicy.ExecuteAsync(async () => string url,
CancellationToken cancellationToken = default)
{
return await Http.ResponsePolicy.ExecuteAsync(async innerCancellationToken =>
{ {
using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url)); using var request = new HttpRequestMessage(HttpMethod.Get, new Uri(_baseUri, url));
request.Headers.Authorization = _token.GetAuthenticationHeader(); request.Headers.Authorization = _token.GetAuthenticationHeader();
return await Http.Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead); return await Http.Client.SendAsync(
}); request,
HttpCompletionOption.ResponseHeadersRead,
innerCancellationToken
);
}, cancellationToken);
}
private async ValueTask<JsonElement> GetJsonResponseAsync(string url) private async ValueTask<JsonElement> GetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
{ {
using var response = await GetResponseAsync(url); using var response = await GetResponseAsync(url, cancellationToken);
if (!response.IsSuccessStatusCode) if (!response.IsSuccessStatusCode)
{ {
@ -45,19 +57,22 @@ namespace DiscordChatExporter.Core.Discord
}; };
} }
return await response.Content.ReadAsJsonAsync(); return await response.Content.ReadAsJsonAsync(cancellationToken);
} }
private async ValueTask<JsonElement?> TryGetJsonResponseAsync(string url) private async ValueTask<JsonElement?> TryGetJsonResponseAsync(
string url,
CancellationToken cancellationToken = default)
{ {
using var response = await GetResponseAsync(url); using var response = await GetResponseAsync(url, cancellationToken);
return response.IsSuccessStatusCode return response.IsSuccessStatusCode
? await response.Content.ReadAsJsonAsync() ? await response.Content.ReadAsJsonAsync(cancellationToken)
: null; : null;
} }
public async IAsyncEnumerable<Guild> GetUserGuildsAsync() public async IAsyncEnumerable<Guild> GetUserGuildsAsync(
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{ {
yield return Guild.DirectMessages; yield return Guild.DirectMessages;
@ -71,7 +86,7 @@ namespace DiscordChatExporter.Core.Discord
.SetQueryParameter("after", currentAfter.ToString()) .SetQueryParameter("after", currentAfter.ToString())
.Build(); .Build();
var response = await GetJsonResponseAsync(url); var response = await GetJsonResponseAsync(url, cancellationToken);
var isEmpty = true; var isEmpty = true;
foreach (var guild in response.EnumerateArray().Select(Guild.Parse)) foreach (var guild in response.EnumerateArray().Select(Guild.Parse))
@ -87,26 +102,30 @@ namespace DiscordChatExporter.Core.Discord
} }
} }
public async ValueTask<Guild> GetGuildAsync(Snowflake guildId) public async ValueTask<Guild> GetGuildAsync(
Snowflake guildId,
CancellationToken cancellationToken = default)
{ {
if (guildId == Guild.DirectMessages.Id) if (guildId == Guild.DirectMessages.Id)
return Guild.DirectMessages; return Guild.DirectMessages;
var response = await GetJsonResponseAsync($"guilds/{guildId}"); var response = await GetJsonResponseAsync($"guilds/{guildId}", cancellationToken);
return Guild.Parse(response); return Guild.Parse(response);
} }
public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(Snowflake guildId) public async IAsyncEnumerable<Channel> GetGuildChannelsAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{ {
if (guildId == Guild.DirectMessages.Id) if (guildId == Guild.DirectMessages.Id)
{ {
var response = await GetJsonResponseAsync("users/@me/channels"); var response = await GetJsonResponseAsync("users/@me/channels", cancellationToken);
foreach (var channelJson in response.EnumerateArray()) foreach (var channelJson in response.EnumerateArray())
yield return Channel.Parse(channelJson); yield return Channel.Parse(channelJson);
} }
else else
{ {
var response = await GetJsonResponseAsync($"guilds/{guildId}/channels"); var response = await GetJsonResponseAsync($"guilds/{guildId}/channels", cancellationToken);
var responseOrdered = response var responseOrdered = response
.EnumerateArray() .EnumerateArray()
@ -138,31 +157,38 @@ namespace DiscordChatExporter.Core.Discord
} }
} }
public async IAsyncEnumerable<Role> GetGuildRolesAsync(Snowflake guildId) public async IAsyncEnumerable<Role> GetGuildRolesAsync(
Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{ {
if (guildId == Guild.DirectMessages.Id) if (guildId == Guild.DirectMessages.Id)
yield break; yield break;
var response = await GetJsonResponseAsync($"guilds/{guildId}/roles"); var response = await GetJsonResponseAsync($"guilds/{guildId}/roles", cancellationToken);
foreach (var roleJson in response.EnumerateArray()) foreach (var roleJson in response.EnumerateArray())
yield return Role.Parse(roleJson); yield return Role.Parse(roleJson);
} }
public async ValueTask<Member> GetGuildMemberAsync(Snowflake guildId, User user) public async ValueTask<Member> GetGuildMemberAsync(
Snowflake guildId,
User user,
CancellationToken cancellationToken = default)
{ {
if (guildId == Guild.DirectMessages.Id) if (guildId == Guild.DirectMessages.Id)
return Member.CreateForUser(user); return Member.CreateForUser(user);
var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{user.Id}"); var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{user.Id}", cancellationToken);
return response?.Pipe(Member.Parse) ?? Member.CreateForUser(user); return response?.Pipe(Member.Parse) ?? Member.CreateForUser(user);
} }
public async ValueTask<ChannelCategory> GetChannelCategoryAsync(Snowflake channelId) public async ValueTask<ChannelCategory> GetChannelCategoryAsync(
Snowflake channelId,
CancellationToken cancellationToken = default)
{ {
try try
{ {
var response = await GetJsonResponseAsync($"channels/{channelId}"); var response = await GetJsonResponseAsync($"channels/{channelId}", cancellationToken);
return ChannelCategory.Parse(response); return ChannelCategory.Parse(response);
} }
// In some cases, the Discord API returns an empty body when requesting channel category. // In some cases, the Discord API returns an empty body when requesting channel category.
@ -173,20 +199,25 @@ namespace DiscordChatExporter.Core.Discord
} }
} }
public async ValueTask<Channel> GetChannelAsync(Snowflake channelId) public async ValueTask<Channel> GetChannelAsync(
Snowflake channelId,
CancellationToken cancellationToken = default)
{ {
var response = await GetJsonResponseAsync($"channels/{channelId}"); var response = await GetJsonResponseAsync($"channels/{channelId}", cancellationToken);
var parentId = response.GetPropertyOrNull("parent_id")?.GetString().Pipe(Snowflake.Parse); var parentId = response.GetPropertyOrNull("parent_id")?.GetString().Pipe(Snowflake.Parse);
var category = parentId is not null var category = parentId is not null
? await GetChannelCategoryAsync(parentId.Value) ? await GetChannelCategoryAsync(parentId.Value, cancellationToken)
: null; : null;
return Channel.Parse(response, category); return Channel.Parse(response, category);
} }
private async ValueTask<Message?> TryGetLastMessageAsync(Snowflake channelId, Snowflake? before = null) private async ValueTask<Message?> TryGetLastMessageAsync(
Snowflake channelId,
Snowflake? before = null,
CancellationToken cancellationToken = default)
{ {
var url = new UrlBuilder() var url = new UrlBuilder()
.SetPath($"channels/{channelId}/messages") .SetPath($"channels/{channelId}/messages")
@ -194,7 +225,7 @@ namespace DiscordChatExporter.Core.Discord
.SetQueryParameter("before", before?.ToString()) .SetQueryParameter("before", before?.ToString())
.Build(); .Build();
var response = await GetJsonResponseAsync(url); var response = await GetJsonResponseAsync(url, cancellationToken);
return response.EnumerateArray().Select(Message.Parse).LastOrDefault(); return response.EnumerateArray().Select(Message.Parse).LastOrDefault();
} }
@ -202,13 +233,14 @@ namespace DiscordChatExporter.Core.Discord
Snowflake channelId, Snowflake channelId,
Snowflake? after = null, Snowflake? after = null,
Snowflake? before = null, Snowflake? before = null,
IProgress<double>? progress = null) IProgress<double>? progress = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default)
{ {
// Get the last message in the specified range. // Get the last message in the specified range.
// This snapshots the boundaries, which means that messages posted after the export started // This snapshots the boundaries, which means that messages posted after the export started
// will not appear in the output. // will not appear in the output.
// Additionally, it provides the date of the last message, which is used to calculate progress. // Additionally, it provides the date of the last message, which is used to calculate progress.
var lastMessage = await TryGetLastMessageAsync(channelId, before); var lastMessage = await TryGetLastMessageAsync(channelId, before, cancellationToken);
if (lastMessage is null || lastMessage.Timestamp < after?.ToDate()) if (lastMessage is null || lastMessage.Timestamp < after?.ToDate())
yield break; yield break;
@ -224,7 +256,7 @@ namespace DiscordChatExporter.Core.Discord
.SetQueryParameter("after", currentAfter.ToString()) .SetQueryParameter("after", currentAfter.ToString())
.Build(); .Build();
var response = await GetJsonResponseAsync(url); var response = await GetJsonResponseAsync(url, cancellationToken);
var messages = response var messages = response
.EnumerateArray() .EnumerateArray()

2
DiscordChatExporter.Core/DiscordChatExporter.Core.csproj
Unescape View File

@ -6,7 +6,7 @@
<ItemGroup> <ItemGroup>
<PackageReference Include="JsonExtensions" Version="1.1.0" /> <PackageReference Include="JsonExtensions" Version="1.1.0" />
<PackageReference Include="MiniRazor.CodeGen" Version="2.1.4" /> <PackageReference Include="MiniRazor.CodeGen" Version="2.2.0" />
<PackageReference Include="Polly" Version="7.2.2" /> <PackageReference Include="Polly" Version="7.2.2" />
<PackageReference Include="Superpower" Version="3.0.0" /> <PackageReference Include="Superpower" Version="3.0.0" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" /> <PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />

29
DiscordChatExporter.Core/Exporting/ChannelExporter.cs
Unescape View File

@ -1,6 +1,7 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
@ -18,12 +19,15 @@ namespace DiscordChatExporter.Core.Exporting
public ChannelExporter(AuthToken token) : this(new DiscordClient(token)) {} public ChannelExporter(AuthToken token) : this(new DiscordClient(token)) {}
public async ValueTask ExportChannelAsync(ExportRequest request, IProgress<double>? progress = null) public async ValueTask ExportChannelAsync(
ExportRequest request,
IProgress<double>? progress = null,
CancellationToken cancellationToken = default)
{ {
// Build context // Build context
var contextMembers = new HashSet<Member>(IdBasedEqualityComparer.Instance); var contextMembers = new HashSet<Member>(IdBasedEqualityComparer.Instance);
var contextChannels = await _discord.GetGuildChannelsAsync(request.Guild.Id); var contextChannels = await _discord.GetGuildChannelsAsync(request.Guild.Id, cancellationToken);
var contextRoles = await _discord.GetGuildRolesAsync(request.Guild.Id); var contextRoles = await _discord.GetGuildRolesAsync(request.Guild.Id, cancellationToken);
var context = new ExportContext( var context = new ExportContext(
request, request,
@ -37,8 +41,16 @@ namespace DiscordChatExporter.Core.Exporting
var exportedAnything = false; var exportedAnything = false;
var encounteredUsers = new HashSet<User>(IdBasedEqualityComparer.Instance); var encounteredUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
await foreach (var message in _discord.GetMessagesAsync(request.Channel.Id, request.After, request.Before, progress))
await foreach (var message in _discord.GetMessagesAsync(
request.Channel.Id,
request.After,
request.Before,
progress,
cancellationToken))
{ {
cancellationToken.ThrowIfCancellationRequested();
// Skips any messages that fail to pass the supplied filter // Skips any messages that fail to pass the supplied filter
if (!request.MessageFilter.IsMatch(message)) if (!request.MessageFilter.IsMatch(message))
continue; continue;
@ -49,12 +61,17 @@ namespace DiscordChatExporter.Core.Exporting
if (!encounteredUsers.Add(referencedUser)) if (!encounteredUsers.Add(referencedUser))
continue; continue;
var member = await _discord.GetGuildMemberAsync(request.Guild.Id, referencedUser); var member = await _discord.GetGuildMemberAsync(
request.Guild.Id,
referencedUser,
cancellationToken
);
contextMembers.Add(member); contextMembers.Add(member);
} }
// Export message // Export message
await messageExporter.ExportMessageAsync(message); await messageExporter.ExportMessageAsync(message, cancellationToken);
exportedAnything = true; exportedAnything = true;
} }

16
DiscordChatExporter.Core/Exporting/ExportContext.cs
Unescape View File

@ -4,11 +4,11 @@ using System.Drawing;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Net.Http; using System.Net.Http;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Utils.Extensions; using DiscordChatExporter.Core.Utils.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Exporting namespace DiscordChatExporter.Core.Exporting
{ {
@ -63,14 +63,14 @@ namespace DiscordChatExporter.Core.Exporting
.FirstOrDefault(); .FirstOrDefault();
} }
public async ValueTask<string> ResolveMediaUrlAsync(string url) public async ValueTask<string> ResolveMediaUrlAsync(string url, CancellationToken cancellationToken = default)
{ {
if (!Request.ShouldDownloadMedia) if (!Request.ShouldDownloadMedia)
return url; return url;
try try
{ {
var filePath = await _mediaDownloader.DownloadAsync(url); var filePath = await _mediaDownloader.DownloadAsync(url, cancellationToken);
// We want relative path so that the output files can be copied around without breaking. // We want relative path so that the output files can be copied around without breaking.
// Base directory path may be null if the file is stored at the root or relative to working directory. // Base directory path may be null if the file is stored at the root or relative to working directory.
@ -82,10 +82,12 @@ namespace DiscordChatExporter.Core.Exporting
if (Request.Format is ExportFormat.HtmlDark or ExportFormat.HtmlLight) if (Request.Format is ExportFormat.HtmlDark or ExportFormat.HtmlLight)
{ {
// Need to escape each path segment while keeping the directory separators intact // Need to escape each path segment while keeping the directory separators intact
return relativeFilePath return string.Join(
.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) Path.AltDirectorySeparatorChar,
.Select(Uri.EscapeDataString) relativeFilePath
.JoinToString(Path.AltDirectorySeparatorChar.ToString()); .Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
.Select(Uri.EscapeDataString)
);
} }
return relativeFilePath; return relativeFilePath;

5
DiscordChatExporter.Core/Exporting/MediaDownloader.cs
Unescape View File

@ -5,6 +5,7 @@ using System.IO;
using System.Security.Cryptography; using System.Security.Cryptography;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Utils; using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions; using DiscordChatExporter.Core.Utils.Extensions;
@ -25,7 +26,7 @@ namespace DiscordChatExporter.Core.Exporting
_reuseMedia = reuseMedia; _reuseMedia = reuseMedia;
} }
public async ValueTask<string> DownloadAsync(string url) public async ValueTask<string> DownloadAsync(string url, CancellationToken cancellationToken = default)
{ {
if (_pathCache.TryGetValue(url, out var cachedFilePath)) if (_pathCache.TryGetValue(url, out var cachedFilePath))
return cachedFilePath; return cachedFilePath;
@ -43,7 +44,7 @@ namespace DiscordChatExporter.Core.Exporting
await Http.ExceptionPolicy.ExecuteAsync(async () => await Http.ExceptionPolicy.ExecuteAsync(async () =>
{ {
// Download the file // Download the file
using var response = await Http.Client.GetAsync(url); using var response = await Http.Client.GetAsync(url, cancellationToken);
await using (var output = File.Create(filePath)) await using (var output = File.Create(filePath))
{ {
await response.Content.CopyToAsync(output); await response.Content.CopyToAsync(output);

17
DiscordChatExporter.Core/Exporting/MessageExporter.cs
Unescape View File

@ -1,5 +1,6 @@
using System; using System;
using System.IO; using System.IO;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers; using DiscordChatExporter.Core.Exporting.Writers;
@ -18,23 +19,23 @@ namespace DiscordChatExporter.Core.Exporting
_context = context; _context = context;
} }
private async ValueTask ResetWriterAsync() private async ValueTask ResetWriterAsync(CancellationToken cancellationToken = default)
{ {
if (_writer is not null) if (_writer is not null)
{ {
await _writer.WritePostambleAsync(); await _writer.WritePostambleAsync(cancellationToken);
await _writer.DisposeAsync(); await _writer.DisposeAsync();
_writer = null; _writer = null;
} }
} }
private async ValueTask<MessageWriter> GetWriterAsync() private async ValueTask<MessageWriter> GetWriterAsync(CancellationToken cancellationToken = default)
{ {
// Ensure partition limit has not been reached // Ensure partition limit has not been reached
if (_writer is not null && if (_writer is not null &&
_context.Request.PartitionLimit.IsReached(_writer.MessagesWritten, _writer.BytesWritten)) _context.Request.PartitionLimit.IsReached(_writer.MessagesWritten, _writer.BytesWritten))
{ {
await ResetWriterAsync(); await ResetWriterAsync(cancellationToken);
_partitionIndex++; _partitionIndex++;
} }
@ -49,15 +50,15 @@ namespace DiscordChatExporter.Core.Exporting
Directory.CreateDirectory(dirPath); Directory.CreateDirectory(dirPath);
var writer = CreateMessageWriter(filePath, _context.Request.Format, _context); var writer = CreateMessageWriter(filePath, _context.Request.Format, _context);
await writer.WritePreambleAsync(); await writer.WritePreambleAsync(cancellationToken);
return _writer = writer; return _writer = writer;
} }
public async ValueTask ExportMessageAsync(Message message) public async ValueTask ExportMessageAsync(Message message, CancellationToken cancellationToken = default)
{ {
var writer = await GetWriterAsync(); var writer = await GetWriterAsync(cancellationToken);
await writer.WriteMessageAsync(message); await writer.WriteMessageAsync(message, cancellationToken);
} }
public async ValueTask DisposeAsync() => await ResetWriterAsync(); public async ValueTask DisposeAsync() => await ResetWriterAsync();

27
DiscordChatExporter.Core/Exporting/Writers/CsvMessageWriter.cs
Unescape View File

@ -1,6 +1,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Text; using System.Text;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors; using DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors;
@ -21,29 +22,37 @@ namespace DiscordChatExporter.Core.Exporting.Writers
private string FormatMarkdown(string? markdown) => private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? ""); PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
public override async ValueTask WritePreambleAsync() => public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions"); await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
private async ValueTask WriteAttachmentsAsync(IReadOnlyList<Attachment> attachments) private async ValueTask WriteAttachmentsAsync(
IReadOnlyList<Attachment> attachments,
CancellationToken cancellationToken = default)
{ {
var buffer = new StringBuilder(); var buffer = new StringBuilder();
foreach (var attachment in attachments) foreach (var attachment in attachments)
{ {
cancellationToken.ThrowIfCancellationRequested();
buffer buffer
.AppendIfNotEmpty(',') .AppendIfNotEmpty(',')
.Append(await Context.ResolveMediaUrlAsync(attachment.Url)); .Append(await Context.ResolveMediaUrlAsync(attachment.Url, cancellationToken));
} }
await _writer.WriteAsync(CsvEncode(buffer.ToString())); await _writer.WriteAsync(CsvEncode(buffer.ToString()));
} }
private async ValueTask WriteReactionsAsync(IReadOnlyList<Reaction> reactions) private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
{ {
var buffer = new StringBuilder(); var buffer = new StringBuilder();
foreach (var reaction in reactions) foreach (var reaction in reactions)
{ {
cancellationToken.ThrowIfCancellationRequested();
buffer buffer
.AppendIfNotEmpty(',') .AppendIfNotEmpty(',')
.Append(reaction.Emoji.Name) .Append(reaction.Emoji.Name)
@ -56,9 +65,11 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteAsync(CsvEncode(buffer.ToString())); await _writer.WriteAsync(CsvEncode(buffer.ToString()));
} }
public override async ValueTask WriteMessageAsync(Message message) public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
{ {
await base.WriteMessageAsync(message); await base.WriteMessageAsync(message, cancellationToken);
// Author ID // Author ID
await _writer.WriteAsync(CsvEncode(message.Author.Id.ToString())); await _writer.WriteAsync(CsvEncode(message.Author.Id.ToString()));
@ -77,11 +88,11 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteAsync(','); await _writer.WriteAsync(',');
// Attachments // Attachments
await WriteAttachmentsAsync(message.Attachments); await WriteAttachmentsAsync(message.Attachments, cancellationToken);
await _writer.WriteAsync(','); await _writer.WriteAsync(',');
// Reactions // Reactions
await WriteReactionsAsync(message.Reactions); await WriteReactionsAsync(message.Reactions, cancellationToken);
// Finish row // Finish row
await _writer.WriteLineAsync(); await _writer.WriteLineAsync();

2
DiscordChatExporter.Core/Exporting/Writers/Html/PreambleTemplate.cshtml
Unescape View File

@ -13,7 +13,7 @@
string FormatDate(DateTimeOffset date) => Model.ExportContext.FormatDate(date); string FormatDate(DateTimeOffset date) => Model.ExportContext.FormatDate(date);
ValueTask<string> ResolveUrlAsync(string url) => Model.ExportContext.ResolveMediaUrlAsync(url); ValueTask<string> ResolveUrlAsync(string url) => Model.ExportContext.ResolveMediaUrlAsync(url, CancellationToken);
} }
<!DOCTYPE html> <!DOCTYPE html>

30
DiscordChatExporter.Core/Exporting/Writers/HtmlMessageWriter.cs
Unescape View File

@ -1,6 +1,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Writers.Html; using DiscordChatExporter.Core.Exporting.Writers.Html;
@ -21,31 +22,35 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_themeName = themeName; _themeName = themeName;
} }
public override async ValueTask WritePreambleAsync() public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
{ {
var templateContext = new PreambleTemplateContext(Context, _themeName); var templateContext = new PreambleTemplateContext(Context, _themeName);
// We are not writing directly to output because Razor // We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream. // does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync( await _writer.WriteLineAsync(
await PreambleTemplate.RenderAsync(templateContext) await PreambleTemplate.RenderAsync(templateContext, cancellationToken)
); );
} }
private async ValueTask WriteMessageGroupAsync(MessageGroup messageGroup) private async ValueTask WriteMessageGroupAsync(
MessageGroup messageGroup,
CancellationToken cancellationToken = default)
{ {
var templateContext = new MessageGroupTemplateContext(Context, messageGroup); var templateContext = new MessageGroupTemplateContext(Context, messageGroup);
// We are not writing directly to output because Razor // We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream. // does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync( await _writer.WriteLineAsync(
await MessageGroupTemplate.RenderAsync(templateContext) await MessageGroupTemplate.RenderAsync(templateContext, cancellationToken)
); );
} }
public override async ValueTask WriteMessageAsync(Message message) public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
{ {
await base.WriteMessageAsync(message); await base.WriteMessageAsync(message, cancellationToken);
// If message group is empty or the given message can be grouped, buffer the given message // If message group is empty or the given message can be grouped, buffer the given message
if (!_messageGroupBuffer.Any() || MessageGroup.CanJoin(_messageGroupBuffer.Last(), message)) if (!_messageGroupBuffer.Any() || MessageGroup.CanJoin(_messageGroupBuffer.Last(), message))
@ -55,25 +60,30 @@ namespace DiscordChatExporter.Core.Exporting.Writers
// Otherwise, flush the group and render messages // Otherwise, flush the group and render messages
else else
{ {
await WriteMessageGroupAsync(MessageGroup.Join(_messageGroupBuffer)); await WriteMessageGroupAsync(MessageGroup.Join(_messageGroupBuffer), cancellationToken);
_messageGroupBuffer.Clear(); _messageGroupBuffer.Clear();
_messageGroupBuffer.Add(message); _messageGroupBuffer.Add(message);
} }
} }
public override async ValueTask WritePostambleAsync() public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
{ {
// Flush current message group // Flush current message group
if (_messageGroupBuffer.Any()) if (_messageGroupBuffer.Any())
await WriteMessageGroupAsync(MessageGroup.Join(_messageGroupBuffer)); {
await WriteMessageGroupAsync(
MessageGroup.Join(_messageGroupBuffer),
cancellationToken
);
}
var templateContext = new PostambleTemplateContext(Context, MessagesWritten); var templateContext = new PostambleTemplateContext(Context, MessagesWritten);
// We are not writing directly to output because Razor // We are not writing directly to output because Razor
// does not actually do asynchronous writes to stream. // does not actually do asynchronous writes to stream.
await _writer.WriteLineAsync( await _writer.WriteLineAsync(
await PostambleTemplate.RenderAsync(templateContext) await PostambleTemplate.RenderAsync(templateContext, cancellationToken)
); );
} }

105
DiscordChatExporter.Core/Exporting/Writers/JsonMessageWriter.cs
Unescape View File

@ -1,6 +1,7 @@
using System.IO; using System.IO;
using System.Text.Encodings.Web; using System.Text.Encodings.Web;
using System.Text.Json; using System.Text.Json;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Data.Embeds; using DiscordChatExporter.Core.Discord.Data.Embeds;
@ -30,20 +31,24 @@ namespace DiscordChatExporter.Core.Exporting.Writers
private string FormatMarkdown(string? markdown) => private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? ""); PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private async ValueTask WriteAttachmentAsync(Attachment attachment) private async ValueTask WriteAttachmentAsync(
Attachment attachment,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject(); _writer.WriteStartObject();
_writer.WriteString("id", attachment.Id.ToString()); _writer.WriteString("id", attachment.Id.ToString());
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(attachment.Url)); _writer.WriteString("url", await Context.ResolveMediaUrlAsync(attachment.Url, cancellationToken));
_writer.WriteString("fileName", attachment.FileName); _writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes); _writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
private async ValueTask WriteEmbedAuthorAsync(EmbedAuthor embedAuthor) private async ValueTask WriteEmbedAuthorAsync(
EmbedAuthor embedAuthor,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject("author"); _writer.WriteStartObject("author");
@ -51,54 +56,62 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteString("url", embedAuthor.Url); _writer.WriteString("url", embedAuthor.Url);
if (!string.IsNullOrWhiteSpace(embedAuthor.IconUrl)) if (!string.IsNullOrWhiteSpace(embedAuthor.IconUrl))
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl)); _writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedAuthor.IconProxyUrl ?? embedAuthor.IconUrl, cancellationToken));
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
private async ValueTask WriteEmbedThumbnailAsync(EmbedImage embedThumbnail) private async ValueTask WriteEmbedThumbnailAsync(
EmbedImage embedThumbnail,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject("thumbnail"); _writer.WriteStartObject("thumbnail");
if (!string.IsNullOrWhiteSpace(embedThumbnail.Url)) if (!string.IsNullOrWhiteSpace(embedThumbnail.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedThumbnail.ProxyUrl ?? embedThumbnail.Url)); _writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedThumbnail.ProxyUrl ?? embedThumbnail.Url, cancellationToken));
_writer.WriteNumber("width", embedThumbnail.Width); _writer.WriteNumber("width", embedThumbnail.Width);
_writer.WriteNumber("height", embedThumbnail.Height); _writer.WriteNumber("height", embedThumbnail.Height);
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
private async ValueTask WriteEmbedImageAsync(EmbedImage embedImage) private async ValueTask WriteEmbedImageAsync(
EmbedImage embedImage,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject("image"); _writer.WriteStartObject("image");
if (!string.IsNullOrWhiteSpace(embedImage.Url)) if (!string.IsNullOrWhiteSpace(embedImage.Url))
_writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedImage.ProxyUrl ?? embedImage.Url)); _writer.WriteString("url", await Context.ResolveMediaUrlAsync(embedImage.ProxyUrl ?? embedImage.Url, cancellationToken));
_writer.WriteNumber("width", embedImage.Width); _writer.WriteNumber("width", embedImage.Width);
_writer.WriteNumber("height", embedImage.Height); _writer.WriteNumber("height", embedImage.Height);
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
private async ValueTask WriteEmbedFooterAsync(EmbedFooter embedFooter) private async ValueTask WriteEmbedFooterAsync(
EmbedFooter embedFooter,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject("footer"); _writer.WriteStartObject("footer");
_writer.WriteString("text", embedFooter.Text); _writer.WriteString("text", embedFooter.Text);
if (!string.IsNullOrWhiteSpace(embedFooter.IconUrl)) if (!string.IsNullOrWhiteSpace(embedFooter.IconUrl))
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedFooter.IconProxyUrl ?? embedFooter.IconUrl)); _writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(embedFooter.IconProxyUrl ?? embedFooter.IconUrl, cancellationToken));
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
private async ValueTask WriteEmbedFieldAsync(EmbedField embedField) private async ValueTask WriteEmbedFieldAsync(
EmbedField embedField,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject(); _writer.WriteStartObject();
@ -107,10 +120,12 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteBoolean("isInline", embedField.IsInline); _writer.WriteBoolean("isInline", embedField.IsInline);
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
private async ValueTask WriteEmbedAsync(Embed embed) private async ValueTask WriteEmbedAsync(
Embed embed,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject(); _writer.WriteStartObject();
@ -123,30 +138,32 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteString("color", embed.Color.Value.ToHex()); _writer.WriteString("color", embed.Color.Value.ToHex());
if (embed.Author is not null) if (embed.Author is not null)
await WriteEmbedAuthorAsync(embed.Author); await WriteEmbedAuthorAsync(embed.Author, cancellationToken);
if (embed.Thumbnail is not null) if (embed.Thumbnail is not null)
await WriteEmbedThumbnailAsync(embed.Thumbnail); await WriteEmbedThumbnailAsync(embed.Thumbnail, cancellationToken);
if (embed.Image is not null) if (embed.Image is not null)
await WriteEmbedImageAsync(embed.Image); await WriteEmbedImageAsync(embed.Image, cancellationToken);
if (embed.Footer is not null) if (embed.Footer is not null)
await WriteEmbedFooterAsync(embed.Footer); await WriteEmbedFooterAsync(embed.Footer, cancellationToken);
// Fields // Fields
_writer.WriteStartArray("fields"); _writer.WriteStartArray("fields");
foreach (var field in embed.Fields) foreach (var field in embed.Fields)
await WriteEmbedFieldAsync(field); await WriteEmbedFieldAsync(field, cancellationToken);
_writer.WriteEndArray(); _writer.WriteEndArray();
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
private async ValueTask WriteReactionAsync(Reaction reaction) private async ValueTask WriteReactionAsync(
Reaction reaction,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject(); _writer.WriteStartObject();
@ -155,16 +172,18 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteString("id", reaction.Emoji.Id); _writer.WriteString("id", reaction.Emoji.Id);
_writer.WriteString("name", reaction.Emoji.Name); _writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated); _writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", await Context.ResolveMediaUrlAsync(reaction.Emoji.ImageUrl)); _writer.WriteString("imageUrl", await Context.ResolveMediaUrlAsync(reaction.Emoji.ImageUrl, cancellationToken));
_writer.WriteEndObject(); _writer.WriteEndObject();
_writer.WriteNumber("count", reaction.Count); _writer.WriteNumber("count", reaction.Count);
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
private async ValueTask WriteMentionAsync(User mentionedUser) private async ValueTask WriteMentionAsync(
User mentionedUser,
CancellationToken cancellationToken = default)
{ {
_writer.WriteStartObject(); _writer.WriteStartObject();
@ -175,10 +194,10 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteBoolean("isBot", mentionedUser.IsBot); _writer.WriteBoolean("isBot", mentionedUser.IsBot);
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
public override async ValueTask WritePreambleAsync() public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
{ {
// Root object (start) // Root object (start)
_writer.WriteStartObject(); _writer.WriteStartObject();
@ -187,7 +206,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteStartObject("guild"); _writer.WriteStartObject("guild");
_writer.WriteString("id", Context.Request.Guild.Id.ToString()); _writer.WriteString("id", Context.Request.Guild.Id.ToString());
_writer.WriteString("name", Context.Request.Guild.Name); _writer.WriteString("name", Context.Request.Guild.Name);
_writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(Context.Request.Guild.IconUrl)); _writer.WriteString("iconUrl", await Context.ResolveMediaUrlAsync(Context.Request.Guild.IconUrl, cancellationToken));
_writer.WriteEndObject(); _writer.WriteEndObject();
// Channel // Channel
@ -208,12 +227,14 @@ namespace DiscordChatExporter.Core.Exporting.Writers
// Message array (start) // Message array (start)
_writer.WriteStartArray("messages"); _writer.WriteStartArray("messages");
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
public override async ValueTask WriteMessageAsync(Message message) public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
{ {
await base.WriteMessageAsync(message); await base.WriteMessageAsync(message, cancellationToken);
_writer.WriteStartObject(); _writer.WriteStartObject();
@ -236,14 +257,14 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteString("nickname", Context.TryGetMember(message.Author.Id)?.Nick ?? message.Author.Name); _writer.WriteString("nickname", Context.TryGetMember(message.Author.Id)?.Nick ?? message.Author.Name);
_writer.WriteString("color", Context.TryGetUserColor(message.Author.Id)?.ToHex()); _writer.WriteString("color", Context.TryGetUserColor(message.Author.Id)?.ToHex());
_writer.WriteBoolean("isBot", message.Author.IsBot); _writer.WriteBoolean("isBot", message.Author.IsBot);
_writer.WriteString("avatarUrl", await Context.ResolveMediaUrlAsync(message.Author.AvatarUrl)); _writer.WriteString("avatarUrl", await Context.ResolveMediaUrlAsync(message.Author.AvatarUrl, cancellationToken));
_writer.WriteEndObject(); _writer.WriteEndObject();
// Attachments // Attachments
_writer.WriteStartArray("attachments"); _writer.WriteStartArray("attachments");
foreach (var attachment in message.Attachments) foreach (var attachment in message.Attachments)
await WriteAttachmentAsync(attachment); await WriteAttachmentAsync(attachment, cancellationToken);
_writer.WriteEndArray(); _writer.WriteEndArray();
@ -251,7 +272,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteStartArray("embeds"); _writer.WriteStartArray("embeds");
foreach (var embed in message.Embeds) foreach (var embed in message.Embeds)
await WriteEmbedAsync(embed); await WriteEmbedAsync(embed, cancellationToken);
_writer.WriteEndArray(); _writer.WriteEndArray();
@ -259,7 +280,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteStartArray("reactions"); _writer.WriteStartArray("reactions");
foreach (var reaction in message.Reactions) foreach (var reaction in message.Reactions)
await WriteReactionAsync(reaction); await WriteReactionAsync(reaction, cancellationToken);
_writer.WriteEndArray(); _writer.WriteEndArray();
@ -267,7 +288,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
_writer.WriteStartArray("mentions"); _writer.WriteStartArray("mentions");
foreach (var mention in message.MentionedUsers) foreach (var mention in message.MentionedUsers)
await WriteMentionAsync(mention); await WriteMentionAsync(mention, cancellationToken);
_writer.WriteEndArray(); _writer.WriteEndArray();
@ -282,10 +303,10 @@ namespace DiscordChatExporter.Core.Exporting.Writers
} }
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
public override async ValueTask WritePostambleAsync() public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
{ {
// Message array (end) // Message array (end)
_writer.WriteEndArray(); _writer.WriteEndArray();
@ -294,7 +315,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
// Root object (end) // Root object (end)
_writer.WriteEndObject(); _writer.WriteEndObject();
await _writer.FlushAsync(); await _writer.FlushAsync(cancellationToken);
} }
public override async ValueTask DisposeAsync() public override async ValueTask DisposeAsync()

7
DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs
Unescape View File

@ -1,5 +1,6 @@
using System; using System;
using System.IO; using System.IO;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
@ -21,15 +22,15 @@ namespace DiscordChatExporter.Core.Exporting.Writers
Context = context; Context = context;
} }
public virtual ValueTask WritePreambleAsync() => default; public virtual ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) => default;
public virtual ValueTask WriteMessageAsync(Message message) public virtual ValueTask WriteMessageAsync(Message message, CancellationToken cancellationToken = default)
{ {
MessagesWritten++; MessagesWritten++;
return default; return default;
} }
public virtual ValueTask WritePostambleAsync() => default; public virtual ValueTask WritePostambleAsync(CancellationToken cancellationToken = default) => default;
public virtual async ValueTask DisposeAsync() => await Stream.DisposeAsync(); public virtual async ValueTask DisposeAsync() => await Stream.DisposeAsync();
} }

43
DiscordChatExporter.Core/Exporting/Writers/PlainTextMessageWriter.cs
Unescape View File

@ -1,6 +1,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Data.Embeds; using DiscordChatExporter.Core.Discord.Data.Embeds;
@ -35,7 +36,9 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync(); await _writer.WriteLineAsync();
} }
private async ValueTask WriteAttachmentsAsync(IReadOnlyList<Attachment> attachments) private async ValueTask WriteAttachmentsAsync(
IReadOnlyList<Attachment> attachments,
CancellationToken cancellationToken = default)
{ {
if (!attachments.Any()) if (!attachments.Any())
return; return;
@ -43,15 +46,23 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync("{Attachments}"); await _writer.WriteLineAsync("{Attachments}");
foreach (var attachment in attachments) foreach (var attachment in attachments)
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(attachment.Url)); {
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(attachment.Url, cancellationToken));
}
await _writer.WriteLineAsync(); await _writer.WriteLineAsync();
} }
private async ValueTask WriteEmbedsAsync(IReadOnlyList<Embed> embeds) private async ValueTask WriteEmbedsAsync(
IReadOnlyList<Embed> embeds,
CancellationToken cancellationToken = default)
{ {
foreach (var embed in embeds) foreach (var embed in embeds)
{ {
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteLineAsync("{Embed}"); await _writer.WriteLineAsync("{Embed}");
if (!string.IsNullOrWhiteSpace(embed.Author?.Name)) if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
@ -76,10 +87,10 @@ namespace DiscordChatExporter.Core.Exporting.Writers
} }
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url)) if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Thumbnail.ProxyUrl ?? embed.Thumbnail.Url)); await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Thumbnail.ProxyUrl ?? embed.Thumbnail.Url, cancellationToken));
if (!string.IsNullOrWhiteSpace(embed.Image?.Url)) if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url)); await _writer.WriteLineAsync(await Context.ResolveMediaUrlAsync(embed.Image.ProxyUrl ?? embed.Image.Url, cancellationToken));
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text)) if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
await _writer.WriteLineAsync(embed.Footer.Text); await _writer.WriteLineAsync(embed.Footer.Text);
@ -88,7 +99,9 @@ namespace DiscordChatExporter.Core.Exporting.Writers
} }
} }
private async ValueTask WriteReactionsAsync(IReadOnlyList<Reaction> reactions) private async ValueTask WriteReactionsAsync(
IReadOnlyList<Reaction> reactions,
CancellationToken cancellationToken = default)
{ {
if (!reactions.Any()) if (!reactions.Any())
return; return;
@ -97,6 +110,8 @@ namespace DiscordChatExporter.Core.Exporting.Writers
foreach (var reaction in reactions) foreach (var reaction in reactions)
{ {
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteAsync(reaction.Emoji.Name); await _writer.WriteAsync(reaction.Emoji.Name);
if (reaction.Count > 1) if (reaction.Count > 1)
@ -108,7 +123,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync(); await _writer.WriteLineAsync();
} }
public override async ValueTask WritePreambleAsync() public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default)
{ {
await _writer.WriteLineAsync('='.Repeat(62)); await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync($"Guild: {Context.Request.Guild.Name}"); await _writer.WriteLineAsync($"Guild: {Context.Request.Guild.Name}");
@ -127,9 +142,11 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync(); await _writer.WriteLineAsync();
} }
public override async ValueTask WriteMessageAsync(Message message) public override async ValueTask WriteMessageAsync(
Message message,
CancellationToken cancellationToken = default)
{ {
await base.WriteMessageAsync(message); await base.WriteMessageAsync(message, cancellationToken);
// Header // Header
await WriteMessageHeaderAsync(message); await WriteMessageHeaderAsync(message);
@ -141,14 +158,14 @@ namespace DiscordChatExporter.Core.Exporting.Writers
await _writer.WriteLineAsync(); await _writer.WriteLineAsync();
// Attachments, embeds, reactions // Attachments, embeds, reactions
await WriteAttachmentsAsync(message.Attachments); await WriteAttachmentsAsync(message.Attachments, cancellationToken);
await WriteEmbedsAsync(message.Embeds); await WriteEmbedsAsync(message.Embeds, cancellationToken);
await WriteReactionsAsync(message.Reactions); await WriteReactionsAsync(message.Reactions, cancellationToken);
await _writer.WriteLineAsync(); await _writer.WriteLineAsync();
} }
public override async ValueTask WritePostambleAsync() public override async ValueTask WritePostambleAsync(CancellationToken cancellationToken = default)
{ {
await _writer.WriteLineAsync('='.Repeat(62)); await _writer.WriteLineAsync('='.Repeat(62));
await _writer.WriteLineAsync($"Exported {MessagesWritten:N0} message(s)"); await _writer.WriteLineAsync($"Exported {MessagesWritten:N0} message(s)");

5
DiscordChatExporter.Core/Utils/Extensions/AsyncExtensions.cs
Unescape View File

@ -27,14 +27,15 @@ namespace DiscordChatExporter.Core.Utils.Extensions
public static async ValueTask ParallelForEachAsync<T>( public static async ValueTask ParallelForEachAsync<T>(
this IEnumerable<T> source, this IEnumerable<T> source,
Func<T, ValueTask> handleAsync, Func<T, ValueTask> handleAsync,
int degreeOfParallelism) int degreeOfParallelism,
CancellationToken cancellationToken = default)
{ {
using var semaphore = new SemaphoreSlim(degreeOfParallelism); using var semaphore = new SemaphoreSlim(degreeOfParallelism);
await Task.WhenAll(source.Select(async item => await Task.WhenAll(source.Select(async item =>
{ {
// ReSharper disable once AccessToDisposedClosure // ReSharper disable once AccessToDisposedClosure
await semaphore.WaitAsync(); await semaphore.WaitAsync(cancellationToken);
try try
{ {

2
DiscordChatExporter.Gui/DiscordChatExporter.Gui.csproj
Unescape View File

@ -19,7 +19,7 @@
<PackageReference Include="Gress" Version="1.2.0" /> <PackageReference Include="Gress" Version="1.2.0" />
<PackageReference Include="MaterialDesignColors" Version="2.0.1" /> <PackageReference Include="MaterialDesignColors" Version="2.0.1" />
<PackageReference Include="MaterialDesignThemes" Version="4.0.0" /> <PackageReference Include="MaterialDesignThemes" Version="4.0.0" />
<PackageReference Include="Microsoft.Xaml.Behaviors.Wpf" Version="1.1.31" /> <PackageReference Include="Microsoft.Xaml.Behaviors.Wpf" Version="1.1.37" />
<PackageReference Include="Ookii.Dialogs.Wpf" Version="4.0.0" /> <PackageReference Include="Ookii.Dialogs.Wpf" Version="4.0.0" />
<PackageReference Include="Onova" Version="2.6.2" /> <PackageReference Include="Onova" Version="2.6.2" />
<PackageReference Include="Stylet" Version="1.3.6" /> <PackageReference Include="Stylet" Version="1.3.6" />

Write Preview
Loading…
Cancel
Save