Optimize fetching threads (#1125)

pull/1130/head
Adam Slatinský 1 year ago committed by GitHub
parent d24550f521
commit 9583e2684d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -76,6 +76,8 @@ public class ExportAllCommand : ExportCommandBase
var thread in Discord.GetGuildThreadsAsync( var thread in Discord.GetGuildThreadsAsync(
guild.Id, guild.Id,
ThreadInclusionMode == ThreadInclusionMode.All, ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken cancellationToken
) )
) )

@ -54,6 +54,8 @@ public class ExportGuildCommand : ExportCommandBase
var thread in Discord.GetGuildThreadsAsync( var thread in Discord.GetGuildThreadsAsync(
GuildId, GuildId,
ThreadInclusionMode == ThreadInclusionMode.All, ThreadInclusionMode == ThreadInclusionMode.All,
Before,
After,
cancellationToken cancellationToken
) )
) )

@ -52,6 +52,8 @@ public class GetChannelsCommand : DiscordCommandBase
await Discord.GetGuildThreadsAsync( await Discord.GetGuildThreadsAsync(
GuildId, GuildId,
ThreadInclusionMode == ThreadInclusionMode.All, ThreadInclusionMode == ThreadInclusionMode.All,
null,
null,
cancellationToken cancellationToken
) )
) )

@ -277,6 +277,8 @@ public class DiscordClient
public async IAsyncEnumerable<Channel> GetGuildThreadsAsync( public async IAsyncEnumerable<Channel> GetGuildThreadsAsync(
Snowflake guildId, Snowflake guildId,
bool includeArchived = false, bool includeArchived = false,
Snowflake? before = null,
Snowflake? after = null,
[EnumeratorCancellation] CancellationToken cancellationToken = default [EnumeratorCancellation] CancellationToken cancellationToken = default
) )
{ {
@ -286,17 +288,29 @@ public class DiscordClient
var tokenKind = _resolvedTokenKind ??= await GetTokenKindAsync(cancellationToken); var tokenKind = _resolvedTokenKind ??= await GetTokenKindAsync(cancellationToken);
var channels = await GetGuildChannelsAsync(guildId, cancellationToken); var channels = await GetGuildChannelsAsync(guildId, cancellationToken);
var filteredChannels = channels
// Categories cannot have threads
.Where(c => c.Kind != ChannelKind.GuildCategory)
// Voice channels cannot have threads
.Where(c => !c.Kind.IsVoice())
// Ordinary channel or forum channel without LastMessageId cannot have threads
.Where(c => c.LastMessageId != null)
// Ff --before is specified, skip channels created after the specified date
.Where(c => before == null || before > c.Id);
// User accounts can only fetch threads using the search endpoint // User accounts can only fetch threads using the search endpoint
if (tokenKind == TokenKind.User) if (tokenKind == TokenKind.User)
{ {
// Active threads // Active threads
foreach (var channel in channels) foreach (var channel in filteredChannels)
{ {
var currentOffset = 0; var currentOffset = 0;
while (true) while (true)
{ {
var url = new UrlBuilder() var url = new UrlBuilder()
.SetPath($"channels/{channel.Id}/threads/search") .SetPath($"channels/{channel.Id}/threads/search")
.SetQueryParameter("sort_by", "last_message_time")
.SetQueryParameter("sort_order", "desc")
.SetQueryParameter("archived", "false") .SetQueryParameter("archived", "false")
.SetQueryParameter("offset", currentOffset.ToString()) .SetQueryParameter("offset", currentOffset.ToString())
.Build(); .Build();
@ -306,14 +320,29 @@ public class DiscordClient
if (response is null) if (response is null)
break; break;
var containsOlder = false;
foreach ( foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray() var threadJson in response.Value.GetProperty("threads").EnumerateArray()
) )
{ {
yield return Channel.Parse(threadJson, channel); var thread = Channel.Parse(threadJson, channel);
// if --after is specified, we can break early, because the threads are sorted by last message time
if (after is not null && after > thread.LastMessageId)
{
containsOlder = true;
break;
}
yield return thread;
currentOffset++; currentOffset++;
} }
if (containsOlder)
break;
if (!response.Value.GetProperty("has_more").GetBoolean()) if (!response.Value.GetProperty("has_more").GetBoolean())
break; break;
} }
@ -322,13 +351,15 @@ public class DiscordClient
// Archived threads // Archived threads
if (includeArchived) if (includeArchived)
{ {
foreach (var channel in channels) foreach (var channel in filteredChannels)
{ {
var currentOffset = 0; var currentOffset = 0;
while (true) while (true)
{ {
var url = new UrlBuilder() var url = new UrlBuilder()
.SetPath($"channels/{channel.Id}/threads/search") .SetPath($"channels/{channel.Id}/threads/search")
.SetQueryParameter("sort_by", "last_message_time")
.SetQueryParameter("sort_order", "desc")
.SetQueryParameter("archived", "true") .SetQueryParameter("archived", "true")
.SetQueryParameter("offset", currentOffset.ToString()) .SetQueryParameter("offset", currentOffset.ToString())
.Build(); .Build();
@ -338,14 +369,29 @@ public class DiscordClient
if (response is null) if (response is null)
break; break;
var containsOlder = false;
foreach ( foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray() var threadJson in response.Value.GetProperty("threads").EnumerateArray()
) )
{ {
yield return Channel.Parse(threadJson, channel); var thread = Channel.Parse(threadJson, channel);
// if --after is specified, we can break early, because the threads are sorted by last message time
if (after is not null && after > thread.LastMessageId)
{
containsOlder = true;
break;
}
yield return thread;
currentOffset++; currentOffset++;
} }
if (containsOlder)
break;
if (!response.Value.GetProperty("has_more").GetBoolean()) if (!response.Value.GetProperty("has_more").GetBoolean())
break; break;
} }
@ -357,7 +403,7 @@ public class DiscordClient
{ {
// Active threads // Active threads
{ {
var parentsById = channels.ToDictionary(c => c.Id); var parentsById = filteredChannels.ToDictionary(c => c.Id);
var response = await GetJsonResponseAsync( var response = await GetJsonResponseAsync(
$"guilds/{guildId}/threads/active", $"guilds/{guildId}/threads/active",
@ -379,7 +425,7 @@ public class DiscordClient
// Archived threads // Archived threads
if (includeArchived) if (includeArchived)
{ {
foreach (var channel in channels) foreach (var channel in filteredChannels)
{ {
// Public archived threads // Public archived threads
{ {

@ -2,6 +2,7 @@
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions; using DiscordChatExporter.Core.Exceptions;
using Gress; using Gress;
@ -33,6 +34,20 @@ public class ChannelExporter
); );
} }
// Check if the 'before' boundary is valid
if (request.Before is not null && request.Channel.Id > request.Before)
{
throw new DiscordChatExporterException(
"Channel does not contain any messages within the specified period."
);
}
// Skip forum channels, they are exported as threads
if (request.Channel.Kind == ChannelKind.GuildForum)
{
throw new DiscordChatExporterException("Channel is a forum.");
}
// Build context // Build context
var context = new ExportContext(_discord, request); var context = new ExportContext(_discord, request);
await context.PopulateChannelsAndRolesAsync(cancellationToken); await context.PopulateChannelsAndRolesAsync(cancellationToken);

Loading…
Cancel
Save