From 290729158bb7b7816d22f93894e459b461bf0946 Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Tue, 14 Feb 2023 19:21:44 +0200 Subject: [PATCH] Resolve mentioned members on demand inside of markdown Closes #304 --- .../Discord/DiscordClient.cs | 11 ++-- .../Exporting/ChannelExporter.cs | 37 ++---------- .../Exporting/ExportContext.cs | 57 ++++++++++++++----- .../Exporting/HtmlMarkdownVisitor.cs | 6 ++ .../Exporting/PlainTextMarkdownVisitor.cs | 6 ++ 5 files changed, 67 insertions(+), 50 deletions(-) diff --git a/DiscordChatExporter.Core/Discord/DiscordClient.cs b/DiscordChatExporter.Core/Discord/DiscordClient.cs index ef7b946..8b7f1bc 100644 --- a/DiscordChatExporter.Core/Discord/DiscordClient.cs +++ b/DiscordChatExporter.Core/Discord/DiscordClient.cs @@ -251,16 +251,17 @@ public class DiscordClient yield return Role.Parse(roleJson); } - public async ValueTask GetGuildMemberAsync( + public async ValueTask TryGetGuildMemberAsync( Snowflake guildId, - User user, + Snowflake memberId, CancellationToken cancellationToken = default) { if (guildId == Guild.DirectMessages.Id) - return Member.CreateForUser(user); + return null; - var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{user.Id}", cancellationToken); - return response?.Pipe(Member.Parse) ?? Member.CreateForUser(user); + var response = await TryGetJsonResponseAsync($"guilds/{guildId}/members/{memberId}", cancellationToken); + + return response?.Pipe(Member.Parse); } public async ValueTask GetChannelCategoryAsync( diff --git a/DiscordChatExporter.Core/Exporting/ChannelExporter.cs b/DiscordChatExporter.Core/Exporting/ChannelExporter.cs index dd2fa4b..0ec5114 100644 --- a/DiscordChatExporter.Core/Exporting/ChannelExporter.cs +++ b/DiscordChatExporter.Core/Exporting/ChannelExporter.cs @@ -1,12 +1,9 @@ using System; -using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; using DiscordChatExporter.Core.Discord; -using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Exceptions; -using DiscordChatExporter.Core.Utils.Extensions; using Gress; namespace DiscordChatExporter.Core.Exporting; @@ -23,21 +20,8 @@ public class ChannelExporter CancellationToken cancellationToken = default) { // Build context - var contextMembers = new Dictionary(); - - var contextChannels = (await _discord.GetGuildChannelsAsync(request.Guild.Id, cancellationToken)) - .ToDictionary(c => c.Id); - - var contextRoles = (await _discord.GetGuildRolesAsync(request.Guild.Id, cancellationToken)) - .ToDictionary(r => r.Id); - - var context = new ExportContext( - _discord, - request, - contextMembers, - contextChannels, - contextRoles - ); + var context = new ExportContext(_discord, request); + await context.PopulateChannelsAndRolesAsync(cancellationToken); // Export messages await using var messageExporter = new MessageExporter(context); @@ -49,20 +33,9 @@ public class ChannelExporter progress, cancellationToken)) { - // Resolve members for referenced users - foreach (var referencedUser in message.MentionedUsers.Prepend(message.Author)) - { - if (contextMembers.ContainsKey(referencedUser.Id)) - continue; - - var member = await _discord.GetGuildMemberAsync( - request.Guild.Id, - referencedUser, - cancellationToken - ); - - contextMembers[member.Id] = member; - } + // Resolve members for the author and mentioned users + foreach (var user in message.MentionedUsers.Prepend(message.Author)) + await context.PopulateMemberAsync(user.Id, cancellationToken); // Export the message if (request.MessageFilter.IsMatch(message)) diff --git a/DiscordChatExporter.Core/Exporting/ExportContext.cs b/DiscordChatExporter.Core/Exporting/ExportContext.cs index 7b84e63..39ed360 100644 --- a/DiscordChatExporter.Core/Exporting/ExportContext.cs +++ b/DiscordChatExporter.Core/Exporting/ExportContext.cs @@ -12,17 +12,48 @@ using DiscordChatExporter.Core.Utils.Extensions; namespace DiscordChatExporter.Core.Exporting; -internal record ExportContext( - DiscordClient Discord, - ExportRequest Request, - IReadOnlyDictionary Members, - IReadOnlyDictionary Channels, - IReadOnlyDictionary Roles) +internal class ExportContext { - private readonly ExportAssetDownloader _assetDownloader = new( - Request.OutputAssetsDirPath, - Request.ShouldReuseAssets - ); + private readonly Dictionary _members = new(); + private readonly Dictionary _channels = new(); + private readonly Dictionary _roles = new(); + private readonly ExportAssetDownloader _assetDownloader; + + public DiscordClient Discord { get; } + public ExportRequest Request { get; } + + public ExportContext(DiscordClient discord, + ExportRequest request) + { + Discord = discord; + Request = request; + + _assetDownloader = new ExportAssetDownloader( + request.OutputAssetsDirPath, + request.ShouldReuseAssets + ); + } + + public async ValueTask PopulateChannelsAndRolesAsync(CancellationToken cancellationToken = default) + { + await foreach (var channel in Discord.GetGuildChannelsAsync(Request.Guild.Id, cancellationToken)) + _channels[channel.Id] = channel; + + await foreach (var role in Discord.GetGuildRolesAsync(Request.Guild.Id, cancellationToken)) + _roles[role.Id] = role; + } + + // Because members are not pulled in bulk, we need to populate them on demand + public async ValueTask PopulateMemberAsync(Snowflake id, CancellationToken cancellationToken = default) + { + if (_members.ContainsKey(id)) + return; + + var member = await Discord.TryGetGuildMemberAsync(Request.Guild.Id, id, cancellationToken); + + // Store the result even if it's null, to avoid re-fetching non-existing members + _members[id] = member; + } public string FormatDate(DateTimeOffset instant) => Request.DateFormat switch { @@ -31,11 +62,11 @@ internal record ExportContext( var format => instant.ToLocalString(format) }; - public Member? TryGetMember(Snowflake id) => Members.GetValueOrDefault(id); + public Member? TryGetMember(Snowflake id) => _members.GetValueOrDefault(id); - public Channel? TryGetChannel(Snowflake id) => Channels.GetValueOrDefault(id); + public Channel? TryGetChannel(Snowflake id) => _channels.GetValueOrDefault(id); - public Role? TryGetRole(Snowflake id) => Roles.GetValueOrDefault(id); + public Role? TryGetRole(Snowflake id) => _roles.GetValueOrDefault(id); public Color? TryGetUserColor(Snowflake id) { diff --git a/DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs b/DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs index edafa6b..005567b 100644 --- a/DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs +++ b/DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs @@ -195,6 +195,12 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor } else if (mention.Kind == MentionKind.User) { + // User mentions are not always included in the message object, + // which means they need to be populated on demand. + // https://github.com/Tyrrrz/DiscordChatExporter/issues/304 + if (mention.TargetId is not null) + await _context.PopulateMemberAsync(mention.TargetId.Value, cancellationToken); + var member = mention.TargetId?.Pipe(_context.TryGetMember); var fullName = member?.User.FullName ?? "Unknown"; var nick = member?.Nick ?? "Unknown"; diff --git a/DiscordChatExporter.Core/Exporting/PlainTextMarkdownVisitor.cs b/DiscordChatExporter.Core/Exporting/PlainTextMarkdownVisitor.cs index 463ce0a..6b4accc 100644 --- a/DiscordChatExporter.Core/Exporting/PlainTextMarkdownVisitor.cs +++ b/DiscordChatExporter.Core/Exporting/PlainTextMarkdownVisitor.cs @@ -53,6 +53,12 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor } else if (mention.Kind == MentionKind.User) { + // User mentions are not always included in the message object, + // which means they need to be populated on demand. + // https://github.com/Tyrrrz/DiscordChatExporter/issues/304 + if (mention.TargetId is not null) + await _context.PopulateMemberAsync(mention.TargetId.Value, cancellationToken); + var member = mention.TargetId?.Pipe(_context.TryGetMember); var name = member?.User.Name ?? "Unknown";