From 789e5af8ba413edf194bbecef55f5e36174d233b Mon Sep 17 00:00:00 2001 From: Oleksii Holub <1935960+Tyrrrz@users.noreply.github.com> Date: Wed, 6 Nov 2024 19:23:40 +0200 Subject: [PATCH] Include inline emoji in JSON export (#1311) --- .../Infra/ChannelIds.cs | 2 + .../Specs/JsonEmojiSpecs.cs | 69 ++++++++++ .../Discord/Data/Emoji.cs | 31 ++--- .../Exporting/HtmlMarkdownVisitor.cs | 4 +- .../Exporting/JsonMessageWriter.cs | 118 ++++++++++++------ .../Markdown/EmojiNode.cs | 16 ++- .../Markdown/Parsing/MarkdownParser.cs | 51 +++++--- 7 files changed, 204 insertions(+), 87 deletions(-) create mode 100644 DiscordChatExporter.Cli.Tests/Specs/JsonEmojiSpecs.cs diff --git a/DiscordChatExporter.Cli.Tests/Infra/ChannelIds.cs b/DiscordChatExporter.Cli.Tests/Infra/ChannelIds.cs index f48f9b0..b3f92fe 100644 --- a/DiscordChatExporter.Cli.Tests/Infra/ChannelIds.cs +++ b/DiscordChatExporter.Cli.Tests/Infra/ChannelIds.cs @@ -10,6 +10,8 @@ public static class ChannelIds public static Snowflake EmbedTestCases { get; } = Snowflake.Parse("866472452459462687"); + public static Snowflake EmojiTestCases { get; } = Snowflake.Parse("866768438290415636"); + public static Snowflake GroupingTestCases { get; } = Snowflake.Parse("992092091545034842"); public static Snowflake FilterTestCases { get; } = Snowflake.Parse("866744075033641020"); diff --git a/DiscordChatExporter.Cli.Tests/Specs/JsonEmojiSpecs.cs b/DiscordChatExporter.Cli.Tests/Specs/JsonEmojiSpecs.cs new file mode 100644 index 0000000..7ffd71a --- /dev/null +++ b/DiscordChatExporter.Cli.Tests/Specs/JsonEmojiSpecs.cs @@ -0,0 +1,69 @@ +using System.Linq; +using System.Threading.Tasks; +using DiscordChatExporter.Cli.Tests.Infra; +using DiscordChatExporter.Core.Discord; +using FluentAssertions; +using Xunit; + +namespace DiscordChatExporter.Cli.Tests.Specs; + +public class JsonEmojiSpecs +{ + [Fact] + public async Task I_can_export_a_channel_that_contains_a_message_with_inline_emoji_and_have_them_listed_separately() + { + // Act + var message = await ExportWrapper.GetMessageAsJsonAsync( + ChannelIds.EmojiTestCases, + Snowflake.Parse("866768521052553216") + ); + + // Assert + var inlineEmojis = message.GetProperty("inlineEmojis").EnumerateArray().ToArray(); + inlineEmojis.Should().HaveCount(4); + + inlineEmojis[0].GetProperty("id").GetString().Should().BeNullOrEmpty(); + inlineEmojis[0].GetProperty("name").GetString().Should().Be("🙂"); + inlineEmojis[0].GetProperty("code").GetString().Should().Be("slight_smile"); + inlineEmojis[0].GetProperty("isAnimated").GetBoolean().Should().BeFalse(); + inlineEmojis[0].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace(); + + inlineEmojis[1].GetProperty("id").GetString().Should().BeNullOrEmpty(); + inlineEmojis[1].GetProperty("name").GetString().Should().Be("😦"); + inlineEmojis[1].GetProperty("code").GetString().Should().Be("frowning"); + inlineEmojis[1].GetProperty("isAnimated").GetBoolean().Should().BeFalse(); + inlineEmojis[1].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace(); + + inlineEmojis[2].GetProperty("id").GetString().Should().BeNullOrEmpty(); + inlineEmojis[2].GetProperty("name").GetString().Should().Be("😔"); + inlineEmojis[2].GetProperty("code").GetString().Should().Be("pensive"); + inlineEmojis[2].GetProperty("isAnimated").GetBoolean().Should().BeFalse(); + inlineEmojis[2].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace(); + + inlineEmojis[3].GetProperty("id").GetString().Should().BeNullOrEmpty(); + inlineEmojis[3].GetProperty("name").GetString().Should().Be("😂"); + inlineEmojis[3].GetProperty("code").GetString().Should().Be("joy"); + inlineEmojis[3].GetProperty("isAnimated").GetBoolean().Should().BeFalse(); + inlineEmojis[3].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace(); + } + + [Fact] + public async Task I_can_export_a_channel_that_contains_a_message_with_custom_inline_emoji_and_have_them_listed_separately() + { + // Act + var message = await ExportWrapper.GetMessageAsJsonAsync( + ChannelIds.EmojiTestCases, + Snowflake.Parse("1299804867447230594") + ); + + // Assert + var inlineEmojis = message.GetProperty("inlineEmojis").EnumerateArray().ToArray(); + inlineEmojis.Should().HaveCount(1); + + inlineEmojis[0].GetProperty("id").GetString().Should().Be("754441880066064584"); + inlineEmojis[0].GetProperty("name").GetString().Should().Be("lemon_blush"); + inlineEmojis[0].GetProperty("code").GetString().Should().Be("lemon_blush"); + inlineEmojis[0].GetProperty("isAnimated").GetBoolean().Should().BeFalse(); + inlineEmojis[0].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace(); + } +} diff --git a/DiscordChatExporter.Core/Discord/Data/Emoji.cs b/DiscordChatExporter.Core/Discord/Data/Emoji.cs index 838a371..31aafff 100644 --- a/DiscordChatExporter.Core/Discord/Data/Emoji.cs +++ b/DiscordChatExporter.Core/Discord/Data/Emoji.cs @@ -1,5 +1,4 @@ -using System; -using System.Text.Json; +using System.Text.Json; using DiscordChatExporter.Core.Discord.Data.Common; using DiscordChatExporter.Core.Utils; using DiscordChatExporter.Core.Utils.Extensions; @@ -13,29 +12,22 @@ public partial record Emoji( Snowflake? Id, // Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂) string Name, - bool IsAnimated, - string ImageUrl + bool IsAnimated ) { + public bool IsCustomEmoji { get; } = Id is not null; + // Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile) - public string Code => Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name; + public string Code { get; } = Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name; + + public string ImageUrl { get; } = + Id is not null + ? ImageCdn.GetCustomEmojiUrl(Id.Value, IsAnimated) + : ImageCdn.GetStandardEmojiUrl(Name); } public partial record Emoji { - public static string GetImageUrl(Snowflake? id, string? name, bool isAnimated) - { - // Custom emoji - if (id is not null) - return ImageCdn.GetCustomEmojiUrl(id.Value, isAnimated); - - // Standard emoji - if (!string.IsNullOrWhiteSpace(name)) - return ImageCdn.GetStandardEmojiUrl(name); - - throw new InvalidOperationException("Either the emoji ID or name should be provided."); - } - public static Emoji Parse(JsonElement json) { var id = json.GetPropertyOrNull("id") @@ -47,8 +39,7 @@ public partial record Emoji json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji"; var isAnimated = json.GetPropertyOrNull("animated")?.GetBooleanOrNull() ?? false; - var imageUrl = GetImageUrl(id, name, isAnimated); - return new Emoji(id, name, isAnimated, imageUrl); + return new Emoji(id, name, isAnimated); } } diff --git a/DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs b/DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs index fa11d97..8445d9b 100644 --- a/DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs +++ b/DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs @@ -5,7 +5,6 @@ using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Markdown; using DiscordChatExporter.Core.Markdown.Parsing; using DiscordChatExporter.Core.Utils.Extensions; @@ -210,7 +209,6 @@ internal partial class HtmlMarkdownVisitor( CancellationToken cancellationToken = default ) { - var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated); var jumboClass = isJumbo ? "chatlog__emoji--large" : ""; buffer.Append( @@ -221,7 +219,7 @@ internal partial class HtmlMarkdownVisitor( class="chatlog__emoji {jumboClass}" alt="{emoji.Name}" title="{emoji.Code}" - src="{await context.ResolveAssetUrlAsync(emojiImageUrl, cancellationToken)}"> + src="{await context.ResolveAssetUrlAsync(emoji.ImageUrl, cancellationToken)}"> """ ); } diff --git a/DiscordChatExporter.Core/Exporting/JsonMessageWriter.cs b/DiscordChatExporter.Core/Exporting/JsonMessageWriter.cs index 60ba01f..78a42b9 100644 --- a/DiscordChatExporter.Core/Exporting/JsonMessageWriter.cs +++ b/DiscordChatExporter.Core/Exporting/JsonMessageWriter.cs @@ -1,12 +1,14 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.Encodings.Web; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Discord.Data.Embeds; +using DiscordChatExporter.Core.Markdown.Parsing; using DiscordChatExporter.Core.Utils.Extensions; using JsonExtensions.Writing; @@ -37,22 +39,31 @@ internal class JsonMessageWriter(Stream stream, ExportContext context) ? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken) : markdown; - private async ValueTask WriteUserAsync(User user, CancellationToken cancellationToken = default) + private async ValueTask WriteUserAsync( + User user, + bool includeRoles = true, + CancellationToken cancellationToken = default + ) { _writer.WriteStartObject(); _writer.WriteString("id", user.Id.ToString()); _writer.WriteString("name", user.Name); _writer.WriteString("discriminator", user.DiscriminatorFormatted); + _writer.WriteString( "nickname", Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName ); + _writer.WriteString("color", Context.TryGetUserColor(user.Id)?.ToHex()); _writer.WriteBoolean("isBot", user.IsBot); - _writer.WritePropertyName("roles"); - await WriteRolesAsync(Context.GetUserRoles(user.Id), cancellationToken); + if (includeRoles) + { + _writer.WritePropertyName("roles"); + await WriteRolesAsync(Context.GetUserRoles(user.Id), cancellationToken); + } _writer.WriteString( "avatarUrl", @@ -66,6 +77,26 @@ internal class JsonMessageWriter(Stream stream, ExportContext context) await _writer.FlushAsync(cancellationToken); } + private async ValueTask WriteEmojiAsync( + Emoji emoji, + CancellationToken cancellationToken = default + ) + { + _writer.WriteStartObject(); + + _writer.WriteString("id", emoji.Id.ToString()); + _writer.WriteString("name", emoji.Name); + _writer.WriteString("code", emoji.Code); + _writer.WriteBoolean("isAnimated", emoji.IsAnimated); + _writer.WriteString( + "imageUrl", + await Context.ResolveAssetUrlAsync(emoji.ImageUrl, cancellationToken) + ); + + _writer.WriteEndObject(); + await _writer.FlushAsync(cancellationToken); + } + private async ValueTask WriteRolesAsync( IReadOnlyList roles, CancellationToken cancellationToken = default @@ -273,6 +304,26 @@ internal class JsonMessageWriter(Stream stream, ExportContext context) _writer.WriteEndArray(); + // Inline emoji + _writer.WriteStartArray("inlineEmojis"); + + if (!string.IsNullOrWhiteSpace(embed.Description)) + { + foreach ( + var emoji in MarkdownParser + .ExtractEmojis(embed.Description) + .DistinctBy(e => e.Name, StringComparer.Ordinal) + ) + { + await WriteEmojiAsync( + new Emoji(emoji.Id, emoji.Name, emoji.IsAnimated), + cancellationToken + ); + } + } + + _writer.WriteEndArray(); + _writer.WriteEndObject(); await _writer.FlushAsync(cancellationToken); } @@ -373,7 +424,7 @@ internal class JsonMessageWriter(Stream stream, ExportContext context) // Author _writer.WritePropertyName("author"); - await WriteUserAsync(message.Author, cancellationToken); + await WriteUserAsync(message.Author, true, cancellationToken); // Attachments _writer.WriteStartArray("attachments"); @@ -431,20 +482,14 @@ internal class JsonMessageWriter(Stream stream, ExportContext context) _writer.WriteStartObject(); // Emoji - _writer.WriteStartObject("emoji"); - _writer.WriteString("id", reaction.Emoji.Id.ToString()); - _writer.WriteString("name", reaction.Emoji.Name); - _writer.WriteString("code", reaction.Emoji.Code); - _writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated); - _writer.WriteString( - "imageUrl", - await Context.ResolveAssetUrlAsync(reaction.Emoji.ImageUrl, cancellationToken) - ); - _writer.WriteEndObject(); + _writer.WritePropertyName("emoji"); + await WriteEmojiAsync(reaction.Emoji, cancellationToken); _writer.WriteNumber("count", reaction.Count); + // Reaction authors _writer.WriteStartArray("users"); + await foreach ( var user in Context.Discord.GetMessageReactionsAsync( Context.Request.Channel.Id, @@ -454,28 +499,7 @@ internal class JsonMessageWriter(Stream stream, ExportContext context) ) ) { - _writer.WriteStartObject(); - - // Write limited user information without color and roles, - // so we can avoid fetching guild member information for each user. - _writer.WriteString("id", user.Id.ToString()); - _writer.WriteString("name", user.Name); - _writer.WriteString("discriminator", user.DiscriminatorFormatted); - _writer.WriteString( - "nickname", - Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName - ); - _writer.WriteBoolean("isBot", user.IsBot); - - _writer.WriteString( - "avatarUrl", - await Context.ResolveAssetUrlAsync( - Context.TryGetMember(user.Id)?.AvatarUrl ?? user.AvatarUrl, - cancellationToken - ) - ); - - _writer.WriteEndObject(); + await WriteUserAsync(user, false, cancellationToken); } _writer.WriteEndArray(); @@ -487,9 +511,8 @@ internal class JsonMessageWriter(Stream stream, ExportContext context) // Mentions _writer.WriteStartArray("mentions"); - foreach (var user in message.MentionedUsers) - await WriteUserAsync(user, cancellationToken); + await WriteUserAsync(user, true, cancellationToken); _writer.WriteEndArray(); @@ -512,11 +535,28 @@ internal class JsonMessageWriter(Stream stream, ExportContext context) _writer.WriteString("name", message.Interaction.Name); _writer.WritePropertyName("user"); - await WriteUserAsync(message.Interaction.User, cancellationToken); + await WriteUserAsync(message.Interaction.User, true, cancellationToken); _writer.WriteEndObject(); } + // Inline emoji + _writer.WriteStartArray("inlineEmojis"); + + foreach ( + var emoji in MarkdownParser + .ExtractEmojis(message.Content) + .DistinctBy(e => e.Name, StringComparer.Ordinal) + ) + { + await WriteEmojiAsync( + new Emoji(emoji.Id, emoji.Name, emoji.IsAnimated), + cancellationToken + ); + } + + _writer.WriteEndArray(); + _writer.WriteEndObject(); await _writer.FlushAsync(cancellationToken); } diff --git a/DiscordChatExporter.Core/Markdown/EmojiNode.cs b/DiscordChatExporter.Core/Markdown/EmojiNode.cs index cc2d2cc..d397063 100644 --- a/DiscordChatExporter.Core/Markdown/EmojiNode.cs +++ b/DiscordChatExporter.Core/Markdown/EmojiNode.cs @@ -1,5 +1,5 @@ using DiscordChatExporter.Core.Discord; -using DiscordChatExporter.Core.Utils; +using DiscordChatExporter.Core.Discord.Data; namespace DiscordChatExporter.Core.Markdown; @@ -11,11 +11,17 @@ internal record EmojiNode( bool IsAnimated ) : MarkdownNode { - public bool IsCustomEmoji => Id is not null; - - // Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile) - public string Code => IsCustomEmoji ? Name : EmojiIndex.TryGetCode(Name) ?? Name; + // This coupling is unsound from the domain-design perspective, but it helps us reuse + // some code for now. We can refactor this later, if the coupling becomes a problem. + private readonly Emoji _emoji = new(Id, Name, IsAnimated); public EmojiNode(string name) : this(null, name, false) { } + + public bool IsCustomEmoji => _emoji.IsCustomEmoji; + + // Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile) + public string Code => _emoji.Code; + + public string ImageUrl => _emoji.ImageUrl; } diff --git a/DiscordChatExporter.Core/Markdown/Parsing/MarkdownParser.cs b/DiscordChatExporter.Core/Markdown/Parsing/MarkdownParser.cs index db67c4f..9ff0946 100644 --- a/DiscordChatExporter.Core/Markdown/Parsing/MarkdownParser.cs +++ b/DiscordChatExporter.Core/Markdown/Parsing/MarkdownParser.cs @@ -484,6 +484,37 @@ internal static partial class MarkdownParser internal static partial class MarkdownParser { + private static void Extract( + IEnumerable nodes, + ICollection extractedNodes + ) + where TNode : MarkdownNode + { + foreach (var node in nodes) + { + if (node is TNode extractedNode) + extractedNodes.Add(extractedNode); + + if (node is IContainerNode containerNode) + Extract(containerNode.Children, extractedNodes); + } + } + + public static IReadOnlyList Extract(string markdown) + where TNode : MarkdownNode + { + var extractedNodes = new List(); + Extract(Parse(markdown), extractedNodes); + + return extractedNodes; + } + + public static IReadOnlyList ExtractLinks(string markdown) => + Extract(markdown); + + public static IReadOnlyList ExtractEmojis(string markdown) => + Extract(markdown); + private static IReadOnlyList Parse( MarkdownContext context, StringSegment segment @@ -499,24 +530,4 @@ internal static partial class MarkdownParser public static IReadOnlyList ParseMinimal(string markdown) => ParseMinimal(new MarkdownContext(), new StringSegment(markdown)); - - private static void ExtractLinks(IEnumerable nodes, ICollection links) - { - foreach (var node in nodes) - { - if (node is LinkNode linkNode) - links.Add(linkNode); - - if (node is IContainerNode containerNode) - ExtractLinks(containerNode.Children, links); - } - } - - public static IReadOnlyList ExtractLinks(string markdown) - { - var links = new List(); - ExtractLinks(Parse(markdown), links); - - return links; - } }