open-source-mirrors
/
DiscordChatExporter
mirror of https://github.com/Tyrrrz/DiscordChatExporter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Include inline emoji in JSON export (#1311)

Browse Source
pull/1314/head
Oleksii Holub 2 weeks ago committed by GitHub
parent 9c15baf799
commit 789e5af8ba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 204 additions and 87 deletions
Show Stats Download Patch File Download Diff File

2
DiscordChatExporter.Cli.Tests/Infra/ChannelIds.cs
Unescape View File

@ -10,6 +10,8 @@ public static class ChannelIds
public static Snowflake EmbedTestCases { get; } = Snowflake.Parse("866472452459462687");
public static Snowflake EmojiTestCases { get; } = Snowflake.Parse("866768438290415636");
public static Snowflake GroupingTestCases { get; } = Snowflake.Parse("992092091545034842");
public static Snowflake FilterTestCases { get; } = Snowflake.Parse("866744075033641020");

69
DiscordChatExporter.Cli.Tests/Specs/JsonEmojiSpecs.cs
Unescape View File

@ -0,0 +1,69 @@
using System.Linq;
using System.Threading.Tasks;
using DiscordChatExporter.Cli.Tests.Infra;
using DiscordChatExporter.Core.Discord;
using FluentAssertions;
using Xunit;
namespace DiscordChatExporter.Cli.Tests.Specs;
public class JsonEmojiSpecs
{
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_inline_emoji_and_have_them_listed_separately()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
ChannelIds.EmojiTestCases,
Snowflake.Parse("866768521052553216")
);
// Assert
var inlineEmojis = message.GetProperty("inlineEmojis").EnumerateArray().ToArray();
inlineEmojis.Should().HaveCount(4);
inlineEmojis[0].GetProperty("id").GetString().Should().BeNullOrEmpty();
inlineEmojis[0].GetProperty("name").GetString().Should().Be("🙂");
inlineEmojis[0].GetProperty("code").GetString().Should().Be("slight_smile");
inlineEmojis[0].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
inlineEmojis[0].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
inlineEmojis[1].GetProperty("id").GetString().Should().BeNullOrEmpty();
inlineEmojis[1].GetProperty("name").GetString().Should().Be("😦");
inlineEmojis[1].GetProperty("code").GetString().Should().Be("frowning");
inlineEmojis[1].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
inlineEmojis[1].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
inlineEmojis[2].GetProperty("id").GetString().Should().BeNullOrEmpty();
inlineEmojis[2].GetProperty("name").GetString().Should().Be("😔");
inlineEmojis[2].GetProperty("code").GetString().Should().Be("pensive");
inlineEmojis[2].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
inlineEmojis[2].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
inlineEmojis[3].GetProperty("id").GetString().Should().BeNullOrEmpty();
inlineEmojis[3].GetProperty("name").GetString().Should().Be("😂");
inlineEmojis[3].GetProperty("code").GetString().Should().Be("joy");
inlineEmojis[3].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
inlineEmojis[3].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
}
[Fact]
public async Task I_can_export_a_channel_that_contains_a_message_with_custom_inline_emoji_and_have_them_listed_separately()
{
// Act
var message = await ExportWrapper.GetMessageAsJsonAsync(
ChannelIds.EmojiTestCases,
Snowflake.Parse("1299804867447230594")
);
// Assert
var inlineEmojis = message.GetProperty("inlineEmojis").EnumerateArray().ToArray();
inlineEmojis.Should().HaveCount(1);
inlineEmojis[0].GetProperty("id").GetString().Should().Be("754441880066064584");
inlineEmojis[0].GetProperty("name").GetString().Should().Be("lemon_blush");
inlineEmojis[0].GetProperty("code").GetString().Should().Be("lemon_blush");
inlineEmojis[0].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
inlineEmojis[0].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
}
}

31
DiscordChatExporter.Core/Discord/Data/Emoji.cs
Unescape View File

@ -1,5 +1,4 @@
using System;
using System.Text.Json;
using System.Text.Json;
using DiscordChatExporter.Core.Discord.Data.Common;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Utils.Extensions;
@ -13,29 +12,22 @@ public partial record Emoji(
Snowflake? Id,
// Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
string Name,
bool IsAnimated,
string ImageUrl
bool IsAnimated
)
{
public bool IsCustomEmoji { get; } = Id is not null;
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
public string Code { get; } = Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
public string ImageUrl { get; } =
Id is not null
? ImageCdn.GetCustomEmojiUrl(Id.Value, IsAnimated)
: ImageCdn.GetStandardEmojiUrl(Name);
}
public partial record Emoji
{
public static string GetImageUrl(Snowflake? id, string? name, bool isAnimated)
{
// Custom emoji
if (id is not null)
return ImageCdn.GetCustomEmojiUrl(id.Value, isAnimated);
// Standard emoji
if (!string.IsNullOrWhiteSpace(name))
return ImageCdn.GetStandardEmojiUrl(name);
throw new InvalidOperationException("Either the emoji ID or name should be provided.");
}
public static Emoji Parse(JsonElement json)
{
var id = json.GetPropertyOrNull("id")
@ -47,8 +39,7 @@ public partial record Emoji
json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji";
var isAnimated = json.GetPropertyOrNull("animated")?.GetBooleanOrNull() ?? false;
var imageUrl = GetImageUrl(id, name, isAnimated);
return new Emoji(id, name, isAnimated, imageUrl);
return new Emoji(id, name, isAnimated);
}
}

4
DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs
Unescape View File

@ -5,7 +5,6 @@ using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Parsing;
using DiscordChatExporter.Core.Utils.Extensions;
@ -210,7 +209,6 @@ internal partial class HtmlMarkdownVisitor(
CancellationToken cancellationToken = default
)
{
var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated);
var jumboClass = isJumbo ? "chatlog__emoji--large" : "";
buffer.Append(
@ -221,7 +219,7 @@ internal partial class HtmlMarkdownVisitor(
class="chatlog__emoji {jumboClass}"
alt="{emoji.Name}"
title="{emoji.Code}"
src="{await context.ResolveAssetUrlAsync(emojiImageUrl, cancellationToken)}">
src="{await context.ResolveAssetUrlAsync(emoji.ImageUrl, cancellationToken)}">
"""
);
}

118
DiscordChatExporter.Core/Exporting/JsonMessageWriter.cs
Unescape View File

@ -1,12 +1,14 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Data.Embeds;
using DiscordChatExporter.Core.Markdown.Parsing;
using DiscordChatExporter.Core.Utils.Extensions;
using JsonExtensions.Writing;
@ -37,22 +39,31 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken)
: markdown;
private async ValueTask WriteUserAsync(User user, CancellationToken cancellationToken = default)
private async ValueTask WriteUserAsync(
User user,
bool includeRoles = true,
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
_writer.WriteString("id", user.Id.ToString());
_writer.WriteString("name", user.Name);
_writer.WriteString("discriminator", user.DiscriminatorFormatted);
_writer.WriteString(
"nickname",
Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName
);
_writer.WriteString("color", Context.TryGetUserColor(user.Id)?.ToHex());
_writer.WriteBoolean("isBot", user.IsBot);
_writer.WritePropertyName("roles");
await WriteRolesAsync(Context.GetUserRoles(user.Id), cancellationToken);
if (includeRoles)
{
_writer.WritePropertyName("roles");
await WriteRolesAsync(Context.GetUserRoles(user.Id), cancellationToken);
}
_writer.WriteString(
"avatarUrl",
@ -66,6 +77,26 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteEmojiAsync(
Emoji emoji,
CancellationToken cancellationToken = default
)
{
_writer.WriteStartObject();
_writer.WriteString("id", emoji.Id.ToString());
_writer.WriteString("name", emoji.Name);
_writer.WriteString("code", emoji.Code);
_writer.WriteBoolean("isAnimated", emoji.IsAnimated);
_writer.WriteString(
"imageUrl",
await Context.ResolveAssetUrlAsync(emoji.ImageUrl, cancellationToken)
);
_writer.WriteEndObject();
await _writer.FlushAsync(cancellationToken);
}
private async ValueTask WriteRolesAsync(
IReadOnlyList<Role> roles,
CancellationToken cancellationToken = default
@ -273,6 +304,26 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
_writer.WriteEndArray();
// Inline emoji
_writer.WriteStartArray("inlineEmojis");
if (!string.IsNullOrWhiteSpace(embed.Description))
{
foreach (
var emoji in MarkdownParser
.ExtractEmojis(embed.Description)
.DistinctBy(e => e.Name, StringComparer.Ordinal)
)
{
await WriteEmojiAsync(
new Emoji(emoji.Id, emoji.Name, emoji.IsAnimated),
cancellationToken
);
}
}
_writer.WriteEndArray();
_writer.WriteEndObject();
await _writer.FlushAsync(cancellationToken);
}
@ -373,7 +424,7 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
// Author
_writer.WritePropertyName("author");
await WriteUserAsync(message.Author, cancellationToken);
await WriteUserAsync(message.Author, true, cancellationToken);
// Attachments
_writer.WriteStartArray("attachments");
@ -431,20 +482,14 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
_writer.WriteStartObject();
// Emoji
_writer.WriteStartObject("emoji");
_writer.WriteString("id", reaction.Emoji.Id.ToString());
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteString("code", reaction.Emoji.Code);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString(
"imageUrl",
await Context.ResolveAssetUrlAsync(reaction.Emoji.ImageUrl, cancellationToken)
);
_writer.WriteEndObject();
_writer.WritePropertyName("emoji");
await WriteEmojiAsync(reaction.Emoji, cancellationToken);
_writer.WriteNumber("count", reaction.Count);
// Reaction authors
_writer.WriteStartArray("users");
await foreach (
var user in Context.Discord.GetMessageReactionsAsync(
Context.Request.Channel.Id,
@ -454,28 +499,7 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
)
)
{
_writer.WriteStartObject();
// Write limited user information without color and roles,
// so we can avoid fetching guild member information for each user.
_writer.WriteString("id", user.Id.ToString());
_writer.WriteString("name", user.Name);
_writer.WriteString("discriminator", user.DiscriminatorFormatted);
_writer.WriteString(
"nickname",
Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName
);
_writer.WriteBoolean("isBot", user.IsBot);
_writer.WriteString(
"avatarUrl",
await Context.ResolveAssetUrlAsync(
Context.TryGetMember(user.Id)?.AvatarUrl ?? user.AvatarUrl,
cancellationToken
)
);
_writer.WriteEndObject();
await WriteUserAsync(user, false, cancellationToken);
}
_writer.WriteEndArray();
@ -487,9 +511,8 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
// Mentions
_writer.WriteStartArray("mentions");
foreach (var user in message.MentionedUsers)
await WriteUserAsync(user, cancellationToken);
await WriteUserAsync(user, true, cancellationToken);
_writer.WriteEndArray();
@ -512,11 +535,28 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
_writer.WriteString("name", message.Interaction.Name);
_writer.WritePropertyName("user");
await WriteUserAsync(message.Interaction.User, cancellationToken);
await WriteUserAsync(message.Interaction.User, true, cancellationToken);
_writer.WriteEndObject();
}
// Inline emoji
_writer.WriteStartArray("inlineEmojis");
foreach (
var emoji in MarkdownParser
.ExtractEmojis(message.Content)
.DistinctBy(e => e.Name, StringComparer.Ordinal)
)
{
await WriteEmojiAsync(
new Emoji(emoji.Id, emoji.Name, emoji.IsAnimated),
cancellationToken
);
}
_writer.WriteEndArray();
_writer.WriteEndObject();
await _writer.FlushAsync(cancellationToken);
}

16
DiscordChatExporter.Core/Markdown/EmojiNode.cs
Unescape View File

@ -1,5 +1,5 @@
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Utils;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Core.Markdown;
@ -11,11 +11,17 @@ internal record EmojiNode(
bool IsAnimated
) : MarkdownNode
{
public bool IsCustomEmoji => Id is not null;
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => IsCustomEmoji ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
// This coupling is unsound from the domain-design perspective, but it helps us reuse
// some code for now. We can refactor this later, if the coupling becomes a problem.
private readonly Emoji _emoji = new(Id, Name, IsAnimated);
public EmojiNode(string name)
: this(null, name, false) { }
public bool IsCustomEmoji => _emoji.IsCustomEmoji;
// Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
public string Code => _emoji.Code;
public string ImageUrl => _emoji.ImageUrl;
}

51
DiscordChatExporter.Core/Markdown/Parsing/MarkdownParser.cs
Unescape View File

@ -484,6 +484,37 @@ internal static partial class MarkdownParser
internal static partial class MarkdownParser
{
private static void Extract<TNode>(
IEnumerable<MarkdownNode> nodes,
ICollection<TNode> extractedNodes
)
where TNode : MarkdownNode
{
foreach (var node in nodes)
{
if (node is TNode extractedNode)
extractedNodes.Add(extractedNode);
if (node is IContainerNode containerNode)
Extract(containerNode.Children, extractedNodes);
}
}
public static IReadOnlyList<TNode> Extract<TNode>(string markdown)
where TNode : MarkdownNode
{
var extractedNodes = new List<TNode>();
Extract(Parse(markdown), extractedNodes);
return extractedNodes;
}
public static IReadOnlyList<LinkNode> ExtractLinks(string markdown) =>
Extract<LinkNode>(markdown);
public static IReadOnlyList<EmojiNode> ExtractEmojis(string markdown) =>
Extract<EmojiNode>(markdown);
private static IReadOnlyList<MarkdownNode> Parse(
MarkdownContext context,
StringSegment segment
@ -499,24 +530,4 @@ internal static partial class MarkdownParser
public static IReadOnlyList<MarkdownNode> ParseMinimal(string markdown) =>
ParseMinimal(new MarkdownContext(), new StringSegment(markdown));
private static void ExtractLinks(IEnumerable<MarkdownNode> nodes, ICollection<LinkNode> links)
{
foreach (var node in nodes)
{
if (node is LinkNode linkNode)
links.Add(linkNode);
if (node is IContainerNode containerNode)
ExtractLinks(containerNode.Children, links);
}
}
public static IReadOnlyList<LinkNode> ExtractLinks(string markdown)
{
var links = new List<LinkNode>();
ExtractLinks(Parse(markdown), links);
return links;
}
}

Write Preview
Loading…
Cancel
Save