open-source-mirrors
/
DiscordChatExporter
mirror of https://github.com/Tyrrrz/DiscordChatExporter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pass cancellation token to markdown visitor

Browse Source
pull/965/head
Oleksii Holub 2 years ago
parent e752269467
commit b672c30071
8 changed files with 186 additions and 75 deletions
Show Stats Download Patch File Download Diff File

8
DiscordChatExporter.Core/Exporting/Writers/CsvMessageWriter.cs
Unescape View File

@ -19,8 +19,10 @@ internal partial class CsvMessageWriter : MessageWriter
_writer = new StreamWriter(stream);
}
private ValueTask<string> FormatMarkdownAsync(string? markdown) =>
PlainTextMarkdownVisitor.FormatAsync(Context, markdown ?? "");
private ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken);
public override async ValueTask WritePreambleAsync(CancellationToken cancellationToken = default) =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
@ -84,7 +86,7 @@ internal partial class CsvMessageWriter : MessageWriter
await _writer.WriteAsync(',');
// Message content
await _writer.WriteAsync(CsvEncode(await FormatMarkdownAsync(message.Content)));
await _writer.WriteAsync(CsvEncode(await FormatMarkdownAsync(message.Content, cancellationToken)));
await _writer.WriteAsync(',');
// Attachments

4
DiscordChatExporter.Core/Exporting/Writers/Html/MessageGroupTemplate.cshtml
Unescape View File

@ -30,10 +30,10 @@
ExportContext.FormatDate(date);
ValueTask<string> FormatMarkdownAsync(string markdown) =>
HtmlMarkdownVisitor.FormatAsync(ExportContext, markdown);
HtmlMarkdownVisitor.FormatAsync(ExportContext, markdown, true, CancellationToken);
ValueTask<string> FormatEmbedMarkdownAsync(string markdown) =>
HtmlMarkdownVisitor.FormatAsync(ExportContext, markdown, false);
HtmlMarkdownVisitor.FormatAsync(ExportContext, markdown, false, CancellationToken);
var firstMessage = Messages.First();

2
DiscordChatExporter.Core/Exporting/Writers/Html/PreambleTemplate.cshtml
Unescape View File

@ -32,7 +32,7 @@
ExportContext.FormatDate(date);
ValueTask<string> FormatMarkdownAsync(string markdown) =>
HtmlMarkdownVisitor.FormatAsync(ExportContext, markdown);
HtmlMarkdownVisitor.FormatAsync(ExportContext, markdown, true, CancellationToken);
}
<!DOCTYPE html>

16
DiscordChatExporter.Core/Exporting/Writers/JsonMessageWriter.cs
Unescape View File

@ -29,8 +29,10 @@ internal class JsonMessageWriter : MessageWriter
});
}
private ValueTask<string> FormatMarkdownAsync(string? markdown) =>
PlainTextMarkdownVisitor.FormatAsync(Context, markdown ?? "");
private ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken);
private async ValueTask WriteAttachmentAsync(
Attachment attachment,
@ -115,8 +117,8 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteStartObject();
_writer.WriteString("name", await FormatMarkdownAsync(embedField.Name));
_writer.WriteString("value", await FormatMarkdownAsync(embedField.Value));
_writer.WriteString("name", await FormatMarkdownAsync(embedField.Name, cancellationToken));
_writer.WriteString("value", await FormatMarkdownAsync(embedField.Value, cancellationToken));
_writer.WriteBoolean("isInline", embedField.IsInline);
_writer.WriteEndObject();
@ -129,10 +131,10 @@ internal class JsonMessageWriter : MessageWriter
{
_writer.WriteStartObject();
_writer.WriteString("title", await FormatMarkdownAsync(embed.Title));
_writer.WriteString("title", await FormatMarkdownAsync(embed.Title ?? "", cancellationToken));
_writer.WriteString("url", embed.Url);
_writer.WriteString("timestamp", embed.Timestamp);
_writer.WriteString("description", await FormatMarkdownAsync(embed.Description));
_writer.WriteString("description", await FormatMarkdownAsync(embed.Description ?? "", cancellationToken));
if (embed.Color is not null)
_writer.WriteString("color", embed.Color.Value.ToHex());
@ -283,7 +285,7 @@ internal class JsonMessageWriter : MessageWriter
_writer.WriteBoolean("isPinned", message.IsPinned);
// Content
_writer.WriteString("content", await FormatMarkdownAsync(message.Content));
_writer.WriteString("content", await FormatMarkdownAsync(message.Content, cancellationToken));
// Author
_writer.WriteStartObject("author");

57
DiscordChatExporter.Core/Exporting/Writers/MarkdownVisitors/HtmlMarkdownVisitor.cs
Unescape View File

@ -3,6 +3,7 @@ using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Markdown;
@ -24,13 +25,17 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
_isJumbo = isJumbo;
}
protected override async ValueTask<MarkdownNode> VisitTextAsync(TextNode text)
protected override async ValueTask<MarkdownNode> VisitTextAsync(
TextNode text,
CancellationToken cancellationToken = default)
{
_buffer.Append(HtmlEncode(text.Text));
return await base.VisitTextAsync(text);
return await base.VisitTextAsync(text, cancellationToken);
}
protected override async ValueTask<MarkdownNode> VisitFormattingAsync(FormattingNode formatting)
protected override async ValueTask<MarkdownNode> VisitFormattingAsync(
FormattingNode formatting,
CancellationToken cancellationToken = default)
{
var (openingTag, closingTag) = formatting.Kind switch
{
@ -68,23 +73,27 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
};
_buffer.Append(openingTag);
var result = await base.VisitFormattingAsync(formatting);
var result = await base.VisitFormattingAsync(formatting, cancellationToken);
_buffer.Append(closingTag);
return result;
}
protected override async ValueTask<MarkdownNode> VisitInlineCodeBlockAsync(InlineCodeBlockNode inlineCodeBlock)
protected override async ValueTask<MarkdownNode> VisitInlineCodeBlockAsync(
InlineCodeBlockNode inlineCodeBlock,
CancellationToken cancellationToken = default)
{
_buffer
.Append("<code class=\"chatlog__markdown-pre chatlog__markdown-pre--inline\">")
.Append(HtmlEncode(inlineCodeBlock.Code))
.Append("</code>");
return await base.VisitInlineCodeBlockAsync(inlineCodeBlock);
return await base.VisitInlineCodeBlockAsync(inlineCodeBlock, cancellationToken);
}
protected override async ValueTask<MarkdownNode> VisitMultiLineCodeBlockAsync(MultiLineCodeBlockNode multiLineCodeBlock)
protected override async ValueTask<MarkdownNode> VisitMultiLineCodeBlockAsync(
MultiLineCodeBlockNode multiLineCodeBlock,
CancellationToken cancellationToken = default)
{
var highlightCssClass = !string.IsNullOrWhiteSpace(multiLineCodeBlock.Language)
? $"language-{multiLineCodeBlock.Language}"
@ -95,10 +104,12 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
.Append(HtmlEncode(multiLineCodeBlock.Code))
.Append("</code>");
return await base.VisitMultiLineCodeBlockAsync(multiLineCodeBlock);
return await base.VisitMultiLineCodeBlockAsync(multiLineCodeBlock, cancellationToken);
}
protected override async ValueTask<MarkdownNode> VisitLinkAsync(LinkNode link)
protected override async ValueTask<MarkdownNode> VisitLinkAsync(
LinkNode link,
CancellationToken cancellationToken = default)
{
// Try to extract message ID if the link refers to a Discord message
var linkedMessageId = Regex.Match(
@ -112,13 +123,15 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
: $"<a href=\"{HtmlEncode(link.Url)}\">"
);
var result = await base.VisitLinkAsync(link);
var result = await base.VisitLinkAsync(link, cancellationToken);
_buffer.Append("</a>");
return result;
}
protected override async ValueTask<MarkdownNode> VisitEmojiAsync(EmojiNode emoji)
protected override async ValueTask<MarkdownNode> VisitEmojiAsync(
EmojiNode emoji,
CancellationToken cancellationToken = default)
{
var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated);
var jumboClass = _isJumbo ? "chatlog__emoji--large" : "";
@ -129,14 +142,16 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
$"class=\"chatlog__emoji {jumboClass}\" " +
$"alt=\"{emoji.Name}\" " +
$"title=\"{emoji.Code}\" " +
$"src=\"{await _context.ResolveAssetUrlAsync(emojiImageUrl)}\"" +
$"src=\"{await _context.ResolveAssetUrlAsync(emojiImageUrl, cancellationToken)}\"" +
$">"
);
return await base.VisitEmojiAsync(emoji);
return await base.VisitEmojiAsync(emoji, cancellationToken);
}
protected override async ValueTask<MarkdownNode> VisitMentionAsync(MentionNode mention)
protected override async ValueTask<MarkdownNode> VisitMentionAsync(
MentionNode mention,
CancellationToken cancellationToken = default)
{
if (mention.Kind == MentionKind.Everyone)
{
@ -191,10 +206,12 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
.Append("</span>");
}
return await base.VisitMentionAsync(mention);
return await base.VisitMentionAsync(mention, cancellationToken);
}
protected override async ValueTask<MarkdownNode> VisitUnixTimestampAsync(UnixTimestampNode timestamp)
protected override async ValueTask<MarkdownNode> VisitUnixTimestampAsync(
UnixTimestampNode timestamp,
CancellationToken cancellationToken = default)
{
var dateString = timestamp.Date is not null
? _context.FormatDate(timestamp.Date.Value)
@ -210,7 +227,7 @@ internal partial class HtmlMarkdownVisitor : MarkdownVisitor
.Append(HtmlEncode(dateString))
.Append("</span>");
return await base.VisitUnixTimestampAsync(timestamp);
return await base.VisitUnixTimestampAsync(timestamp, cancellationToken);
}
}
@ -221,7 +238,8 @@ internal partial class HtmlMarkdownVisitor
public static async ValueTask<string> FormatAsync(
ExportContext context,
string markdown,
bool isJumboAllowed = true)
bool isJumboAllowed = true,
CancellationToken cancellationToken = default)
{
var nodes = MarkdownParser.Parse(markdown);
@ -231,7 +249,8 @@ internal partial class HtmlMarkdownVisitor
var buffer = new StringBuilder();
await new HtmlMarkdownVisitor(context, buffer, isJumbo).VisitAsync(nodes);
await new HtmlMarkdownVisitor(context, buffer, isJumbo)
.VisitAsync(nodes, cancellationToken);
return buffer.ToString();
}

33
DiscordChatExporter.Core/Exporting/Writers/MarkdownVisitors/PlainTextMarkdownVisitor.cs
Unescape View File

@ -1,4 +1,5 @@
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Parsing;
@ -17,13 +18,17 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
_buffer = buffer;
}
protected override async ValueTask<MarkdownNode> VisitTextAsync(TextNode text)
protected override async ValueTask<MarkdownNode> VisitTextAsync(
TextNode text,
CancellationToken cancellationToken = default)
{
_buffer.Append(text.Text);
return await base.VisitTextAsync(text);
return await base.VisitTextAsync(text, cancellationToken);
}
protected override async ValueTask<MarkdownNode> VisitEmojiAsync(EmojiNode emoji)
protected override async ValueTask<MarkdownNode> VisitEmojiAsync(
EmojiNode emoji,
CancellationToken cancellationToken = default)
{
_buffer.Append(
emoji.IsCustomEmoji
@ -31,10 +36,12 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
: emoji.Name
);
return await base.VisitEmojiAsync(emoji);
return await base.VisitEmojiAsync(emoji, cancellationToken);
}
protected override async ValueTask<MarkdownNode> VisitMentionAsync(MentionNode mention)
protected override async ValueTask<MarkdownNode> VisitMentionAsync(
MentionNode mention,
CancellationToken cancellationToken = default)
{
if (mention.Kind == MentionKind.Everyone)
{
@ -70,10 +77,12 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
_buffer.Append($"@{name}");
}
return await base.VisitMentionAsync(mention);
return await base.VisitMentionAsync(mention, cancellationToken);
}
protected override async ValueTask<MarkdownNode> VisitUnixTimestampAsync(UnixTimestampNode timestamp)
protected override async ValueTask<MarkdownNode> VisitUnixTimestampAsync(
UnixTimestampNode timestamp,
CancellationToken cancellationToken = default)
{
_buffer.Append(
timestamp.Date is not null
@ -81,18 +90,22 @@ internal partial class PlainTextMarkdownVisitor : MarkdownVisitor
: "Invalid date"
);
return await base.VisitUnixTimestampAsync(timestamp);
return await base.VisitUnixTimestampAsync(timestamp, cancellationToken);
}
}
internal partial class PlainTextMarkdownVisitor
{
public static async ValueTask<string> FormatAsync(ExportContext context, string markdown)
public static async ValueTask<string> FormatAsync(
ExportContext context,
string markdown,
CancellationToken cancellationToken = default)
{
var nodes = MarkdownParser.ParseMinimal(markdown);
var buffer = new StringBuilder();
await new PlainTextMarkdownVisitor(context, buffer).VisitAsync(nodes);
await new PlainTextMarkdownVisitor(context, buffer)
.VisitAsync(nodes, cancellationToken);
return buffer.ToString();
}

54
DiscordChatExporter.Core/Exporting/Writers/PlainTextMessageWriter.cs
Unescape View File

@ -19,8 +19,10 @@ internal class PlainTextMessageWriter : MessageWriter
_writer = new StreamWriter(stream);
}
private ValueTask<string> FormatMarkdownAsync(string? markdown) =>
PlainTextMarkdownVisitor.FormatAsync(Context, markdown ?? "");
private ValueTask<string> FormatMarkdownAsync(
string markdown,
CancellationToken cancellationToken = default) =>
PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken);
private async ValueTask WriteMessageHeaderAsync(Message message)
{
@ -48,7 +50,9 @@ internal class PlainTextMessageWriter : MessageWriter
{
cancellationToken.ThrowIfCancellationRequested();
await _writer.WriteLineAsync(await Context.ResolveAssetUrlAsync(attachment.Url, cancellationToken));
await _writer.WriteLineAsync(
await Context.ResolveAssetUrlAsync(attachment.Url, cancellationToken)
);
}
await _writer.WriteLineAsync();
@ -65,24 +69,44 @@ internal class PlainTextMessageWriter : MessageWriter
await _writer.WriteLineAsync("{Embed}");
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
{
await _writer.WriteLineAsync(embed.Author.Name);
}
if (!string.IsNullOrWhiteSpace(embed.Url))
{
await _writer.WriteLineAsync(embed.Url);
}
if (!string.IsNullOrWhiteSpace(embed.Title))
await _writer.WriteLineAsync(await FormatMarkdownAsync(embed.Title));
{
await _writer.WriteLineAsync(
await FormatMarkdownAsync(embed.Title, cancellationToken)
);
}
if (!string.IsNullOrWhiteSpace(embed.Description))
await _writer.WriteLineAsync(await FormatMarkdownAsync(embed.Description));
{
await _writer.WriteLineAsync(
await FormatMarkdownAsync(embed.Description, cancellationToken)
);
}
foreach (var field in embed.Fields)
{
if (!string.IsNullOrWhiteSpace(field.Name))
await _writer.WriteLineAsync(await FormatMarkdownAsync(field.Name));
{
await _writer.WriteLineAsync(
await FormatMarkdownAsync(field.Name, cancellationToken)
);
}
if (!string.IsNullOrWhiteSpace(field.Value))
await _writer.WriteLineAsync(await FormatMarkdownAsync(field.Value));
{
await _writer.WriteLineAsync(
await FormatMarkdownAsync(field.Value, cancellationToken)
);
}
}
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
@ -109,7 +133,9 @@ internal class PlainTextMessageWriter : MessageWriter
}
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
{
await _writer.WriteLineAsync(embed.Footer.Text);
}
await _writer.WriteLineAsync();
}
@ -152,7 +178,9 @@ internal class PlainTextMessageWriter : MessageWriter
await _writer.WriteAsync(reaction.Emoji.Name);
if (reaction.Count > 1)
{
await _writer.WriteAsync($" ({reaction.Count})");
}
await _writer.WriteAsync(' ');
}
@ -167,13 +195,19 @@ internal class PlainTextMessageWriter : MessageWriter
await _writer.WriteLineAsync($"Channel: {Context.Request.Channel.Category.Name} / {Context.Request.Channel.Name}");
if (!string.IsNullOrWhiteSpace(Context.Request.Channel.Topic))
{
await _writer.WriteLineAsync($"Topic: {Context.Request.Channel.Topic}");
}
if (Context.Request.After is not null)
{
await _writer.WriteLineAsync($"After: {Context.FormatDate(Context.Request.After.Value.ToDate())}");
}
if (Context.Request.Before is not null)
{
await _writer.WriteLineAsync($"Before: {Context.FormatDate(Context.Request.Before.Value.ToDate())}");
}
await _writer.WriteLineAsync(new string('=', 62));
await _writer.WriteLineAsync();
@ -190,7 +224,11 @@ internal class PlainTextMessageWriter : MessageWriter
// Content
if (!string.IsNullOrWhiteSpace(message.Content))
await _writer.WriteLineAsync(await FormatMarkdownAsync(message.Content));
{
await _writer.WriteLineAsync(
await FormatMarkdownAsync(message.Content, cancellationToken)
);
}
await _writer.WriteLineAsync();

87
DiscordChatExporter.Core/Markdown/Parsing/MarkdownVisitor.cs
Unescape View File

@ -1,57 +1,94 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace DiscordChatExporter.Core.Markdown.Parsing;
internal abstract class MarkdownVisitor
{
protected virtual ValueTask<MarkdownNode> VisitTextAsync(TextNode text) =>
protected virtual ValueTask<MarkdownNode> VisitTextAsync(
TextNode text,
CancellationToken cancellationToken = default) =>
new(text);
protected virtual async ValueTask<MarkdownNode> VisitFormattingAsync(FormattingNode formatting)
protected virtual async ValueTask<MarkdownNode> VisitFormattingAsync(
FormattingNode formatting,
CancellationToken cancellationToken = default)
{
await VisitAsync(formatting.Children);
await VisitAsync(formatting.Children, cancellationToken);
return formatting;
}
protected virtual ValueTask<MarkdownNode> VisitInlineCodeBlockAsync(InlineCodeBlockNode inlineCodeBlock) =>
protected virtual ValueTask<MarkdownNode> VisitInlineCodeBlockAsync(
InlineCodeBlockNode inlineCodeBlock,
CancellationToken cancellationToken = default) =>
new(inlineCodeBlock);
protected virtual ValueTask<MarkdownNode> VisitMultiLineCodeBlockAsync(MultiLineCodeBlockNode multiLineCodeBlock) =>
protected virtual ValueTask<MarkdownNode> VisitMultiLineCodeBlockAsync(
MultiLineCodeBlockNode multiLineCodeBlock,
CancellationToken cancellationToken = default) =>
new(multiLineCodeBlock);
protected virtual async ValueTask<MarkdownNode> VisitLinkAsync(LinkNode link)
protected virtual async ValueTask<MarkdownNode> VisitLinkAsync(
LinkNode link,
CancellationToken cancellationToken = default)
{
await VisitAsync(link.Children);
await VisitAsync(link.Children, cancellationToken);
return link;
}
protected virtual ValueTask<MarkdownNode> VisitEmojiAsync(EmojiNode emoji) =>
protected virtual ValueTask<MarkdownNode> VisitEmojiAsync(
EmojiNode emoji,
CancellationToken cancellationToken = default) =>
new(emoji);
protected virtual ValueTask<MarkdownNode> VisitMentionAsync(MentionNode mention) =>
protected virtual ValueTask<MarkdownNode> VisitMentionAsync(
MentionNode mention,
CancellationToken cancellationToken = default) =>
new(mention);
protected virtual ValueTask<MarkdownNode> VisitUnixTimestampAsync(UnixTimestampNode timestamp) =>
protected virtual ValueTask<MarkdownNode> VisitUnixTimestampAsync(
UnixTimestampNode timestamp,
CancellationToken cancellationToken = default) =>
new(timestamp);
public async ValueTask<MarkdownNode> VisitAsync(MarkdownNode node) => node switch
{
TextNode text => await VisitTextAsync(text),
FormattingNode formatting => await VisitFormattingAsync(formatting),
InlineCodeBlockNode inlineCodeBlock => await VisitInlineCodeBlockAsync(inlineCodeBlock),
MultiLineCodeBlockNode multiLineCodeBlock => await VisitMultiLineCodeBlockAsync(multiLineCodeBlock),
LinkNode link => await VisitLinkAsync(link),
EmojiNode emoji => await VisitEmojiAsync(emoji),
MentionNode mention => await VisitMentionAsync(mention),
UnixTimestampNode timestamp => await VisitUnixTimestampAsync(timestamp),
_ => throw new ArgumentOutOfRangeException(nameof(node))
};
public async ValueTask VisitAsync(IEnumerable<MarkdownNode> nodes)
public async ValueTask<MarkdownNode> VisitAsync(
MarkdownNode node,
CancellationToken cancellationToken = default) => node switch
{
TextNode text =>
await VisitTextAsync(text, cancellationToken),
FormattingNode formatting =>
await VisitFormattingAsync(formatting, cancellationToken),
InlineCodeBlockNode inlineCodeBlock =>
await VisitInlineCodeBlockAsync(inlineCodeBlock, cancellationToken),
MultiLineCodeBlockNode multiLineCodeBlock =>
await VisitMultiLineCodeBlockAsync(multiLineCodeBlock, cancellationToken),
LinkNode link =>
await VisitLinkAsync(link, cancellationToken),
EmojiNode emoji =>
await VisitEmojiAsync(emoji, cancellationToken),
MentionNode mention =>
await VisitMentionAsync(mention, cancellationToken),
UnixTimestampNode timestamp =>
await VisitUnixTimestampAsync(timestamp, cancellationToken),
_ => throw new ArgumentOutOfRangeException(nameof(node))
};
public async ValueTask VisitAsync(
IEnumerable<MarkdownNode> nodes,
CancellationToken cancellationToken = default)
{
foreach (var node in nodes)
await VisitAsync(node);
await VisitAsync(node, cancellationToken);
}
}
Write Preview
Loading…
Cancel
Save