pull/678/head
Tyrrrz 3 years ago
parent e1726683f8
commit 650c55bbd1

@ -33,11 +33,11 @@ namespace DiscordChatExporter.Cli.Commands.Base
[CommandOption("before", Description = "Only include messages sent before this date or message ID.")]
public Snowflake? Before { get; init; }
[CommandOption("partition", 'p', Description = "Split output into partitions, each limited to this number of messages (e.g. 100) or file size (e.g. 10mb).")]
public PartitionLimit PartitionLimit { get; init; } = NullPartitionLimit.Instance;
[CommandOption("partition", 'p', Description = "Split output into partitions, each limited to this number of messages (e.g. '100') or file size (e.g. '10mb').")]
public PartitionLimit PartitionLimit { get; init; } = PartitionLimit.Null;
[CommandOption("filter", Description = "Only include messages that satisfy this filter (e.g. from:foo#1234).")]
public MessageFilter MessageFilter { get; init; } = NullMessageFilter.Instance;
[CommandOption("filter", Description = "Only include messages that satisfy this filter (e.g. 'from:foo#1234' or 'has:image').")]
public MessageFilter MessageFilter { get; init; } = MessageFilter.Null;
[CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")]
public int ParallelLimit { get; init; } = 1;
@ -133,8 +133,6 @@ namespace DiscordChatExporter.Cli.Commands.Base
{
throw new CommandException("Export failed.");
}
await console.Output.WriteLineAsync("Done.");
}
public override ValueTask ExecuteAsync(IConsole console)

@ -6,8 +6,8 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="CliFx" Version="2.0.5" />
<PackageReference Include="Spectre.Console" Version="0.40.0" />
<PackageReference Include="CliFx" Version="2.0.6" />
<PackageReference Include="Spectre.Console" Version="0.41.0" />
<PackageReference Include="Gress" Version="1.2.0" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
</ItemGroup>

@ -1,6 +1,6 @@
namespace DiscordChatExporter.Core.Exporting.Filtering
{
public enum BinaryExpressionKind
internal enum BinaryExpressionKind
{
Or,
And

@ -1,9 +1,9 @@
using DiscordChatExporter.Core.Discord.Data;
using System;
using System;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Core.Exporting.Filtering
{
public class BinaryExpressionMessageFilter : MessageFilter
internal class BinaryExpressionMessageFilter : MessageFilter
{
private readonly MessageFilter _first;
private readonly MessageFilter _second;
@ -19,8 +19,8 @@ namespace DiscordChatExporter.Core.Exporting.Filtering
public override bool Filter(Message message) => _kind switch
{
BinaryExpressionKind.Or => _first.Filter(message) || _second.Filter(message),
BinaryExpressionKind.And => _first.Filter(message) && _second.Filter(message),
BinaryExpressionKind.And => _first.Filter(message) && _second.Filter(message),
_ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.")
};
}
}
}

@ -1,15 +1,18 @@
using DiscordChatExporter.Core.Discord.Data;
using System.Text.RegularExpressions;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Core.Exporting.Filtering
{
public class ContainsMessageFilter : MessageFilter
internal class ContainsMessageFilter : MessageFilter
{
private readonly string _value;
private readonly string _text;
public ContainsMessageFilter(string value) => _value = value;
public ContainsMessageFilter(string text) => _text = text;
public override bool Filter(Message message) =>
Regex.IsMatch(message.Content, $@"\b{Regex.Escape(_value)}\b", RegexOptions.IgnoreCase | DefaultRegexOptions);
public override bool Filter(Message message) => Regex.IsMatch(
message.Content,
"\\b" + _text + "\\b",
RegexOptions.IgnoreCase | RegexOptions.CultureInvariant
);
}
}

@ -1,9 +1,9 @@
using DiscordChatExporter.Core.Discord.Data;
using System;
using System;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Core.Exporting.Filtering
{
public class FromMessageFilter : MessageFilter
internal class FromMessageFilter : MessageFilter
{
private readonly string _value;
@ -14,4 +14,4 @@ namespace DiscordChatExporter.Core.Exporting.Filtering
string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase);
}
}
}

@ -1,26 +1,25 @@
using DiscordChatExporter.Core.Discord.Data;
using System;
using System;
using System.Linq;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Core.Exporting.Filtering
{
public class HasMessageFilter : MessageFilter
internal class HasMessageFilter : MessageFilter
{
private readonly string _value;
private readonly MessageContentMatchKind _kind;
public HasMessageFilter(string value) => _value = value;
public HasMessageFilter(MessageContentMatchKind kind) => _kind = kind;
public override bool Filter(Message message) =>
_value switch
{
"link" => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]", DefaultRegexOptions),
"embed" => message.Embeds.Any(),
"file" => message.Attachments.Any(),
"video" => message.Attachments.Any(file => file.IsVideo),
"image" => message.Attachments.Any(file => file.IsImage),
"sound" => message.Attachments.Any(file => file.IsAudio),
_ => throw new InvalidOperationException($"Invalid value provided for the 'has' message filter: '{_value}'")
};
public override bool Filter(Message message) => _kind switch
{
MessageContentMatchKind.Link => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]"),
MessageContentMatchKind.Embed => message.Embeds.Any(),
MessageContentMatchKind.File => message.Attachments.Any(),
MessageContentMatchKind.Video => message.Attachments.Any(file => file.IsVideo),
MessageContentMatchKind.Image => message.Attachments.Any(file => file.IsImage),
MessageContentMatchKind.Sound => message.Attachments.Any(file => file.IsAudio),
_ => throw new InvalidOperationException($"Unknown message content match kind '{_kind}'.")
};
}
}
}

@ -1,19 +1,19 @@
using DiscordChatExporter.Core.Discord.Data;
using System;
using System;
using System.Linq;
using DiscordChatExporter.Core.Discord.Data;
namespace DiscordChatExporter.Core.Exporting.Filtering
{
public class MentionsMessageFilter : MessageFilter
internal class MentionsMessageFilter : MessageFilter
{
private readonly string _value;
public MentionsMessageFilter(string value) => _value = value;
public override bool Filter(Message message) =>
message.MentionedUsers.Any(user =>
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase));
public override bool Filter(Message message) => message.MentionedUsers.Any(user =>
string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) ||
string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)
);
}
}

@ -0,0 +1,12 @@
namespace DiscordChatExporter.Core.Exporting.Filtering
{
internal enum MessageContentMatchKind
{
Link,
Embed,
File,
Video,
Image,
Sound
}
}

@ -1,6 +1,4 @@
using System;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exporting.Filtering.Parsing;
using Superpower;
@ -13,26 +11,8 @@ namespace DiscordChatExporter.Core.Exporting.Filtering
public partial class MessageFilter
{
protected const RegexOptions DefaultRegexOptions = RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.Multiline;
public static MessageFilter Null { get; } = new NullMessageFilter();
internal static MessageFilter CreateFilter(string text) => new ContainsMessageFilter(text);
internal static MessageFilter CreateFilter(string key, string value)
{
return key.ToLowerInvariant() switch
{
"from" => new FromMessageFilter(value),
"has" => new HasMessageFilter(value),
"mentions" => new MentionsMessageFilter(value),
_ => throw new ArgumentException($"Invalid filter type '{key}'.", nameof(key))
};
}
public static MessageFilter Parse(string value, IFormatProvider? formatProvider = null)
{
var tokens = FilterTokenizer.Instance.Tokenize(value);
var parsed = FilterParser.Instance.Parse(tokens);
return parsed;
}
public static MessageFilter Parse(string value) => FilterGrammar.Filter.Parse(value);
}
}
}

@ -2,7 +2,7 @@
namespace DiscordChatExporter.Core.Exporting.Filtering
{
public class NegatedMessageFilter : MessageFilter
internal class NegatedMessageFilter : MessageFilter
{
private readonly MessageFilter _filter;

@ -2,10 +2,8 @@
namespace DiscordChatExporter.Core.Exporting.Filtering
{
public class NullMessageFilter : MessageFilter
internal class NullMessageFilter : MessageFilter
{
public static NullMessageFilter Instance { get; } = new();
public override bool Filter(Message message) => true;
}
}

@ -0,0 +1,106 @@
using System.Linq;
using DiscordChatExporter.Core.Utils.Extensions;
using Superpower;
using Superpower.Parsers;
namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
{
internal static class FilterGrammar
{
// Choice(a, b) looks cleaner than a.Or(b)
private static TextParser<T> Choice<T>(params TextParser<T>[] parsers) =>
parsers.Aggregate((current, next) => current.Or(next));
private static readonly TextParser<char> EscapedCharacter =
Character.EqualTo('\\').IgnoreThen(Character.AnyChar);
private static readonly TextParser<string> QuotedString =
from open in Character.In('"', '\'')
from value in Choice(EscapedCharacter, Character.Except(open)).Many().Text()
from close in Character.EqualTo(open)
select value;
private static readonly TextParser<char> FreeCharacter =
Character.Matching(c =>
!char.IsWhiteSpace(c) &&
// Avoid all special tokens used by the grammar
c is not ('(' or ')' or '"' or '\'' or '-' or '|' or '&'),
"any character except whitespace or `(`, `)`, `\"`, `'`, `-`, `|`, `&`"
);
private static readonly TextParser<string> UnquotedString =
Choice(EscapedCharacter, FreeCharacter).AtLeastOnce().Text();
private static readonly TextParser<string> String =
Choice(QuotedString, UnquotedString).Named("text string");
private static readonly TextParser<MessageFilter> ContainsFilter =
String.Select(v => (MessageFilter) new ContainsMessageFilter(v));
private static readonly TextParser<MessageFilter> FromFilter = Span
.EqualToIgnoreCase("from:")
.IgnoreThen(String)
.Select(v => (MessageFilter) new FromMessageFilter(v))
.Named("from:<value>");
private static readonly TextParser<MessageFilter> MentionsFilter = Span
.EqualToIgnoreCase("mentions:")
.IgnoreThen(String)
.Select(v => (MessageFilter) new MentionsMessageFilter(v))
.Named("mentions:<value>");
private static readonly TextParser<MessageFilter> HasFilter = Span
.EqualToIgnoreCase("has:")
.IgnoreThen(Choice(
Span.EqualToIgnoreCase("link").IgnoreThen(Parse.Return(MessageContentMatchKind.Link)),
Span.EqualToIgnoreCase("embed").IgnoreThen(Parse.Return(MessageContentMatchKind.Embed)),
Span.EqualToIgnoreCase("video").IgnoreThen(Parse.Return(MessageContentMatchKind.Video)),
Span.EqualToIgnoreCase("image").IgnoreThen(Parse.Return(MessageContentMatchKind.Image)),
Span.EqualToIgnoreCase("sound").IgnoreThen(Parse.Return(MessageContentMatchKind.Sound))
))
.Select(k => (MessageFilter) new HasMessageFilter(k))
.Named("has:<value>");
private static readonly TextParser<MessageFilter> NegatedFilter = Character
.EqualTo('-')
.IgnoreThen(Parse.Ref(() => StandaloneFilter))
.Select(f => (MessageFilter) new NegatedMessageFilter(f));
private static readonly TextParser<MessageFilter> GroupedFilter =
from open in Character.EqualTo('(')
from content in Parse.Ref(() => BinaryExpressionFilter).Token()
from close in Character.EqualTo(')')
select content;
private static readonly TextParser<MessageFilter> StandaloneFilter = Choice(
GroupedFilter,
FromFilter,
MentionsFilter,
HasFilter,
ContainsFilter
);
private static readonly TextParser<MessageFilter> UnaryExpressionFilter = Choice(
NegatedFilter,
StandaloneFilter
);
private static readonly TextParser<MessageFilter> BinaryExpressionFilter = Parse.Chain(
Choice(
// Explicit operator
Character.In('|', '&').Token().Try(),
// Implicit operator (resolves to 'and')
Character.WhiteSpace.AtLeastOnce().IgnoreThen(Parse.Return(' '))
),
UnaryExpressionFilter,
(op, left, right) => op switch
{
'|' => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.Or),
_ => new BinaryExpressionMessageFilter(left, right, BinaryExpressionKind.And)
}
);
public static readonly TextParser<MessageFilter> Filter =
BinaryExpressionFilter.Token().AtEnd();
}
}

@ -1,68 +0,0 @@
using Superpower;
using Superpower.Model;
using Superpower.Parsers;
using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using System.Text;
namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
{
public static class FilterParser
{
public static TextParser<string> QuotedString { get; } =
from open in Character.EqualTo('"')
from content in Character.EqualTo('\\').IgnoreThen(Character.AnyChar).Try()
.Or(Character.Except('"'))
.Many()
from close in Character.EqualTo('"')
select new string(content);
public static TextParser<string> UnquotedString { get; } =
from content in Character.EqualTo('\\').IgnoreThen(Character.In('"', '/')).Try()
.Or(Character.Except(c => char.IsWhiteSpace(c) || "():-|\"".Contains(c), "non-whitespace character except for (, ), :, -, |, and \""))
.AtLeastOnce()
select new string(content);
public static TokenListParser<FilterToken, string> AnyString { get; } =
Token.EqualTo(FilterToken.QuotedString).Apply(QuotedString)
.Or(Token.EqualTo(FilterToken.UnquotedString).Apply(UnquotedString));
public static TokenListParser<FilterToken, MessageFilter> AnyFilter { get; } =
from minus in Token.EqualTo(FilterToken.Minus).Optional()
from content in KeyValueFilter.Or(TextFilter).Or(GroupedFilter)
select minus.HasValue ? new NegatedMessageFilter(content) : content;
public static TokenListParser<FilterToken, MessageFilter> TextFilter { get; } =
from value in AnyString
select MessageFilter.CreateFilter(value);
public static TokenListParser<FilterToken, MessageFilter> KeyValueFilter { get; } =
from key in AnyString.Try()
from colon in Token.EqualTo(FilterToken.Colon).Try()
from value in AnyString
select MessageFilter.CreateFilter(key, value);
public static TokenListParser<FilterToken, MessageFilter> GroupedFilter { get; } =
from open in Token.EqualTo(FilterToken.LParen)
from content in BinaryExpression
from close in Token.EqualTo(FilterToken.RParen)
select content;
public static TokenListParser<FilterToken, MessageFilter> OrBinaryExpression { get; } =
from first in AnyFilter
from vbar in Token.EqualTo(FilterToken.VBar)
from rest in BinaryExpression
select (MessageFilter)new BinaryExpressionMessageFilter(first, rest, BinaryExpressionKind.Or);
public static TokenListParser<FilterToken, MessageFilter> AndBinaryExpression { get; } =
from first in AnyFilter
from rest in BinaryExpression
select (MessageFilter)new BinaryExpressionMessageFilter(first, rest, BinaryExpressionKind.And);
public static TokenListParser<FilterToken, MessageFilter> BinaryExpression { get; } = OrBinaryExpression.Try().Or(AndBinaryExpression.Try()).Or(AnyFilter);
public static TokenListParser<FilterToken, MessageFilter> Instance { get; } = BinaryExpression.AtEnd();
}
}

@ -1,18 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
{
public enum FilterToken
{
None,
LParen,
RParen,
Colon,
Minus,
VBar,
UnquotedString,
QuotedString
}
}

@ -1,23 +0,0 @@
using Superpower;
using Superpower.Parsers;
using Superpower.Tokenizers;
using System;
using System.Collections.Generic;
using System.Text;
namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing
{
public static class FilterTokenizer
{
public static Tokenizer<FilterToken> Instance { get; } = new TokenizerBuilder<FilterToken>()
.Ignore(Span.WhiteSpace)
.Match(Character.EqualTo('('), FilterToken.LParen)
.Match(Character.EqualTo(')'), FilterToken.RParen)
.Match(Character.EqualTo(':'), FilterToken.Colon)
.Match(Character.EqualTo('-'), FilterToken.Minus)
.Match(Character.EqualTo('|'), FilterToken.VBar)
.Match(FilterParser.QuotedString, FilterToken.QuotedString)
.Match(FilterParser.UnquotedString, FilterToken.UnquotedString)
.Build();
}
}

@ -1,6 +1,6 @@
namespace DiscordChatExporter.Core.Exporting.Partitioning
{
public class FileSizePartitionLimit : PartitionLimit
internal class FileSizePartitionLimit : PartitionLimit
{
private readonly long _limit;

@ -1,6 +1,6 @@
namespace DiscordChatExporter.Core.Exporting.Partitioning
{
public class MessageCountPartitionLimit : PartitionLimit
internal class MessageCountPartitionLimit : PartitionLimit
{
private readonly long _limit;

@ -1,9 +1,7 @@
namespace DiscordChatExporter.Core.Exporting.Partitioning
{
public class NullPartitionLimit : PartitionLimit
internal class NullPartitionLimit : PartitionLimit
{
public static NullPartitionLimit Instance { get; } = new();
public override bool IsReached(long messagesWritten, long bytesWritten) => false;
}
}

@ -11,6 +11,8 @@ namespace DiscordChatExporter.Core.Exporting.Partitioning
public partial class PartitionLimit
{
public static PartitionLimit Null { get; } = new NullPartitionLimit();
private static long? TryParseFileSizeBytes(string value, IFormatProvider? formatProvider = null)
{
var match = Regex.Match(value, @"^\s*(\d+[\.,]?\d*)\s*(\w)?b\s*$", RegexOptions.IgnoreCase);

@ -6,7 +6,7 @@ using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Markdown.Parsing;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
@ -78,14 +78,14 @@ namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
protected override MarkdownNode VisitMention(MentionNode mention)
{
var mentionId = Snowflake.TryParse(mention.Id);
if (mention.Type == MentionType.Meta)
if (mention.Kind == MentionKind.Meta)
{
_buffer
.Append("<span class=\"mention\">")
.Append("@").Append(HtmlEncode(mention.Id))
.Append("</span>");
}
else if (mention.Type == MentionType.User)
else if (mention.Kind == MentionKind.User)
{
var member = mentionId?.Pipe(_context.TryGetMember);
var fullName = member?.User.FullName ?? "Unknown";
@ -96,7 +96,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
.Append("@").Append(HtmlEncode(nick))
.Append("</span>");
}
else if (mention.Type == MentionType.Channel)
else if (mention.Kind == MentionKind.Channel)
{
var channel = mentionId?.Pipe(_context.TryGetChannel);
var name = channel?.Name ?? "deleted-channel";
@ -106,7 +106,7 @@ namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
.Append("#").Append(HtmlEncode(name))
.Append("</span>");
}
else if (mention.Type == MentionType.Role)
else if (mention.Kind == MentionKind.Role)
{
var role = mentionId?.Pipe(_context.TryGetRole);
var name = role?.Name ?? "deleted-role";

@ -1,7 +1,7 @@
using System.Text;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Markdown.Parsing;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
@ -26,25 +26,25 @@ namespace DiscordChatExporter.Core.Exporting.Writers.MarkdownVisitors
protected override MarkdownNode VisitMention(MentionNode mention)
{
var mentionId = Snowflake.TryParse(mention.Id);
if (mention.Type == MentionType.Meta)
if (mention.Kind == MentionKind.Meta)
{
_buffer.Append($"@{mention.Id}");
}
else if (mention.Type == MentionType.User)
else if (mention.Kind == MentionKind.User)
{
var member = mentionId?.Pipe(_context.TryGetMember);
var name = member?.User.Name ?? "Unknown";
_buffer.Append($"@{name}");
}
else if (mention.Type == MentionType.Channel)
else if (mention.Kind == MentionKind.Channel)
{
var channel = mentionId?.Pipe(_context.TryGetChannel);
var name = channel?.Name ?? "deleted-channel";
_buffer.Append($"#{name}");
}
else if (mention.Type == MentionType.Role)
else if (mention.Kind == MentionKind.Role)
{
var role = mentionId?.Pipe(_context.TryGetRole);
var name = role?.Name ?? "deleted-role";

@ -1,17 +0,0 @@
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal class MentionNode : MarkdownNode
{
public string Id { get; }
public MentionType Type { get; }
public MentionNode(string id, MentionType type)
{
Id = id;
Type = type;
}
public override string ToString() => $"<{Type} mention> {Id}";
}
}

@ -1,10 +0,0 @@
namespace DiscordChatExporter.Core.Markdown.Ast
{
internal enum MentionType
{
Meta,
User,
Channel,
Role
}
}

@ -1,6 +1,6 @@
using DiscordChatExporter.Core.Utils;
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown
{
internal class EmojiNode : MarkdownNode
{

@ -1,6 +1,6 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown
{
internal class FormattedNode : MarkdownNode
{

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown
{
internal class InlineCodeBlockNode : MarkdownNode
{

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown
{
internal class LinkNode : MarkdownNode
{

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown
{
internal abstract class MarkdownNode
{

@ -0,0 +1,10 @@
namespace DiscordChatExporter.Core.Markdown
{
internal enum MentionKind
{
Meta,
User,
Channel,
Role
}
}

@ -0,0 +1,17 @@
namespace DiscordChatExporter.Core.Markdown
{
internal class MentionNode : MarkdownNode
{
public string Id { get; }
public MentionKind Kind { get; }
public MentionNode(string id, MentionKind kind)
{
Id = id;
Kind = kind;
}
public override string ToString() => $"<{Kind} mention> {Id}";
}
}

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown
{
internal class MultiLineCodeBlockNode : MarkdownNode
{

@ -1,6 +1,6 @@
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Parsing
{
internal class AggregateMatcher<T> : IMatcher<T>
{

@ -1,7 +1,7 @@
using System;
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Parsing
{
internal interface IMatcher<T>
{

@ -1,11 +1,9 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Markdown.Matching;
using DiscordChatExporter.Core.Utils;
namespace DiscordChatExporter.Core.Markdown
namespace DiscordChatExporter.Core.Markdown.Parsing
{
// The following parsing logic is meant to replicate Discord's markdown grammar as close as possible
internal static partial class MarkdownParser
@ -120,31 +118,31 @@ namespace DiscordChatExporter.Core.Markdown
// Capture @everyone
private static readonly IMatcher<MarkdownNode> EveryoneMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@everyone",
_ => new MentionNode("everyone", MentionType.Meta)
_ => new MentionNode("everyone", MentionKind.Meta)
);
// Capture @here
private static readonly IMatcher<MarkdownNode> HereMentionNodeMatcher = new StringMatcher<MarkdownNode>(
"@here",
_ => new MentionNode("here", MentionType.Meta)
_ => new MentionNode("here", MentionKind.Meta)
);
// Capture <@123456> or <@!123456>
private static readonly IMatcher<MarkdownNode> UserMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<@!?(\\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.User)
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.User)
);
// Capture <#123456>
private static readonly IMatcher<MarkdownNode> ChannelMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<#(\\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.Channel)
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.Channel)
);
// Capture <@&123456>
private static readonly IMatcher<MarkdownNode> RoleMentionNodeMatcher = new RegexMatcher<MarkdownNode>(
new Regex("<@&(\\d+)>", DefaultRegexOptions),
(_, m) => new MentionNode(m.Groups[1].Value, MentionType.Role)
(_, m) => new MentionNode(m.Groups[1].Value, MentionKind.Role)
);
/* Emojis */
@ -293,12 +291,16 @@ namespace DiscordChatExporter.Core.Markdown
internal static partial class MarkdownParser
{
private static IReadOnlyList<MarkdownNode> Parse(StringPart stringPart) => Parse(stringPart, AggregateNodeMatcher);
private static IReadOnlyList<MarkdownNode> Parse(StringPart stringPart) =>
Parse(stringPart, AggregateNodeMatcher);
private static IReadOnlyList<MarkdownNode> ParseMinimal(StringPart stringPart) => Parse(stringPart, MinimalAggregateNodeMatcher);
private static IReadOnlyList<MarkdownNode> ParseMinimal(StringPart stringPart) =>
Parse(stringPart, MinimalAggregateNodeMatcher);
public static IReadOnlyList<MarkdownNode> Parse(string input) => Parse(new StringPart(input));
public static IReadOnlyList<MarkdownNode> Parse(string input) =>
Parse(new StringPart(input));
public static IReadOnlyList<MarkdownNode> ParseMinimal(string input) => ParseMinimal(new StringPart(input));
public static IReadOnlyList<MarkdownNode> ParseMinimal(string input) =>
ParseMinimal(new StringPart(input));
}
}

@ -1,8 +1,7 @@
using System;
using System.Collections.Generic;
using DiscordChatExporter.Core.Markdown.Ast;
namespace DiscordChatExporter.Core.Markdown
namespace DiscordChatExporter.Core.Markdown.Parsing
{
internal abstract class MarkdownVisitor
{

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Parsing
{
internal class ParsedMatch<T>
{

@ -1,7 +1,7 @@
using System;
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Core.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Parsing
{
internal class RegexMatcher<T> : IMatcher<T>
{

@ -1,6 +1,6 @@
using System;
namespace DiscordChatExporter.Core.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Parsing
{
internal class StringMatcher<T> : IMatcher<T>
{

@ -1,6 +1,6 @@
using System.Text.RegularExpressions;
namespace DiscordChatExporter.Core.Markdown.Matching
namespace DiscordChatExporter.Core.Markdown.Parsing
{
internal readonly struct StringPart
{

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown
{
internal enum TextFormatting
{

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Ast
namespace DiscordChatExporter.Core.Markdown
{
internal class TextNode : MarkdownNode
{

@ -0,0 +1,24 @@
using System;
using Superpower;
using Superpower.Parsers;
namespace DiscordChatExporter.Core.Utils.Extensions
{
public static class SuperpowerExtensions
{
public static TextParser<string> Text(this TextParser<char[]> parser) =>
parser.Select(chars => new string(chars));
public static TextParser<T> Token<T>(this TextParser<T> parser) =>
parser.Between(Character.WhiteSpace.IgnoreMany(), Character.WhiteSpace.IgnoreMany());
// From: https://twitter.com/nblumhardt/status/1389349059786264578
public static TextParser<T> Log<T>(this TextParser<T> parser, string description) => i =>
{
Console.WriteLine($"Trying {description} ->");
var r = parser(i);
Console.WriteLine($"Result was {r}");
return r;
};
}
}

@ -21,14 +21,15 @@ namespace DiscordChatExporter.Core.Utils
.OrResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
.OrResult(m => m.StatusCode == HttpStatusCode.RequestTimeout)
.OrResult(m => m.StatusCode >= HttpStatusCode.InternalServerError)
.WaitAndRetryAsync(8,
.WaitAndRetryAsync(
8,
(i, result, _) =>
{
// If rate-limited, use retry-after as a guide
if (result.Result?.StatusCode == HttpStatusCode.TooManyRequests)
{
// Only start respecting retry-after after a few attempts.
// The reason is that Discord often sends unreasonable (20+ minutes) retry-after
// Only start respecting retry-after after a few attempts, because
// Discord often sends unreasonable (20+ minutes) retry-after
// on the very first request.
if (i > 3)
{
@ -40,7 +41,8 @@ namespace DiscordChatExporter.Core.Utils
return TimeSpan.FromSeconds(Math.Pow(2, i) + 1);
},
(_, _, _, _) => Task.CompletedTask);
(_, _, _, _) => Task.CompletedTask
);
private static HttpStatusCode? TryGetStatusCodeFromException(HttpRequestException ex) =>
// This is extremely frail, but there's no other way

@ -52,13 +52,13 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
public PartitionLimit PartitionLimit => !string.IsNullOrWhiteSpace(PartitionLimitValue)
? PartitionLimit.Parse(PartitionLimitValue)
: NullPartitionLimit.Instance;
: PartitionLimit.Null;
public string? MessageFilterValue { get; set; }
public MessageFilter MessageFilter => !string.IsNullOrWhiteSpace(MessageFilterValue)
? MessageFilter.Parse(MessageFilterValue)
: NullMessageFilter.Instance;
: MessageFilter.Null;
public bool ShouldDownloadMedia { get; set; }

@ -131,7 +131,7 @@
materialDesign:HintAssist.Hint="Partition limit"
materialDesign:HintAssist.IsFloating="True"
Text="{Binding PartitionLimitValue}"
ToolTip="Split output into partitions, each limited to this number of messages (e.g. 100) or file size (e.g. 10mb)" />
ToolTip="Split output into partitions, each limited to this number of messages (e.g. '100') or file size (e.g. '10mb')" />
<!-- Filtering -->
<TextBox
@ -139,7 +139,7 @@
materialDesign:HintAssist.Hint="Message filter"
materialDesign:HintAssist.IsFloating="True"
Text="{Binding MessageFilterValue}"
ToolTip="Only include messages that satisfy this filter (e.g. from:foo#1234)." />
ToolTip="Only include messages that satisfy this filter (e.g. 'from:foo#1234' or 'has:image')." />
<!-- Download media -->
<Grid Margin="16,16" ToolTip="Download referenced media content (user avatars, attached files, embedded images, etc)">

@ -1,4 +1,4 @@
<Window
<Window
x:Class="DiscordChatExporter.Gui.Views.RootView"
xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation"
xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml"

Loading…
Cancel
Save