diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs index 6cc6c4c..e99fc98 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs @@ -12,6 +12,7 @@ using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Exceptions; using DiscordChatExporter.Core.Exporting; +using DiscordChatExporter.Core.Exporting.Filtering; using DiscordChatExporter.Core.Exporting.Partitioning; using DiscordChatExporter.Core.Utils.Extensions; using Tyrrrz.Extensions; @@ -35,6 +36,9 @@ namespace DiscordChatExporter.Cli.Commands.Base [CommandOption("partition", 'p', Description = "Split output into partitions, each limited to this number of messages (e.g. 100) or file size (e.g. 10mb).")] public PartitionLimit PartitionLimit { get; init; } = NullPartitionLimit.Instance; + [CommandOption("filter", Description = "Only include messages that satisfy this filter (e.g. from:foo#1234).")] + public MessageFilter MessageFilter { get; init; } = NullMessageFilter.Instance; + [CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")] public int ParallelLimit { get; init; } = 1; @@ -76,6 +80,7 @@ namespace DiscordChatExporter.Cli.Commands.Base After, Before, PartitionLimit, + MessageFilter, ShouldDownloadMedia, ShouldReuseMedia, DateFormat diff --git a/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj b/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj index 9651a8f..1d46eca 100644 --- a/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj +++ b/DiscordChatExporter.Core/DiscordChatExporter.Core.csproj @@ -8,6 +8,7 @@ + diff --git a/DiscordChatExporter.Core/Exporting/ChannelExporter.cs b/DiscordChatExporter.Core/Exporting/ChannelExporter.cs index 7e5944f..7a8f262 100644 --- a/DiscordChatExporter.Core/Exporting/ChannelExporter.cs +++ b/DiscordChatExporter.Core/Exporting/ChannelExporter.cs @@ -39,6 +39,10 @@ namespace DiscordChatExporter.Core.Exporting var encounteredUsers = new HashSet(IdBasedEqualityComparer.Instance); await foreach (var message in _discord.GetMessagesAsync(request.Channel.Id, request.After, request.Before, progress)) { + // Skips any messages that fail to pass the supplied filter + if (!request.MessageFilter.Filter(message)) + continue; + // Resolve members for referenced users foreach (var referencedUser in message.MentionedUsers.Prepend(message.Author)) { diff --git a/DiscordChatExporter.Core/Exporting/ExportRequest.cs b/DiscordChatExporter.Core/Exporting/ExportRequest.cs index 319d451..6de136e 100644 --- a/DiscordChatExporter.Core/Exporting/ExportRequest.cs +++ b/DiscordChatExporter.Core/Exporting/ExportRequest.cs @@ -4,6 +4,7 @@ using System.Text; using System.Text.RegularExpressions; using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord.Data; +using DiscordChatExporter.Core.Exporting.Filtering; using DiscordChatExporter.Core.Exporting.Partitioning; using DiscordChatExporter.Core.Utils; @@ -31,6 +32,8 @@ namespace DiscordChatExporter.Core.Exporting public PartitionLimit PartitionLimit { get; } + public MessageFilter MessageFilter { get; } + public bool ShouldDownloadMedia { get; } public bool ShouldReuseMedia { get; } @@ -45,6 +48,7 @@ namespace DiscordChatExporter.Core.Exporting Snowflake? after, Snowflake? before, PartitionLimit partitionLimit, + MessageFilter messageFilter, bool shouldDownloadMedia, bool shouldReuseMedia, string dateFormat) @@ -56,6 +60,7 @@ namespace DiscordChatExporter.Core.Exporting After = after; Before = before; PartitionLimit = partitionLimit; + MessageFilter = messageFilter; ShouldDownloadMedia = shouldDownloadMedia; ShouldReuseMedia = shouldReuseMedia; DateFormat = dateFormat; diff --git a/DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionKind.cs b/DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionKind.cs new file mode 100644 index 0000000..962ee06 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionKind.cs @@ -0,0 +1,8 @@ +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public enum BinaryExpressionKind + { + Or, + And + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionMessageFilter.cs b/DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionMessageFilter.cs new file mode 100644 index 0000000..1946726 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/BinaryExpressionMessageFilter.cs @@ -0,0 +1,26 @@ +using DiscordChatExporter.Core.Discord.Data; +using System; + +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public class BinaryExpressionMessageFilter : MessageFilter + { + private readonly MessageFilter _first; + private readonly MessageFilter _second; + private readonly BinaryExpressionKind _kind; + + public BinaryExpressionMessageFilter(MessageFilter first, MessageFilter second, BinaryExpressionKind kind) + { + _first = first; + _second = second; + _kind = kind; + } + + public override bool Filter(Message message) => _kind switch + { + BinaryExpressionKind.Or => _first.Filter(message) || _second.Filter(message), + BinaryExpressionKind.And => _first.Filter(message) && _second.Filter(message), + _ => throw new InvalidOperationException($"Unknown binary expression kind '{_kind}'.") + }; + } +} diff --git a/DiscordChatExporter.Core/Exporting/Filtering/ContainsMessageFilter.cs b/DiscordChatExporter.Core/Exporting/Filtering/ContainsMessageFilter.cs new file mode 100644 index 0000000..9b59460 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/ContainsMessageFilter.cs @@ -0,0 +1,15 @@ +using DiscordChatExporter.Core.Discord.Data; +using System.Text.RegularExpressions; + +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public class ContainsMessageFilter : MessageFilter + { + private readonly string _value; + + public ContainsMessageFilter(string value) => _value = value; + + public override bool Filter(Message message) => + Regex.IsMatch(message.Content, $@"\b{Regex.Escape(_value)}\b", RegexOptions.IgnoreCase | DefaultRegexOptions); + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core/Exporting/Filtering/FromMessageFilter.cs b/DiscordChatExporter.Core/Exporting/Filtering/FromMessageFilter.cs new file mode 100644 index 0000000..fb2f863 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/FromMessageFilter.cs @@ -0,0 +1,17 @@ +using DiscordChatExporter.Core.Discord.Data; +using System; + +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public class FromMessageFilter : MessageFilter + { + private readonly string _value; + + public FromMessageFilter(string value) => _value = value; + + public override bool Filter(Message message) => + string.Equals(_value, message.Author.Name, StringComparison.OrdinalIgnoreCase) || + string.Equals(_value, message.Author.FullName, StringComparison.OrdinalIgnoreCase) || + string.Equals(_value, message.Author.Id.ToString(), StringComparison.OrdinalIgnoreCase); + } +} diff --git a/DiscordChatExporter.Core/Exporting/Filtering/HasMessageFilter.cs b/DiscordChatExporter.Core/Exporting/Filtering/HasMessageFilter.cs new file mode 100644 index 0000000..53c8fca --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/HasMessageFilter.cs @@ -0,0 +1,26 @@ +using DiscordChatExporter.Core.Discord.Data; +using System; +using System.Linq; +using System.Text.RegularExpressions; + +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public class HasMessageFilter : MessageFilter + { + private readonly string _value; + + public HasMessageFilter(string value) => _value = value; + + public override bool Filter(Message message) => + _value switch + { + "link" => Regex.IsMatch(message.Content, "https?://\\S*[^\\.,:;\"\'\\s]", DefaultRegexOptions), + "embed" => message.Embeds.Any(), + "file" => message.Attachments.Any(), + "video" => message.Attachments.Any(file => file.IsVideo), + "image" => message.Attachments.Any(file => file.IsImage), + "sound" => message.Attachments.Any(file => file.IsAudio), + _ => throw new InvalidOperationException($"Invalid value provided for the 'has' message filter: '{_value}'") + }; + } +} diff --git a/DiscordChatExporter.Core/Exporting/Filtering/MentionsMessageFilter.cs b/DiscordChatExporter.Core/Exporting/Filtering/MentionsMessageFilter.cs new file mode 100644 index 0000000..9bc2179 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/MentionsMessageFilter.cs @@ -0,0 +1,19 @@ +using DiscordChatExporter.Core.Discord.Data; +using System; +using System.Linq; + +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public class MentionsMessageFilter : MessageFilter + { + private readonly string _value; + + public MentionsMessageFilter(string value) => _value = value; + + public override bool Filter(Message message) => + message.MentionedUsers.Any(user => + string.Equals(_value, user.Name, StringComparison.OrdinalIgnoreCase) || + string.Equals(_value, user.FullName, StringComparison.OrdinalIgnoreCase) || + string.Equals(_value, user.Id.ToString(), StringComparison.OrdinalIgnoreCase)); + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core/Exporting/Filtering/MessageFilter.cs b/DiscordChatExporter.Core/Exporting/Filtering/MessageFilter.cs new file mode 100644 index 0000000..6593f05 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/MessageFilter.cs @@ -0,0 +1,38 @@ +using System; +using System.Text.RegularExpressions; +using DiscordChatExporter.Core.Discord.Data; +using DiscordChatExporter.Core.Exporting.Filtering.Parsing; +using Superpower; + +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public abstract partial class MessageFilter + { + public abstract bool Filter(Message message); + } + + public partial class MessageFilter + { + protected const RegexOptions DefaultRegexOptions = RegexOptions.Compiled | RegexOptions.CultureInvariant | RegexOptions.Multiline; + + internal static MessageFilter CreateFilter(string text) => new ContainsMessageFilter(text); + + internal static MessageFilter CreateFilter(string key, string value) + { + return key.ToLowerInvariant() switch + { + "from" => new FromMessageFilter(value), + "has" => new HasMessageFilter(value), + "mentions" => new MentionsMessageFilter(value), + _ => throw new ArgumentException($"Invalid filter type '{key}'.", nameof(key)) + }; + } + + public static MessageFilter Parse(string value, IFormatProvider? formatProvider = null) + { + var tokens = FilterTokenizer.Instance.Tokenize(value); + var parsed = FilterParser.Instance.Parse(tokens); + return parsed; + } + } +} diff --git a/DiscordChatExporter.Core/Exporting/Filtering/NegatedMessageFilter.cs b/DiscordChatExporter.Core/Exporting/Filtering/NegatedMessageFilter.cs new file mode 100644 index 0000000..e2a554c --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/NegatedMessageFilter.cs @@ -0,0 +1,13 @@ +using DiscordChatExporter.Core.Discord.Data; + +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public class NegatedMessageFilter : MessageFilter + { + private readonly MessageFilter _filter; + + public NegatedMessageFilter(MessageFilter filter) => _filter = filter; + + public override bool Filter(Message message) => !_filter.Filter(message); + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core/Exporting/Filtering/NullMessageFilter.cs b/DiscordChatExporter.Core/Exporting/Filtering/NullMessageFilter.cs new file mode 100644 index 0000000..d213d19 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/NullMessageFilter.cs @@ -0,0 +1,11 @@ +using DiscordChatExporter.Core.Discord.Data; + +namespace DiscordChatExporter.Core.Exporting.Filtering +{ + public class NullMessageFilter : MessageFilter + { + public static NullMessageFilter Instance { get; } = new(); + + public override bool Filter(Message message) => true; + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterParser.cs b/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterParser.cs new file mode 100644 index 0000000..79b5785 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterParser.cs @@ -0,0 +1,68 @@ +using Superpower; +using Superpower.Model; +using Superpower.Parsers; +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Text; + +namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing +{ + public static class FilterParser + { + public static TextParser QuotedString { get; } = + from open in Character.EqualTo('"') + from content in Character.EqualTo('\\').IgnoreThen(Character.AnyChar).Try() + .Or(Character.Except('"')) + .Many() + from close in Character.EqualTo('"') + select new string(content); + + public static TextParser UnquotedString { get; } = + from content in Character.EqualTo('\\').IgnoreThen(Character.In('"', '/')).Try() + .Or(Character.Except(c => char.IsWhiteSpace(c) || "():-|\"".Contains(c), "non-whitespace character except for (, ), :, -, |, and \"")) + .AtLeastOnce() + select new string(content); + + public static TokenListParser AnyString { get; } = + Token.EqualTo(FilterToken.QuotedString).Apply(QuotedString) + .Or(Token.EqualTo(FilterToken.UnquotedString).Apply(UnquotedString)); + + public static TokenListParser AnyFilter { get; } = + from minus in Token.EqualTo(FilterToken.Minus).Optional() + from content in KeyValueFilter.Or(TextFilter).Or(GroupedFilter) + select minus.HasValue ? new NegatedMessageFilter(content) : content; + + public static TokenListParser TextFilter { get; } = + from value in AnyString + select MessageFilter.CreateFilter(value); + + public static TokenListParser KeyValueFilter { get; } = + from key in AnyString.Try() + from colon in Token.EqualTo(FilterToken.Colon).Try() + from value in AnyString + select MessageFilter.CreateFilter(key, value); + + public static TokenListParser GroupedFilter { get; } = + from open in Token.EqualTo(FilterToken.LParen) + from content in BinaryExpression + from close in Token.EqualTo(FilterToken.RParen) + select content; + + public static TokenListParser OrBinaryExpression { get; } = + from first in AnyFilter + from vbar in Token.EqualTo(FilterToken.VBar) + from rest in BinaryExpression + select (MessageFilter)new BinaryExpressionMessageFilter(first, rest, BinaryExpressionKind.Or); + + public static TokenListParser AndBinaryExpression { get; } = + from first in AnyFilter + from rest in BinaryExpression + select (MessageFilter)new BinaryExpressionMessageFilter(first, rest, BinaryExpressionKind.And); + + public static TokenListParser BinaryExpression { get; } = OrBinaryExpression.Try().Or(AndBinaryExpression.Try()).Or(AnyFilter); + + public static TokenListParser Instance { get; } = BinaryExpression.AtEnd(); + } +} diff --git a/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterToken.cs b/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterToken.cs new file mode 100644 index 0000000..222525a --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterToken.cs @@ -0,0 +1,18 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing +{ + public enum FilterToken + { + None, + LParen, + RParen, + Colon, + Minus, + VBar, + UnquotedString, + QuotedString + } +} diff --git a/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterTokenizer.cs b/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterTokenizer.cs new file mode 100644 index 0000000..11bd3e3 --- /dev/null +++ b/DiscordChatExporter.Core/Exporting/Filtering/Parsing/FilterTokenizer.cs @@ -0,0 +1,23 @@ +using Superpower; +using Superpower.Parsers; +using Superpower.Tokenizers; +using System; +using System.Collections.Generic; +using System.Text; + +namespace DiscordChatExporter.Core.Exporting.Filtering.Parsing +{ + public static class FilterTokenizer + { + public static Tokenizer Instance { get; } = new TokenizerBuilder() + .Ignore(Span.WhiteSpace) + .Match(Character.EqualTo('('), FilterToken.LParen) + .Match(Character.EqualTo(')'), FilterToken.RParen) + .Match(Character.EqualTo(':'), FilterToken.Colon) + .Match(Character.EqualTo('-'), FilterToken.Minus) + .Match(Character.EqualTo('|'), FilterToken.VBar) + .Match(FilterParser.QuotedString, FilterToken.QuotedString) + .Match(FilterParser.UnquotedString, FilterToken.UnquotedString) + .Build(); + } +} diff --git a/DiscordChatExporter.Core/Utils/Polyfills.cs b/DiscordChatExporter.Core/Utils/Polyfills.cs new file mode 100644 index 0000000..2e592b8 --- /dev/null +++ b/DiscordChatExporter.Core/Utils/Polyfills.cs @@ -0,0 +1,9 @@ +// ReSharper disable CheckNamespace +// TODO: remove after moving to .NET 5 + +namespace System.Runtime.CompilerServices +{ + internal static class IsExternalInit + { + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Gui/Services/SettingsService.cs b/DiscordChatExporter.Gui/Services/SettingsService.cs index 06b65f7..852020f 100644 --- a/DiscordChatExporter.Gui/Services/SettingsService.cs +++ b/DiscordChatExporter.Gui/Services/SettingsService.cs @@ -25,6 +25,8 @@ namespace DiscordChatExporter.Gui.Services public string? LastPartitionLimitValue { get; set; } + public string? LastMessageFilterValue { get; set; } + public bool LastShouldDownloadMedia { get; set; } public SettingsService() diff --git a/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs index 13f0702..a1d5cc5 100644 --- a/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs @@ -4,6 +4,7 @@ using System.Linq; using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord.Data; using DiscordChatExporter.Core.Exporting; +using DiscordChatExporter.Core.Exporting.Filtering; using DiscordChatExporter.Core.Exporting.Partitioning; using DiscordChatExporter.Core.Utils.Extensions; using DiscordChatExporter.Gui.Services; @@ -53,6 +54,12 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs ? PartitionLimit.Parse(PartitionLimitValue) : NullPartitionLimit.Instance; + public string? MessageFilterValue { get; set; } + + public MessageFilter MessageFilter => !string.IsNullOrWhiteSpace(MessageFilterValue) + ? MessageFilter.Parse(MessageFilterValue) + : NullMessageFilter.Instance; + public bool ShouldDownloadMedia { get; set; } // Whether to show the "advanced options" by default when the dialog opens. @@ -61,6 +68,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs After != default || Before != default || !string.IsNullOrWhiteSpace(PartitionLimitValue) || + !string.IsNullOrWhiteSpace(MessageFilterValue) || ShouldDownloadMedia != default; public ExportSetupViewModel(DialogManager dialogManager, SettingsService settingsService) @@ -71,6 +79,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs // Persist preferences SelectedFormat = _settingsService.LastExportFormat; PartitionLimitValue = _settingsService.LastPartitionLimitValue; + MessageFilterValue = _settingsService.LastMessageFilterValue; ShouldDownloadMedia = _settingsService.LastShouldDownloadMedia; } @@ -79,6 +88,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs // Persist preferences _settingsService.LastExportFormat = SelectedFormat; _settingsService.LastPartitionLimitValue = PartitionLimitValue; + _settingsService.LastMessageFilterValue = MessageFilterValue; _settingsService.LastShouldDownloadMedia = ShouldDownloadMedia; // If single channel - prompt file path diff --git a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs index 83eb3dd..a822b29 100644 --- a/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/RootViewModel.cs @@ -212,6 +212,7 @@ namespace DiscordChatExporter.Gui.ViewModels dialog.After?.Pipe(Snowflake.FromDate), dialog.Before?.Pipe(Snowflake.FromDate), dialog.PartitionLimit, + dialog.MessageFilter, dialog.ShouldDownloadMedia, _settingsService.ShouldReuseMedia, _settingsService.DateFormat diff --git a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml index 2bc0b3c..6a0d7eb 100644 --- a/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml +++ b/DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml @@ -133,6 +133,14 @@ Text="{Binding PartitionLimitValue}" ToolTip="Split output into partitions, each limited to this number of messages (e.g. 100) or file size (e.g. 10mb)" /> + + +