From 0e1c3e4c76a1631ea2abe00f26efcc350bce91d2 Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Sat, 18 Feb 2023 19:45:31 +0200 Subject: [PATCH] Add support for extracting channels from data dump in `exportall` Closes #597 --- ...enCommandBase.cs => DiscordCommandBase.cs} | 2 +- .../Commands/Base/ExportCommandBase.cs | 8 +- .../Commands/ExportAllCommand.cs | 78 +++++++++++++++++-- .../Commands/ExportChannelsCommand.cs | 2 +- .../Commands/ExportDirectMessagesCommand.cs | 8 +- .../Commands/GetChannelsCommand.cs | 2 +- ...Command.cs => GetDirectChannelsCommand.cs} | 2 +- .../Commands/GetGuildsCommand.cs | 2 +- .../Discord/Data/ChannelKind.cs | 9 +++ .../Components/DashboardViewModel.cs | 14 ++-- .../Views/Components/DashboardView.xaml | 4 +- 11 files changed, 99 insertions(+), 32 deletions(-) rename DiscordChatExporter.Cli/Commands/Base/{TokenCommandBase.cs => DiscordCommandBase.cs} (94%) rename DiscordChatExporter.Cli/Commands/{GetDirectMessageChannelsCommand.cs => GetDirectChannelsCommand.cs} (95%) diff --git a/DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs similarity index 94% rename from DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs rename to DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs index 01a8327..f24b4e0 100644 --- a/DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs @@ -7,7 +7,7 @@ using DiscordChatExporter.Core.Discord; namespace DiscordChatExporter.Cli.Commands.Base; -public abstract class TokenCommandBase : ICommand +public abstract class DiscordCommandBase : ICommand { [CommandOption( "token", diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs index a98e020..40255b3 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs @@ -20,7 +20,7 @@ using Gress; namespace DiscordChatExporter.Cli.Commands.Base; -public abstract class ExportCommandBase : TokenCommandBase +public abstract class ExportCommandBase : DiscordCommandBase { private readonly string _outputPath = Directory.GetCurrentDirectory(); @@ -268,7 +268,7 @@ public abstract class ExportCommandBase : TokenCommandBase await console.Output.WriteLineAsync("Resolving channel(s)..."); var channels = new List(); - var guildChannelMap = new Dictionary>(); + var channelsByGuild = new Dictionary>(); foreach (var channelId in channelIds) { @@ -278,7 +278,7 @@ public abstract class ExportCommandBase : TokenCommandBase if (channel.Kind == ChannelKind.GuildCategory) { var guildChannels = - guildChannelMap.GetValueOrDefault(channel.GuildId) ?? + channelsByGuild.GetValueOrDefault(channel.GuildId) ?? await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken); foreach (var guildChannel in guildChannels) @@ -288,7 +288,7 @@ public abstract class ExportCommandBase : TokenCommandBase } // Cache the guild channels to avoid redundant work - guildChannelMap[channel.GuildId] = guildChannels; + channelsByGuild[channel.GuildId] = guildChannels; } else { diff --git a/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs b/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs index 5000888..4b58df3 100644 --- a/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs @@ -1,9 +1,15 @@ using System.Collections.Generic; +using System.IO.Compression; +using System.Text.Json; using System.Threading.Tasks; using CliFx.Attributes; +using CliFx.Exceptions; using CliFx.Infrastructure; using DiscordChatExporter.Cli.Commands.Base; +using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord.Data; +using DiscordChatExporter.Core.Exceptions; +using JsonExtensions.Reading; namespace DiscordChatExporter.Cli.Commands; @@ -14,7 +20,19 @@ public class ExportAllCommand : ExportCommandBase "include-dm", Description = "Include direct message channels." )] - public bool IncludeDirectMessages { get; init; } = true; + public bool IncludeDirectChannels { get; init; } = true; + + [CommandOption( + "include-guilds", + Description = "Include guild channels." + )] + public bool IncludeGuildChannels { get; init; } = true; + + [CommandOption( + "data-package", + Description = "Path to the personal data package (ZIP file) requested from Discord. If provided, only channels referenced in the dump will be exported." + )] + public string? DataPackageFilePath { get; init; } public override async ValueTask ExecuteAsync(IConsole console) { @@ -23,16 +41,60 @@ public class ExportAllCommand : ExportCommandBase var cancellationToken = console.RegisterCancellationHandler(); var channels = new List(); - await console.Output.WriteLineAsync("Fetching channels..."); - await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken)) + // Pull from the API + if (string.IsNullOrWhiteSpace(DataPackageFilePath)) { - // Skip DMs if instructed to - if (!IncludeDirectMessages && guild.Id == Guild.DirectMessages.Id) - continue; + await console.Output.WriteLineAsync("Fetching channels..."); - await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken)) - channels.Add(channel); + await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken)) + { + await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken)) + { + channels.Add(channel); + } + } } + // Pull from the data package + else + { + await console.Output.WriteLineAsync("Extracting channels..."); + using var archive = ZipFile.OpenRead(DataPackageFilePath); + + var entry = archive.GetEntry("messages/index.json"); + if (entry is null) + throw new CommandException("Cannot find channel index inside the data package."); + + await using var stream = entry.Open(); + using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken); + + foreach (var property in document.RootElement.EnumerateObjectOrEmpty()) + { + var channelId = Snowflake.Parse(property.Name); + var channelName = property.Value.GetString(); + + // Null items refer to deleted channels + if (channelName is null) + continue; + + await console.Output.WriteLineAsync($"Fetching channel '{channelName}' ({channelId})..."); + + try + { + var channel = await Discord.GetChannelAsync(channelId, cancellationToken); + channels.Add(channel); + } + catch (DiscordChatExporterException) + { + await console.Output.WriteLineAsync($"Channel '{channelName}' ({channelId}) is inaccessible."); + } + } + } + + // Filter out unwanted channels + if (!IncludeDirectChannels) + channels.RemoveAll(c => c.Kind.IsDirect()); + if (!IncludeGuildChannels) + channels.RemoveAll(c => c.Kind.IsGuild()); await base.ExecuteAsync(console, channels); } diff --git a/DiscordChatExporter.Cli/Commands/ExportChannelsCommand.cs b/DiscordChatExporter.Cli/Commands/ExportChannelsCommand.cs index a0f9b89..1e76d79 100644 --- a/DiscordChatExporter.Cli/Commands/ExportChannelsCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportChannelsCommand.cs @@ -14,7 +14,7 @@ public class ExportChannelsCommand : ExportCommandBase [CommandOption( "channel", 'c', - Description = "Channel ID(s). If provided with a category ID, all channels in that category will be exported." + Description = "Channel ID(s). If provided with category IDs, all channels inside those categories will be exported." )] public required IReadOnlyList ChannelIds { get; init; } diff --git a/DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs b/DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs index a7ab35d..608adcb 100644 --- a/DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs @@ -1,5 +1,4 @@ -using System.Linq; -using System.Threading.Tasks; +using System.Threading.Tasks; using CliFx.Attributes; using CliFx.Infrastructure; using DiscordChatExporter.Cli.Commands.Base; @@ -18,10 +17,7 @@ public class ExportDirectMessagesCommand : ExportCommandBase var cancellationToken = console.RegisterCancellationHandler(); await console.Output.WriteLineAsync("Fetching channels..."); - - var channels = (await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken)) - .Where(c => c.Kind != ChannelKind.GuildCategory) - .ToArray(); + var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken); await base.ExecuteAsync(console, channels); } diff --git a/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs b/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs index d1940ac..fcc95ce 100644 --- a/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs +++ b/DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs @@ -11,7 +11,7 @@ using DiscordChatExporter.Core.Utils.Extensions; namespace DiscordChatExporter.Cli.Commands; [Command("channels", Description = "Get the list of channels in a guild.")] -public class GetChannelsCommand : TokenCommandBase +public class GetChannelsCommand : DiscordCommandBase { [CommandOption( "guild", diff --git a/DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs b/DiscordChatExporter.Cli/Commands/GetDirectChannelsCommand.cs similarity index 95% rename from DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs rename to DiscordChatExporter.Cli/Commands/GetDirectChannelsCommand.cs index 2d7742a..7bb0304 100644 --- a/DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs +++ b/DiscordChatExporter.Cli/Commands/GetDirectChannelsCommand.cs @@ -10,7 +10,7 @@ using DiscordChatExporter.Core.Utils.Extensions; namespace DiscordChatExporter.Cli.Commands; [Command("dm", Description = "Get the list of direct message channels.")] -public class GetDirectMessageChannelsCommand : TokenCommandBase +public class GetDirectChannelsCommand : DiscordCommandBase { public override async ValueTask ExecuteAsync(IConsole console) { diff --git a/DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs b/DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs index a0024b7..4bd38c0 100644 --- a/DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs +++ b/DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs @@ -10,7 +10,7 @@ using DiscordChatExporter.Core.Utils.Extensions; namespace DiscordChatExporter.Cli.Commands; [Command("guilds", Description = "Get the list of accessible guilds.")] -public class GetGuildsCommand : TokenCommandBase +public class GetGuildsCommand : DiscordCommandBase { public override async ValueTask ExecuteAsync(IConsole console) { diff --git a/DiscordChatExporter.Core/Discord/Data/ChannelKind.cs b/DiscordChatExporter.Core/Discord/Data/ChannelKind.cs index 35908e4..f6b73c3 100644 --- a/DiscordChatExporter.Core/Discord/Data/ChannelKind.cs +++ b/DiscordChatExporter.Core/Discord/Data/ChannelKind.cs @@ -16,4 +16,13 @@ public enum ChannelKind GuildStageVoice = 13, GuildDirectory = 14, GuildForum = 15 +} + +public static class ChannelKindExtensions +{ + public static bool IsDirect(this ChannelKind kind) => + kind is ChannelKind.DirectTextChat or ChannelKind.DirectGroupTextChat; + + public static bool IsGuild(this ChannelKind kind) => + !kind.IsDirect(); } \ No newline at end of file diff --git a/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs b/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs index 07eb062..ffc5b02 100644 --- a/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs @@ -38,16 +38,16 @@ public class DashboardViewModel : PropertyChangedBase public string? Token { get; set; } - private IReadOnlyDictionary>? GuildChannelMap { get; set; } + private IReadOnlyDictionary>? ChannelsByGuild { get; set; } - public IReadOnlyList? AvailableGuilds => GuildChannelMap?.Keys.ToArray(); + public IReadOnlyList? AvailableGuilds => ChannelsByGuild?.Keys.ToArray(); public Guild? SelectedGuild { get; set; } public bool IsDirectMessageGuildSelected => SelectedGuild?.Id == Guild.DirectMessages.Id; public IReadOnlyList? AvailableChannels => SelectedGuild is not null - ? GuildChannelMap?[SelectedGuild] + ? ChannelsByGuild?[SelectedGuild] : null; public IReadOnlyList? SelectedChannels { get; set; } @@ -103,17 +103,17 @@ public class DashboardViewModel : PropertyChangedBase var discord = new DiscordClient(token); - var guildChannelMap = new Dictionary>(); + var channelsByGuild = new Dictionary>(); await foreach (var guild in discord.GetUserGuildsAsync()) { - guildChannelMap[guild] = (await discord.GetGuildChannelsAsync(guild.Id)) + channelsByGuild[guild] = (await discord.GetGuildChannelsAsync(guild.Id)) .Where(c => c.Kind != ChannelKind.GuildCategory) .ToArray(); } _discord = discord; - GuildChannelMap = guildChannelMap; - SelectedGuild = guildChannelMap.Keys.FirstOrDefault(); + ChannelsByGuild = channelsByGuild; + SelectedGuild = channelsByGuild.Keys.FirstOrDefault(); } catch (DiscordChatExporterException ex) when (!ex.IsFatal) { diff --git a/DiscordChatExporter.Gui/Views/Components/DashboardView.xaml b/DiscordChatExporter.Gui/Views/Components/DashboardView.xaml index 5c190c9..497c373 100644 --- a/DiscordChatExporter.Gui/Views/Components/DashboardView.xaml +++ b/DiscordChatExporter.Gui/Views/Components/DashboardView.xaml @@ -19,7 +19,7 @@ mc:Ignorable="d"> - + @@ -300,7 +300,7 @@