Add support for extracting channels from data dump in `exportall`

Closes #597

DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs

@@ -7,7 +7,7 @@ using DiscordChatExporter.Core.Discord;
 
 namespace DiscordChatExporter.Cli.Commands.Base;
 
-public abstract class TokenCommandBase : ICommand
+public abstract class DiscordCommandBase : ICommand
 {
     [CommandOption(
         "token",

DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs

@@ -20,7 +20,7 @@ using Gress;
 
 namespace DiscordChatExporter.Cli.Commands.Base;
 
-public abstract class ExportCommandBase : TokenCommandBase
+public abstract class ExportCommandBase : DiscordCommandBase
 {
     private readonly string _outputPath = Directory.GetCurrentDirectory();
 
@@ -268,7 +268,7 @@ public abstract class ExportCommandBase : TokenCommandBase
         await console.Output.WriteLineAsync("Resolving channel(s)...");
 
         var channels = new List<Channel>();
-        var guildChannelMap = new Dictionary<Snowflake, IReadOnlyList<Channel>>();
+        var channelsByGuild = new Dictionary<Snowflake, IReadOnlyList<Channel>>();
 
         foreach (var channelId in channelIds)
         {
@@ -278,7 +278,7 @@ public abstract class ExportCommandBase : TokenCommandBase
             if (channel.Kind == ChannelKind.GuildCategory)
             {
                 var guildChannels =
-                    guildChannelMap.GetValueOrDefault(channel.GuildId) ??
+                    channelsByGuild.GetValueOrDefault(channel.GuildId) ??
                     await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken);
 
                 foreach (var guildChannel in guildChannels)
@@ -288,7 +288,7 @@ public abstract class ExportCommandBase : TokenCommandBase
                 }
 
                 // Cache the guild channels to avoid redundant work
-                guildChannelMap[channel.GuildId] = guildChannels;
+                channelsByGuild[channel.GuildId] = guildChannels;
             }
             else
             {

DiscordChatExporter.Cli/Commands/ExportAllCommand.cs

@@ -1,9 +1,15 @@
 using System.Collections.Generic;
+using System.IO.Compression;
+using System.Text.Json;
 using System.Threading.Tasks;
 using CliFx.Attributes;
+using CliFx.Exceptions;
 using CliFx.Infrastructure;
 using DiscordChatExporter.Cli.Commands.Base;
+using DiscordChatExporter.Core.Discord;
 using DiscordChatExporter.Core.Discord.Data;
+using DiscordChatExporter.Core.Exceptions;
+using JsonExtensions.Reading;
 
 namespace DiscordChatExporter.Cli.Commands;
 
@@ -14,7 +20,19 @@ public class ExportAllCommand : ExportCommandBase
         "include-dm",
         Description = "Include direct message channels."
     )]
-    public bool IncludeDirectMessages { get; init; } = true;
+    public bool IncludeDirectChannels { get; init; } = true;
+
+    [CommandOption(
+        "include-guilds",
+        Description = "Include guild channels."
+    )]
+    public bool IncludeGuildChannels { get; init; } = true;
+
+    [CommandOption(
+        "data-package",
+        Description = "Path to the personal data package (ZIP file) requested from Discord. If provided, only channels referenced in the dump will be exported."
+    )]
+    public string? DataPackageFilePath { get; init; }
 
     public override async ValueTask ExecuteAsync(IConsole console)
     {
@@ -23,16 +41,60 @@ public class ExportAllCommand : ExportCommandBase
         var cancellationToken = console.RegisterCancellationHandler();
         var channels = new List<Channel>();
 
+        // Pull from the API
+        if (string.IsNullOrWhiteSpace(DataPackageFilePath))
+        {
             await console.Output.WriteLineAsync("Fetching channels...");
+
             await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
             {
-            // Skip DMs if instructed to
-            if (!IncludeDirectMessages && guild.Id == Guild.DirectMessages.Id)
+                await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
+                {
+                    channels.Add(channel);
+                }
+            }
+        }
+        // Pull from the data package
+        else
+        {
+            await console.Output.WriteLineAsync("Extracting channels...");
+            using var archive = ZipFile.OpenRead(DataPackageFilePath);
+
+            var entry = archive.GetEntry("messages/index.json");
+            if (entry is null)
+                throw new CommandException("Cannot find channel index inside the data package.");
+
+            await using var stream = entry.Open();
+            using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
+
+            foreach (var property in document.RootElement.EnumerateObjectOrEmpty())
+            {
+                var channelId = Snowflake.Parse(property.Name);
+                var channelName = property.Value.GetString();
+
+                // Null items refer to deleted channels
+                if (channelName is null)
                     continue;
 
-            await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
+                await console.Output.WriteLineAsync($"Fetching channel '{channelName}' ({channelId})...");
+
+                try
+                {
+                    var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
                     channels.Add(channel);
                 }
+                catch (DiscordChatExporterException)
+                {
+                    await console.Output.WriteLineAsync($"Channel '{channelName}' ({channelId}) is inaccessible.");
+                }
+            }
+        }
+
+        // Filter out unwanted channels
+        if (!IncludeDirectChannels)
+            channels.RemoveAll(c => c.Kind.IsDirect());
+        if (!IncludeGuildChannels)
+            channels.RemoveAll(c => c.Kind.IsGuild());
 
         await base.ExecuteAsync(console, channels);
     }

DiscordChatExporter.Cli/Commands/ExportChannelsCommand.cs

@@ -14,7 +14,7 @@ public class ExportChannelsCommand : ExportCommandBase
     [CommandOption(
         "channel",
         'c',
-        Description = "Channel ID(s). If provided with a category ID, all channels in that category will be exported."
+        Description = "Channel ID(s). If provided with category IDs, all channels inside those categories will be exported."
     )]
     public required IReadOnlyList<Snowflake> ChannelIds { get; init; }
 

DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs

@@ -1,5 +1,4 @@
-using System.Linq;
-using System.Threading.Tasks;
+using System.Threading.Tasks;
 using CliFx.Attributes;
 using CliFx.Infrastructure;
 using DiscordChatExporter.Cli.Commands.Base;
@@ -18,10 +17,7 @@ public class ExportDirectMessagesCommand : ExportCommandBase
         var cancellationToken = console.RegisterCancellationHandler();
 
         await console.Output.WriteLineAsync("Fetching channels...");
-
-        var channels = (await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken))
-            .Where(c => c.Kind != ChannelKind.GuildCategory)
-            .ToArray();
+        var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
 
         await base.ExecuteAsync(console, channels);
     }

DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs

@@ -11,7 +11,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
 namespace DiscordChatExporter.Cli.Commands;
 
 [Command("channels", Description = "Get the list of channels in a guild.")]
-public class GetChannelsCommand : TokenCommandBase
+public class GetChannelsCommand : DiscordCommandBase
 {
     [CommandOption(
         "guild",

DiscordChatExporter.Cli/Commands/GetDirectChannelsCommand.cs

@@ -10,7 +10,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
 namespace DiscordChatExporter.Cli.Commands;
 
 [Command("dm", Description = "Get the list of direct message channels.")]
-public class GetDirectMessageChannelsCommand : TokenCommandBase
+public class GetDirectChannelsCommand : DiscordCommandBase
 {
     public override async ValueTask ExecuteAsync(IConsole console)
     {

DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs

@@ -10,7 +10,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
 namespace DiscordChatExporter.Cli.Commands;
 
 [Command("guilds", Description = "Get the list of accessible guilds.")]
-public class GetGuildsCommand : TokenCommandBase
+public class GetGuildsCommand : DiscordCommandBase
 {
     public override async ValueTask ExecuteAsync(IConsole console)
     {

DiscordChatExporter.Core/Discord/Data/ChannelKind.cs

@@ -17,3 +17,12 @@ public enum ChannelKind
     GuildDirectory = 14,
     GuildForum = 15
 }
+
+public static class ChannelKindExtensions
+{
+    public static bool IsDirect(this ChannelKind kind) =>
+        kind is ChannelKind.DirectTextChat or ChannelKind.DirectGroupTextChat;
+
+    public static bool IsGuild(this ChannelKind kind) =>
+        !kind.IsDirect();
+}

DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs

@@ -38,16 +38,16 @@ public class DashboardViewModel : PropertyChangedBase
 
     public string? Token { get; set; }
 
-    private IReadOnlyDictionary<Guild, IReadOnlyList<Channel>>? GuildChannelMap { get; set; }
+    private IReadOnlyDictionary<Guild, IReadOnlyList<Channel>>? ChannelsByGuild { get; set; }
 
-    public IReadOnlyList<Guild>? AvailableGuilds => GuildChannelMap?.Keys.ToArray();
+    public IReadOnlyList<Guild>? AvailableGuilds => ChannelsByGuild?.Keys.ToArray();
 
     public Guild? SelectedGuild { get; set; }
 
     public bool IsDirectMessageGuildSelected => SelectedGuild?.Id == Guild.DirectMessages.Id;
 
     public IReadOnlyList<Channel>? AvailableChannels => SelectedGuild is not null
-        ? GuildChannelMap?[SelectedGuild]
+        ? ChannelsByGuild?[SelectedGuild]
         : null;
 
     public IReadOnlyList<Channel>? SelectedChannels { get; set; }
@@ -103,17 +103,17 @@ public class DashboardViewModel : PropertyChangedBase
 
             var discord = new DiscordClient(token);
 
-            var guildChannelMap = new Dictionary<Guild, IReadOnlyList<Channel>>();
+            var channelsByGuild = new Dictionary<Guild, IReadOnlyList<Channel>>();
             await foreach (var guild in discord.GetUserGuildsAsync())
             {
-                guildChannelMap[guild] = (await discord.GetGuildChannelsAsync(guild.Id))
+                channelsByGuild[guild] = (await discord.GetGuildChannelsAsync(guild.Id))
                     .Where(c => c.Kind != ChannelKind.GuildCategory)
                     .ToArray();
             }
 
             _discord = discord;
-            GuildChannelMap = guildChannelMap;
-            SelectedGuild = guildChannelMap.Keys.FirstOrDefault();
+            ChannelsByGuild = channelsByGuild;
+            SelectedGuild = channelsByGuild.Keys.FirstOrDefault();
         }
         catch (DiscordChatExporterException ex) when (!ex.IsFatal)
         {

DiscordChatExporter.Gui/Views/Components/DashboardView.xaml

@@ -19,7 +19,7 @@
     mc:Ignorable="d">
     <UserControl.Resources>
         <!--  Sort DMs by last message  -->
-        <CollectionViewSource x:Key="AvailableDirectMessageChannelsViewSource" Source="{Binding AvailableChannels, Mode=OneWay}">
+        <CollectionViewSource x:Key="AvailableDirectChannelsViewSource" Source="{Binding AvailableChannels, Mode=OneWay}">
             <CollectionViewSource.SortDescriptions>
                 <componentModel:SortDescription Direction="Descending" PropertyName="LastMessageId" />
                 <componentModel:SortDescription Direction="Ascending" PropertyName="Name" />
@@ -300,7 +300,7 @@
                             <Style BasedOn="{StaticResource {x:Type ListBox}}" TargetType="{x:Type ListBox}">
                                 <Style.Triggers>
                                     <DataTrigger Binding="{Binding IsDirectMessageGuildSelected}" Value="True">
-                                        <Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableDirectMessageChannelsViewSource}}" />
+                                        <Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableDirectChannelsViewSource}}" />
                                     </DataTrigger>
                                     <DataTrigger Binding="{Binding IsDirectMessageGuildSelected}" Value="False">
                                         <Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableChannelsViewSource}}" />