open-source-mirrors
/
DiscordChatExporter
mirror of https://github.com/Tyrrrz/DiscordChatExporter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add support for extracting channels from data dump in `exportall`

Browse Source 
Closes #597
pull/1003/head
Tyrrrz 2 years ago
parent 83e3289ead
commit 0e1c3e4c76
11 changed files with 99 additions and 32 deletions
Show Stats Download Patch File Download Diff File

2
DiscordChatExporter.Cli/Commands/Base/TokenCommandBase.cs → DiscordChatExporter.Cli/Commands/Base/DiscordCommandBase.cs
Unescape View File

@ -7,7 +7,7 @@ using DiscordChatExporter.Core.Discord;
namespace DiscordChatExporter.Cli.Commands.Base;
public abstract class TokenCommandBase : ICommand
public abstract class DiscordCommandBase : ICommand
{
[CommandOption(
"token",

8
DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
Unescape View File

@ -20,7 +20,7 @@ using Gress;
namespace DiscordChatExporter.Cli.Commands.Base;
public abstract class ExportCommandBase : TokenCommandBase
public abstract class ExportCommandBase : DiscordCommandBase
{
private readonly string _outputPath = Directory.GetCurrentDirectory();
@ -268,7 +268,7 @@ public abstract class ExportCommandBase : TokenCommandBase
await console.Output.WriteLineAsync("Resolving channel(s)...");
var channels = new List<Channel>();
var guildChannelMap = new Dictionary<Snowflake, IReadOnlyList<Channel>>();
var channelsByGuild = new Dictionary<Snowflake, IReadOnlyList<Channel>>();
foreach (var channelId in channelIds)
{
@ -278,7 +278,7 @@ public abstract class ExportCommandBase : TokenCommandBase
if (channel.Kind == ChannelKind.GuildCategory)
{
var guildChannels =
guildChannelMap.GetValueOrDefault(channel.GuildId) ??
channelsByGuild.GetValueOrDefault(channel.GuildId) ??
await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken);
foreach (var guildChannel in guildChannels)
@ -288,7 +288,7 @@ public abstract class ExportCommandBase : TokenCommandBase
}
// Cache the guild channels to avoid redundant work
guildChannelMap[channel.GuildId] = guildChannels;
channelsByGuild[channel.GuildId] = guildChannels;
}
else
{

78
DiscordChatExporter.Cli/Commands/ExportAllCommand.cs
Unescape View File

@ -1,9 +1,15 @@
using System.Collections.Generic;
using System.IO.Compression;
using System.Text.Json;
using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Exceptions;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Exceptions;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Cli.Commands;
@ -14,7 +20,19 @@ public class ExportAllCommand : ExportCommandBase
"include-dm",
Description = "Include direct message channels."
)]
public bool IncludeDirectMessages { get; init; } = true;
public bool IncludeDirectChannels { get; init; } = true;
[CommandOption(
"include-guilds",
Description = "Include guild channels."
)]
public bool IncludeGuildChannels { get; init; } = true;
[CommandOption(
"data-package",
Description = "Path to the personal data package (ZIP file) requested from Discord. If provided, only channels referenced in the dump will be exported."
)]
public string? DataPackageFilePath { get; init; }
public override async ValueTask ExecuteAsync(IConsole console)
{
@ -23,16 +41,60 @@ public class ExportAllCommand : ExportCommandBase
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>();
await console.Output.WriteLineAsync("Fetching channels...");
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
// Pull from the API
if (string.IsNullOrWhiteSpace(DataPackageFilePath))
{
// Skip DMs if instructed to
if (!IncludeDirectMessages && guild.Id == Guild.DirectMessages.Id)
continue;
await console.Output.WriteLineAsync("Fetching channels...");
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
channels.Add(channel);
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
{
await foreach (var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken))
{
channels.Add(channel);
}
}
}
// Pull from the data package
else
{
await console.Output.WriteLineAsync("Extracting channels...");
using var archive = ZipFile.OpenRead(DataPackageFilePath);
var entry = archive.GetEntry("messages/index.json");
if (entry is null)
throw new CommandException("Cannot find channel index inside the data package.");
await using var stream = entry.Open();
using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
foreach (var property in document.RootElement.EnumerateObjectOrEmpty())
{
var channelId = Snowflake.Parse(property.Name);
var channelName = property.Value.GetString();
// Null items refer to deleted channels
if (channelName is null)
continue;
await console.Output.WriteLineAsync($"Fetching channel '{channelName}' ({channelId})...");
try
{
var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
channels.Add(channel);
}
catch (DiscordChatExporterException)
{
await console.Output.WriteLineAsync($"Channel '{channelName}' ({channelId}) is inaccessible.");
}
}
}
// Filter out unwanted channels
if (!IncludeDirectChannels)
channels.RemoveAll(c => c.Kind.IsDirect());
if (!IncludeGuildChannels)
channels.RemoveAll(c => c.Kind.IsGuild());
await base.ExecuteAsync(console, channels);
}

2
DiscordChatExporter.Cli/Commands/ExportChannelsCommand.cs
Unescape View File

@ -14,7 +14,7 @@ public class ExportChannelsCommand : ExportCommandBase
[CommandOption(
"channel",
'c',
Description = "Channel ID(s). If provided with a category ID, all channels in that category will be exported."
Description = "Channel ID(s). If provided with category IDs, all channels inside those categories will be exported."
)]
public required IReadOnlyList<Snowflake> ChannelIds { get; init; }

8
DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs
Unescape View File

@ -1,5 +1,4 @@
using System.Linq;
using System.Threading.Tasks;
using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
@ -18,10 +17,7 @@ public class ExportDirectMessagesCommand : ExportCommandBase
var cancellationToken = console.RegisterCancellationHandler();
await console.Output.WriteLineAsync("Fetching channels...");
var channels = (await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken))
.Where(c => c.Kind != ChannelKind.GuildCategory)
.ToArray();
var channels = await Discord.GetGuildChannelsAsync(Guild.DirectMessages.Id, cancellationToken);
await base.ExecuteAsync(console, channels);
}

2
DiscordChatExporter.Cli/Commands/GetChannelsCommand.cs
Unescape View File

@ -11,7 +11,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands;
[Command("channels", Description = "Get the list of channels in a guild.")]
public class GetChannelsCommand : TokenCommandBase
public class GetChannelsCommand : DiscordCommandBase
{
[CommandOption(
"guild",

2
DiscordChatExporter.Cli/Commands/GetDirectMessageChannelsCommand.cs → DiscordChatExporter.Cli/Commands/GetDirectChannelsCommand.cs
Unescape View File

@ -10,7 +10,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands;
[Command("dm", Description = "Get the list of direct message channels.")]
public class GetDirectMessageChannelsCommand : TokenCommandBase
public class GetDirectChannelsCommand : DiscordCommandBase
{
public override async ValueTask ExecuteAsync(IConsole console)
{

2
DiscordChatExporter.Cli/Commands/GetGuildsCommand.cs
Unescape View File

@ -10,7 +10,7 @@ using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands;
[Command("guilds", Description = "Get the list of accessible guilds.")]
public class GetGuildsCommand : TokenCommandBase
public class GetGuildsCommand : DiscordCommandBase
{
public override async ValueTask ExecuteAsync(IConsole console)
{

9
DiscordChatExporter.Core/Discord/Data/ChannelKind.cs
Unescape View File

@ -16,4 +16,13 @@ public enum ChannelKind
GuildStageVoice = 13,
GuildDirectory = 14,
GuildForum = 15
}
public static class ChannelKindExtensions
{
public static bool IsDirect(this ChannelKind kind) =>
kind is ChannelKind.DirectTextChat or ChannelKind.DirectGroupTextChat;
public static bool IsGuild(this ChannelKind kind) =>
!kind.IsDirect();
}

14
DiscordChatExporter.Gui/ViewModels/Components/DashboardViewModel.cs
Unescape View File

@ -38,16 +38,16 @@ public class DashboardViewModel : PropertyChangedBase
public string? Token { get; set; }
private IReadOnlyDictionary<Guild, IReadOnlyList<Channel>>? GuildChannelMap { get; set; }
private IReadOnlyDictionary<Guild, IReadOnlyList<Channel>>? ChannelsByGuild { get; set; }
public IReadOnlyList<Guild>? AvailableGuilds => GuildChannelMap?.Keys.ToArray();
public IReadOnlyList<Guild>? AvailableGuilds => ChannelsByGuild?.Keys.ToArray();
public Guild? SelectedGuild { get; set; }
public bool IsDirectMessageGuildSelected => SelectedGuild?.Id == Guild.DirectMessages.Id;
public IReadOnlyList<Channel>? AvailableChannels => SelectedGuild is not null
? GuildChannelMap?[SelectedGuild]
? ChannelsByGuild?[SelectedGuild]
: null;
public IReadOnlyList<Channel>? SelectedChannels { get; set; }
@ -103,17 +103,17 @@ public class DashboardViewModel : PropertyChangedBase
var discord = new DiscordClient(token);
var guildChannelMap = new Dictionary<Guild, IReadOnlyList<Channel>>();
var channelsByGuild = new Dictionary<Guild, IReadOnlyList<Channel>>();
await foreach (var guild in discord.GetUserGuildsAsync())
{
guildChannelMap[guild] = (await discord.GetGuildChannelsAsync(guild.Id))
channelsByGuild[guild] = (await discord.GetGuildChannelsAsync(guild.Id))
.Where(c => c.Kind != ChannelKind.GuildCategory)
.ToArray();
}
_discord = discord;
GuildChannelMap = guildChannelMap;
SelectedGuild = guildChannelMap.Keys.FirstOrDefault();
ChannelsByGuild = channelsByGuild;
SelectedGuild = channelsByGuild.Keys.FirstOrDefault();
}
catch (DiscordChatExporterException ex) when (!ex.IsFatal)
{

4
DiscordChatExporter.Gui/Views/Components/DashboardView.xaml
Unescape View File

@ -19,7 +19,7 @@
mc:Ignorable="d">
<UserControl.Resources>
<!-- Sort DMs by last message -->
<CollectionViewSource x:Key="AvailableDirectMessageChannelsViewSource" Source="{Binding AvailableChannels, Mode=OneWay}">
<CollectionViewSource x:Key="AvailableDirectChannelsViewSource" Source="{Binding AvailableChannels, Mode=OneWay}">
<CollectionViewSource.SortDescriptions>
<componentModel:SortDescription Direction="Descending" PropertyName="LastMessageId" />
<componentModel:SortDescription Direction="Ascending" PropertyName="Name" />
@ -300,7 +300,7 @@
<Style BasedOn="{StaticResource {x:Type ListBox}}" TargetType="{x:Type ListBox}">
<Style.Triggers>
<DataTrigger Binding="{Binding IsDirectMessageGuildSelected}" Value="True">
<Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableDirectMessageChannelsViewSource}}" />
<Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableDirectChannelsViewSource}}" />
</DataTrigger>
<DataTrigger Binding="{Binding IsDirectMessageGuildSelected}" Value="False">
<Setter Property="ItemsSource" Value="{Binding Source={StaticResource AvailableChannelsViewSource}}" />

Write Preview
Loading…
Cancel
Save