open-source-mirrors
/
DiscordChatExporter
mirror of https://github.com/Tyrrrz/DiscordChatExporter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor

Browse Source
pull/1138/head^2
Tyrrrz 1 year ago
parent ad2dab2157
commit 09f8937d99
6 changed files with 193 additions and 75 deletions
Show Stats Download Patch File Download Diff File

6
DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs
Unescape View File

@ -186,7 +186,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1124
ParallelLimit > 1
)
.StartAsync(async progressContext =>
.StartAsync(async ctx =>
{
await Parallel.ForEachAsync(
channels,
@ -199,7 +199,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
{
try
{
await progressContext.StartTaskAsync(
await ctx.StartTaskAsync(
channel.GetHierarchicalName(),
async progress =>
{
@ -257,7 +257,7 @@ public abstract class ExportCommandBase : DiscordCommandBase
using (console.WithForegroundColor(ConsoleColor.Red))
{
await console.Error.WriteLineAsync(
$"Failed to export {errorsByChannel.Count} channel(s):"
$"Failed to export {errorsByChannel.Count} the following channel(s):"
);
}

110
DiscordChatExporter.Cli/Commands/ExportAllCommand.cs
Unescape View File

@ -1,18 +1,16 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Linq;
using System.Text.Json;
using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Exceptions;
using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Cli.Commands.Converters;
using DiscordChatExporter.Cli.Commands.Shared;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Cli.Utils.Extensions;
using DiscordChatExporter.Core.Discord.Data;
using DiscordChatExporter.Core.Discord.Dump;
using DiscordChatExporter.Core.Exceptions;
using JsonExtensions.Reading;
using Spectre.Console;
namespace DiscordChatExporter.Cli.Commands;
@ -55,11 +53,23 @@ public class ExportAllCommand : ExportCommandBase
{
await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken))
{
await console.Output.WriteLineAsync($"Fetching channels for guild '{guild.Name}'...");
// Regular channels
await console.Output.WriteLineAsync(
$"Fetching channels for guild '{guild.Name}'..."
);
var fetchedChannelsCount = 0;
await console
.CreateStatusTicker()
.StartAsync(
"...",
async ctx =>
{
await foreach (
var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken)
var channel in Discord.GetGuildChannelsAsync(
guild.Id,
cancellationToken
)
)
{
if (channel.IsCategory)
@ -69,18 +79,27 @@ public class ExportAllCommand : ExportCommandBase
continue;
channels.Add(channel);
ctx.Status($"Fetched '{channel.GetHierarchicalName()}'.");
fetchedChannelsCount++;
}
}
);
await console.Output.WriteLineAsync($" Found {channels.Count} channels.");
await console.Output.WriteLineAsync($"Fetched {fetchedChannelsCount} channel(s).");
// Threads
if (ThreadInclusionMode != ThreadInclusionMode.None)
{
AnsiConsole.MarkupLine("Fetching threads...");
await AnsiConsole
.Status()
await console.Output.WriteLineAsync(
$"Fetching threads for guild '{guild.Name}'..."
);
var fetchedThreadsCount = 0;
await console
.CreateStatusTicker()
.StartAsync(
"Found 0 threads.",
"...",
async ctx =>
{
await foreach (
@ -94,14 +113,15 @@ public class ExportAllCommand : ExportCommandBase
)
{
channels.Add(thread);
ctx.Status(
$"Found {channels.Count(channel => channel.IsThread)} threads: {thread.GetHierarchicalName()}"
);
ctx.Status($"Fetched '{thread.GetHierarchicalName()}'.");
fetchedThreadsCount++;
}
}
);
await console.Output.WriteLineAsync(
$" Found {channels.Count(channel => channel.IsThread)} threads."
$"Fetched {fetchedThreadsCount} thread(s)."
);
}
}
@ -110,39 +130,55 @@ public class ExportAllCommand : ExportCommandBase
else
{
await console.Output.WriteLineAsync("Extracting channels...");
using var archive = ZipFile.OpenRead(DataPackageFilePath);
var entry = archive.GetEntry("messages/index.json");
if (entry is null)
throw new CommandException("Could not find channel index inside the data package.");
var dump = await DataDump.LoadAsync(DataPackageFilePath, cancellationToken);
var inaccessibleChannels = new List<DataDumpChannel>();
await using var stream = entry.Open();
using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
foreach (var property in document.RootElement.EnumerateObjectOrEmpty())
await console
.CreateStatusTicker()
.StartAsync(
"...",
async ctx =>
{
var channelId = Snowflake.Parse(property.Name);
var channelName = property.Value.GetString();
// Null items refer to deleted channels
if (channelName is null)
continue;
await console.Output.WriteLineAsync(
$"Fetching channel '{channelName}' ({channelId})..."
);
foreach (var dumpChannel in dump.Channels)
{
ctx.Status($"Fetching '{dumpChannel.Name}' ({dumpChannel.Id})...");
try
{
var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
var channel = await Discord.GetChannelAsync(
dumpChannel.Id,
cancellationToken
);
channels.Add(channel);
}
catch (DiscordChatExporterException)
{
inaccessibleChannels.Add(dumpChannel);
}
}
}
);
await console.Output.WriteLineAsync($"Fetched {channels} channel(s).");
// Print inaccessible channels
if (inaccessibleChannels.Any())
{
await console.Output.WriteLineAsync();
using (console.WithForegroundColor(ConsoleColor.Red))
{
await console.Error.WriteLineAsync(
$"Channel '{channelName}' ({channelId}) is inaccessible."
"Failed to access the following channel(s):"
);
}
foreach (var dumpChannel in inaccessibleChannels)
await console.Error.WriteLineAsync($"{dumpChannel.Name} ({dumpChannel.Id})");
await console.Error.WriteLineAsync();
}
}

42
DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs
Unescape View File

@ -6,6 +6,7 @@ using CliFx.Infrastructure;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Cli.Commands.Converters;
using DiscordChatExporter.Cli.Commands.Shared;
using DiscordChatExporter.Cli.Utils.Extensions;
using DiscordChatExporter.Core.Discord;
using DiscordChatExporter.Core.Discord.Data;
using Spectre.Console;
@ -35,10 +36,19 @@ public class ExportGuildCommand : ExportCommandBase
var cancellationToken = console.RegisterCancellationHandler();
var channels = new List<Channel>();
// Regular channels
await console.Output.WriteLineAsync("Fetching channels...");
// Regular channels
await foreach (var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken))
var fetchedChannelsCount = 0;
await console
.CreateStatusTicker()
.StartAsync(
"...",
async ctx =>
{
await foreach (
var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken)
)
{
if (channel.IsCategory)
continue;
@ -47,18 +57,25 @@ public class ExportGuildCommand : ExportCommandBase
continue;
channels.Add(channel);
ctx.Status($"Fetched '{channel.GetHierarchicalName()}'.");
fetchedChannelsCount++;
}
}
);
await console.Output.WriteLineAsync($" Found {channels.Count} channels.");
await console.Output.WriteLineAsync($"Fetched {fetchedChannelsCount} channel(s).");
// Threads
if (ThreadInclusionMode != ThreadInclusionMode.None)
{
AnsiConsole.MarkupLine("Fetching threads...");
await AnsiConsole
.Status()
await console.Output.WriteLineAsync("Fetching threads...");
var fetchedThreadsCount = 0;
await console
.CreateStatusTicker()
.StartAsync(
"Found 0 threads.",
"...",
async ctx =>
{
await foreach (
@ -72,15 +89,14 @@ public class ExportGuildCommand : ExportCommandBase
)
{
channels.Add(thread);
ctx.Status(
$"Found {channels.Count(channel => channel.IsThread)} threads: {thread.GetHierarchicalName()}"
);
ctx.Status($"Fetched '{thread.GetHierarchicalName()}'.");
fetchedThreadsCount++;
}
}
);
await console.Output.WriteLineAsync(
$" Found {channels.Count(channel => channel.IsThread)} threads."
);
await console.Output.WriteLineAsync($"Fetched {fetchedThreadsCount} thread(s).");
}
await ExportAsync(console, channels);

9
DiscordChatExporter.Cli/Utils/Extensions/ConsoleExtensions.cs
Unescape View File

@ -17,6 +17,9 @@ internal static class ConsoleExtensions
}
);
public static Status CreateStatusTicker(this IConsole console) =>
console.CreateAnsiConsole().Status().AutoRefresh(true);
public static Progress CreateProgressTicker(this IConsole console) =>
console
.CreateAnsiConsole()
@ -31,16 +34,16 @@ internal static class ConsoleExtensions
);
public static async ValueTask StartTaskAsync(
this ProgressContext progressContext,
this ProgressContext context,
string description,
Func<ProgressTask, ValueTask> performOperationAsync
)
{
// Description cannot be empty
// https://github.com/Tyrrrz/DiscordChatExporter/issues/1133
var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "?";
var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "...";
var progressTask = progressContext.AddTask(
var progressTask = context.AddTask(
// Don't recognize random square brackets as style tags
Markup.Escape(actualDescription),
new ProgressTaskSettings { MaxValue = 1 }

60
DiscordChatExporter.Core/Discord/Dump/DataDump.cs
Unescape View File

@ -0,0 +1,60 @@
using System;
using System.Collections.Generic;
using System.IO.Compression;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using JsonExtensions.Reading;
namespace DiscordChatExporter.Core.Discord.Dump;
public partial class DataDump
{
public IReadOnlyList<DataDumpChannel> Channels { get; }
public DataDump(IReadOnlyList<DataDumpChannel> channels) => Channels = channels;
}
public partial class DataDump
{
public static DataDump Parse(JsonElement json)
{
var channels = new List<DataDumpChannel>();
foreach (var property in json.EnumerateObjectOrEmpty())
{
var channelId = Snowflake.Parse(property.Name);
var channelName = property.Value.GetString();
// Null items refer to deleted channels
if (channelName is null)
continue;
var channel = new DataDumpChannel(channelId, channelName);
channels.Add(channel);
}
return new DataDump(channels);
}
public static async ValueTask<DataDump> LoadAsync(
string zipFilePath,
CancellationToken cancellationToken = default
)
{
using var archive = ZipFile.OpenRead(zipFilePath);
var entry = archive.GetEntry("messages/index.json");
if (entry is null)
{
throw new InvalidOperationException(
"Could not find the channel index inside the data package."
);
}
await using var stream = entry.Open();
using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken);
return Parse(document.RootElement);
}
}

3
DiscordChatExporter.Core/Discord/Dump/DataDumpChannel.cs
Unescape View File

@ -0,0 +1,3 @@
namespace DiscordChatExporter.Core.Discord.Dump;
public record DataDumpChannel(Snowflake Id, string Name);
Write Preview
Loading…
Cancel
Save