From 09f8937d99edadead5ec438afbbe88ea26badf16 Mon Sep 17 00:00:00 2001 From: Tyrrrz <1935960+Tyrrrz@users.noreply.github.com> Date: Mon, 9 Oct 2023 16:18:56 +0300 Subject: [PATCH] Refactor --- .../Commands/Base/ExportCommandBase.cs | 6 +- .../Commands/ExportAllCommand.cs | 134 +++++++++++------- .../Commands/ExportGuildCommand.cs | 56 +++++--- .../Utils/Extensions/ConsoleExtensions.cs | 9 +- .../Discord/Dump/DataDump.cs | 60 ++++++++ .../Discord/Dump/DataDumpChannel.cs | 3 + 6 files changed, 193 insertions(+), 75 deletions(-) create mode 100644 DiscordChatExporter.Core/Discord/Dump/DataDump.cs create mode 100644 DiscordChatExporter.Core/Discord/Dump/DataDumpChannel.cs diff --git a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs index 5ec0a53..87e0b25 100644 --- a/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs +++ b/DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs @@ -186,7 +186,7 @@ public abstract class ExportCommandBase : DiscordCommandBase // https://github.com/Tyrrrz/DiscordChatExporter/issues/1124 ParallelLimit > 1 ) - .StartAsync(async progressContext => + .StartAsync(async ctx => { await Parallel.ForEachAsync( channels, @@ -199,7 +199,7 @@ public abstract class ExportCommandBase : DiscordCommandBase { try { - await progressContext.StartTaskAsync( + await ctx.StartTaskAsync( channel.GetHierarchicalName(), async progress => { @@ -257,7 +257,7 @@ public abstract class ExportCommandBase : DiscordCommandBase using (console.WithForegroundColor(ConsoleColor.Red)) { await console.Error.WriteLineAsync( - $"Failed to export {errorsByChannel.Count} channel(s):" + $"Failed to export {errorsByChannel.Count} the following channel(s):" ); } diff --git a/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs b/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs index 7c58117..7d6393f 100644 --- a/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportAllCommand.cs @@ -1,18 +1,16 @@ +using System; using System.Collections.Generic; -using System.IO.Compression; using System.Linq; -using System.Text.Json; using System.Threading.Tasks; using CliFx.Attributes; -using CliFx.Exceptions; using CliFx.Infrastructure; using DiscordChatExporter.Cli.Commands.Base; using DiscordChatExporter.Cli.Commands.Converters; using DiscordChatExporter.Cli.Commands.Shared; -using DiscordChatExporter.Core.Discord; +using DiscordChatExporter.Cli.Utils.Extensions; using DiscordChatExporter.Core.Discord.Data; +using DiscordChatExporter.Core.Discord.Dump; using DiscordChatExporter.Core.Exceptions; -using JsonExtensions.Reading; using Spectre.Console; namespace DiscordChatExporter.Cli.Commands; @@ -55,32 +53,53 @@ public class ExportAllCommand : ExportCommandBase { await foreach (var guild in Discord.GetUserGuildsAsync(cancellationToken)) { - await console.Output.WriteLineAsync($"Fetching channels for guild '{guild.Name}'..."); - // Regular channels - await foreach ( - var channel in Discord.GetGuildChannelsAsync(guild.Id, cancellationToken) - ) - { - if (channel.IsCategory) - continue; + await console.Output.WriteLineAsync( + $"Fetching channels for guild '{guild.Name}'..." + ); - if (!IncludeVoiceChannels && channel.IsVoice) - continue; + var fetchedChannelsCount = 0; + await console + .CreateStatusTicker() + .StartAsync( + "...", + async ctx => + { + await foreach ( + var channel in Discord.GetGuildChannelsAsync( + guild.Id, + cancellationToken + ) + ) + { + if (channel.IsCategory) + continue; - channels.Add(channel); - } + if (!IncludeVoiceChannels && channel.IsVoice) + continue; - await console.Output.WriteLineAsync($" Found {channels.Count} channels."); + channels.Add(channel); + + ctx.Status($"Fetched '{channel.GetHierarchicalName()}'."); + fetchedChannelsCount++; + } + } + ); + + await console.Output.WriteLineAsync($"Fetched {fetchedChannelsCount} channel(s)."); // Threads if (ThreadInclusionMode != ThreadInclusionMode.None) { - AnsiConsole.MarkupLine("Fetching threads..."); - await AnsiConsole - .Status() + await console.Output.WriteLineAsync( + $"Fetching threads for guild '{guild.Name}'..." + ); + + var fetchedThreadsCount = 0; + await console + .CreateStatusTicker() .StartAsync( - "Found 0 threads.", + "...", async ctx => { await foreach ( @@ -94,14 +113,15 @@ public class ExportAllCommand : ExportCommandBase ) { channels.Add(thread); - ctx.Status( - $"Found {channels.Count(channel => channel.IsThread)} threads: {thread.GetHierarchicalName()}" - ); + + ctx.Status($"Fetched '{thread.GetHierarchicalName()}'."); + fetchedThreadsCount++; } } ); + await console.Output.WriteLineAsync( - $" Found {channels.Count(channel => channel.IsThread)} threads." + $"Fetched {fetchedThreadsCount} thread(s)." ); } } @@ -110,39 +130,55 @@ public class ExportAllCommand : ExportCommandBase else { await console.Output.WriteLineAsync("Extracting channels..."); - using var archive = ZipFile.OpenRead(DataPackageFilePath); - - var entry = archive.GetEntry("messages/index.json"); - if (entry is null) - throw new CommandException("Could not find channel index inside the data package."); - await using var stream = entry.Open(); - using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken); + var dump = await DataDump.LoadAsync(DataPackageFilePath, cancellationToken); + var inaccessibleChannels = new List(); - foreach (var property in document.RootElement.EnumerateObjectOrEmpty()) - { - var channelId = Snowflake.Parse(property.Name); - var channelName = property.Value.GetString(); + await console + .CreateStatusTicker() + .StartAsync( + "...", + async ctx => + { + foreach (var dumpChannel in dump.Channels) + { + ctx.Status($"Fetching '{dumpChannel.Name}' ({dumpChannel.Id})..."); - // Null items refer to deleted channels - if (channelName is null) - continue; + try + { + var channel = await Discord.GetChannelAsync( + dumpChannel.Id, + cancellationToken + ); - await console.Output.WriteLineAsync( - $"Fetching channel '{channelName}' ({channelId})..." + channels.Add(channel); + } + catch (DiscordChatExporterException) + { + inaccessibleChannels.Add(dumpChannel); + } + } + } ); - try - { - var channel = await Discord.GetChannelAsync(channelId, cancellationToken); - channels.Add(channel); - } - catch (DiscordChatExporterException) + await console.Output.WriteLineAsync($"Fetched {channels} channel(s)."); + + // Print inaccessible channels + if (inaccessibleChannels.Any()) + { + await console.Output.WriteLineAsync(); + + using (console.WithForegroundColor(ConsoleColor.Red)) { await console.Error.WriteLineAsync( - $"Channel '{channelName}' ({channelId}) is inaccessible." + "Failed to access the following channel(s):" ); } + + foreach (var dumpChannel in inaccessibleChannels) + await console.Error.WriteLineAsync($"{dumpChannel.Name} ({dumpChannel.Id})"); + + await console.Error.WriteLineAsync(); } } diff --git a/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs b/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs index 7e316d5..c9e6088 100644 --- a/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs +++ b/DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs @@ -6,6 +6,7 @@ using CliFx.Infrastructure; using DiscordChatExporter.Cli.Commands.Base; using DiscordChatExporter.Cli.Commands.Converters; using DiscordChatExporter.Cli.Commands.Shared; +using DiscordChatExporter.Cli.Utils.Extensions; using DiscordChatExporter.Core.Discord; using DiscordChatExporter.Core.Discord.Data; using Spectre.Console; @@ -35,30 +36,46 @@ public class ExportGuildCommand : ExportCommandBase var cancellationToken = console.RegisterCancellationHandler(); var channels = new List(); + // Regular channels await console.Output.WriteLineAsync("Fetching channels..."); - // Regular channels - await foreach (var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken)) - { - if (channel.IsCategory) - continue; + var fetchedChannelsCount = 0; + await console + .CreateStatusTicker() + .StartAsync( + "...", + async ctx => + { + await foreach ( + var channel in Discord.GetGuildChannelsAsync(GuildId, cancellationToken) + ) + { + if (channel.IsCategory) + continue; - if (!IncludeVoiceChannels && channel.IsVoice) - continue; + if (!IncludeVoiceChannels && channel.IsVoice) + continue; - channels.Add(channel); - } + channels.Add(channel); + + ctx.Status($"Fetched '{channel.GetHierarchicalName()}'."); + fetchedChannelsCount++; + } + } + ); - await console.Output.WriteLineAsync($" Found {channels.Count} channels."); + await console.Output.WriteLineAsync($"Fetched {fetchedChannelsCount} channel(s)."); // Threads if (ThreadInclusionMode != ThreadInclusionMode.None) { - AnsiConsole.MarkupLine("Fetching threads..."); - await AnsiConsole - .Status() + await console.Output.WriteLineAsync("Fetching threads..."); + + var fetchedThreadsCount = 0; + await console + .CreateStatusTicker() .StartAsync( - "Found 0 threads.", + "...", async ctx => { await foreach ( @@ -72,15 +89,14 @@ public class ExportGuildCommand : ExportCommandBase ) { channels.Add(thread); - ctx.Status( - $"Found {channels.Count(channel => channel.IsThread)} threads: {thread.GetHierarchicalName()}" - ); + + ctx.Status($"Fetched '{thread.GetHierarchicalName()}'."); + fetchedThreadsCount++; } } ); - await console.Output.WriteLineAsync( - $" Found {channels.Count(channel => channel.IsThread)} threads." - ); + + await console.Output.WriteLineAsync($"Fetched {fetchedThreadsCount} thread(s)."); } await ExportAsync(console, channels); diff --git a/DiscordChatExporter.Cli/Utils/Extensions/ConsoleExtensions.cs b/DiscordChatExporter.Cli/Utils/Extensions/ConsoleExtensions.cs index 0d6570c..345fb81 100644 --- a/DiscordChatExporter.Cli/Utils/Extensions/ConsoleExtensions.cs +++ b/DiscordChatExporter.Cli/Utils/Extensions/ConsoleExtensions.cs @@ -17,6 +17,9 @@ internal static class ConsoleExtensions } ); + public static Status CreateStatusTicker(this IConsole console) => + console.CreateAnsiConsole().Status().AutoRefresh(true); + public static Progress CreateProgressTicker(this IConsole console) => console .CreateAnsiConsole() @@ -31,16 +34,16 @@ internal static class ConsoleExtensions ); public static async ValueTask StartTaskAsync( - this ProgressContext progressContext, + this ProgressContext context, string description, Func performOperationAsync ) { // Description cannot be empty // https://github.com/Tyrrrz/DiscordChatExporter/issues/1133 - var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "?"; + var actualDescription = !string.IsNullOrWhiteSpace(description) ? description : "..."; - var progressTask = progressContext.AddTask( + var progressTask = context.AddTask( // Don't recognize random square brackets as style tags Markup.Escape(actualDescription), new ProgressTaskSettings { MaxValue = 1 } diff --git a/DiscordChatExporter.Core/Discord/Dump/DataDump.cs b/DiscordChatExporter.Core/Discord/Dump/DataDump.cs new file mode 100644 index 0000000..28ae9f5 --- /dev/null +++ b/DiscordChatExporter.Core/Discord/Dump/DataDump.cs @@ -0,0 +1,60 @@ +using System; +using System.Collections.Generic; +using System.IO.Compression; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using JsonExtensions.Reading; + +namespace DiscordChatExporter.Core.Discord.Dump; + +public partial class DataDump +{ + public IReadOnlyList Channels { get; } + + public DataDump(IReadOnlyList channels) => Channels = channels; +} + +public partial class DataDump +{ + public static DataDump Parse(JsonElement json) + { + var channels = new List(); + + foreach (var property in json.EnumerateObjectOrEmpty()) + { + var channelId = Snowflake.Parse(property.Name); + var channelName = property.Value.GetString(); + + // Null items refer to deleted channels + if (channelName is null) + continue; + + var channel = new DataDumpChannel(channelId, channelName); + channels.Add(channel); + } + + return new DataDump(channels); + } + + public static async ValueTask LoadAsync( + string zipFilePath, + CancellationToken cancellationToken = default + ) + { + using var archive = ZipFile.OpenRead(zipFilePath); + + var entry = archive.GetEntry("messages/index.json"); + if (entry is null) + { + throw new InvalidOperationException( + "Could not find the channel index inside the data package." + ); + } + + await using var stream = entry.Open(); + using var document = await JsonDocument.ParseAsync(stream, default, cancellationToken); + + return Parse(document.RootElement); + } +} diff --git a/DiscordChatExporter.Core/Discord/Dump/DataDumpChannel.cs b/DiscordChatExporter.Core/Discord/Dump/DataDumpChannel.cs new file mode 100644 index 0000000..01bdf2d --- /dev/null +++ b/DiscordChatExporter.Core/Discord/Dump/DataDumpChannel.cs @@ -0,0 +1,3 @@ +namespace DiscordChatExporter.Core.Discord.Dump; + +public record DataDumpChannel(Snowflake Id, string Name);