Use .NET 6's `ParallelForEachAsync(...)`

pull/811/head
Tyrrrz 3 years ago
parent b8567d384f
commit 008bb2f591

@ -14,7 +14,6 @@ using DiscordChatExporter.Core.Exceptions;
using DiscordChatExporter.Core.Exporting; using DiscordChatExporter.Core.Exporting;
using DiscordChatExporter.Core.Exporting.Filtering; using DiscordChatExporter.Core.Exporting.Filtering;
using DiscordChatExporter.Core.Exporting.Partitioning; using DiscordChatExporter.Core.Exporting.Partitioning;
using DiscordChatExporter.Core.Utils.Extensions;
namespace DiscordChatExporter.Cli.Commands.Base; namespace DiscordChatExporter.Cli.Commands.Base;
@ -68,13 +67,22 @@ public abstract class ExportCommandBase : TokenCommandBase
await console.Output.WriteLineAsync($"Exporting {channels.Count} channel(s)..."); await console.Output.WriteLineAsync($"Exporting {channels.Count} channel(s)...");
await console.CreateProgressTicker().StartAsync(async progressContext => await console.CreateProgressTicker().StartAsync(async progressContext =>
{ {
await channels.ParallelForEachAsync(async channel => await Parallel.ForEachAsync(
channels,
new ParallelOptions
{
MaxDegreeOfParallelism = Math.Max(1, ParallelLimit),
CancellationToken = cancellationToken
},
async (channel, innerCancellationToken) =>
{ {
try try
{ {
await progressContext.StartTaskAsync($"{channel.Category.Name} / {channel.Name}", async progress => await progressContext.StartTaskAsync(
$"{channel.Category.Name} / {channel.Name}",
async progress =>
{ {
var guild = await Discord.GetGuildAsync(channel.GuildId, cancellationToken); var guild = await Discord.GetGuildAsync(channel.GuildId, innerCancellationToken);
var request = new ExportRequest( var request = new ExportRequest(
guild, guild,
@ -90,14 +98,16 @@ public abstract class ExportCommandBase : TokenCommandBase
DateFormat DateFormat
); );
await Exporter.ExportChannelAsync(request, progress, cancellationToken); await Exporter.ExportChannelAsync(request, progress, innerCancellationToken);
}); }
);
} }
catch (DiscordChatExporterException ex) when (!ex.IsFatal) catch (DiscordChatExporterException ex) when (!ex.IsFatal)
{ {
errors[channel] = ex.Message; errors[channel] = ex.Message;
} }
}, Math.Max(ParallelLimit, 1), cancellationToken); }
);
}); });
// Print result // Print result

@ -48,10 +48,9 @@ public partial record ExportRequest
Snowflake? after = null, Snowflake? after = null,
Snowflake? before = null) Snowflake? before = null)
{ {
// Formats path // Formats path
outputPath = Regex.Replace(outputPath, "%.", m => outputPath = Regex.Replace(outputPath, "%.", m =>
PathEx.EscapePath(m.Value switch PathEx.EscapeFileName(m.Value switch
{ {
"%g" => guild.Id.ToString(), "%g" => guild.Id.ToString(),
"%G" => guild.Name, "%G" => guild.Name,
@ -118,9 +117,6 @@ public partial record ExportRequest
// File extension // File extension
buffer.Append($".{format.GetFileExtension()}"); buffer.Append($".{format.GetFileExtension()}");
// Replace invalid chars return PathEx.EscapeFileName(buffer.ToString());
PathEx.EscapePath(buffer);
return buffer.ToString();
} }
} }

@ -104,6 +104,6 @@ internal partial class MediaDownloader
var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName); var fileNameWithoutExtension = Path.GetFileNameWithoutExtension(fileName);
var fileExtension = Path.GetExtension(fileName); var fileExtension = Path.GetExtension(fileName);
return PathEx.EscapePath(fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension); return PathEx.EscapeFileName(fileNameWithoutExtension.Truncate(42) + '-' + urlHash + fileExtension);
} }
} }

@ -1,8 +1,5 @@
using System; using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace DiscordChatExporter.Core.Utils.Extensions; namespace DiscordChatExporter.Core.Utils.Extensions;
@ -23,29 +20,4 @@ public static class AsyncExtensions
public static ValueTaskAwaiter<IReadOnlyList<T>> GetAwaiter<T>( public static ValueTaskAwaiter<IReadOnlyList<T>> GetAwaiter<T>(
this IAsyncEnumerable<T> asyncEnumerable) => this IAsyncEnumerable<T> asyncEnumerable) =>
asyncEnumerable.AggregateAsync().GetAwaiter(); asyncEnumerable.AggregateAsync().GetAwaiter();
public static async ValueTask ParallelForEachAsync<T>(
this IEnumerable<T> source,
Func<T, ValueTask> handleAsync,
int degreeOfParallelism,
CancellationToken cancellationToken = default)
{
using var semaphore = new SemaphoreSlim(degreeOfParallelism);
await Task.WhenAll(source.Select(async item =>
{
// ReSharper disable once AccessToDisposedClosure
await semaphore.WaitAsync(cancellationToken);
try
{
await handleAsync(item);
}
finally
{
// ReSharper disable once AccessToDisposedClosure
semaphore.Release();
}
}));
}
} }

@ -1,17 +1,20 @@
using System.IO; using System.Collections.Generic;
using System.IO;
using System.Text; using System.Text;
namespace DiscordChatExporter.Core.Utils; namespace DiscordChatExporter.Core.Utils;
public static class PathEx public static class PathEx
{ {
public static StringBuilder EscapePath(StringBuilder pathBuffer) private static readonly HashSet<char> InvalidFileNameChars = new(Path.GetInvalidFileNameChars());
public static string EscapeFileName(string path)
{ {
foreach (var invalidChar in Path.GetInvalidFileNameChars()) var buffer = new StringBuilder(path.Length);
pathBuffer.Replace(invalidChar, '_');
return pathBuffer; foreach (var c in path)
} buffer.Append(!InvalidFileNameChars.Contains(c) ? c : '_');
public static string EscapePath(string path) => EscapePath(new StringBuilder(path)).ToString(); return buffer.ToString();
}
} }

@ -210,7 +210,13 @@ public class RootViewModel : Screen
var operations = ProgressManager.CreateOperations(dialog.Channels!.Count); var operations = ProgressManager.CreateOperations(dialog.Channels!.Count);
var successfulExportCount = 0; var successfulExportCount = 0;
await dialog.Channels.Zip(operations).ParallelForEachAsync(async tuple => await Parallel.ForEachAsync(
dialog.Channels.Zip(operations),
new ParallelOptions
{
MaxDegreeOfParallelism = Math.Max(1, _settingsService.ParallelLimit)
},
async (tuple, cancellationToken) =>
{ {
var (channel, operation) = tuple; var (channel, operation) = tuple;
@ -218,7 +224,7 @@ public class RootViewModel : Screen
{ {
var request = new ExportRequest( var request = new ExportRequest(
dialog.Guild!, dialog.Guild!,
channel!, channel,
dialog.OutputPath!, dialog.OutputPath!,
dialog.SelectedFormat, dialog.SelectedFormat,
dialog.After?.Pipe(Snowflake.FromDate), dialog.After?.Pipe(Snowflake.FromDate),
@ -230,7 +236,7 @@ public class RootViewModel : Screen
_settingsService.DateFormat _settingsService.DateFormat
); );
await exporter.ExportChannelAsync(request, operation); await exporter.ExportChannelAsync(request, operation, cancellationToken);
Interlocked.Increment(ref successfulExportCount); Interlocked.Increment(ref successfulExportCount);
} }
@ -242,7 +248,8 @@ public class RootViewModel : Screen
{ {
operation.Dispose(); operation.Dispose();
} }
}, Math.Max(1, _settingsService.ParallelLimit)); }
);
// Notify of overall completion // Notify of overall completion
if (successfulExportCount > 0) if (successfulExportCount > 0)

Loading…
Cancel
Save