Add partition by file size (#497)

DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs

@@ -31,8 +31,9 @@ namespace DiscordChatExporter.Cli.Commands.Base
         [CommandOption("before", Description = "Only include messages sent before this date or message ID.")]
         public Snowflake? Before { get; init; }
 
-        [CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")]
+        [CommandOption("partition", 'p', Converter = typeof(PartitionConverter),
-        public int? PartitionLimit { get; init; }
+            Description = "Split output into partitions limited to this number of messages or a maximum file size (e.g. \"25mb\").")]
+        public IPartitioner Partitoner { get; init; } = new NullPartitioner();
 
         [CommandOption("parallel", Description = "Limits how many channels can be exported in parallel.")]
         public int ParallelLimit { get; init; } = 1;
@@ -74,7 +75,7 @@ namespace DiscordChatExporter.Cli.Commands.Base
                                 ExportFormat,
                                 After,
                                 Before,
-                                PartitionLimit,
+                                Partitoner,
                                 ShouldDownloadMedia,
                                 ShouldReuseMedia,
                                 DateFormat

DiscordChatExporter.Cli/Commands/Base/PartitionConverter.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using ByteSizeLib;
+using CliFx.Extensibility;
+using DiscordChatExporter.Core;
+using DiscordChatExporter.Core.Exporting;
+
+namespace DiscordChatExporter.Cli.Commands.Base
+{
+    public class PartitionConverter : BindingConverter<IPartitioner>
+    {
+        public override IPartitioner Convert(string? rawValue)
+        {
+            if (rawValue == null) return new NullPartitioner();
+
+            if (ByteSize.TryParse(rawValue, out ByteSize filesize))
+            {
+                return new FileSizePartitioner((long)filesize.Bytes);
+            }
+            else
+            {
+                int messageLimit = int.Parse(rawValue);
+                return new MessageCountPartitioner(messageLimit);
+            }
+        }
+    }
+}

DiscordChatExporter.Cli/DiscordChatExporter.Cli.csproj

@@ -8,6 +8,8 @@
   <ItemGroup>
     <PackageReference Include="CliFx" Version="2.0.1" />
     <PackageReference Include="Spectre.Console" Version="0.38.0" />
+    <PackageReference Include="Gress" Version="1.2.0" />
+    <PackageReference Include="OneOf" Version="3.0.174" />
     <PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
   </ItemGroup>
 

DiscordChatExporter.Core/Discord/Data/Attachment.cs

@@ -5,6 +5,7 @@ using System.Text.Json;
 using DiscordChatExporter.Core.Discord.Data.Common;
 using DiscordChatExporter.Core.Utils.Extensions;
 using JsonExtensions.Reading;
+using FileSize = DiscordChatExporter.Core.Discord.Data.Common.FileSize;
 
 namespace DiscordChatExporter.Core.Discord.Data
 {

DiscordChatExporter.Core/Discord/Data/Common/FileSize.cs

@@ -62,4 +62,4 @@ namespace DiscordChatExporter.Core.Discord.Data.Common
     {
         public static FileSize FromBytes(long bytes) => new(bytes);
     }
-}
+}

DiscordChatExporter.Core/DiscordChatExporter.Core.csproj

@@ -5,6 +5,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="ByteSize" Version="2.0.0" />
     <PackageReference Include="JsonExtensions" Version="1.0.1" />
     <PackageReference Include="MiniRazor.CodeGen" Version="2.1.2" />
     <PackageReference Include="Polly" Version="7.2.1" />

DiscordChatExporter.Core/Exporting/ExportRequest.cs

@@ -28,7 +28,7 @@ namespace DiscordChatExporter.Core.Exporting
 
         public Snowflake? Before { get; }
 
-        public int? PartitionLimit { get; }
+        public IPartitioner Partitoner { get; }
 
         public bool ShouldDownloadMedia { get; }
 
@@ -43,7 +43,7 @@ namespace DiscordChatExporter.Core.Exporting
             ExportFormat format,
             Snowflake? after,
             Snowflake? before,
-            int? partitionLimit,
+            IPartitioner partitioner,
             bool shouldDownloadMedia,
             bool shouldReuseMedia,
             string dateFormat)
@@ -54,7 +54,7 @@ namespace DiscordChatExporter.Core.Exporting
             Format = format;
             After = after;
             Before = before;
-            PartitionLimit = partitionLimit;
+            Partitoner = partitioner;
             ShouldDownloadMedia = shouldDownloadMedia;
             ShouldReuseMedia = shouldReuseMedia;
             DateFormat = dateFormat;

DiscordChatExporter.Core/Exporting/MessageExporter.cs

@@ -1,13 +1,18 @@
 using System;
 using System.IO;
+using System.Text.RegularExpressions;
 using System.Threading.Tasks;
+using ByteSizeLib;
 using DiscordChatExporter.Core.Discord.Data;
+using DiscordChatExporter.Core.Exporting;
+using DiscordChatExporter.Core.Exporting.Partitioners;
 using DiscordChatExporter.Core.Exporting.Writers;
 
 namespace DiscordChatExporter.Core.Exporting
 {
     internal partial class MessageExporter : IAsyncDisposable
     {
+
         private readonly ExportContext _context;
 
         private long _messageCount;
@@ -19,11 +24,16 @@ namespace DiscordChatExporter.Core.Exporting
             _context = context;
         }
 
-        private bool IsPartitionLimitReached() =>
+        private bool IsPartitionLimitReached()
-            _messageCount > 0 &&
+        {
-            _context.Request.PartitionLimit is not null &&
+            if (_writer is null)
-            _context.Request.PartitionLimit != 0 &&
+            {
-            _messageCount % _context.Request.PartitionLimit == 0;
+                return false;
+            }
+
+            return _context.Request.Partitoner.IsLimitReached(
+                new ExportPartitioningContext(_messageCount, _writer.SizeInBytes));
+        }
 
         private async ValueTask ResetWriterAsync()
         {
@@ -38,7 +48,7 @@ namespace DiscordChatExporter.Core.Exporting
         private async ValueTask<MessageWriter> GetWriterAsync()
         {
             // Ensure partition limit has not been exceeded
-            if (IsPartitionLimitReached())
+            if (_writer != null && IsPartitionLimitReached())
             {
                 await ResetWriterAsync();
                 _partitionIndex++;

DiscordChatExporter.Core/Exporting/Partitioners/ExportPartitioningContext.cs

@@ -0,0 +1,18 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DiscordChatExporter.Core.Exporting.Partitioners
+{
+    public class ExportPartitioningContext
+    {
+        public long MessageCount { get; } 
+        public long SizeInBytes { get; }
+
+        public ExportPartitioningContext(long messageCount, long sizeInBytes)
+        {
+            MessageCount = messageCount;
+            SizeInBytes = sizeInBytes;
+        }
+    }
+}

DiscordChatExporter.Core/Exporting/Partitioners/FileSizePartitioner.cs

@@ -0,0 +1,21 @@
+using DiscordChatExporter.Core.Exporting.Partitioners;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DiscordChatExporter.Core.Exporting
+{
+    public class FileSizePartitioner : IPartitioner
+    {
+        private long _bytesPerFile;
+
+        public FileSizePartitioner(long bytesPerFile)
+        {
+            _bytesPerFile = bytesPerFile;
+        }
+        public bool IsLimitReached(ExportPartitioningContext context)
+        {
+            return context.SizeInBytes >= _bytesPerFile;
+        }
+    }
+}

DiscordChatExporter.Core/Exporting/Partitioners/IPartitioner.cs

@@ -0,0 +1,12 @@
+using DiscordChatExporter.Core.Exporting.Partitioners;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DiscordChatExporter.Core.Exporting
+{
+    public interface IPartitioner
+    {
+        bool IsLimitReached(ExportPartitioningContext context);
+    }
+}

DiscordChatExporter.Core/Exporting/Partitioners/MessageCountPartitioner.cs

@@ -0,0 +1,25 @@
+using DiscordChatExporter.Core.Exporting.Partitioners;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DiscordChatExporter.Core.Exporting
+{
+    public class MessageCountPartitioner : IPartitioner
+    {
+
+        private int _messagesPerPartition;
+
+        public MessageCountPartitioner(int messagesPerPartition)
+        {
+            _messagesPerPartition = messagesPerPartition;
+        }
+
+        public bool IsLimitReached(ExportPartitioningContext context)
+        {
+            return context.MessageCount > 0 &&
+            _messagesPerPartition != 0 &&
+            context.MessageCount % _messagesPerPartition == 0;
+        }
+    }
+}

DiscordChatExporter.Core/Exporting/Partitioners/NullPartitioner.cs

@@ -0,0 +1,16 @@
+using DiscordChatExporter.Core.Exporting;
+using DiscordChatExporter.Core.Exporting.Partitioners;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DiscordChatExporter.Core.Exporting
+{
+    public class NullPartitioner : IPartitioner
+    {
+        public bool IsLimitReached(ExportPartitioningContext context)
+        {
+            return false;
+        }
+    }
+}

DiscordChatExporter.Core/Exporting/Writers/MessageWriter.cs

@@ -17,6 +17,8 @@ namespace DiscordChatExporter.Core.Exporting.Writers
             Context = context;
         }
 
+        public long SizeInBytes => Stream.Length;
+
         public virtual ValueTask WritePreambleAsync() => default;
 
         public abstract ValueTask WriteMessageAsync(Message message);

DiscordChatExporter.Gui/Converters/PartitionFormatToStringConverter.cs

@@ -0,0 +1,27 @@
+using DiscordChatExporter.Core.Exporting;
+using DiscordChatExporter.Gui.Internal;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Text;
+using System.Windows.Data;
+
+namespace DiscordChatExporter.Gui.Converters
+{
+    [ValueConversion(typeof(ExportFormat), typeof(string))]
+    public class PartitionFormatToStringConverter : IValueConverter
+    {
+        public static PartitionFormatToStringConverter Instance { get; } = new();
+
+        public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
+        {
+            if (value is PartitionFormat partitionFormatValue)
+                return partitionFormatValue.GetDisplayName();
+
+            return default(string);
+        }
+
+        public object ConvertBack(object value, Type targetType, object parameter, CultureInfo culture) =>
+            throw new NotSupportedException();
+    }
+}

DiscordChatExporter.Gui/Converters/PartitionFormatToTextBoxHintConverter.cs

@@ -0,0 +1,33 @@
+using DiscordChatExporter.Gui.Internal;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Text;
+using System.Windows.Data;
+
+namespace DiscordChatExporter.Gui.Converters
+{
+    [ValueConversion(typeof(DateTimeOffset?), typeof(DateTime?))]
+    public class PartitionFormatToTextBoxHintConverter : IValueConverter
+    {
+        public static PartitionFormatToTextBoxHintConverter Instance { get; } = new();
+
+        public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
+        {
+            if (value is PartitionFormat partitionFormat)
+                return partitionFormat switch
+                {
+                    PartitionFormat.FileSize => "MB per partition",
+                    PartitionFormat.MessageCount => "Messages per partition",
+                    _ => default(string)
+                };
+
+            return default(DateTime?);
+        }
+
+        public object? ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
+        {
+            throw new NotSupportedException();
+        }
+    }
+}

DiscordChatExporter.Gui/Converters/PartitionFormatToTooltipConverter.cs

@@ -0,0 +1,33 @@
+using DiscordChatExporter.Gui.Internal;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Text;
+using System.Windows.Data;
+
+namespace DiscordChatExporter.Gui.Converters
+{
+    [ValueConversion(typeof(DateTimeOffset?), typeof(DateTime?))]
+    public class PartitionFormatToTooltipConverter : IValueConverter
+    {
+        public static PartitionFormatToTextBoxHintConverter Instance { get; } = new();
+
+        public object? Convert(object value, Type targetType, object parameter, CultureInfo culture)
+        {
+            if (value is PartitionFormat partitionFormat)
+                return partitionFormat switch
+                {
+                    PartitionFormat.FileSize => "Split output into partitions close to this file size",
+                    PartitionFormat.MessageCount => "Split output into partitions limited to this number of messages",
+                    _ => default(string)
+                };
+
+            return default(DateTime?);
+        }
+
+        public object? ConvertBack(object value, Type targetType, object parameter, CultureInfo culture)
+        {
+            throw new NotSupportedException();
+        }
+    }
+}

DiscordChatExporter.Gui/Services/SettingsService.cs

@@ -1,5 +1,6 @@
 using DiscordChatExporter.Core.Discord;
 using DiscordChatExporter.Core.Exporting;
+using DiscordChatExporter.Gui.Internal;
 using Tyrrrz.Settings;
 
 namespace DiscordChatExporter.Gui.Services
@@ -22,6 +23,8 @@ namespace DiscordChatExporter.Gui.Services
 
         public ExportFormat LastExportFormat { get; set; } = ExportFormat.HtmlDark;
 
+        public PartitionFormat LastPartitionFormat { get; set; } = PartitionFormat.MessageCount;
+
         public int? LastPartitionLimit { get; set; }
 
         public bool LastShouldDownloadMedia { get; set; }

DiscordChatExporter.Gui/Utils/PartitionFormat.cs

@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace DiscordChatExporter.Gui.Internal
+{
+    public enum PartitionFormat
+    {
+        MessageCount,
+        FileSize,
+    }
+
+    public static class PartitionFormatExtensions
+    {
+        public static string GetDisplayName(this PartitionFormat format) => format switch
+        {
+            PartitionFormat.MessageCount => "Message count",
+            PartitionFormat.FileSize => "File size (MB)",
+            _ => throw new ArgumentOutOfRangeException(nameof(format))
+        };
+    }
+}

DiscordChatExporter.Gui/ViewModels/Dialogs/ExportSetupViewModel.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using DiscordChatExporter.Gui.Internal;
 using DiscordChatExporter.Core.Discord;
 using DiscordChatExporter.Core.Discord.Data;
 using DiscordChatExporter.Core.Exporting;
@@ -46,6 +47,11 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
 
         public DateTimeOffset? Before => BeforeDate?.Add(BeforeTime ?? TimeSpan.Zero);
 
+        public IReadOnlyList<PartitionFormat> AvailablePartitionFormats =>
+            Enum.GetValues(typeof(PartitionFormat)).Cast<PartitionFormat>().ToArray();
+
+        public PartitionFormat SelectedPartitionFormat { get; set; }
+
         public int? PartitionLimit { get; set; }
 
         public bool ShouldDownloadMedia { get; set; }
@@ -67,6 +73,8 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
             SelectedFormat = _settingsService.LastExportFormat;
             PartitionLimit = _settingsService.LastPartitionLimit;
             ShouldDownloadMedia = _settingsService.LastShouldDownloadMedia;
+            SelectedPartitionFormat = _settingsService.LastPartitionFormat;
+
         }
 
         public void Confirm()
@@ -74,6 +82,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
             // Persist preferences
             _settingsService.LastExportFormat = SelectedFormat;
             _settingsService.LastPartitionLimit = PartitionLimit;
+            _settingsService.LastPartitionFormat = SelectedPartitionFormat;
             _settingsService.LastShouldDownloadMedia = ShouldDownloadMedia;
 
             // If single channel - prompt file path

DiscordChatExporter.Gui/ViewModels/RootViewModel.cs

@@ -8,6 +8,7 @@ using DiscordChatExporter.Core.Discord.Data;
 using DiscordChatExporter.Core.Exceptions;
 using DiscordChatExporter.Core.Exporting;
 using DiscordChatExporter.Core.Utils.Extensions;
+using DiscordChatExporter.Gui.Internal;
 using DiscordChatExporter.Gui.Services;
 using DiscordChatExporter.Gui.Utils;
 using DiscordChatExporter.Gui.ViewModels.Dialogs;
@@ -213,7 +214,7 @@ namespace DiscordChatExporter.Gui.ViewModels
                         dialog.SelectedFormat,
                         dialog.After?.Pipe(Snowflake.FromDate),
                         dialog.Before?.Pipe(Snowflake.FromDate),
-                        dialog.PartitionLimit,
+                        CreatePartitioner(),
                         dialog.ShouldDownloadMedia,
                         _settingsService.ShouldReuseMedia,
                         _settingsService.DateFormat
@@ -236,6 +237,19 @@ namespace DiscordChatExporter.Gui.ViewModels
             // Notify of overall completion
             if (successfulExportCount > 0)
                 Notifications.Enqueue($"Successfully exported {successfulExportCount} channel(s)");
+
+            IPartitioner CreatePartitioner()
+            {
+                var partitionFormat = dialog.SelectedPartitionFormat;
+                var partitionLimit = dialog.PartitionLimit;
+
+                return (partitionFormat, partitionLimit) switch
+                {
+                    (PartitionFormat.MessageCount, int messageLimit) => new MessageCountPartitioner(messageLimit),
+                    (PartitionFormat.FileSize, int fileSizeLimit) => new FileSizePartitioner(fileSizeLimit),
+                    _ => new NullPartitioner()
+                };
+            }
         }
     }
 }

DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml

@@ -126,12 +126,43 @@
             </Grid>
 
             <!--  Partitioning  -->
-            <TextBox
+            <Grid Name="PartitioningGrid">
-                Margin="16,8"
+                <Grid.ColumnDefinitions>
-                materialDesign:HintAssist.Hint="Messages per partition"
+                    <ColumnDefinition Width="1*" />
-                materialDesign:HintAssist.IsFloating="True"
+                    <ColumnDefinition Width="1*" />
-                Text="{Binding PartitionLimit, TargetNullValue=''}"
+                </Grid.ColumnDefinitions>
-                ToolTip="Split output into partitions limited to this number of messages" />
+                <Grid.RowDefinitions>
+                    <RowDefinition Height="*" />
+                </Grid.RowDefinitions>
+
+                <ComboBox
+                    Name="PartitionFormatComboBox"
+                    Grid.Row="0"
+                    Grid.Column="0"
+                    Margin="16,8"
+                    materialDesign:HintAssist.Hint="Partition by"
+                    materialDesign:HintAssist.IsFloating="True"
+                    IsReadOnly="True"
+                    ItemsSource="{Binding AvailablePartitionFormats}"
+                    SelectedItem="{Binding SelectedPartitionFormat}">
+                    <ComboBox.ItemTemplate>
+                        <DataTemplate>
+                            <TextBlock Text="{Binding Converter={x:Static converters:PartitionFormatToStringConverter.Instance}}" />
+                        </DataTemplate>
+                    </ComboBox.ItemTemplate>
+                </ComboBox>
+
+                <TextBox
+                    Name="PartitionTextBox"
+                    Grid.Row="0"
+                    Grid.Column="1"
+                    Margin="16,8"
+                    materialDesign:HintAssist.Hint="{Binding SelectedPartitionFormat, Converter={x:Static converters:PartitionFormatToTooltipConverter.Instance}}"
+                    materialDesign:HintAssist.IsFloating="True"
+                    Text="{Binding PartitionLimit, TargetNullValue=''}"
+                    ToolTip="{Binding SelectedPartitionFormat, Converter={x:Static converters:PartitionFormatToTooltipConverter.Instance}}" />
+
+            </Grid>
 
             <!--  Download media  -->
             <Grid Margin="16,16" ToolTip="Download referenced media content (user avatars, attached files, embedded images, etc)">

DiscordChatExporter.Gui/Views/Dialogs/ExportSetupView.xaml.cs

@@ -1,4 +1,4 @@
-namespace DiscordChatExporter.Gui.Views.Dialogs
+namespace DiscordChatExporter.Gui.Views.Dialogs
 {
     public partial class ExportSetupView
     {
@@ -7,4 +7,4 @@
             InitializeComponent();
         }
     }
-}
+}