open-source-mirrors
/
DiscordChatExporter
mirror of https://github.com/Tyrrrz/DiscordChatExporter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactor partitioning, don't create empty files

Browse Source 
Closes #246
pull/278/head
Alexey Golub 5 years ago
parent b4df267372
commit bf56902134
11 changed files with 193 additions and 90 deletions
Show Stats Download Patch File Download Diff File

5
DiscordChatExporter.Cli/Commands/ExportDirectMessagesCommand.cs
Unescape View File

@ -3,6 +3,7 @@ using System.Net;
using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Services;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions;
@ -39,6 +40,10 @@ namespace DiscordChatExporter.Cli.Commands
{
console.Error.WriteLine("This channel doesn't exist.");
}
catch (DomainException ex)
{
console.Error.WriteLine(ex.Message);
}
}
}
}

5
DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs
Unescape View File

@ -4,6 +4,7 @@ using System.Threading.Tasks;
using CliFx.Attributes;
using CliFx.Services;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions;
@ -43,6 +44,10 @@ namespace DiscordChatExporter.Cli.Commands
{
console.Error.WriteLine("This channel doesn't exist.");
}
catch (DomainException ex)
{
console.Error.WriteLine(ex.Message);
}
}
}
}

12
DiscordChatExporter.Core.Models/Exceptions/DomainException.cs
Unescape View File

@ -0,0 +1,12 @@
using System;
namespace DiscordChatExporter.Core.Models.Exceptions
{
public class DomainException : Exception
{
public DomainException(string message)
: base(message)
{
}
}
}

7
DiscordChatExporter.Core.Rendering/CsvMessageRenderer.cs
Unescape View File

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
@ -8,8 +9,8 @@ namespace DiscordChatExporter.Core.Rendering
{
private bool _isHeaderRendered;
public CsvMessageRenderer(string filePath, RenderContext context)
: base(filePath, context)
public CsvMessageRenderer(TextWriter writer, RenderContext context)
: base(writer, context)
{
}

112
DiscordChatExporter.Core.Rendering/FacadeMessageRenderer.cs
Unescape View File

@ -0,0 +1,112 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public partial class FacadeMessageRenderer : IMessageRenderer
{
private readonly string _baseFilePath;
private readonly ExportFormat _format;
private readonly RenderContext _context;
private int _partitionIndex;
private TextWriter _writer;
private IMessageRenderer _innerRenderer;
public FacadeMessageRenderer(string baseFilePath, ExportFormat format, RenderContext context)
{
_baseFilePath = baseFilePath;
_format = format;
_context = context;
}
private void EnsureInnerRendererInitialized()
{
if (_writer != null && _innerRenderer != null)
return;
// Get partition file path
var filePath = GetPartitionFilePath(_baseFilePath, _partitionIndex);
// Create output directory
var dirPath = Path.GetDirectoryName(_baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
Directory.CreateDirectory(dirPath);
// Create writer (will be disposed by renderer)
_writer = File.CreateText(filePath);
// Create inner renderer
if (_format == ExportFormat.PlainText)
{
_innerRenderer = new PlainTextMessageRenderer(_writer, _context);
}
else if (_format == ExportFormat.Csv)
{
_innerRenderer = new CsvMessageRenderer(_writer, _context);
}
else if (_format == ExportFormat.HtmlDark)
{
_innerRenderer = new HtmlMessageRenderer(_writer, _context, "Dark");
}
else if (_format == ExportFormat.HtmlLight)
{
_innerRenderer = new HtmlMessageRenderer(_writer, _context, "Light");
}
else
{
throw new InvalidOperationException($"Unknown export format [{_format}].");
}
}
public async Task NextPartitionAsync()
{
// Dispose writer and inner renderer
await DisposeAsync();
_writer = null;
_innerRenderer = null;
// Increment partition index
_partitionIndex++;
}
public async Task RenderMessageAsync(Message message)
{
EnsureInnerRendererInitialized();
await _innerRenderer.RenderMessageAsync(message);
}
public async ValueTask DisposeAsync()
{
if (_innerRenderer != null)
await _innerRenderer.DisposeAsync();
if (_writer != null)
await _writer.DisposeAsync();
}
}
public partial class FacadeMessageRenderer
{
private static string GetPartitionFilePath(string baseFilePath, int partitionIndex)
{
// First partition - no changes
if (partitionIndex <= 0)
return baseFilePath;
// Inject partition index into file name
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath);
var fileExt = Path.GetExtension(baseFilePath);
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
// Generate new path
var dirPath = Path.GetDirectoryName(baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
return Path.Combine(dirPath, fileName);
return fileName;
}
}
}

5
DiscordChatExporter.Core.Rendering/HtmlMessageRenderer.cs
Unescape View File

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
@ -22,8 +23,8 @@ namespace DiscordChatExporter.Core.Rendering
private bool _isLeadingBlockRendered;
public HtmlMessageRenderer(string filePath, RenderContext context, string themeName)
: base(filePath, context)
public HtmlMessageRenderer(TextWriter writer, RenderContext context, string themeName)
: base(writer, context)
{
_themeName = themeName;

6
DiscordChatExporter.Core.Rendering/MessageRendererBase.cs
Unescape View File

@ -10,14 +10,14 @@ namespace DiscordChatExporter.Core.Rendering
protected RenderContext Context { get; }
protected MessageRendererBase(string filePath, RenderContext context)
protected MessageRendererBase(TextWriter writer, RenderContext context)
{
Writer = File.CreateText(filePath);
Writer = writer;
Context = context;
}
public abstract Task RenderMessageAsync(Message message);
public virtual ValueTask DisposeAsync() => Writer.DisposeAsync();
public virtual ValueTask DisposeAsync() => default;
}
}

7
DiscordChatExporter.Core.Rendering/PlainTextMessageRenderer.cs
Unescape View File

@ -1,4 +1,5 @@
using System.Threading.Tasks;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
@ -8,8 +9,8 @@ namespace DiscordChatExporter.Core.Rendering
{
private bool _isPreambleRendered;
public PlainTextMessageRenderer(string filePath, RenderContext context)
: base(filePath, context)
public PlainTextMessageRenderer(TextWriter writer, RenderContext context)
: base(writer, context)
{
}

93
DiscordChatExporter.Core.Services/ExportService.cs
Unescape View File

@ -3,13 +3,14 @@ using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Rendering;
using DiscordChatExporter.Core.Services.Logic;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Services
{
public class ExportService
public partial class ExportService
{
private readonly SettingsService _settingsService;
private readonly DataService _dataService;
@ -20,47 +21,6 @@ namespace DiscordChatExporter.Core.Services
_dataService = dataService;
}
private string GetFilePathFromOutputPath(string outputPath, ExportFormat format, RenderContext context)
{
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
var fileName = ExportLogic.GetDefaultExportFileName(format, context.Guild, context.Channel, context.After, context.Before);
return Path.Combine(outputPath, fileName);
}
// Output is a file
return outputPath;
}
private IMessageRenderer CreateRenderer(string outputPath, int partitionIndex, ExportFormat format, RenderContext context)
{
var filePath = ExportLogic.GetExportPartitionFilePath(
GetFilePathFromOutputPath(outputPath, format, context),
partitionIndex);
// Create output directory
var dirPath = Path.GetDirectoryName(filePath);
if (!string.IsNullOrWhiteSpace(dirPath))
Directory.CreateDirectory(dirPath);
// Create renderer
if (format == ExportFormat.PlainText)
return new PlainTextMessageRenderer(filePath, context);
if (format == ExportFormat.Csv)
return new CsvMessageRenderer(filePath, context);
if (format == ExportFormat.HtmlDark)
return new HtmlMessageRenderer(filePath, context, "Dark");
if (format == ExportFormat.HtmlLight)
return new HtmlMessageRenderer(filePath, context, "Light");
throw new InvalidOperationException($"Unknown export format [{format}].");
}
public async Task ExportChatLogAsync(AuthToken token, Guild guild, Channel channel,
string outputPath, ExportFormat format, int? partitionLimit,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
@ -76,35 +36,50 @@ namespace DiscordChatExporter.Core.Services
mentionableUsers, mentionableChannels, mentionableRoles
);
// Render messages
var partitionIndex = 0;
var partitionMessageCount = 0;
var renderer = CreateRenderer(outputPath, partitionIndex, format, context);
// Create renderer
var baseFilePath = GetFilePathFromOutputPath(outputPath, format, context);
await using var renderer = new FacadeMessageRenderer(baseFilePath, format, context);
// Render messages
var messageCount = 0L;
await foreach (var message in _dataService.GetMessagesAsync(token, channel.Id, after, before, progress))
{
// Add encountered users to the list of mentionable users
mentionableUsers.Add(message.Author);
mentionableUsers.AddRange(message.MentionedUsers);
// If new partition is required, reset renderer
if (partitionLimit != null && partitionLimit > 0 && partitionMessageCount >= partitionLimit)
{
partitionIndex++;
partitionMessageCount = 0;
// Render message
await renderer.RenderMessageAsync(message);
messageCount++;
// Flush old renderer and create a new one
await renderer.DisposeAsync();
renderer = CreateRenderer(outputPath, partitionIndex, format, context);
// Trigger next partition when needed
if (partitionLimit != null &&
partitionLimit != 0 &&
messageCount % partitionLimit.Value == 0)
{
await renderer.NextPartitionAsync();
}
}
// Render message
await renderer.RenderMessageAsync(message);
partitionMessageCount++;
// Throw if no messages were rendered
if (messageCount == 0)
throw new DomainException($"Channel [{channel.Name}] contains no messages for specified period");
}
}
public partial class ExportService
{
private static string GetFilePathFromOutputPath(string outputPath, ExportFormat format, RenderContext context)
{
// Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
var fileName = ExportLogic.GetDefaultExportFileName(format, context.Guild, context.Channel, context.After, context.Before);
return Path.Combine(outputPath, fileName);
}
// Flush last renderer
await renderer.DisposeAsync();
// Output is a file
return outputPath;
}
}
}

19
DiscordChatExporter.Core.Services/Logic/ExportLogic.cs
Unescape View File

@ -49,24 +49,5 @@ namespace DiscordChatExporter.Core.Services.Logic
return buffer.ToString();
}
public static string GetExportPartitionFilePath(string baseFilePath, int partitionIndex)
{
// First partition - no changes
if (partitionIndex <= 0)
return baseFilePath;
// Inject partition index into file name
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath);
var fileExt = Path.GetExtension(baseFilePath);
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
// Generate new path
var dirPath = Path.GetDirectoryName(baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
return Path.Combine(dirPath, fileName);
return fileName;
}
}
}

12
DiscordChatExporter.Gui/ViewModels/RootViewModel.cs
Unescape View File

@ -4,6 +4,7 @@ using System.Linq;
using System.Net;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Models.Exceptions;
using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions;
using DiscordChatExporter.Gui.Services;
@ -230,6 +231,10 @@ namespace DiscordChatExporter.Gui.ViewModels
{
Notifications.Enqueue("Forbidden – account may be locked by 2FA");
}
catch (DomainException ex)
{
Notifications.Enqueue(ex.Message);
}
finally
{
operation.Dispose();
@ -276,6 +281,10 @@ namespace DiscordChatExporter.Gui.ViewModels
{
Notifications.Enqueue($"Channel [{channel.Model.Name}] doesn't exist");
}
catch (DomainException ex)
{
Notifications.Enqueue(ex.Message);
}
finally
{
operation.Dispose();
@ -283,7 +292,8 @@ namespace DiscordChatExporter.Gui.ViewModels
}
// Notify of overall completion
Notifications.Enqueue($"Successfully exported {successfulExportCount} channel(s)");
if (successfulExportCount > 0)
Notifications.Enqueue($"Successfully exported {successfulExportCount} channel(s)");
}
}
}
Write Preview
Loading…
Cancel
Save