pull/278/head
Alexey Golub 5 years ago
parent 5c2e725739
commit 26d713a17c

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public class EmojiNode : Node public class EmojiNode : Node
{ {

@ -1,6 +1,6 @@
using System.Collections.Generic; using System.Collections.Generic;
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public class FormattedNode : Node public class FormattedNode : Node
{ {

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public class InlineCodeBlockNode : Node public class InlineCodeBlockNode : Node
{ {

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public class LinkNode : Node public class LinkNode : Node
{ {

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public class MentionNode : Node public class MentionNode : Node
{ {

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public enum MentionType public enum MentionType
{ {

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public class MultiLineCodeBlockNode : Node public class MultiLineCodeBlockNode : Node
{ {

@ -0,0 +1,6 @@
namespace DiscordChatExporter.Core.Markdown.Ast
{
public abstract class Node
{
}
}

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public enum TextFormatting public enum TextFormatting
{ {

@ -1,4 +1,4 @@
namespace DiscordChatExporter.Core.Markdown.Nodes namespace DiscordChatExporter.Core.Markdown.Ast
{ {
public class TextNode : Node public class TextNode : Node
{ {

@ -1,8 +1,8 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Markdown.Internal; using DiscordChatExporter.Core.Markdown.Internal;
using DiscordChatExporter.Core.Markdown.Nodes;
namespace DiscordChatExporter.Core.Markdown namespace DiscordChatExporter.Core.Markdown
{ {

@ -1,6 +0,0 @@
namespace DiscordChatExporter.Core.Markdown.Nodes
{
public abstract class Node
{
}
}

@ -1,29 +0,0 @@
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
namespace DiscordChatExporter.Core.Rendering
{
public class CsvMessageRenderer : MessageRendererBase
{
private bool _isHeaderRendered;
public CsvMessageRenderer(TextWriter writer, RenderContext context)
: base(writer, context)
{
}
public override async Task RenderMessageAsync(Message message)
{
// Render header if it's the first entry
if (!_isHeaderRendered)
{
await Writer.WriteLineAsync(CsvRenderingLogic.FormatHeader(Context));
_isHeaderRendered = true;
}
await Writer.WriteLineAsync(CsvRenderingLogic.FormatMessage(Context, message));
}
}
}

@ -1,125 +0,0 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public partial class FacadeMessageRenderer : IMessageRenderer
{
private readonly string _baseFilePath;
private readonly ExportFormat _format;
private readonly RenderContext _context;
private readonly int? _partitionLimit;
private long _renderedMessageCount;
private int _partitionIndex;
private TextWriter _writer;
private IMessageRenderer _innerRenderer;
public FacadeMessageRenderer(string baseFilePath, ExportFormat format, RenderContext context, int? partitionLimit)
{
_baseFilePath = baseFilePath;
_format = format;
_context = context;
_partitionLimit = partitionLimit;
}
private void EnsureInnerRendererInitialized()
{
if (_writer != null && _innerRenderer != null)
return;
// Get partition file path
var filePath = GetPartitionFilePath(_baseFilePath, _partitionIndex);
// Create output directory
var dirPath = Path.GetDirectoryName(_baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
Directory.CreateDirectory(dirPath);
// Create writer
_writer = File.CreateText(filePath);
// Create inner renderer
if (_format == ExportFormat.PlainText)
{
_innerRenderer = new PlainTextMessageRenderer(_writer, _context);
}
else if (_format == ExportFormat.Csv)
{
_innerRenderer = new CsvMessageRenderer(_writer, _context);
}
else if (_format == ExportFormat.HtmlDark)
{
_innerRenderer = new HtmlMessageRenderer(_writer, _context, "Dark");
}
else if (_format == ExportFormat.HtmlLight)
{
_innerRenderer = new HtmlMessageRenderer(_writer, _context, "Light");
}
else
{
throw new InvalidOperationException($"Unknown export format [{_format}].");
}
}
private async Task ResetInnerRendererAsync()
{
if (_innerRenderer != null)
{
await _innerRenderer.DisposeAsync();
_innerRenderer = null;
}
if (_writer != null)
{
await _writer.DisposeAsync();
_writer = null;
}
}
public async Task RenderMessageAsync(Message message)
{
// Ensure underlying writer and renderer are initialized
EnsureInnerRendererInitialized();
// Render the actual message
await _innerRenderer.RenderMessageAsync(message);
// Increment count
_renderedMessageCount++;
// Update partition if necessary
if (_partitionLimit != null && _partitionLimit != 0 && _renderedMessageCount % _partitionLimit == 0)
{
await ResetInnerRendererAsync();
_partitionIndex++;
}
}
public async ValueTask DisposeAsync() => await ResetInnerRendererAsync();
}
public partial class FacadeMessageRenderer
{
private static string GetPartitionFilePath(string baseFilePath, int partitionIndex)
{
// First partition - no changes
if (partitionIndex <= 0)
return baseFilePath;
// Inject partition index into file name
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath);
var fileExt = Path.GetExtension(baseFilePath);
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
// Generate new path
var dirPath = Path.GetDirectoryName(baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
return Path.Combine(dirPath, fileName);
return fileName;
}
}
}

@ -0,0 +1,25 @@
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
namespace DiscordChatExporter.Core.Rendering.Formatters
{
public class CsvMessageWriter : MessageWriterBase
{
public CsvMessageWriter(TextWriter writer, RenderContext context)
: base(writer, context)
{
}
public override async Task WritePreambleAsync()
{
await Writer.WriteLineAsync(CsvRenderingLogic.FormatHeader(Context));
}
public override async Task WriteMessageAsync(Message message)
{
await Writer.WriteLineAsync(CsvRenderingLogic.FormatMessage(Context, message));
}
}
}

@ -10,27 +10,25 @@ using Scriban;
using Scriban.Runtime; using Scriban.Runtime;
using Tyrrrz.Extensions; using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering namespace DiscordChatExporter.Core.Rendering.Formatters
{ {
public partial class HtmlMessageRenderer : MessageRendererBase public partial class HtmlMessageWriter : MessageWriterBase
{ {
private readonly string _themeName; private readonly string _themeName;
private readonly List<Message> _messageGroupBuffer = new List<Message>(); private readonly List<Message> _messageGroupBuffer = new List<Message>();
private readonly Template _leadingBlockTemplate; private readonly Template _preambleTemplate;
private readonly Template _messageGroupTemplate; private readonly Template _messageGroupTemplate;
private readonly Template _trailingBlockTemplate; private readonly Template _postambleTemplate;
private bool _isLeadingBlockRendered; public HtmlMessageWriter(TextWriter writer, RenderContext context, string themeName)
public HtmlMessageRenderer(TextWriter writer, RenderContext context, string themeName)
: base(writer, context) : base(writer, context)
{ {
_themeName = themeName; _themeName = themeName;
_leadingBlockTemplate = Template.Parse(GetLeadingBlockTemplateCode()); _preambleTemplate = Template.Parse(GetPreambleTemplateCode());
_messageGroupTemplate = Template.Parse(GetMessageGroupTemplateCode()); _messageGroupTemplate = Template.Parse(GetMessageGroupTemplateCode());
_trailingBlockTemplate = Template.Parse(GetTrailingBlockTemplateCode()); _postambleTemplate = Template.Parse(GetPostambleTemplateCode());
} }
private MessageGroup GetCurrentMessageGroup() private MessageGroup GetCurrentMessageGroup()
@ -82,12 +80,6 @@ namespace DiscordChatExporter.Core.Rendering
return templateContext; return templateContext;
} }
private async Task RenderLeadingBlockAsync()
{
var templateContext = CreateTemplateContext();
await templateContext.EvaluateAsync(_leadingBlockTemplate.Page);
}
private async Task RenderCurrentMessageGroupAsync() private async Task RenderCurrentMessageGroupAsync()
{ {
var templateContext = CreateTemplateContext(new Dictionary<string, object> var templateContext = CreateTemplateContext(new Dictionary<string, object>
@ -98,21 +90,14 @@ namespace DiscordChatExporter.Core.Rendering
await templateContext.EvaluateAsync(_messageGroupTemplate.Page); await templateContext.EvaluateAsync(_messageGroupTemplate.Page);
} }
private async Task RenderTrailingBlockAsync() public override async Task WritePreambleAsync()
{ {
var templateContext = CreateTemplateContext(); var templateContext = CreateTemplateContext();
await templateContext.EvaluateAsync(_trailingBlockTemplate.Page); await templateContext.EvaluateAsync(_preambleTemplate.Page);
} }
public override async Task RenderMessageAsync(Message message) public override async Task WriteMessageAsync(Message message)
{ {
// Render leading block if it's the first entry
if (!_isLeadingBlockRendered)
{
await RenderLeadingBlockAsync();
_isLeadingBlockRendered = true;
}
// If message group is empty or the given message can be grouped, buffer the given message // If message group is empty or the given message can be grouped, buffer the given message
if (!_messageGroupBuffer.Any() || HtmlRenderingLogic.CanBeGrouped(_messageGroupBuffer.Last(), message)) if (!_messageGroupBuffer.Any() || HtmlRenderingLogic.CanBeGrouped(_messageGroupBuffer.Last(), message))
{ {
@ -128,25 +113,18 @@ namespace DiscordChatExporter.Core.Rendering
} }
} }
public override async ValueTask DisposeAsync() public override async Task WritePostambleAsync()
{ {
// Leading block (can happen if no message were rendered)
if (!_isLeadingBlockRendered)
await RenderLeadingBlockAsync();
// Flush current message group // Flush current message group
if (_messageGroupBuffer.Any()) if (_messageGroupBuffer.Any())
await RenderCurrentMessageGroupAsync(); await RenderCurrentMessageGroupAsync();
// Trailing block var templateContext = CreateTemplateContext();
await RenderTrailingBlockAsync(); await templateContext.EvaluateAsync(_postambleTemplate.Page);
// Dispose stream
await base.DisposeAsync();
} }
} }
public partial class HtmlMessageRenderer public partial class HtmlMessageWriter
{ {
private static readonly Assembly ResourcesAssembly = typeof(HtmlRenderingLogic).Assembly; private static readonly Assembly ResourcesAssembly = typeof(HtmlRenderingLogic).Assembly;
private static readonly string ResourcesNamespace = $"{ResourcesAssembly.GetName().Name}.Resources"; private static readonly string ResourcesNamespace = $"{ResourcesAssembly.GetName().Name}.Resources";
@ -159,18 +137,18 @@ namespace DiscordChatExporter.Core.Rendering
ResourcesAssembly ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.Html{themeName}.css"); .GetManifestResourceString($"{ResourcesNamespace}.Html{themeName}.css");
private static string GetLeadingBlockTemplateCode() => private static string GetPreambleTemplateCode() =>
ResourcesAssembly ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html") .GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.SubstringUntil("{{~ %SPLIT% ~}}"); .SubstringUntil("{{~ %SPLIT% ~}}");
private static string GetTrailingBlockTemplateCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.SubstringAfter("{{~ %SPLIT% ~}}");
private static string GetMessageGroupTemplateCode() => private static string GetMessageGroupTemplateCode() =>
ResourcesAssembly ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlMessageGroupTemplate.html"); .GetManifestResourceString($"{ResourcesNamespace}.HtmlMessageGroupTemplate.html");
private static string GetPostambleTemplateCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.SubstringAfter("{{~ %SPLIT% ~}}");
} }
} }

@ -0,0 +1,28 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering.Formatters
{
public abstract class MessageWriterBase : IAsyncDisposable
{
protected TextWriter Writer { get; }
protected RenderContext Context { get; }
protected MessageWriterBase(TextWriter writer, RenderContext context)
{
Writer = writer;
Context = context;
}
public virtual Task WritePreambleAsync() => Task.CompletedTask;
public abstract Task WriteMessageAsync(Message message);
public virtual Task WritePostambleAsync() => Task.CompletedTask;
public async ValueTask DisposeAsync() => await Writer.DisposeAsync();
}
}

@ -0,0 +1,26 @@
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
namespace DiscordChatExporter.Core.Rendering.Formatters
{
public class PlainTextMessageWriter : MessageWriterBase
{
public PlainTextMessageWriter(TextWriter writer, RenderContext context)
: base(writer, context)
{
}
public override async Task WritePreambleAsync()
{
await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatPreamble(Context));
}
public override async Task WriteMessageAsync(Message message)
{
await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatMessage(Context, message));
await Writer.WriteLineAsync();
}
}
}

@ -1,11 +0,0 @@
using System;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public interface IMessageRenderer : IAsyncDisposable
{
Task RenderMessageAsync(Message message);
}
}

@ -4,7 +4,7 @@ using System.Linq;
using System.Net; using System.Net;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Markdown; using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Nodes; using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions; using Tyrrrz.Extensions;

@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using System.Text; using System.Text;
using DiscordChatExporter.Core.Markdown; using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Nodes; using DiscordChatExporter.Core.Markdown.Ast;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Internal; using DiscordChatExporter.Core.Rendering.Internal;
using Tyrrrz.Extensions; using Tyrrrz.Extensions;

@ -0,0 +1,121 @@
using System;
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Formatters;
namespace DiscordChatExporter.Core.Rendering
{
public partial class MessageRenderer : IAsyncDisposable
{
private readonly RenderOptions _options;
private readonly RenderContext _context;
private long _renderedMessageCount;
private int _partitionIndex;
private MessageWriterBase? _writer;
public MessageRenderer(RenderOptions options, RenderContext context)
{
_options = options;
_context = context;
}
private async Task InitializeWriterAsync()
{
// Get partition file path
var filePath = GetPartitionFilePath(_options.BaseFilePath, _partitionIndex);
// Create output directory
var dirPath = Path.GetDirectoryName(_options.BaseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
Directory.CreateDirectory(dirPath);
// Create writer
_writer = CreateMessageWriter(filePath, _options.Format, _context);
// Write preamble
await _writer.WritePreambleAsync();
}
private async Task ResetWriterAsync()
{
if (_writer != null)
{
// Write postamble
await _writer.WritePostambleAsync();
// Flush
await _writer.DisposeAsync();
_writer = null;
}
}
public async Task RenderMessageAsync(Message message)
{
// Ensure underlying writer is initialized
if (_writer == null)
await InitializeWriterAsync();
// Render the actual message
await _writer!.WriteMessageAsync(message);
// Increment count
_renderedMessageCount++;
// Shift partition if necessary
if (_options.PartitionLimit != null &&
_options.PartitionLimit != 0 &&
_renderedMessageCount % _options.PartitionLimit == 0)
{
await ResetWriterAsync();
_partitionIndex++;
}
}
public async ValueTask DisposeAsync() => await ResetWriterAsync();
}
public partial class MessageRenderer
{
private static string GetPartitionFilePath(string baseFilePath, int partitionIndex)
{
// First partition - no changes
if (partitionIndex <= 0)
return baseFilePath;
// Inject partition index into file name
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath);
var fileExt = Path.GetExtension(baseFilePath);
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
// Generate new path
var dirPath = Path.GetDirectoryName(baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
return Path.Combine(dirPath, fileName);
return fileName;
}
private static MessageWriterBase CreateMessageWriter(string filePath, ExportFormat format, RenderContext context)
{
// Create inner writer (it will get disposed by the wrapper)
var writer = File.CreateText(filePath);
// Create formatter
if (format == ExportFormat.PlainText)
return new PlainTextMessageWriter(writer, context);
if (format == ExportFormat.Csv)
return new CsvMessageWriter(writer, context);
if (format == ExportFormat.HtmlDark)
return new HtmlMessageWriter(writer, context, "Dark");
if (format == ExportFormat.HtmlLight)
return new HtmlMessageWriter(writer, context, "Light");
throw new InvalidOperationException($"Unknown export format [{format}].");
}
}
}

@ -1,23 +0,0 @@
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public abstract class MessageRendererBase : IMessageRenderer
{
protected TextWriter Writer { get; }
protected RenderContext Context { get; }
protected MessageRendererBase(TextWriter writer, RenderContext context)
{
Writer = writer;
Context = context;
}
public abstract Task RenderMessageAsync(Message message);
public virtual ValueTask DisposeAsync() => default;
}
}

@ -1,30 +0,0 @@
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
namespace DiscordChatExporter.Core.Rendering
{
public class PlainTextMessageRenderer : MessageRendererBase
{
private bool _isPreambleRendered;
public PlainTextMessageRenderer(TextWriter writer, RenderContext context)
: base(writer, context)
{
}
public override async Task RenderMessageAsync(Message message)
{
// Render preamble if it's the first entry
if (!_isPreambleRendered)
{
await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatPreamble(Context));
_isPreambleRendered = true;
}
await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatMessage(Context, message));
await Writer.WriteLineAsync();
}
}
}

@ -0,0 +1,20 @@
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public class RenderOptions
{
public string BaseFilePath { get; }
public ExportFormat Format { get; }
public int? PartitionLimit { get; }
public RenderOptions(string baseFilePath, ExportFormat format, int? partitionLimit)
{
BaseFilePath = baseFilePath;
Format = format;
PartitionLimit = partitionLimit;
}
}
}

@ -25,6 +25,12 @@ namespace DiscordChatExporter.Core.Services
string outputPath, ExportFormat format, int? partitionLimit, string outputPath, ExportFormat format, int? partitionLimit,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null) DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
{ {
// Get base file path from output path
var baseFilePath = GetFilePathFromOutputPath(outputPath, format, guild, channel, after, before);
// Create options
var options = new RenderOptions(baseFilePath, format, partitionLimit);
// Create context // Create context
var mentionableUsers = new HashSet<User>(IdBasedEqualityComparer.Instance); var mentionableUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
var mentionableChannels = await _dataService.GetGuildChannelsAsync(token, guild.Id); var mentionableChannels = await _dataService.GetGuildChannelsAsync(token, guild.Id);
@ -37,8 +43,7 @@ namespace DiscordChatExporter.Core.Services
); );
// Create renderer // Create renderer
var baseFilePath = GetFilePathFromOutputPath(outputPath, format, context); await using var renderer = new MessageRenderer(options, context);
await using var renderer = new FacadeMessageRenderer(baseFilePath, format, context, partitionLimit);
// Render messages // Render messages
var renderedAnything = false; var renderedAnything = false;
@ -61,12 +66,13 @@ namespace DiscordChatExporter.Core.Services
public partial class ExportService public partial class ExportService
{ {
private static string GetFilePathFromOutputPath(string outputPath, ExportFormat format, RenderContext context) private static string GetFilePathFromOutputPath(string outputPath, ExportFormat format, Guild guild, Channel channel,
DateTimeOffset? after, DateTimeOffset? before)
{ {
// Output is a directory // Output is a directory
if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath))) if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{ {
var fileName = ExportLogic.GetDefaultExportFileName(format, context.Guild, context.Channel, context.After, context.Before); var fileName = ExportLogic.GetDefaultExportFileName(format, guild, channel, after, before);
return Path.Combine(outputPath, fileName); return Path.Combine(outputPath, fileName);
} }

Loading…
Cancel
Save