Streaming exporter

Fixes #125
Closes #177
pull/278/head
Alexey Golub 5 years ago
parent fc38afe6a0
commit 2a223599f9

@ -16,13 +16,6 @@ namespace DiscordChatExporter.Cli.Commands
{ {
} }
public override async Task ExecuteAsync(IConsole console) public override async Task ExecuteAsync(IConsole console) => await ExportAsync(console, ChannelId);
{
// Get channel
var channel = await DataService.GetChannelAsync(GetToken(), ChannelId);
// Export
await ExportChannelAsync(console, channel);
}
} }
} }

@ -6,7 +6,6 @@ using CliFx.Services;
using CliFx.Utilities; using CliFx.Utilities;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services; using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Helpers;
namespace DiscordChatExporter.Cli.Commands namespace DiscordChatExporter.Cli.Commands
{ {
@ -16,17 +15,16 @@ namespace DiscordChatExporter.Cli.Commands
protected ExportService ExportService { get; } protected ExportService ExportService { get; }
[CommandOption("format", 'f', Description = "Output file format.")] [CommandOption("format", 'f', Description = "Output file format.")]
public ExportFormat ExportFormat { get; set; } = ExportFormat.HtmlDark; public ExportFormat ExportFormat { get; set; } = ExportFormat.HtmlDark;
[CommandOption("output", 'o', Description = "Output file or directory path.")] [CommandOption("output", 'o', Description = "Output file or directory path.")]
public string? OutputPath { get; set; } public string? OutputPath { get; set; }
[CommandOption("after",Description = "Limit to messages sent after this date.")] [CommandOption("after", Description = "Limit to messages sent after this date.")]
public DateTimeOffset? After { get; set; } public DateTimeOffset? After { get; set; }
[CommandOption("before",Description = "Limit to messages sent before this date.")] [CommandOption("before", Description = "Limit to messages sent before this date.")]
public DateTimeOffset? Before { get; set; } public DateTimeOffset? Before { get; set; }
[CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")] [CommandOption("partition", 'p', Description = "Split output into partitions limited to this number of messages.")]
@ -42,34 +40,32 @@ namespace DiscordChatExporter.Cli.Commands
ExportService = exportService; ExportService = exportService;
} }
protected async Task ExportChannelAsync(IConsole console, Channel channel) protected async Task ExportAsync(IConsole console, Guild guild, Channel channel)
{ {
// Configure settings
if (!string.IsNullOrWhiteSpace(DateFormat)) if (!string.IsNullOrWhiteSpace(DateFormat))
SettingsService.DateFormat = DateFormat!; SettingsService.DateFormat = DateFormat;
console.Output.Write($"Exporting channel [{channel.Name}]... "); console.Output.Write($"Exporting channel [{channel.Name}]... ");
var progress = console.CreateProgressTicker(); var progress = console.CreateProgressTicker();
// Get chat log var outputPath = OutputPath ?? Directory.GetCurrentDirectory();
var chatLog = await DataService.GetChatLogAsync(GetToken(), channel, After, Before, progress); await ExportService.ExportChatLogAsync(GetToken(), guild, channel,
outputPath, ExportFormat, PartitionLimit,
// Generate file path if not set or is a directory After, Before, progress);
var filePath = OutputPath;
if (string.IsNullOrWhiteSpace(filePath) || ExportHelper.IsDirectoryPath(filePath))
{
// Generate default file name
var fileName = ExportHelper.GetDefaultExportFileName(ExportFormat, chatLog.Guild,
chatLog.Channel, After, Before);
// Combine paths console.Output.WriteLine();
filePath = Path.Combine(filePath ?? "", fileName); }
}
// Export protected async Task ExportAsync(IConsole console, Channel channel)
await ExportService.ExportChatLogAsync(chatLog, filePath, ExportFormat, PartitionLimit); {
var guild = await DataService.GetGuildAsync(GetToken(), channel.GuildId);
await ExportAsync(console, guild, channel);
}
console.Output.WriteLine(); protected async Task ExportAsync(IConsole console, string channelId)
{
var channel = await DataService.GetChannelAsync(GetToken(), channelId);
await ExportAsync(console, channel);
} }
} }
} }

@ -29,7 +29,7 @@ namespace DiscordChatExporter.Cli.Commands
{ {
try try
{ {
await ExportChannelAsync(console, channel); await ExportAsync(console, channel);
} }
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden) catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
{ {

@ -33,7 +33,7 @@ namespace DiscordChatExporter.Cli.Commands
{ {
try try
{ {
await ExportChannelAsync(console, channel); await ExportAsync(console, channel);
} }
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden) catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
{ {

@ -210,6 +210,7 @@ namespace DiscordChatExporter.Core.Markdown
StandardEmojiNodeMatcher, StandardEmojiNodeMatcher,
CustomEmojiNodeMatcher); CustomEmojiNodeMatcher);
// Minimal set of matchers for non-multimedia formats (e.g. plain text)
private static readonly IMatcher<Node> MinimalAggregateNodeMatcher = new AggregateMatcher<Node>( private static readonly IMatcher<Node> MinimalAggregateNodeMatcher = new AggregateMatcher<Node>(
// Mentions // Mentions
EveryoneMentionNodeMatcher, EveryoneMentionNodeMatcher,
@ -219,7 +220,6 @@ namespace DiscordChatExporter.Core.Markdown
RoleMentionNodeMatcher, RoleMentionNodeMatcher,
// Emoji // Emoji
StandardEmojiNodeMatcher,
CustomEmojiNodeMatcher); CustomEmojiNodeMatcher);
private static IReadOnlyList<Node> Parse(StringPart stringPart, IMatcher<Node> matcher) => private static IReadOnlyList<Node> Parse(StringPart stringPart, IMatcher<Node> matcher) =>

@ -1,33 +0,0 @@
using System;
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Models
{
public class ChatLog
{
public Guild Guild { get; }
public Channel Channel { get; }
public DateTimeOffset? After { get; }
public DateTimeOffset? Before { get; }
public IReadOnlyList<Message> Messages { get; }
public Mentionables Mentionables { get; }
public ChatLog(Guild guild, Channel channel, DateTimeOffset? after, DateTimeOffset? before,
IReadOnlyList<Message> messages, Mentionables mentionables)
{
Guild = guild;
Channel = channel;
After = after;
Before = before;
Messages = messages;
Mentionables = mentionables;
}
public override string ToString() => $"{Guild.Name} | {Channel.Name}";
}
}

@ -6,7 +6,7 @@ namespace DiscordChatExporter.Core.Models
{ {
// https://discordapp.com/developers/docs/resources/emoji#emoji-object // https://discordapp.com/developers/docs/resources/emoji#emoji-object
public partial class Emoji : IHasId public partial class Emoji
{ {
public string? Id { get; } public string? Id { get; }

@ -27,7 +27,7 @@ namespace DiscordChatExporter.Core.Models
ExportFormat.PlainText => "Plain Text", ExportFormat.PlainText => "Plain Text",
ExportFormat.HtmlDark => "HTML (Dark)", ExportFormat.HtmlDark => "HTML (Dark)",
ExportFormat.HtmlLight => "HTML (Light)", ExportFormat.HtmlLight => "HTML (Light)",
ExportFormat.Csv => "Comma Seperated Values (CSV)", ExportFormat.Csv => "Comma Separated Values (CSV)",
_ => throw new ArgumentOutOfRangeException(nameof(format)) _ => throw new ArgumentOutOfRangeException(nameof(format))
}; };
} }

@ -1,30 +0,0 @@
using System.Collections.Generic;
using System.Linq;
namespace DiscordChatExporter.Core.Models
{
public class Mentionables
{
public IReadOnlyList<User> Users { get; }
public IReadOnlyList<Channel> Channels { get; }
public IReadOnlyList<Role> Roles { get; }
public Mentionables(IReadOnlyList<User> users, IReadOnlyList<Channel> channels, IReadOnlyList<Role> roles)
{
Users = users;
Channels = channels;
Roles = roles;
}
public User GetUser(string id) =>
Users.FirstOrDefault(u => u.Id == id) ?? User.CreateUnknownUser(id);
public Channel GetChannel(string id) =>
Channels.FirstOrDefault(c => c.Id == id) ?? Channel.CreateDeletedChannel(id);
public Role GetRole(string id) =>
Roles.FirstOrDefault(r => r.Id == id) ?? Role.CreateDeletedRole(id);
}
}

@ -19,6 +19,8 @@ namespace DiscordChatExporter.Core.Models
public DateTimeOffset? EditedTimestamp { get; } public DateTimeOffset? EditedTimestamp { get; }
public bool IsPinned { get; }
public string? Content { get; } public string? Content { get; }
public IReadOnlyList<Attachment> Attachments { get; } public IReadOnlyList<Attachment> Attachments { get; }
@ -29,12 +31,11 @@ namespace DiscordChatExporter.Core.Models
public IReadOnlyList<User> MentionedUsers { get; } public IReadOnlyList<User> MentionedUsers { get; }
public bool IsPinned { get; } public Message(string id, string channelId, MessageType type, User author,
DateTimeOffset timestamp, DateTimeOffset? editedTimestamp, bool isPinned,
public Message(string id, string channelId, MessageType type, User author, DateTimeOffset timestamp, string content,
DateTimeOffset? editedTimestamp, string? content, IReadOnlyList<Attachment> attachments, IReadOnlyList<Attachment> attachments,IReadOnlyList<Embed> embeds, IReadOnlyList<Reaction> reactions,
IReadOnlyList<Embed> embeds, IReadOnlyList<Reaction> reactions, IReadOnlyList<User> mentionedUsers, IReadOnlyList<User> mentionedUsers)
bool isPinned)
{ {
Id = id; Id = id;
ChannelId = channelId; ChannelId = channelId;
@ -42,12 +43,12 @@ namespace DiscordChatExporter.Core.Models
Author = author; Author = author;
Timestamp = timestamp; Timestamp = timestamp;
EditedTimestamp = editedTimestamp; EditedTimestamp = editedTimestamp;
IsPinned = isPinned;
Content = content; Content = content;
Attachments = attachments; Attachments = attachments;
Embeds = embeds; Embeds = embeds;
Reactions = reactions; Reactions = reactions;
MentionedUsers = mentionedUsers; MentionedUsers = mentionedUsers;
IsPinned = isPinned;
} }
public override string ToString() => Content ?? "<message without content>"; public override string ToString() => Content ?? "<message without content>";

@ -0,0 +1,23 @@
using System;
using System.Collections.Generic;
namespace DiscordChatExporter.Core.Models
{
// Used for grouping contiguous messages in HTML export
public class MessageGroup
{
public User Author { get; }
public DateTimeOffset Timestamp { get; }
public IReadOnlyList<Message> Messages { get; }
public MessageGroup(User author, DateTimeOffset timestamp, IReadOnlyList<Message> messages)
{
Author = author;
Timestamp = timestamp;
Messages = messages;
}
}
}

@ -1,123 +0,0 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Nodes;
using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering
{
public class CsvChatLogRenderer : IChatLogRenderer
{
private readonly ChatLog _chatLog;
private readonly string _dateFormat;
public CsvChatLogRenderer(ChatLog chatLog, string dateFormat)
{
_chatLog = chatLog;
_dateFormat = dateFormat;
}
private string FormatDate(DateTimeOffset date) =>
date.ToLocalTime().ToString(_dateFormat, CultureInfo.InvariantCulture);
private string FormatMarkdown(Node node)
{
// Text node
if (node is TextNode textNode)
{
return textNode.Text;
}
// Mention node
if (node is MentionNode mentionNode)
{
// Meta mention node
if (mentionNode.Type == MentionType.Meta)
{
return mentionNode.Id;
}
// User mention node
if (mentionNode.Type == MentionType.User)
{
var user = _chatLog.Mentionables.GetUser(mentionNode.Id);
return $"@{user.Name}";
}
// Channel mention node
if (mentionNode.Type == MentionType.Channel)
{
var channel = _chatLog.Mentionables.GetChannel(mentionNode.Id);
return $"#{channel.Name}";
}
// Role mention node
if (mentionNode.Type == MentionType.Role)
{
var role = _chatLog.Mentionables.GetRole(mentionNode.Id);
return $"@{role.Name}";
}
}
// Emoji node
if (node is EmojiNode emojiNode)
{
return emojiNode.IsCustomEmoji ? $":{emojiNode.Name}:" : emojiNode.Name;
}
// Throw on unexpected nodes
throw new InvalidOperationException($"Unexpected node: [{node.GetType()}].");
}
private string FormatMarkdown(IEnumerable<Node> nodes) => nodes.Select(FormatMarkdown).JoinToString("");
private string FormatMarkdown(string markdown) => FormatMarkdown(MarkdownParser.ParseMinimal(markdown));
private async Task RenderFieldAsync(TextWriter writer, string value)
{
var encodedValue = value.Replace("\"", "\"\"");
await writer.WriteAsync($"\"{encodedValue}\",");
}
private async Task RenderMessageAsync(TextWriter writer, Message message)
{
// Author ID
await RenderFieldAsync(writer, message.Author.Id);
// Author
await RenderFieldAsync(writer, message.Author.FullName);
// Timestamp
await RenderFieldAsync(writer, FormatDate(message.Timestamp));
// Content
await RenderFieldAsync(writer, FormatMarkdown(message.Content ?? ""));
// Attachments
var formattedAttachments = message.Attachments.Select(a => a.Url).JoinToString(",");
await RenderFieldAsync(writer, formattedAttachments);
// Reactions
var formattedReactions = message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",");
await RenderFieldAsync(writer, formattedReactions);
// Line break
await writer.WriteLineAsync();
}
public async Task RenderAsync(TextWriter writer)
{
// Headers
await writer.WriteLineAsync("AuthorID;Author;Date;Content;Attachments;Reactions;");
// Log
foreach (var message in _chatLog.Messages)
await RenderMessageAsync(writer, message);
}
}
}

@ -0,0 +1,28 @@
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
namespace DiscordChatExporter.Core.Rendering
{
public class CsvMessageRenderer : MessageRendererBase
{
private bool _isHeaderRendered;
public CsvMessageRenderer(string filePath, RenderContext context)
: base(filePath, context)
{
}
public override async Task RenderMessageAsync(Message message)
{
// Render header if it's the first entry
if (!_isHeaderRendered)
{
await Writer.WriteLineAsync(CsvRenderingLogic.FormatHeader(Context));
_isHeaderRendered = true;
}
await Writer.WriteLineAsync(CsvRenderingLogic.FormatMessage(Context, message));
}
}
}

@ -6,12 +6,11 @@
</PropertyGroup> </PropertyGroup>
<ItemGroup> <ItemGroup>
<EmbeddedResource Include="Resources\HtmlCore.css" />
<EmbeddedResource Include="Resources\HtmlDark.css" /> <EmbeddedResource Include="Resources\HtmlDark.css" />
<EmbeddedResource Include="Resources\HtmlDark.html" />
<EmbeddedResource Include="Resources\HtmlLight.css" /> <EmbeddedResource Include="Resources\HtmlLight.css" />
<EmbeddedResource Include="Resources\HtmlLight.html" /> <EmbeddedResource Include="Resources\HtmlLayoutTemplate.html" />
<EmbeddedResource Include="Resources\HtmlShared.css" /> <EmbeddedResource Include="Resources\HtmlMessageGroupTemplate.html" />
<EmbeddedResource Include="Resources\HtmlShared.html" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>

@ -1,25 +0,0 @@
using System;
using System.Collections.Generic;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public partial class HtmlChatLogRenderer
{
private class MessageGroup
{
public User Author { get; }
public DateTimeOffset Timestamp { get; }
public IReadOnlyList<Message> Messages { get; }
public MessageGroup(User author, DateTimeOffset timestamp, IReadOnlyList<Message> messages)
{
Author = author;
Timestamp = timestamp;
Messages = messages;
}
}
}
}

@ -1,27 +0,0 @@
using System.Reflection;
using System.Threading.Tasks;
using Scriban;
using Scriban.Parsing;
using Scriban.Runtime;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering
{
public partial class HtmlChatLogRenderer
{
private class TemplateLoader : ITemplateLoader
{
private const string ResourceRootNamespace = "DiscordChatExporter.Core.Rendering.Resources";
public string Load(string templatePath) =>
Assembly.GetExecutingAssembly().GetManifestResourceString($"{ResourceRootNamespace}.{templatePath}");
public string GetPath(TemplateContext context, SourceSpan callerSpan, string templateName) => templateName;
public string Load(TemplateContext context, SourceSpan callerSpan, string templatePath) => Load(templatePath);
public ValueTask<string> LoadAsync(TemplateContext context, SourceSpan callerSpan, string templatePath) =>
new ValueTask<string>(Load(templatePath));
}
}
}

@ -0,0 +1,169 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Reflection;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
using Scriban;
using Scriban.Runtime;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering
{
public partial class HtmlMessageRenderer : MessageRendererBase
{
private readonly string _themeName;
private readonly List<Message> _messageGroupBuffer = new List<Message>();
private bool _isLeadingBlockRendered;
public HtmlMessageRenderer(string filePath, RenderContext context, string themeName)
: base(filePath, context)
{
_themeName = themeName;
}
private MessageGroup GetCurrentMessageGroup()
{
var firstMessage = _messageGroupBuffer.First();
return new MessageGroup(firstMessage.Author, firstMessage.Timestamp, _messageGroupBuffer);
}
private async Task RenderLeadingBlockAsync()
{
var template = Template.Parse(GetLeadingBlockTemplateCode());
var templateContext = CreateTemplateContext();
var scriptObject = CreateScriptObject(Context, _themeName);
templateContext.PushGlobal(scriptObject);
templateContext.PushOutput(new TextWriterOutput(Writer));
await templateContext.EvaluateAsync(template.Page);
}
private async Task RenderTrailingBlockAsync()
{
var template = Template.Parse(GetTrailingBlockTemplateCode());
var templateContext = CreateTemplateContext();
var scriptObject = CreateScriptObject(Context, _themeName);
templateContext.PushGlobal(scriptObject);
templateContext.PushOutput(new TextWriterOutput(Writer));
await templateContext.EvaluateAsync(template.Page);
}
private async Task RenderCurrentMessageGroupAsync()
{
var template = Template.Parse(GetMessageGroupTemplateCode());
var templateContext = CreateTemplateContext();
var scriptObject = CreateScriptObject(Context, _themeName);
scriptObject.SetValue("MessageGroup", GetCurrentMessageGroup(), true);
templateContext.PushGlobal(scriptObject);
templateContext.PushOutput(new TextWriterOutput(Writer));
await templateContext.EvaluateAsync(template.Page);
}
public override async Task RenderMessageAsync(Message message)
{
// Render leading block if it's the first entry
if (!_isLeadingBlockRendered)
{
await RenderLeadingBlockAsync();
_isLeadingBlockRendered = true;
}
// If message group is empty or the given message can be grouped, buffer the given message
if (!_messageGroupBuffer.Any() || HtmlRenderingLogic.CanBeGrouped(_messageGroupBuffer.Last(), message))
{
_messageGroupBuffer.Add(message);
}
// Otherwise, flush the group and render messages
else
{
await RenderCurrentMessageGroupAsync();
_messageGroupBuffer.Clear();
_messageGroupBuffer.Add(message);
}
}
public override async ValueTask DisposeAsync()
{
// Leading block (can happen if no message were rendered)
if (!_isLeadingBlockRendered)
await RenderLeadingBlockAsync();
// Flush current message group
if (_messageGroupBuffer.Any())
await RenderCurrentMessageGroupAsync();
// Trailing block
await RenderTrailingBlockAsync();
await base.DisposeAsync();
}
}
public partial class HtmlMessageRenderer
{
private static readonly Assembly ResourcesAssembly = typeof(HtmlRenderingLogic).Assembly;
private static readonly string ResourcesNamespace = $"{ResourcesAssembly.GetName().Name}.Resources";
private static string GetCoreStyleSheetCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlCore.css");
private static string GetThemeStyleSheetCode(string themeName) =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.Html{themeName}.css");
private static string GetLeadingBlockTemplateCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.SubstringUntil("{{~ %SPLIT% ~}}");
private static string GetTrailingBlockTemplateCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.SubstringAfter("{{~ %SPLIT% ~}}");
private static string GetMessageGroupTemplateCode() =>
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlMessageGroupTemplate.html");
private static ScriptObject CreateScriptObject(RenderContext context, string themeName)
{
var scriptObject = new ScriptObject();
// Constants
scriptObject.SetValue("Context", context, true);
scriptObject.SetValue("CoreStyleSheet", GetCoreStyleSheetCode(), true);
scriptObject.SetValue("ThemeStyleSheet", GetThemeStyleSheetCode(themeName), true);
scriptObject.SetValue("HighlightJsStyleName", $"solarized-{themeName.ToLowerInvariant()}", true);
// Functions
scriptObject.Import("FormatDate",
new Func<DateTimeOffset, string>(d => SharedRenderingLogic.FormatDate(d, context.DateFormat)));
scriptObject.Import("FormatMarkdown",
new Func<string, string>(m => HtmlRenderingLogic.FormatMarkdown(context, m)));
return scriptObject;
}
private static TemplateContext CreateTemplateContext() =>
new TemplateContext
{
MemberRenamer = m => m.Name,
MemberFilter = m => true,
LoopLimit = int.MaxValue,
StrictVariables = true
};
}
}

@ -1,10 +0,0 @@
using System.IO;
using System.Threading.Tasks;
namespace DiscordChatExporter.Core.Rendering
{
public interface IChatLogRenderer
{
Task RenderAsync(TextWriter writer);
}
}

@ -0,0 +1,11 @@
using System;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public interface IMessageRenderer : IAsyncDisposable
{
Task RenderMessageAsync(Message message);
}
}

@ -0,0 +1,21 @@
using System.Text;
namespace DiscordChatExporter.Core.Rendering.Internal
{
internal static class Extensions
{
public static StringBuilder AppendLineIfNotEmpty(this StringBuilder builder, string value) =>
!string.IsNullOrWhiteSpace(value) ? builder.AppendLine(value) : builder;
public static StringBuilder Trim(this StringBuilder builder)
{
while (builder.Length > 0 && char.IsWhiteSpace(builder[0]))
builder.Remove(0, 1);
while (builder.Length > 0 && char.IsWhiteSpace(builder[^1]))
builder.Remove(builder.Length - 1, 1);
return builder;
}
}
}

@ -0,0 +1,39 @@
using System.Linq;
using System.Text;
using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions;
using static DiscordChatExporter.Core.Rendering.Logic.SharedRenderingLogic;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class CsvRenderingLogic
{
// Header is always the same
public static string FormatHeader(RenderContext context) => "AuthorID,Author,Date,Content,Attachments,Reactions";
private static string EncodeValue(string value)
{
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
public static string FormatMarkdown(RenderContext context, string markdown) =>
PlainTextRenderingLogic.FormatMarkdown(context, markdown);
public static string FormatMessage(RenderContext context, Message message)
{
var buffer = new StringBuilder();
buffer
.Append(EncodeValue(message.Author.Id)).Append(',')
.Append(EncodeValue(message.Author.FullName)).Append(',')
.Append(EncodeValue(FormatDate(message.Timestamp, context.DateFormat))).Append(',')
.Append(EncodeValue(FormatMarkdown(context, message.Content ?? ""))).Append(',')
.Append(EncodeValue(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(EncodeValue(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
return buffer.ToString();
}
}
}

@ -1,53 +1,31 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Markdown; using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Nodes; using DiscordChatExporter.Core.Markdown.Nodes;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using Scriban;
using Scriban.Runtime;
using Tyrrrz.Extensions; using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering namespace DiscordChatExporter.Core.Rendering.Logic
{ {
public partial class HtmlChatLogRenderer : IChatLogRenderer internal static class HtmlRenderingLogic
{ {
private readonly ChatLog _chatLog; public static bool CanBeGrouped(Message message1, Message message2)
private readonly string _themeName;
private readonly string _dateFormat;
public HtmlChatLogRenderer(ChatLog chatLog, string themeName, string dateFormat)
{ {
_chatLog = chatLog; if (message1.Author.Id != message2.Author.Id)
_themeName = themeName; return false;
_dateFormat = dateFormat;
}
private string HtmlEncode(string s) => WebUtility.HtmlEncode(s);
private string FormatDate(DateTimeOffset date) =>
date.ToLocalTime().ToString(_dateFormat, CultureInfo.InvariantCulture);
private IEnumerable<MessageGroup> GroupMessages(IEnumerable<Message> messages) => if ((message2.Timestamp - message1.Timestamp).Duration().TotalMinutes > 7)
messages.GroupContiguous((buffer, message) => return false;
{
// Break group if the author changed
if (buffer.Last().Author.Id != message.Author.Id)
return false;
// Break group if last message was more than 7 minutes ago return true;
if ((message.Timestamp - buffer.Last().Timestamp).TotalMinutes > 7) }
return false;
return true; private static string HtmlEncode(string s) => WebUtility.HtmlEncode(s);
}).Select(g => new MessageGroup(g.First().Author, g.First().Timestamp, g));
private string FormatMarkdown(Node node, bool isJumbo) private static string FormatMarkdownNode(RenderContext context, Node node, bool isJumbo)
{ {
// Text node // Text node
if (node is TextNode textNode) if (node is TextNode textNode)
@ -60,7 +38,7 @@ namespace DiscordChatExporter.Core.Rendering
if (node is FormattedNode formattedNode) if (node is FormattedNode formattedNode)
{ {
// Recursively get inner html // Recursively get inner html
var innerHtml = FormatMarkdown(formattedNode.Children, false); var innerHtml = FormatMarkdownNodes(context, formattedNode.Children, false);
// Bold // Bold
if (formattedNode.Formatting == TextFormatting.Bold) if (formattedNode.Formatting == TextFormatting.Bold)
@ -116,21 +94,27 @@ namespace DiscordChatExporter.Core.Rendering
// User mention node // User mention node
if (mentionNode.Type == MentionType.User) if (mentionNode.Type == MentionType.User)
{ {
var user = _chatLog.Mentionables.GetUser(mentionNode.Id); var user = context.MentionableUsers.FirstOrDefault(u => u.Id == mentionNode.Id) ??
User.CreateUnknownUser(mentionNode.Id);
return $"<span class=\"mention\" title=\"{HtmlEncode(user.FullName)}\">@{HtmlEncode(user.Name)}</span>"; return $"<span class=\"mention\" title=\"{HtmlEncode(user.FullName)}\">@{HtmlEncode(user.Name)}</span>";
} }
// Channel mention node // Channel mention node
if (mentionNode.Type == MentionType.Channel) if (mentionNode.Type == MentionType.Channel)
{ {
var channel = _chatLog.Mentionables.GetChannel(mentionNode.Id); var channel = context.MentionableChannels.FirstOrDefault(c => c.Id == mentionNode.Id) ??
Channel.CreateDeletedChannel(mentionNode.Id);
return $"<span class=\"mention\">#{HtmlEncode(channel.Name)}</span>"; return $"<span class=\"mention\">#{HtmlEncode(channel.Name)}</span>";
} }
// Role mention node // Role mention node
if (mentionNode.Type == MentionType.Role) if (mentionNode.Type == MentionType.Role)
{ {
var role = _chatLog.Mentionables.GetRole(mentionNode.Id); var role = context.MentionableRoles.FirstOrDefault(r => r.Id == mentionNode.Id) ??
Role.CreateDeletedRole(mentionNode.Id);
return $"<span class=\"mention\">@{HtmlEncode(role.Name)}</span>"; return $"<span class=\"mention\">@{HtmlEncode(role.Name)}</span>";
} }
} }
@ -159,52 +143,18 @@ namespace DiscordChatExporter.Core.Rendering
} }
// Throw on unexpected nodes // Throw on unexpected nodes
throw new InvalidOperationException($"Unexpected node: [{node.GetType()}]."); throw new InvalidOperationException($"Unexpected node [{node.GetType()}].");
} }
private string FormatMarkdown(IReadOnlyList<Node> nodes, bool isTopLevel) private static string FormatMarkdownNodes(RenderContext context, IReadOnlyList<Node> nodes, bool isTopLevel)
{ {
// Emojis are jumbo if all top-level nodes are emoji nodes or whitespace text nodes // Emojis are jumbo if all top-level nodes are emoji nodes or whitespace text nodes
var isJumbo = isTopLevel && nodes.All(n => n is EmojiNode || n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text)); var isJumbo = isTopLevel && nodes.All(n => n is EmojiNode || n is TextNode textNode && string.IsNullOrWhiteSpace(textNode.Text));
return nodes.Select(n => FormatMarkdown(n, isJumbo)).JoinToString(""); return nodes.Select(n => FormatMarkdownNode(context, n, isJumbo)).JoinToString("");
} }
private string FormatMarkdown(string markdown) => FormatMarkdown(MarkdownParser.Parse(markdown), true); public static string FormatMarkdown(RenderContext context, string markdown) =>
FormatMarkdownNodes(context, MarkdownParser.Parse(markdown), true);
public async Task RenderAsync(TextWriter writer)
{
// Create template loader
var loader = new TemplateLoader();
// Get template
var templateCode = loader.Load($"Html{_themeName}.html");
var template = Template.Parse(templateCode);
// Create template context
var context = new TemplateContext
{
TemplateLoader = loader,
MemberRenamer = m => m.Name,
MemberFilter = m => true,
LoopLimit = int.MaxValue,
StrictVariables = true
};
// Create template model
var model = new ScriptObject();
model.SetValue("Model", _chatLog, true);
model.Import(nameof(GroupMessages), new Func<IEnumerable<Message>, IEnumerable<MessageGroup>>(GroupMessages));
model.Import(nameof(FormatDate), new Func<DateTimeOffset, string>(FormatDate));
model.Import(nameof(FormatMarkdown), new Func<string, string>(FormatMarkdown));
context.PushGlobal(model);
// Configure output
context.PushOutput(new TextWriterOutput(writer));
// HACK: Render output in a separate thread
// (even though Scriban has async API, it still makes a lot of blocking CPU-bound calls)
await Task.Run(async () => await context.EvaluateAsync(template.Page));
}
} }
} }

@ -0,0 +1,235 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Nodes;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Internal;
using Tyrrrz.Extensions;
using static DiscordChatExporter.Core.Rendering.Logic.SharedRenderingLogic;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class PlainTextRenderingLogic
{
public static string FormatPreamble(RenderContext context)
{
var buffer = new StringBuilder();
buffer.AppendLine('='.Repeat(62));
buffer.AppendLine($"Guild: {context.Guild.Name}");
buffer.AppendLine($"Channel: {context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(context.Channel.Topic))
buffer.AppendLine($"Topic: {context.Channel.Topic}");
if (context.After != null)
buffer.AppendLine($"After: {FormatDate(context.After.Value, context.DateFormat)}");
if (context.Before != null)
buffer.AppendLine($"Before: {FormatDate(context.Before.Value, context.DateFormat)}");
buffer.AppendLine('='.Repeat(62));
return buffer.ToString();
}
private static string FormatMarkdownNode(RenderContext context, Node node)
{
// Text node
if (node is TextNode textNode)
{
return textNode.Text;
}
// Mention node
if (node is MentionNode mentionNode)
{
// Meta mention node
if (mentionNode.Type == MentionType.Meta)
{
return $"@{mentionNode.Id}";
}
// User mention node
if (mentionNode.Type == MentionType.User)
{
var user = context.MentionableUsers.FirstOrDefault(u => u.Id == mentionNode.Id) ??
User.CreateUnknownUser(mentionNode.Id);
return $"@{user.Name}";
}
// Channel mention node
if (mentionNode.Type == MentionType.Channel)
{
var channel = context.MentionableChannels.FirstOrDefault(c => c.Id == mentionNode.Id) ??
Channel.CreateDeletedChannel(mentionNode.Id);
return $"#{channel.Name}";
}
// Role mention node
if (mentionNode.Type == MentionType.Role)
{
var role = context.MentionableRoles.FirstOrDefault(r => r.Id == mentionNode.Id) ??
Role.CreateDeletedRole(mentionNode.Id);
return $"@{role.Name}";
}
}
// Emoji node
if (node is EmojiNode emojiNode)
{
return emojiNode.IsCustomEmoji ? $":{emojiNode.Name}:" : emojiNode.Name;
}
// Throw on unexpected nodes
throw new InvalidOperationException($"Unexpected node [{node.GetType()}].");
}
public static string FormatMarkdown(RenderContext context, string markdown) =>
MarkdownParser.ParseMinimal(markdown).Select(n => FormatMarkdownNode(context, n)).JoinToString("");
public static string FormatMessageHeader(RenderContext context, Message message)
{
var buffer = new StringBuilder();
// Timestamp & author
buffer
.Append($"[{FormatDate(message.Timestamp, context.DateFormat)}]")
.Append(' ')
.Append($"{message.Author.FullName}");
// Whether the message is pinned
if (message.IsPinned)
{
buffer.Append(' ').Append("(pinned)");
}
return buffer.ToString();
}
public static string FormatMessageContent(RenderContext context, Message message)
{
if (string.IsNullOrWhiteSpace(message.Content))
return "";
return FormatMarkdown(context, message.Content);
}
public static string FormatAttachments(RenderContext context, IReadOnlyList<Attachment> attachments)
{
if (!attachments.Any())
return "";
var buffer = new StringBuilder();
buffer
.AppendLine("{Attachments}")
.AppendJoin(Environment.NewLine, attachments.Select(a => a.Url))
.AppendLine();
return buffer.ToString();
}
public static string FormatEmbeds(RenderContext context, IReadOnlyList<Embed> embeds)
{
if (!embeds.Any())
return "";
var buffer = new StringBuilder();
foreach (var embed in embeds)
{
buffer.AppendLine("{Embed}");
// Author name
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
buffer.AppendLine(embed.Author.Name);
// URL
if (!string.IsNullOrWhiteSpace(embed.Url))
buffer.AppendLine(embed.Url);
// Title
if (!string.IsNullOrWhiteSpace(embed.Title))
buffer.AppendLine(FormatMarkdown(context, embed.Title));
// Description
if (!string.IsNullOrWhiteSpace(embed.Description))
buffer.AppendLine(FormatMarkdown(context, embed.Description));
// Fields
foreach (var field in embed.Fields)
{
// Name
if (!string.IsNullOrWhiteSpace(field.Name))
buffer.AppendLine(field.Name);
// Value
if (!string.IsNullOrWhiteSpace(field.Value))
buffer.AppendLine(field.Value);
}
// Thumbnail URL
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
buffer.AppendLine(embed.Thumbnail?.Url);
// Image URL
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
buffer.AppendLine(embed.Image?.Url);
// Footer text
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
buffer.AppendLine(embed.Footer?.Text);
buffer.AppendLine();
}
return buffer.ToString();
}
public static string FormatReactions(RenderContext context, IReadOnlyList<Reaction> reactions)
{
if (!reactions.Any())
return "";
var buffer = new StringBuilder();
buffer.AppendLine("{Reactions}");
foreach (var reaction in reactions)
{
buffer.Append(reaction.Emoji.Name);
if (reaction.Count > 1)
buffer.Append($" ({reaction.Count})");
buffer.Append(" ");
}
buffer.AppendLine();
return buffer.ToString();
}
public static string FormatMessage(RenderContext context, Message message)
{
var buffer = new StringBuilder();
buffer
.AppendLine(FormatMessageHeader(context, message))
.AppendLineIfNotEmpty(FormatMessageContent(context, message))
.AppendLine()
.AppendLineIfNotEmpty(FormatAttachments(context, message.Attachments))
.AppendLineIfNotEmpty(FormatEmbeds(context, message.Embeds))
.AppendLineIfNotEmpty(FormatReactions(context, message.Reactions));
return buffer.Trim().ToString();
}
}
}

@ -0,0 +1,11 @@
using System;
using System.Globalization;
namespace DiscordChatExporter.Core.Rendering.Logic
{
public static class SharedRenderingLogic
{
public static string FormatDate(DateTimeOffset date, string dateFormat) =>
date.ToLocalTime().ToString(dateFormat, CultureInfo.InvariantCulture);
}
}

@ -0,0 +1,23 @@
using System.IO;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public abstract class MessageRendererBase : IMessageRenderer
{
protected TextWriter Writer { get; }
protected RenderContext Context { get; }
protected MessageRendererBase(string filePath, RenderContext context)
{
Writer = File.CreateText(filePath);
Context = context;
}
public abstract Task RenderMessageAsync(Message message);
public virtual ValueTask DisposeAsync() => Writer.DisposeAsync();
}
}

@ -1,237 +0,0 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Markdown;
using DiscordChatExporter.Core.Markdown.Nodes;
using DiscordChatExporter.Core.Models;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Rendering
{
public class PlainTextChatLogRenderer : IChatLogRenderer
{
private readonly ChatLog _chatLog;
private readonly string _dateFormat;
public PlainTextChatLogRenderer(ChatLog chatLog, string dateFormat)
{
_chatLog = chatLog;
_dateFormat = dateFormat;
}
private string FormatDate(DateTimeOffset date) =>
date.ToLocalTime().ToString(_dateFormat, CultureInfo.InvariantCulture);
private string FormatDateRange(DateTimeOffset? after, DateTimeOffset? before)
{
// Both 'after' and 'before'
if (after != null && before != null)
return $"{FormatDate(after.Value)} to {FormatDate(before.Value)}";
// Just 'after'
if (after != null)
return $"after {FormatDate(after.Value)}";
// Just 'before'
if (before != null)
return $"before {FormatDate(before.Value)}";
// Neither
return "";
}
private string FormatMarkdown(Node node)
{
// Text node
if (node is TextNode textNode)
{
return textNode.Text;
}
// Mention node
if (node is MentionNode mentionNode)
{
// Meta mention node
if (mentionNode.Type == MentionType.Meta)
{
return mentionNode.Id;
}
// User mention node
if (mentionNode.Type == MentionType.User)
{
var user = _chatLog.Mentionables.GetUser(mentionNode.Id);
return $"@{user.Name}";
}
// Channel mention node
if (mentionNode.Type == MentionType.Channel)
{
var channel = _chatLog.Mentionables.GetChannel(mentionNode.Id);
return $"#{channel.Name}";
}
// Role mention node
if (mentionNode.Type == MentionType.Role)
{
var role = _chatLog.Mentionables.GetRole(mentionNode.Id);
return $"@{role.Name}";
}
}
// Emoji node
if (node is EmojiNode emojiNode)
{
return emojiNode.IsCustomEmoji ? $":{emojiNode.Name}:" : emojiNode.Name;
}
// Throw on unexpected nodes
throw new InvalidOperationException($"Unexpected node: [{node.GetType()}].");
}
private string FormatMarkdown(IEnumerable<Node> nodes) => nodes.Select(FormatMarkdown).JoinToString("");
private string FormatMarkdown(string markdown) => FormatMarkdown(MarkdownParser.ParseMinimal(markdown));
private async Task RenderMessageHeaderAsync(TextWriter writer, Message message)
{
// Timestamp
await writer.WriteAsync($"[{FormatDate(message.Timestamp)}]");
// Author
await writer.WriteAsync($" {message.Author.FullName}");
// Whether the message is pinned
if (message.IsPinned)
await writer.WriteAsync(" (pinned)");
await writer.WriteLineAsync();
}
private async Task RenderAttachmentsAsync(TextWriter writer, IReadOnlyList<Attachment> attachments)
{
if (attachments.Any())
{
await writer.WriteLineAsync("{Attachments}");
foreach (var attachment in attachments)
await writer.WriteLineAsync(attachment.Url);
await writer.WriteLineAsync();
}
}
private async Task RenderEmbedsAsync(TextWriter writer, IReadOnlyList<Embed> embeds)
{
foreach (var embed in embeds)
{
await writer.WriteLineAsync("{Embed}");
// Author name
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
await writer.WriteLineAsync(embed.Author?.Name);
// URL
if (!string.IsNullOrWhiteSpace(embed.Url))
await writer.WriteLineAsync(embed.Url);
// Title
if (!string.IsNullOrWhiteSpace(embed.Title))
await writer.WriteLineAsync(FormatMarkdown(embed.Title));
// Description
if (!string.IsNullOrWhiteSpace(embed.Description))
await writer.WriteLineAsync(FormatMarkdown(embed.Description));
// Fields
foreach (var field in embed.Fields)
{
// Name
if (!string.IsNullOrWhiteSpace(field.Name))
await writer.WriteLineAsync(field.Name);
// Value
if (!string.IsNullOrWhiteSpace(field.Value))
await writer.WriteLineAsync(field.Value);
}
// Thumbnail URL
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
await writer.WriteLineAsync(embed.Thumbnail?.Url);
// Image URL
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
await writer.WriteLineAsync(embed.Image?.Url);
// Footer text
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
await writer.WriteLineAsync(embed.Footer?.Text);
await writer.WriteLineAsync();
}
}
private async Task RenderReactionsAsync(TextWriter writer, IReadOnlyList<Reaction> reactions)
{
if (reactions.Any())
{
await writer.WriteLineAsync("{Reactions}");
foreach (var reaction in reactions)
{
await writer.WriteAsync(reaction.Emoji.Name);
if (reaction.Count > 1)
await writer.WriteAsync($" ({reaction.Count})");
await writer.WriteAsync(" ");
}
await writer.WriteLineAsync();
await writer.WriteLineAsync();
}
}
private async Task RenderMessageAsync(TextWriter writer, Message message)
{
// Header
await RenderMessageHeaderAsync(writer, message);
// Content
if (!string.IsNullOrWhiteSpace(message.Content))
await writer.WriteLineAsync(FormatMarkdown(message.Content));
// Separator
await writer.WriteLineAsync();
// Attachments
await RenderAttachmentsAsync(writer, message.Attachments);
// Embeds
await RenderEmbedsAsync(writer, message.Embeds);
// Reactions
await RenderReactionsAsync(writer, message.Reactions);
}
public async Task RenderAsync(TextWriter writer)
{
// Metadata
await writer.WriteLineAsync('='.Repeat(62));
await writer.WriteLineAsync($"Guild: {_chatLog.Guild.Name}");
await writer.WriteLineAsync($"Channel: {_chatLog.Channel.Name}");
await writer.WriteLineAsync($"Topic: {_chatLog.Channel.Topic}");
await writer.WriteLineAsync($"Messages: {_chatLog.Messages.Count:N0}");
await writer.WriteLineAsync($"Range: {FormatDateRange(_chatLog.After, _chatLog.Before)}");
await writer.WriteLineAsync('='.Repeat(62));
await writer.WriteLineAsync();
// Log
foreach (var message in _chatLog.Messages)
await RenderMessageAsync(writer, message);
}
}
}

@ -0,0 +1,29 @@
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering.Logic;
namespace DiscordChatExporter.Core.Rendering
{
public class PlainTextMessageRenderer : MessageRendererBase
{
private bool _isPreambleRendered;
public PlainTextMessageRenderer(string filePath, RenderContext context)
: base(filePath, context)
{
}
public override async Task RenderMessageAsync(Message message)
{
// Render preamble if it's the first entry
if (!_isPreambleRendered)
{
await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatPreamble(Context));
_isPreambleRendered = true;
}
await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatMessage(Context, message));
await Writer.WriteLineAsync();
}
}
}

@ -0,0 +1,38 @@
using System;
using System.Collections.Generic;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Rendering
{
public class RenderContext
{
public Guild Guild { get; }
public Channel Channel { get; }
public DateTimeOffset? After { get; }
public DateTimeOffset? Before { get; }
public string DateFormat { get; }
public IReadOnlyCollection<User> MentionableUsers { get; }
public IReadOnlyCollection<Channel> MentionableChannels { get; }
public IReadOnlyCollection<Role> MentionableRoles { get; }
public RenderContext(Guild guild, Channel channel, DateTimeOffset? after, DateTimeOffset? before, string dateFormat,
IReadOnlyCollection<User> mentionableUsers, IReadOnlyCollection<Channel> mentionableChannels, IReadOnlyCollection<Role> mentionableRoles)
{
Guild = guild;
Channel = channel;
After = after;
Before = before;
DateFormat = dateFormat;
MentionableUsers = mentionableUsers;
MentionableChannels = mentionableChannels;
MentionableRoles = mentionableRoles;
}
}
}

@ -358,6 +358,7 @@ img {
background: #7289da; background: #7289da;
color: #ffffff; color: #ffffff;
font-size: 0.625em; font-size: 0.625em;
font-weight: 500;
padding: 1px 2px; padding: 1px 2px;
border-radius: 3px; border-radius: 3px;
vertical-align: middle; vertical-align: middle;

@ -23,7 +23,7 @@ a {
.pre--multiline { .pre--multiline {
border-color: #282b30 !important; border-color: #282b30 !important;
color: #839496 !important; color: #b9bbbe !important;
} }
.mention { .mention {

@ -1,3 +0,0 @@
{{~ ThemeStyleSheet = include "HtmlDark.css" ~}}
{{~ HighlightJsStyleName = "solarized-dark" ~}}
{{~ include "HtmlShared.html" ~}}

@ -0,0 +1,84 @@
<!DOCTYPE html>
<html lang="en">
<head>
{{~ # Metadata ~}}
<title>{{ Context.Guild.Name | html.escape }} - {{ Context.Channel.Name | html.escape }}</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width" />
{{~ # Styles ~}}
<style>
{{ CoreStyleSheet }}
</style>
<style>
{{ ThemeStyleSheet }}
</style>
{{~ # Syntax highlighting ~}}
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.6/styles/{{HighlightJsStyleName}}.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.6/highlight.min.js"></script>
<script>
document.addEventListener('DOMContentLoaded', () => {
document.querySelectorAll('.pre--multiline').forEach(block => hljs.highlightBlock(block));
});
</script>
{{~ # Local scripts ~}}
<script>
function scrollToMessage(event, id) {
var element = document.getElementById('message-' + id);
if (element) {
event.preventDefault();
element.classList.add('chatlog__message--highlighted');
window.scrollTo({
top: element.getBoundingClientRect().top - document.body.getBoundingClientRect().top - (window.innerHeight / 2),
behavior: 'smooth'
});
window.setTimeout(function() {
element.classList.remove('chatlog__message--highlighted');
}, 2000);
}
}
</script>
</head>
<body>
{{~ # Info ~}}
<div class="info">
<div class="info__guild-icon-container">
<img class="info__guild-icon" src="{{ Context.Guild.IconUrl }}" />
</div>
<div class="info__metadata">
<div class="info__guild-name">{{ Context.Guild.Name | html.escape }}</div>
<div class="info__channel-name">{{ Context.Channel.Name | html.escape }}</div>
{{~ if Context.Channel.Topic ~}}
<div class="info__channel-topic">{{ Context.Channel.Topic | html.escape }}</div>
{{~ end ~}}
{{~ if Context.After || Context.Before ~}}
<div class="info__channel-date-range">
{{~ if Context.After && Context.Before ~}}
Between {{ Context.After | FormatDate | html.escape }} and {{ Context.Before | FormatDate | html.escape }}
{{~ else if Context.After ~}}
After {{ Context.After | FormatDate | html.escape }}
{{~ else if Context.Before ~}}
Before {{ Context.Before | FormatDate | html.escape }}
{{~ end ~}}
</div>
{{~ end ~}}
</div>
</div>
{{~ # Log ~}}
<div class="chatlog">
{{~ %SPLIT% ~}}
</div>
</body>
</html>

@ -1,3 +0,0 @@
{{~ ThemeStyleSheet = include "HtmlLight.css" ~}}
{{~ HighlightJsStyleName = "solarized-light" ~}}
{{~ include "HtmlShared.html" ~}}

@ -0,0 +1,170 @@
<div class="chatlog__message-group">
{{~ # Avatar ~}}
<div class="chatlog__author-avatar-container">
<img class="chatlog__author-avatar" src="{{ MessageGroup.Author.AvatarUrl }}" />
</div>
<div class="chatlog__messages">
{{~ # Author name and timestamp ~}}
<span class="chatlog__author-name" title="{{ MessageGroup.Author.FullName | html.escape }}" data-user-id="{{ MessageGroup.Author.Id | html.escape }}">{{ MessageGroup.Author.Name | html.escape }}</span>
{{~ # Bot tag ~}}
{{~ if MessageGroup.Author.IsBot ~}}
<span class="chatlog__bot-tag">BOT</span>
{{~ end ~}}
<span class="chatlog__timestamp">{{ MessageGroup.Timestamp | FormatDate | html.escape }}</span>
{{~ # Messages ~}}
{{~ for message in MessageGroup.Messages ~}}
<div class="chatlog__message {{if message.IsPinned }}chatlog__message--pinned{{ end }}" data-message-id="{{ message.Id }}" id="message-{{ message.Id }}">
{{~ # Content ~}}
{{~ if message.Content ~}}
<div class="chatlog__content">
<span class="markdown">{{ message.Content | FormatMarkdown }}</span>
{{~ # Edited timestamp ~}}
{{~ if message.EditedTimestamp ~}}
<span class="chatlog__edited-timestamp" title="{{ message.EditedTimestamp | FormatDate | html.escape }}">(edited)</span>
{{~ end ~}}
</div>
{{~ end ~}}
{{~ # Attachments ~}}
{{~ for attachment in message.Attachments ~}}
<div class="chatlog__attachment">
<a href="{{ attachment.Url }}">
{{ # Image }}
{{~ if attachment.IsImage ~}}
<img class="chatlog__attachment-thumbnail" src="{{ attachment.Url }}" />
{{~ # Non-image ~}}
{{~ else ~}}
Attachment: {{ attachment.FileName }} ({{ attachment.FileSize }})
{{~ end ~}}
</a>
</div>
{{~ end ~}}
{{~ # Embeds ~}}
{{~ for embed in message.Embeds ~}}
<div class="chatlog__embed">
{{~ if embed.Color ~}}
<div class="chatlog__embed-color-pill" style="background-color: rgba({{ embed.Color.R }},{{ embed.Color.G }},{{ embed.Color.B }},{{ embed.Color.A }})"></div>
{{~ else ~}}
<div class="chatlog__embed-color-pill chatlog__embed-color-pill--default"></div>
{{~ end ~}}
<div class="chatlog__embed-content-container">
<div class="chatlog__embed-content">
<div class="chatlog__embed-text">
{{~ # Author ~}}
{{~ if embed.Author ~}}
<div class="chatlog__embed-author">
{{~ if embed.Author.IconUrl ~}}
<img class="chatlog__embed-author-icon" src="{{ embed.Author.IconUrl }}" />
{{~ end ~}}
{{~ if embed.Author.Name ~}}
<span class="chatlog__embed-author-name">
{{~ if embed.Author.Url ~}}
<a class="chatlog__embed-author-name-link" href="{{ embed.Author.Url }}">{{ embed.Author.Name | html.escape }}</a>
{{~ else ~}}
{{ embed.Author.Name | html.escape }}
{{~ end ~}}
</span>
{{~ end ~}}
</div>
{{~ end ~}}
{{~ # Title ~}}
{{~ if embed.Title ~}}
<div class="chatlog__embed-title">
{{~ if embed.Url ~}}
<a class="chatlog__embed-title-link" href="{{ embed.Url }}"><span class="markdown">{{ embed.Title | FormatMarkdown }}</span></a>
{{~ else ~}}
<span class="markdown">{{ embed.Title | FormatMarkdown }}</span>
{{~ end ~}}
</div>
{{~ end ~}}
{{~ # Description ~}}
{{~ if embed.Description ~}}
<div class="chatlog__embed-description"><span class="markdown">{{ embed.Description | FormatMarkdown }}</span></div>
{{~ end ~}}
{{~ # Fields ~}}
{{~ if embed.Fields | array.size > 0 ~}}
<div class="chatlog__embed-fields">
{{~ for field in embed.Fields ~}}
<div class="chatlog__embed-field {{ if field.IsInline }} chatlog__embed-field--inline {{ end }}">
{{~ if field.Name ~}}
<div class="chatlog__embed-field-name"><span class="markdown">{{ field.Name | FormatMarkdown }}</span></div>
{{~ end ~}}
{{~ if field.Value ~}}
<div class="chatlog__embed-field-value"><span class="markdown">{{ field.Value | FormatMarkdown }}</span></div>
{{~ end ~}}
</div>
{{~ end ~}}
</div>
{{~ end ~}}
</div>
{{~ # Thumbnail ~}}
{{~ if embed.Thumbnail ~}}
<div class="chatlog__embed-thumbnail-container">
<a class="chatlog__embed-thumbnail-link" href="{{ embed.Thumbnail.Url }}">
<img class="chatlog__embed-thumbnail" src="{{ embed.Thumbnail.Url }}" />
</a>
</div>
{{~ end ~}}
</div>
{{~ # Image ~}}
{{~ if embed.Image ~}}
<div class="chatlog__embed-image-container">
<a class="chatlog__embed-image-link" href="{{ embed.Image.Url }}">
<img class="chatlog__embed-image" src="{{ embed.Image.Url }}" />
</a>
</div>
{{~ end ~}}
{{~ # Footer ~}}
{{~ if embed.Footer || embed.Timestamp ~}}
<div class="chatlog__embed-footer">
{{~ if embed.Footer ~}}
{{~ if embed.Footer.Text && embed.Footer.IconUrl ~}}
<img class="chatlog__embed-footer-icon" src="{{ embed.Footer.IconUrl }}" />
{{~ end ~}}
{{~ end ~}}
<span class="chatlog__embed-footer-text">
{{~ if embed.Footer ~}}
{{~ if embed.Footer.Text ~}}
{{ embed.Footer.Text | html.escape }}
{{ if embed.Timestamp }} • {{ end }}
{{~ end ~}}
{{~ end ~}}
{{~ if embed.Timestamp ~}}
{{ embed.Timestamp | FormatDate | html.escape }}
{{~ end ~}}
</span>
</div>
{{~ end ~}}
</div>
</div>
{{~ end ~}}
{{~ # Reactions ~}}
{{~ if message.Reactions | array.size > 0 ~}}
<div class="chatlog__reactions">
{{~ for reaction in message.Reactions ~}}
<div class="chatlog__reaction">
<img class="emoji emoji--small" alt="{{ reaction.Emoji.Name }}" title="{{ reaction.Emoji.Name }}" src="{{ reaction.Emoji.ImageUrl }}" />
<span class="chatlog__reaction-count">{{ reaction.Count }}</span>
</div>
{{~ end ~}}
</div>
{{~ end ~}}
</div>
{{~ end ~}}
</div>
</div>

@ -1,259 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
{{~ # Metadata ~}}
<title>{{ Model.Guild.Name | html.escape }} - {{ Model.Channel.Name | html.escape }}</title>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width" />
{{~ # Styles ~}}
<style>
{{ include "HtmlShared.css" }}
</style>
<style>
{{ ThemeStyleSheet }}
</style>
{{~ # Syntax highlighting ~}}
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.6/styles/{{HighlightJsStyleName}}.min.css">
<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.15.6/highlight.min.js"></script>
<script>
document.addEventListener('DOMContentLoaded', () => {
document.querySelectorAll('.pre--multiline').forEach((block) => {
hljs.highlightBlock(block);
});
});
</script>
{{~ # Local scripts ~}}
<script>
function scrollToMessage(event, id) {
var element = document.getElementById('message-' + id);
if (element) {
event.preventDefault();
element.classList.add('chatlog__message--highlighted');
window.scrollTo({
top: element.getBoundingClientRect().top - document.body.getBoundingClientRect().top - (window.innerHeight / 2),
behavior: 'smooth'
});
window.setTimeout(function() {
element.classList.remove('chatlog__message--highlighted');
}, 2000);
}
}
</script>
</head>
<body>
{{~ # Info ~}}
<div class="info">
<div class="info__guild-icon-container">
<img class="info__guild-icon" src="{{ Model.Guild.IconUrl }}" />
</div>
<div class="info__metadata">
<div class="info__guild-name">{{ Model.Guild.Name | html.escape }}</div>
<div class="info__channel-name">{{ Model.Channel.Name | html.escape }}</div>
{{~ if Model.Channel.Topic ~}}
<div class="info__channel-topic">{{ Model.Channel.Topic | html.escape }}</div>
{{~ end ~}}
<div class="info__channel-message-count">{{ Model.Messages | array.size | object.format "N0" }} messages</div>
{{~ if Model.After || Model.Before ~}}
<div class="info__channel-date-range">
{{~ if Model.After && Model.Before ~}}
Between {{ Model.After | FormatDate | html.escape }} and {{ Model.Before | FormatDate | html.escape }}
{{~ else if Model.After ~}}
After {{ Model.After | FormatDate | html.escape }}
{{~ else if Model.Before ~}}
Before {{ Model.Before | FormatDate | html.escape }}
{{~ end ~}}
</div>
{{~ end ~}}
</div>
</div>
{{~ # Log ~}}
<div class="chatlog">
{{~ for group in Model.Messages | GroupMessages ~}}
<div class="chatlog__message-group">
{{~ # Avatar ~}}
<div class="chatlog__author-avatar-container">
<img class="chatlog__author-avatar" src="{{ group.Author.AvatarUrl }}" />
</div>
<div class="chatlog__messages">
{{~ # Author name and timestamp ~}}
<span class="chatlog__author-name" title="{{ group.Author.FullName | html.escape }}" data-user-id="{{ group.Author.Id | html.escape }}">{{ group.Author.Name | html.escape }}</span>
{{~ # Bot tag ~}}
{{~ if group.Author.IsBot ~}}
<span class="chatlog__bot-tag">BOT</span>
{{~ end ~}}
<span class="chatlog__timestamp">{{ group.Timestamp | FormatDate | html.escape }}</span>
{{~ # Messages ~}}
{{~ for message in group.Messages ~}}
<div class="chatlog__message {{if message.IsPinned }}chatlog__message--pinned{{ end }}" data-message-id="{{ message.Id }}" id="message-{{ message.Id }}">
{{~ # Content ~}}
{{~ if message.Content ~}}
<div class="chatlog__content">
<span class="markdown">{{ message.Content | FormatMarkdown }}</span>
{{~ # Edited timestamp ~}}
{{~ if message.EditedTimestamp ~}}
<span class="chatlog__edited-timestamp" title="{{ message.EditedTimestamp | FormatDate | html.escape }}">(edited)</span>
{{~ end ~}}
</div>
{{~ end ~}}
{{~ # Attachments ~}}
{{~ for attachment in message.Attachments ~}}
<div class="chatlog__attachment">
<a href="{{ attachment.Url }}">
{{ # Image }}
{{~ if attachment.IsImage ~}}
<img class="chatlog__attachment-thumbnail" src="{{ attachment.Url }}" />
{{~ # Non-image ~}}
{{~ else ~}}
Attachment: {{ attachment.FileName }} ({{ attachment.FileSize }})
{{~ end ~}}
</a>
</div>
{{~ end ~}}
{{~ # Embeds ~}}
{{~ for embed in message.Embeds ~}}
<div class="chatlog__embed">
{{~ if embed.Color ~}}
<div class="chatlog__embed-color-pill" style="background-color: rgba({{ embed.Color.R }},{{ embed.Color.G }},{{ embed.Color.B }},{{ embed.Color.A }})"></div>
{{~ else ~}}
<div class="chatlog__embed-color-pill chatlog__embed-color-pill--default"></div>
{{~ end ~}}
<div class="chatlog__embed-content-container">
<div class="chatlog__embed-content">
<div class="chatlog__embed-text">
{{~ # Author ~}}
{{~ if embed.Author ~}}
<div class="chatlog__embed-author">
{{~ if embed.Author.IconUrl ~}}
<img class="chatlog__embed-author-icon" src="{{ embed.Author.IconUrl }}" />
{{~ end ~}}
{{~ if embed.Author.Name ~}}
<span class="chatlog__embed-author-name">
{{~ if embed.Author.Url ~}}
<a class="chatlog__embed-author-name-link" href="{{ embed.Author.Url }}">{{ embed.Author.Name | html.escape }}</a>
{{~ else ~}}
{{ embed.Author.Name | html.escape }}
{{~ end ~}}
</span>
{{~ end ~}}
</div>
{{~ end ~}}
{{~ # Title ~}}
{{~ if embed.Title ~}}
<div class="chatlog__embed-title">
{{~ if embed.Url ~}}
<a class="chatlog__embed-title-link" href="{{ embed.Url }}"><span class="markdown">{{ embed.Title | FormatMarkdown }}</span></a>
{{~ else ~}}
<span class="markdown">{{ embed.Title | FormatMarkdown }}</span>
{{~ end ~}}
</div>
{{~ end ~}}
{{~ # Description ~}}
{{~ if embed.Description ~}}
<div class="chatlog__embed-description"><span class="markdown">{{ embed.Description | FormatMarkdown }}</span></div>
{{~ end ~}}
{{~ # Fields ~}}
{{~ if embed.Fields | array.size > 0 ~}}
<div class="chatlog__embed-fields">
{{~ for field in embed.Fields ~}}
<div class="chatlog__embed-field {{ if field.IsInline }} chatlog__embed-field--inline {{ end }}">
{{~ if field.Name ~}}
<div class="chatlog__embed-field-name"><span class="markdown">{{ field.Name | FormatMarkdown }}</span></div>
{{~ end ~}}
{{~ if field.Value ~}}
<div class="chatlog__embed-field-value"><span class="markdown">{{ field.Value | FormatMarkdown }}</span></div>
{{~ end ~}}
</div>
{{~ end ~}}
</div>
{{~ end ~}}
</div>
{{~ # Thumbnail ~}}
{{~ if embed.Thumbnail ~}}
<div class="chatlog__embed-thumbnail-container">
<a class="chatlog__embed-thumbnail-link" href="{{ embed.Thumbnail.Url }}">
<img class="chatlog__embed-thumbnail" src="{{ embed.Thumbnail.Url }}" />
</a>
</div>
{{~ end ~}}
</div>
{{~ # Image ~}}
{{~ if embed.Image ~}}
<div class="chatlog__embed-image-container">
<a class="chatlog__embed-image-link" href="{{ embed.Image.Url }}">
<img class="chatlog__embed-image" src="{{ embed.Image.Url }}" />
</a>
</div>
{{~ end ~}}
{{~ # Footer ~}}
{{~ if embed.Footer || embed.Timestamp ~}}
<div class="chatlog__embed-footer">
{{~ if embed.Footer ~}}
{{~ if embed.Footer.Text && embed.Footer.IconUrl ~}}
<img class="chatlog__embed-footer-icon" src="{{ embed.Footer.IconUrl }}" />
{{~ end ~}}
{{~ end ~}}
<span class="chatlog__embed-footer-text">
{{~ if embed.Footer ~}}
{{~ if embed.Footer.Text ~}}
{{ embed.Footer.Text | html.escape }}
{{ if embed.Timestamp }} • {{ end }}
{{~ end ~}}
{{~ end ~}}
{{~ if embed.Timestamp ~}}
{{ embed.Timestamp | FormatDate | html.escape }}
{{~ end ~}}
</span>
</div>
{{~ end ~}}
</div>
</div>
{{~ end ~}}
{{~ # Reactions ~}}
{{~ if message.Reactions | array.size > 0 ~}}
<div class="chatlog__reactions">
{{~ for reaction in message.Reactions ~}}
<div class="chatlog__reaction">
<img class="emoji emoji--small" alt="{{ reaction.Emoji.Name }}" title="{{ reaction.Emoji.Name }}" src="{{ reaction.Emoji.ImageUrl }}" />
<span class="chatlog__reaction-count">{{ reaction.Count }}</span>
</div>
{{~ end ~}}
</div>
{{~ end ~}}
</div>
{{~ end ~}}
</div>
</div>
{{~ end ~}}
</div>
</body>
</html>

@ -203,14 +203,14 @@ namespace DiscordChatExporter.Core.Services
// Get reactions // Get reactions
var reactions = (json["reactions"] ?? Enumerable.Empty<JToken>()).Select(ParseReaction).ToArray(); var reactions = (json["reactions"] ?? Enumerable.Empty<JToken>()).Select(ParseReaction).ToArray();
// Get mentioned users // Get mentions
var mentionedUsers = (json["mentions"] ?? Enumerable.Empty<JToken>()).Select(ParseUser).ToArray(); var mentionedUsers = (json["mentions"] ?? Enumerable.Empty<JToken>()).Select(ParseUser).ToArray();
// Get whether this message is pinned // Get whether this message is pinned
var isPinned = json["pinned"]!.Value<bool>(); var isPinned = json["pinned"]!.Value<bool>();
return new Message(id, channelId, type, author, timestamp, editedTimestamp, content, attachments, embeds, return new Message(id, channelId, type, author, timestamp, editedTimestamp, isPinned, content, attachments, embeds,
reactions, mentionedUsers, isPinned); reactions, mentionedUsers);
} }
} }
} }

@ -82,7 +82,7 @@ namespace DiscordChatExporter.Core.Services
return channel; return channel;
} }
public async IAsyncEnumerable<Guild> EnumerateUserGuildsAsync(AuthToken token) public async IAsyncEnumerable<Guild> GetUserGuildsAsync(AuthToken token)
{ {
var afterId = ""; var afterId = "";
@ -105,8 +105,6 @@ namespace DiscordChatExporter.Core.Services
} }
} }
public Task<IReadOnlyList<Guild>> GetUserGuildsAsync(AuthToken token) => EnumerateUserGuildsAsync(token).AggregateAsync();
public async Task<IReadOnlyList<Channel>> GetDirectMessageChannelsAsync(AuthToken token) public async Task<IReadOnlyList<Channel>> GetDirectMessageChannelsAsync(AuthToken token)
{ {
var response = await GetApiResponseAsync(token, "users/@me/channels"); var response = await GetApiResponseAsync(token, "users/@me/channels");
@ -117,6 +115,10 @@ namespace DiscordChatExporter.Core.Services
public async Task<IReadOnlyList<Channel>> GetGuildChannelsAsync(AuthToken token, string guildId) public async Task<IReadOnlyList<Channel>> GetGuildChannelsAsync(AuthToken token, string guildId)
{ {
// Special case for direct messages pseudo-guild
if (guildId == Guild.DirectMessages.Id)
return Array.Empty<Channel>();
var response = await GetApiResponseAsync(token, $"guilds/{guildId}/channels"); var response = await GetApiResponseAsync(token, $"guilds/{guildId}/channels");
var channels = response.Select(ParseChannel).ToArray(); var channels = response.Select(ParseChannel).ToArray();
@ -125,6 +127,10 @@ namespace DiscordChatExporter.Core.Services
public async Task<IReadOnlyList<Role>> GetGuildRolesAsync(AuthToken token, string guildId) public async Task<IReadOnlyList<Role>> GetGuildRolesAsync(AuthToken token, string guildId)
{ {
// Special case for direct messages pseudo-guild
if (guildId == Guild.DirectMessages.Id)
return Array.Empty<Role>();
var response = await GetApiResponseAsync(token, $"guilds/{guildId}/roles"); var response = await GetApiResponseAsync(token, $"guilds/{guildId}/roles");
var roles = response.Select(ParseRole).ToArray(); var roles = response.Select(ParseRole).ToArray();
@ -142,7 +148,7 @@ namespace DiscordChatExporter.Core.Services
return response.Select(ParseMessage).FirstOrDefault(); return response.Select(ParseMessage).FirstOrDefault();
} }
public async IAsyncEnumerable<Message> EnumerateMessagesAsync(AuthToken token, string channelId, public async IAsyncEnumerable<Message> GetMessagesAsync(AuthToken token, string channelId,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null) DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
{ {
// Get the last message // Get the last message
@ -157,11 +163,11 @@ namespace DiscordChatExporter.Core.Services
// Get other messages // Get other messages
var firstMessage = default(Message); var firstMessage = default(Message);
var offsetId = after?.ToSnowflake() ?? "0"; var afterId = after?.ToSnowflake() ?? "0";
while (true) while (true)
{ {
// Get message batch // Get message batch
var route = $"channels/{channelId}/messages?limit=100&after={offsetId}"; var route = $"channels/{channelId}/messages?limit=100&after={afterId}";
var response = await GetApiResponseAsync(token, route); var response = await GetApiResponseAsync(token, route);
// Parse // Parse
@ -190,7 +196,7 @@ namespace DiscordChatExporter.Core.Services
(lastMessage.Timestamp - firstMessage.Timestamp).TotalSeconds); (lastMessage.Timestamp - firstMessage.Timestamp).TotalSeconds);
yield return message; yield return message;
offsetId = message.Id; afterId = message.Id;
} }
// Break if messages were trimmed (which means the last message was encountered) // Break if messages were trimmed (which means the last message was encountered)
@ -200,67 +206,9 @@ namespace DiscordChatExporter.Core.Services
// Yield last message // Yield last message
yield return lastMessage; yield return lastMessage;
// Report progress
progress?.Report(1); progress?.Report(1);
} }
public Task<IReadOnlyList<Message>> GetMessagesAsync(AuthToken token, string channelId,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null) =>
EnumerateMessagesAsync(token, channelId, after, before, progress).AggregateAsync();
public async Task<Mentionables> GetMentionablesAsync(AuthToken token, string guildId,
IEnumerable<Message> messages)
{
// Get channels and roles
var channels = guildId != Guild.DirectMessages.Id
? await GetGuildChannelsAsync(token, guildId)
: Array.Empty<Channel>();
var roles = guildId != Guild.DirectMessages.Id
? await GetGuildRolesAsync(token, guildId)
: Array.Empty<Role>();
// Get users
var userMap = new Dictionary<string, User>();
foreach (var message in messages)
{
// Author
userMap[message.Author.Id] = message.Author;
// Mentioned users
foreach (var mentionedUser in message.MentionedUsers)
userMap[mentionedUser.Id] = mentionedUser;
}
var users = userMap.Values.ToArray();
return new Mentionables(users, channels, roles);
}
public async Task<ChatLog> GetChatLogAsync(AuthToken token, Guild guild, Channel channel,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
{
// Get messages
var messages = await GetMessagesAsync(token, channel.Id, after, before, progress);
// Get mentionables
var mentionables = await GetMentionablesAsync(token, guild.Id, messages);
return new ChatLog(guild, channel, after, before, messages, mentionables);
}
public async Task<ChatLog> GetChatLogAsync(AuthToken token, Channel channel,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
{
// Get guild
var guild = !string.IsNullOrWhiteSpace(channel.GuildId)
? await GetGuildAsync(token, channel.GuildId)
: Guild.DirectMessages;
// Get the chat log
return await GetChatLogAsync(token, guild, channel, after, before, progress);
}
public void Dispose() => _httpClient.Dispose(); public void Dispose() => _httpClient.Dispose();
} }
} }

@ -1,9 +1,10 @@
using System; using System;
using System.Collections.Generic;
using System.IO; using System.IO;
using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Rendering; using DiscordChatExporter.Core.Rendering;
using DiscordChatExporter.Core.Services.Logic;
using Tyrrrz.Extensions; using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Services namespace DiscordChatExporter.Core.Services
@ -11,79 +12,99 @@ namespace DiscordChatExporter.Core.Services
public class ExportService public class ExportService
{ {
private readonly SettingsService _settingsService; private readonly SettingsService _settingsService;
private readonly DataService _dataService;
public ExportService(SettingsService settingsService) public ExportService(SettingsService settingsService, DataService dataService)
{ {
_settingsService = settingsService; _settingsService = settingsService;
_dataService = dataService;
} }
private IChatLogRenderer CreateRenderer(ChatLog chatLog, ExportFormat format) private string GetFilePathFromOutputPath(string outputPath, ExportFormat format, RenderContext context)
{ {
if (format == ExportFormat.PlainText) // Output is a directory
return new PlainTextChatLogRenderer(chatLog, _settingsService.DateFormat); if (Directory.Exists(outputPath) || string.IsNullOrWhiteSpace(Path.GetExtension(outputPath)))
{
if (format == ExportFormat.HtmlDark) var fileName = ExportLogic.GetDefaultExportFileName(format, context.Guild, context.Channel, context.After, context.Before);
return new HtmlChatLogRenderer(chatLog, "Dark", _settingsService.DateFormat); return Path.Combine(outputPath, fileName);
}
if (format == ExportFormat.HtmlLight)
return new HtmlChatLogRenderer(chatLog, "Light", _settingsService.DateFormat);
if (format == ExportFormat.Csv)
return new CsvChatLogRenderer(chatLog, _settingsService.DateFormat);
throw new ArgumentOutOfRangeException(nameof(format), $"Unknown format [{format}]."); // Output is a file
return outputPath;
} }
private async Task ExportChatLogAsync(ChatLog chatLog, string filePath, ExportFormat format) private IMessageRenderer CreateRenderer(string outputPath, int partitionIndex, ExportFormat format, RenderContext context)
{ {
var filePath = ExportLogic.GetExportPartitionFilePath(
GetFilePathFromOutputPath(outputPath, format, context),
partitionIndex);
// Create output directory // Create output directory
var dirPath = Path.GetDirectoryName(filePath); var dirPath = Path.GetDirectoryName(filePath);
if (!string.IsNullOrWhiteSpace(dirPath)) if (!string.IsNullOrWhiteSpace(dirPath))
Directory.CreateDirectory(dirPath); Directory.CreateDirectory(dirPath);
// Render chat log to output file // Create renderer
await using var writer = File.CreateText(filePath);
await CreateRenderer(chatLog, format).RenderAsync(writer); if (format == ExportFormat.PlainText)
return new PlainTextMessageRenderer(filePath, context);
if (format == ExportFormat.Csv)
return new CsvMessageRenderer(filePath, context);
if (format == ExportFormat.HtmlDark)
return new HtmlMessageRenderer(filePath, context, "Dark");
if (format == ExportFormat.HtmlLight)
return new HtmlMessageRenderer(filePath, context, "Light");
throw new InvalidOperationException($"Unknown export format [{format}].");
} }
public async Task ExportChatLogAsync(ChatLog chatLog, string filePath, ExportFormat format, int? partitionLimit) public async Task ExportChatLogAsync(AuthToken token, Guild guild, Channel channel,
string outputPath, ExportFormat format, int? partitionLimit,
DateTimeOffset? after = null, DateTimeOffset? before = null, IProgress<double>? progress = null)
{ {
// If partitioning is disabled or there are fewer messages in chat log than the limit - process it without partitioning // Create context
if (partitionLimit == null || partitionLimit <= 0 || chatLog.Messages.Count <= partitionLimit) var mentionableUsers = new HashSet<User>(IdBasedEqualityComparer.Instance);
{ var mentionableChannels = await _dataService.GetGuildChannelsAsync(token, guild.Id);
await ExportChatLogAsync(chatLog, filePath, format); var mentionableRoles = await _dataService.GetGuildRolesAsync(token, guild.Id);
}
// Otherwise split into partitions and export separately var context = new RenderContext
else (
{ guild, channel, after, before, _settingsService.DateFormat,
// Create partitions by grouping up to X contiguous messages into separate chat logs mentionableUsers, mentionableChannels, mentionableRoles
var partitions = chatLog.Messages.GroupContiguous(g => g.Count < partitionLimit.Value) );
.Select(g => new ChatLog(chatLog.Guild, chatLog.Channel, chatLog.After, chatLog.Before, g, chatLog.Mentionables))
.ToArray();
// Split file path into components
var dirPath = Path.GetDirectoryName(filePath);
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(filePath);
var fileExt = Path.GetExtension(filePath);
// Export each partition separately
var partitionNumber = 1;
foreach (var partition in partitions)
{
// Compose new file name
var partitionFilePath = $"{fileNameWithoutExt} [{partitionNumber} of {partitions.Length}]{fileExt}";
// Compose full file path // Render messages
if (!string.IsNullOrWhiteSpace(dirPath)) var partitionIndex = 0;
partitionFilePath = Path.Combine(dirPath, partitionFilePath); var partitionMessageCount = 0;
var renderer = CreateRenderer(outputPath, partitionIndex, format, context);
// Export await foreach (var message in _dataService.GetMessagesAsync(token, channel.Id, after, before, progress))
await ExportChatLogAsync(partition, partitionFilePath, format); {
// Add encountered users to the list of mentionable users
mentionableUsers.Add(message.Author);
mentionableUsers.AddRange(message.MentionedUsers);
// Increment partition number // If new partition is required, reset renderer
partitionNumber++; if (partitionLimit != null && partitionLimit > 0 && partitionMessageCount >= partitionLimit)
{
partitionIndex++;
partitionMessageCount = 0;
// Flush old renderer and create a new one
await renderer.DisposeAsync();
renderer = CreateRenderer(outputPath, partitionIndex, format, context);
} }
// Render message
await renderer.RenderMessageAsync(message);
partitionMessageCount++;
} }
// Flush last renderer
await renderer.DisposeAsync();
} }
} }
} }

@ -0,0 +1,22 @@
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
namespace DiscordChatExporter.Core.Services
{
public static class Extensions
{
private static async ValueTask<IReadOnlyList<T>> AggregateAsync<T>(this IAsyncEnumerable<T> asyncEnumerable)
{
var list = new List<T>();
await foreach (var i in asyncEnumerable)
list.Add(i);
return list;
}
public static ValueTaskAwaiter<IReadOnlyList<T>> GetAwaiter<T>(this IAsyncEnumerable<T> asyncEnumerable) =>
asyncEnumerable.AggregateAsync().GetAwaiter();
}
}

@ -1,58 +0,0 @@
using System;
using System.IO;
using System.Linq;
using System.Text;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Services.Helpers
{
public static class ExportHelper
{
public static bool IsDirectoryPath(string path) =>
path.Last() == Path.DirectorySeparatorChar ||
path.Last() == Path.AltDirectorySeparatorChar ||
string.IsNullOrWhiteSpace(Path.GetExtension(path)) && !File.Exists(path);
public static string GetDefaultExportFileName(ExportFormat format, Guild guild, Channel channel,
DateTimeOffset? after = null, DateTimeOffset? before = null)
{
var result = new StringBuilder();
// Append guild and channel names
result.Append($"{guild.Name} - {channel.Name} [{channel.Id}]");
// Append date range
if (after != null || before != null)
{
result.Append(" (");
// Both 'after' and 'before' are set
if (after != null && before != null)
{
result.Append($"{after:yyyy-MM-dd} to {before:yyyy-MM-dd}");
}
// Only 'after' is set
else if (after != null)
{
result.Append($"after {after:yyyy-MM-dd}");
}
// Only 'before' is set
else
{
result.Append($"before {before:yyyy-MM-dd}");
}
result.Append(")");
}
// Append extension
result.Append($".{format.GetFileExtension()}");
// Replace invalid chars
foreach (var invalidChar in Path.GetInvalidFileNameChars())
result.Replace(invalidChar, '_');
return result.ToString();
}
}
}

@ -1,7 +1,5 @@
using System; using System;
using System.Collections.Generic;
using System.Drawing; using System.Drawing;
using System.Threading.Tasks;
namespace DiscordChatExporter.Core.Services.Internal namespace DiscordChatExporter.Core.Services.Internal
{ {
@ -16,15 +14,5 @@ namespace DiscordChatExporter.Core.Services.Internal
} }
public static Color ResetAlpha(this Color color) => Color.FromArgb(1, color); public static Color ResetAlpha(this Color color) => Color.FromArgb(1, color);
public static async Task<IReadOnlyList<T>> AggregateAsync<T>(this IAsyncEnumerable<T> asyncEnumerable)
{
var list = new List<T>();
await foreach (var i in asyncEnumerable)
list.Add(i);
return list;
}
} }
} }

@ -0,0 +1,72 @@
using System;
using System.IO;
using System.Text;
using DiscordChatExporter.Core.Models;
namespace DiscordChatExporter.Core.Services.Logic
{
public static class ExportLogic
{
public static string GetDefaultExportFileName(ExportFormat format,
Guild guild, Channel channel,
DateTimeOffset? after = null, DateTimeOffset? before = null)
{
var buffer = new StringBuilder();
// Append guild and channel names
buffer.Append($"{guild.Name} - {channel.Name} [{channel.Id}]");
// Append date range
if (after != null || before != null)
{
buffer.Append(" (");
// Both 'after' and 'before' are set
if (after != null && before != null)
{
buffer.Append($"{after:yyyy-MM-dd} to {before:yyyy-MM-dd}");
}
// Only 'after' is set
else if (after != null)
{
buffer.Append($"after {after:yyyy-MM-dd}");
}
// Only 'before' is set
else
{
buffer.Append($"before {before:yyyy-MM-dd}");
}
buffer.Append(")");
}
// Append extension
buffer.Append($".{format.GetFileExtension()}");
// Replace invalid chars
foreach (var invalidChar in Path.GetInvalidFileNameChars())
buffer.Replace(invalidChar, '_');
return buffer.ToString();
}
public static string GetExportPartitionFilePath(string baseFilePath, int partitionIndex)
{
// First partition - no changes
if (partitionIndex <= 0)
return baseFilePath;
// Inject partition index into file name
var fileNameWithoutExt = Path.GetFileNameWithoutExtension(baseFilePath);
var fileExt = Path.GetExtension(baseFilePath);
var fileName = $"{fileNameWithoutExt} [part {partitionIndex + 1}]{fileExt}";
// Generate new path
var dirPath = Path.GetDirectoryName(baseFilePath);
if (!string.IsNullOrWhiteSpace(dirPath))
return Path.Combine(dirPath, fileName);
return fileName;
}
}
}

@ -3,7 +3,7 @@ using System.Collections.Generic;
using System.Linq; using System.Linq;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services; using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Helpers; using DiscordChatExporter.Core.Services.Logic;
using DiscordChatExporter.Gui.ViewModels.Components; using DiscordChatExporter.Gui.ViewModels.Components;
using DiscordChatExporter.Gui.ViewModels.Framework; using DiscordChatExporter.Gui.ViewModels.Framework;
@ -62,7 +62,7 @@ namespace DiscordChatExporter.Gui.ViewModels.Dialogs
var channel = Channels.Single(); var channel = Channels.Single();
// Generate default file name // Generate default file name
var defaultFileName = ExportHelper.GetDefaultExportFileName(SelectedFormat, Guild, channel, After, Before); var defaultFileName = ExportLogic.GetDefaultExportFileName(SelectedFormat, Guild, channel, After, Before);
// Generate filter // Generate filter
var ext = SelectedFormat.GetFileExtension(); var ext = SelectedFormat.GetFileExtension();

@ -1,13 +1,11 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.IO;
using System.Linq; using System.Linq;
using System.Net; using System.Net;
using System.Threading.Tasks; using System.Threading.Tasks;
using DiscordChatExporter.Core.Models; using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services; using DiscordChatExporter.Core.Services;
using DiscordChatExporter.Core.Services.Exceptions; using DiscordChatExporter.Core.Services.Exceptions;
using DiscordChatExporter.Core.Services.Helpers;
using DiscordChatExporter.Gui.Services; using DiscordChatExporter.Gui.Services;
using DiscordChatExporter.Gui.ViewModels.Components; using DiscordChatExporter.Gui.ViewModels.Components;
using DiscordChatExporter.Gui.ViewModels.Framework; using DiscordChatExporter.Gui.ViewModels.Framework;
@ -163,10 +161,7 @@ namespace DiscordChatExporter.Gui.ViewModels
// Get direct messages // Get direct messages
{ {
// Get fake guild
var guild = Guild.DirectMessages; var guild = Guild.DirectMessages;
// Get channels
var channels = await _dataService.GetDirectMessageChannelsAsync(token); var channels = await _dataService.GetDirectMessageChannelsAsync(token);
// Create channel view models // Create channel view models
@ -197,13 +192,8 @@ namespace DiscordChatExporter.Gui.ViewModels
var guilds = await _dataService.GetUserGuildsAsync(token); var guilds = await _dataService.GetUserGuildsAsync(token);
foreach (var guild in guilds) foreach (var guild in guilds)
{ {
// Get channels
var channels = await _dataService.GetGuildChannelsAsync(token, guild.Id); var channels = await _dataService.GetGuildChannelsAsync(token, guild.Id);
// Get category channels
var categoryChannels = channels.Where(c => c.Type == ChannelType.GuildCategory).ToArray(); var categoryChannels = channels.Where(c => c.Type == ChannelType.GuildCategory).ToArray();
// Get exportable channels
var exportableChannels = channels.Where(c => c.Type.IsExportable()).ToArray(); var exportableChannels = channels.Where(c => c.Type.IsExportable()).ToArray();
// Create channel view models // Create channel view models
@ -246,7 +236,6 @@ namespace DiscordChatExporter.Gui.ViewModels
} }
finally finally
{ {
// Dispose progress operation
operation.Dispose(); operation.Dispose();
} }
} }
@ -272,33 +261,15 @@ namespace DiscordChatExporter.Gui.ViewModels
var successfulExportCount = 0; var successfulExportCount = 0;
for (var i = 0; i < dialog.Channels.Count; i++) for (var i = 0; i < dialog.Channels.Count; i++)
{ {
// Get operation and channel
var operation = operations[i]; var operation = operations[i];
var channel = dialog.Channels[i]; var channel = dialog.Channels[i];
try try
{ {
// Generate file path if necessary await _exportService.ExportChatLogAsync(token, dialog.Guild, channel,
var filePath = dialog.OutputPath!; dialog.OutputPath!, dialog.SelectedFormat, dialog.PartitionLimit,
if (ExportHelper.IsDirectoryPath(filePath))
{
// Generate default file name
var fileName = ExportHelper.GetDefaultExportFileName(dialog.SelectedFormat, dialog.Guild,
channel, dialog.After, dialog.Before);
// Combine paths
filePath = Path.Combine(filePath, fileName);
}
// Get chat log
var chatLog = await _dataService.GetChatLogAsync(token, dialog.Guild, channel,
dialog.After, dialog.Before, operation); dialog.After, dialog.Before, operation);
// Export
await _exportService.ExportChatLogAsync(chatLog, filePath, dialog.SelectedFormat,
dialog.PartitionLimit);
// Report successful export
successfulExportCount++; successfulExportCount++;
} }
catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden) catch (HttpErrorStatusCodeException ex) when (ex.StatusCode == HttpStatusCode.Forbidden)
@ -311,7 +282,6 @@ namespace DiscordChatExporter.Gui.ViewModels
} }
finally finally
{ {
// Dispose progress operation
operation.Dispose(); operation.Dispose();
} }
} }

Loading…
Cancel
Save