From 9f6090b3af8f947bc0aee95fcc1ce9f4c388c248 Mon Sep 17 00:00:00 2001 From: Alexey Golub Date: Mon, 3 Feb 2020 17:47:42 +0200 Subject: [PATCH] Add JSON message writer Closes #103 --- .../ExportFormat.cs | 3 +- DiscordChatExporter.Core.Models/Extensions.cs | 2 + .../Formatters/CsvMessageWriter.cs | 17 +- .../Formatters/HtmlMessageWriter.cs | 14 +- .../Formatters/JsonMessageWriter.cs | 222 ++++++++++++++++++ .../Formatters/MessageWriterBase.cs | 8 +- .../Formatters/PlainTextMessageWriter.cs | 23 +- .../Internal/Extensions.cs | 24 +- .../MessageRenderer.cs | 15 +- 9 files changed, 302 insertions(+), 26 deletions(-) create mode 100644 DiscordChatExporter.Core.Rendering/Formatters/JsonMessageWriter.cs diff --git a/DiscordChatExporter.Core.Models/ExportFormat.cs b/DiscordChatExporter.Core.Models/ExportFormat.cs index aca8307..302e228 100644 --- a/DiscordChatExporter.Core.Models/ExportFormat.cs +++ b/DiscordChatExporter.Core.Models/ExportFormat.cs @@ -5,6 +5,7 @@ PlainText, HtmlDark, HtmlLight, - Csv + Csv, + Json } } \ No newline at end of file diff --git a/DiscordChatExporter.Core.Models/Extensions.cs b/DiscordChatExporter.Core.Models/Extensions.cs index a07a814..33928ca 100644 --- a/DiscordChatExporter.Core.Models/Extensions.cs +++ b/DiscordChatExporter.Core.Models/Extensions.cs @@ -18,6 +18,7 @@ namespace DiscordChatExporter.Core.Models ExportFormat.HtmlDark => "html", ExportFormat.HtmlLight => "html", ExportFormat.Csv => "csv", + ExportFormat.Json => "json", _ => throw new ArgumentOutOfRangeException(nameof(format)) }; @@ -28,6 +29,7 @@ namespace DiscordChatExporter.Core.Models ExportFormat.HtmlDark => "HTML (Dark)", ExportFormat.HtmlLight => "HTML (Light)", ExportFormat.Csv => "Comma Separated Values (CSV)", + ExportFormat.Json => "JavaScript Object Notation (JSON)", _ => throw new ArgumentOutOfRangeException(nameof(format)) }; } diff --git a/DiscordChatExporter.Core.Rendering/Formatters/CsvMessageWriter.cs b/DiscordChatExporter.Core.Rendering/Formatters/CsvMessageWriter.cs index 715980e..78b353b 100644 --- a/DiscordChatExporter.Core.Rendering/Formatters/CsvMessageWriter.cs +++ b/DiscordChatExporter.Core.Rendering/Formatters/CsvMessageWriter.cs @@ -7,19 +7,28 @@ namespace DiscordChatExporter.Core.Rendering.Formatters { public class CsvMessageWriter : MessageWriterBase { - public CsvMessageWriter(TextWriter writer, RenderContext context) - : base(writer, context) + private readonly TextWriter _writer; + + public CsvMessageWriter(Stream stream, RenderContext context) + : base(stream, context) { + _writer = new StreamWriter(stream); } public override async Task WritePreambleAsync() { - await Writer.WriteLineAsync(CsvRenderingLogic.FormatHeader(Context)); + await _writer.WriteLineAsync(CsvRenderingLogic.FormatHeader(Context)); } public override async Task WriteMessageAsync(Message message) { - await Writer.WriteLineAsync(CsvRenderingLogic.FormatMessage(Context, message)); + await _writer.WriteLineAsync(CsvRenderingLogic.FormatMessage(Context, message)); + } + + public override async ValueTask DisposeAsync() + { + await _writer.DisposeAsync(); + await base.DisposeAsync(); } } } \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/Formatters/HtmlMessageWriter.cs b/DiscordChatExporter.Core.Rendering/Formatters/HtmlMessageWriter.cs index 6509813..ed12e38 100644 --- a/DiscordChatExporter.Core.Rendering/Formatters/HtmlMessageWriter.cs +++ b/DiscordChatExporter.Core.Rendering/Formatters/HtmlMessageWriter.cs @@ -14,6 +14,7 @@ namespace DiscordChatExporter.Core.Rendering.Formatters { public partial class HtmlMessageWriter : MessageWriterBase { + private readonly TextWriter _writer; private readonly string _themeName; private readonly List _messageGroupBuffer = new List(); @@ -23,9 +24,10 @@ namespace DiscordChatExporter.Core.Rendering.Formatters private long _messageCount; - public HtmlMessageWriter(TextWriter writer, RenderContext context, string themeName) - : base(writer, context) + public HtmlMessageWriter(Stream stream, RenderContext context, string themeName) + : base(stream, context) { + _writer = new StreamWriter(stream); _themeName = themeName; _preambleTemplate = Template.Parse(GetPreambleTemplateCode()); @@ -77,7 +79,7 @@ namespace DiscordChatExporter.Core.Rendering.Formatters templateContext.PushGlobal(scriptObject); // Push output - templateContext.PushOutput(new TextWriterOutput(Writer)); + templateContext.PushOutput(new TextWriterOutput(_writer)); return templateContext; } @@ -131,6 +133,12 @@ namespace DiscordChatExporter.Core.Rendering.Formatters await templateContext.EvaluateAsync(_postambleTemplate.Page); } + + public override async ValueTask DisposeAsync() + { + await _writer.DisposeAsync(); + await base.DisposeAsync(); + } } public partial class HtmlMessageWriter diff --git a/DiscordChatExporter.Core.Rendering/Formatters/JsonMessageWriter.cs b/DiscordChatExporter.Core.Rendering/Formatters/JsonMessageWriter.cs new file mode 100644 index 0000000..e5b5f58 --- /dev/null +++ b/DiscordChatExporter.Core.Rendering/Formatters/JsonMessageWriter.cs @@ -0,0 +1,222 @@ +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; +using DiscordChatExporter.Core.Models; +using DiscordChatExporter.Core.Rendering.Internal; +using DiscordChatExporter.Core.Rendering.Logic; + +namespace DiscordChatExporter.Core.Rendering.Formatters +{ + public class JsonMessageWriter : MessageWriterBase + { + private readonly Utf8JsonWriter _writer; + + private long _messageCount; + + public JsonMessageWriter(Stream stream, RenderContext context) + : base(stream, context) + { + _writer = new Utf8JsonWriter(stream, new JsonWriterOptions + { + Indented = true + }); + } + + public override async Task WritePreambleAsync() + { + // Root object (start) + _writer.WriteStartObject(); + + // Guild + _writer.WriteStartObject("guild"); + _writer.WriteString("id", Context.Guild.Id); + _writer.WriteString("name", Context.Guild.Name); + _writer.WriteString("iconUrl", Context.Guild.IconUrl); + _writer.WriteEndObject(); + + // Channel + _writer.WriteStartObject("channel"); + _writer.WriteString("id", Context.Channel.Id); + _writer.WriteString("type", Context.Channel.Type.ToString()); + _writer.WriteString("name", Context.Channel.Name); + _writer.WriteString("topic", Context.Channel.Topic); + _writer.WriteEndObject(); + + // Date range + _writer.WriteStartObject("dateRange"); + _writer.WriteString("after", Context.After); + _writer.WriteString("before", Context.Before); + _writer.WriteEndObject(); + + // Message array (start) + _writer.WriteStartArray("messages"); + + await _writer.FlushAsync(); + } + + public override async Task WriteMessageAsync(Message message) + { + _writer.WriteStartObject(); + + // Metadata + _writer.WriteString("id", message.Id); + _writer.WriteString("type", message.Type.ToString()); + _writer.WriteString("timestamp", message.Timestamp); + _writer.WriteString("timestampEdited", message.EditedTimestamp); + _writer.WriteBoolean("isPinned", message.IsPinned); + + // Content + var content = PlainTextRenderingLogic.FormatMessageContent(Context, message); + _writer.WriteString("content", content); + + // Author + _writer.WriteStartObject("author"); + _writer.WriteString("id", message.Author.Id); + _writer.WriteString("name", message.Author.Name); + _writer.WriteString("discriminator", $"{message.Author.Discriminator:0000}"); + _writer.WriteBoolean("isBot", message.Author.IsBot); + _writer.WriteString("avatarUrl", message.Author.AvatarUrl); + _writer.WriteEndObject(); + + // Attachments + _writer.WriteStartArray("attachments"); + + foreach (var attachment in message.Attachments) + { + _writer.WriteStartObject(); + + _writer.WriteString("id", attachment.Id); + _writer.WriteString("url", attachment.Url); + _writer.WriteString("fileName", attachment.FileName); + _writer.WriteNumber("fileSizeBytes", (long) attachment.FileSize.Bytes); + + _writer.WriteEndObject(); + } + + _writer.WriteEndArray(); + + // Embeds + _writer.WriteStartArray("embeds"); + + foreach (var embed in message.Embeds) + { + _writer.WriteStartObject(); + + _writer.WriteString("title", embed.Title); + _writer.WriteString("url", embed.Url); + _writer.WriteString("timestamp", embed.Timestamp); + _writer.WriteString("description", embed.Description); + + // Author + if (embed.Author != null) + { + _writer.WriteStartObject("author"); + _writer.WriteString("name", embed.Author.Name); + _writer.WriteString("url", embed.Author.Url); + _writer.WriteString("iconUrl", embed.Author.IconUrl); + _writer.WriteEndObject(); + } + + // Thumbnail + if (embed.Thumbnail != null) + { + _writer.WriteStartObject("thumbnail"); + _writer.WriteString("url", embed.Thumbnail.Url); + _writer.WriteNumber("width", embed.Thumbnail.Width); + _writer.WriteNumber("height", embed.Thumbnail.Height); + _writer.WriteEndObject(); + } + + // Image + if (embed.Image != null) + { + _writer.WriteStartObject("image"); + _writer.WriteString("url", embed.Image.Url); + _writer.WriteNumber("width", embed.Image.Width); + _writer.WriteNumber("height", embed.Image.Height); + _writer.WriteEndObject(); + } + + // Footer + if (embed.Footer != null) + { + _writer.WriteStartObject("footer"); + _writer.WriteString("text", embed.Footer.Text); + _writer.WriteString("iconUrl", embed.Footer.IconUrl); + _writer.WriteEndObject(); + } + + // Fields + _writer.WriteStartArray("fields"); + + foreach (var field in embed.Fields) + { + _writer.WriteStartObject(); + + _writer.WriteString("name", field.Name); + _writer.WriteString("value", field.Value); + _writer.WriteBoolean("isInline", field.IsInline); + + _writer.WriteEndObject(); + } + + _writer.WriteEndArray(); + + _writer.WriteEndObject(); + } + + _writer.WriteEndArray(); + + // Reactions + _writer.WriteStartArray("reactions"); + + foreach (var reaction in message.Reactions) + { + _writer.WriteStartObject(); + + // Emoji + _writer.WriteStartObject("emoji"); + _writer.WriteString("id", reaction.Emoji.Id); + _writer.WriteString("name", reaction.Emoji.Name); + _writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated); + _writer.WriteString("imageUrl", reaction.Emoji.ImageUrl); + _writer.WriteEndObject(); + + // Count + _writer.WriteNumber("count", reaction.Count); + + _writer.WriteEndObject(); + } + + _writer.WriteEndArray(); + + _writer.WriteEndObject(); + + _messageCount++; + + // Flush every 100 messages + if (_messageCount % 100 == 0) + await _writer.FlushAsync(); + } + + public override async Task WritePostambleAsync() + { + // Message array (end) + _writer.WriteEndArray(); + + // Message count + _writer.WriteNumber("messageCount", _messageCount); + + // Root object (end) + _writer.WriteEndObject(); + + await _writer.FlushAsync(); + } + + public override async ValueTask DisposeAsync() + { + await _writer.DisposeAsync(); + await base.DisposeAsync(); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/Formatters/MessageWriterBase.cs b/DiscordChatExporter.Core.Rendering/Formatters/MessageWriterBase.cs index fc72104..b637cb6 100644 --- a/DiscordChatExporter.Core.Rendering/Formatters/MessageWriterBase.cs +++ b/DiscordChatExporter.Core.Rendering/Formatters/MessageWriterBase.cs @@ -7,13 +7,13 @@ namespace DiscordChatExporter.Core.Rendering.Formatters { public abstract class MessageWriterBase : IAsyncDisposable { - protected TextWriter Writer { get; } + protected Stream Stream { get; } protected RenderContext Context { get; } - protected MessageWriterBase(TextWriter writer, RenderContext context) + protected MessageWriterBase(Stream stream, RenderContext context) { - Writer = writer; + Stream = stream; Context = context; } @@ -23,6 +23,6 @@ namespace DiscordChatExporter.Core.Rendering.Formatters public virtual Task WritePostambleAsync() => Task.CompletedTask; - public async ValueTask DisposeAsync() => await Writer.DisposeAsync(); + public virtual async ValueTask DisposeAsync() => await Stream.DisposeAsync(); } } \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/Formatters/PlainTextMessageWriter.cs b/DiscordChatExporter.Core.Rendering/Formatters/PlainTextMessageWriter.cs index 8d1fd7c..18466ff 100644 --- a/DiscordChatExporter.Core.Rendering/Formatters/PlainTextMessageWriter.cs +++ b/DiscordChatExporter.Core.Rendering/Formatters/PlainTextMessageWriter.cs @@ -7,30 +7,39 @@ namespace DiscordChatExporter.Core.Rendering.Formatters { public class PlainTextMessageWriter : MessageWriterBase { + private readonly TextWriter _writer; + private long _messageCount; - public PlainTextMessageWriter(TextWriter writer, RenderContext context) - : base(writer, context) + public PlainTextMessageWriter(Stream stream, RenderContext context) + : base(stream, context) { + _writer = new StreamWriter(stream); } public override async Task WritePreambleAsync() { - await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatPreamble(Context)); + await _writer.WriteLineAsync(PlainTextRenderingLogic.FormatPreamble(Context)); } public override async Task WriteMessageAsync(Message message) { - await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatMessage(Context, message)); - await Writer.WriteLineAsync(); + await _writer.WriteLineAsync(PlainTextRenderingLogic.FormatMessage(Context, message)); + await _writer.WriteLineAsync(); _messageCount++; } public override async Task WritePostambleAsync() { - await Writer.WriteLineAsync(); - await Writer.WriteLineAsync(PlainTextRenderingLogic.FormatPostamble(_messageCount)); + await _writer.WriteLineAsync(); + await _writer.WriteLineAsync(PlainTextRenderingLogic.FormatPostamble(_messageCount)); + } + + public override async ValueTask DisposeAsync() + { + await _writer.DisposeAsync(); + await base.DisposeAsync(); } } } \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/Internal/Extensions.cs b/DiscordChatExporter.Core.Rendering/Internal/Extensions.cs index 73cc290..1d8e1e7 100644 --- a/DiscordChatExporter.Core.Rendering/Internal/Extensions.cs +++ b/DiscordChatExporter.Core.Rendering/Internal/Extensions.cs @@ -1,4 +1,6 @@ -using System.Text; +using System; +using System.Text; +using System.Text.Json; namespace DiscordChatExporter.Core.Rendering.Internal { @@ -17,5 +19,25 @@ namespace DiscordChatExporter.Core.Rendering.Internal return builder; } + + public static void WriteString(this Utf8JsonWriter writer, string propertyName, DateTimeOffset? value) + { + writer.WritePropertyName(propertyName); + + if (value != null) + writer.WriteStringValue(value.Value); + else + writer.WriteNullValue(); + } + + public static void WriteNumber(this Utf8JsonWriter writer, string propertyName, int? value) + { + writer.WritePropertyName(propertyName); + + if (value != null) + writer.WriteNumberValue(value.Value); + else + writer.WriteNullValue(); + } } } \ No newline at end of file diff --git a/DiscordChatExporter.Core.Rendering/MessageRenderer.cs b/DiscordChatExporter.Core.Rendering/MessageRenderer.cs index c647e08..8519a60 100644 --- a/DiscordChatExporter.Core.Rendering/MessageRenderer.cs +++ b/DiscordChatExporter.Core.Rendering/MessageRenderer.cs @@ -99,21 +99,24 @@ namespace DiscordChatExporter.Core.Rendering private static MessageWriterBase CreateMessageWriter(string filePath, ExportFormat format, RenderContext context) { - // Create inner writer (it will get disposed by the wrapper) - var writer = File.CreateText(filePath); + // Create a stream (it will get disposed by the writer) + var stream = File.Create(filePath); // Create formatter if (format == ExportFormat.PlainText) - return new PlainTextMessageWriter(writer, context); + return new PlainTextMessageWriter(stream, context); if (format == ExportFormat.Csv) - return new CsvMessageWriter(writer, context); + return new CsvMessageWriter(stream, context); if (format == ExportFormat.HtmlDark) - return new HtmlMessageWriter(writer, context, "Dark"); + return new HtmlMessageWriter(stream, context, "Dark"); if (format == ExportFormat.HtmlLight) - return new HtmlMessageWriter(writer, context, "Light"); + return new HtmlMessageWriter(stream, context, "Light"); + + if (format == ExportFormat.Json) + return new JsonMessageWriter(stream, context); throw new InvalidOperationException($"Unknown export format [{format}]."); }