pull/321/head
Alexey Golub 5 years ago
parent 6a430cdc76
commit dc79813ad7

@ -3,7 +3,7 @@ using System.Threading.Tasks;
using CliFx;
using CliFx.Attributes;
using DiscordChatExporter.Cli.Commands.Base;
using DiscordChatExporter.Domain.Discord;
using DiscordChatExporter.Domain.Utilities;
namespace DiscordChatExporter.Cli.Commands
{

@ -1,22 +0,0 @@
using System.Collections.Generic;
using System.Runtime.CompilerServices;
using System.Threading.Tasks;
namespace DiscordChatExporter.Domain.Discord
{
public static class AccessibilityExtensions
{
private static async ValueTask<IReadOnlyList<T>> AggregateAsync<T>(this IAsyncEnumerable<T> asyncEnumerable)
{
var list = new List<T>();
await foreach (var i in asyncEnumerable)
list.Add(i);
return list;
}
public static ValueTaskAwaiter<IReadOnlyList<T>> GetAwaiter<T>(this IAsyncEnumerable<T> asyncEnumerable) =>
asyncEnumerable.AggregateAsync().GetAwaiter();
}
}

@ -2,11 +2,7 @@
namespace DiscordChatExporter.Domain.Discord
{
public enum AuthTokenType
{
User,
Bot
}
public enum AuthTokenType { User, Bot }
public class AuthToken
{

@ -23,7 +23,9 @@ namespace DiscordChatExporter.Domain.Discord
_token = token;
_httpClient = httpClient;
// Discord seems to always respond 429 on our first request with unreasonable wait time (10+ minutes).
_httpClient.BaseAddress = new Uri("https://discordapp.com/api/v6");
// Discord seems to always respond with 429 on the first request with unreasonable wait time (10+ minutes).
// For that reason the policy will start respecting their retry-after header only after Nth failed response.
_httpRequestPolicy = Policy
.HandleResult<HttpResponseMessage>(m => m.StatusCode == HttpStatusCode.TooManyRequests)
@ -240,10 +242,7 @@ namespace DiscordChatExporter.Domain.Discord
handler.UseCookies = false;
return new HttpClient(handler, true)
{
BaseAddress = new Uri("https://discordapp.com/api/v6")
};
return new HttpClient(handler, true);
});
}
}

@ -31,8 +31,7 @@ namespace DiscordChatExporter.Domain.Exporting
var mentionableChannels = await _discord.GetGuildChannelsAsync(guild.Id);
var mentionableRoles = guild.Roles;
var context = new RenderContext
(
var context = new RenderContext(
guild, channel, after, before, dateFormat,
mentionableUsers, mentionableChannels, mentionableRoles
);

@ -9,7 +9,7 @@ using Tyrrrz.Extensions;
namespace DiscordChatExporter.Domain.Exporting.Writers
{
internal class CsvMessageWriter : MessageWriterBase
internal partial class CsvMessageWriter : MessageWriterBase
{
private readonly TextWriter _writer;
@ -19,40 +19,40 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer = new StreamWriter(stream);
}
private string EncodeValue(string value)
{
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private string FormatMarkdown(string markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown);
public override async Task WritePreambleAsync() =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
private string FormatMessage(Message message)
public override async Task WriteMessageAsync(Message message)
{
var buffer = new StringBuilder();
buffer
.Append(EncodeValue(message.Author.Id)).Append(',')
.Append(EncodeValue(message.Author.FullName)).Append(',')
.Append(EncodeValue(message.Timestamp.ToLocalString(Context.DateFormat))).Append(',')
.Append(EncodeValue(FormatMarkdown(message.Content))).Append(',')
.Append(EncodeValue(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(EncodeValue(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
return buffer.ToString();
.Append(CsvEncode(message.Author.Id)).Append(',')
.Append(CsvEncode(message.Author.FullName)).Append(',')
.Append(CsvEncode(message.Timestamp.ToLocalString(Context.DateFormat))).Append(',')
.Append(CsvEncode(FormatMarkdown(message.Content))).Append(',')
.Append(CsvEncode(message.Attachments.Select(a => a.Url).JoinToString(","))).Append(',')
.Append(CsvEncode(message.Reactions.Select(r => $"{r.Emoji.Name} ({r.Count})").JoinToString(",")));
await _writer.WriteLineAsync(buffer.ToString());
}
public override async Task WritePreambleAsync() =>
await _writer.WriteLineAsync("AuthorID,Author,Date,Content,Attachments,Reactions");
public override async Task WriteMessageAsync(Message message) =>
await _writer.WriteLineAsync(FormatMessage(message));
public override async ValueTask DisposeAsync()
{
await _writer.DisposeAsync();
await base.DisposeAsync();
}
}
internal partial class CsvMessageWriter
{
private static string CsvEncode(string value)
{
value = value.Replace("\"", "\"\"");
return $"\"{value}\"";
}
}
}

@ -1,17 +1,12 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DiscordChatExporter.Domain.Discord.Models;
using DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors;
using DiscordChatExporter.Domain.Internal;
using DiscordChatExporter.Domain.Markdown;
using DiscordChatExporter.Domain.Markdown.Ast;
using Scriban;
using Scriban.Runtime;
using Tyrrrz.Extensions;
@ -79,7 +74,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
new Func<DateTimeOffset, string>(d => d.ToLocalString(Context.DateFormat)));
scriptObject.Import("FormatMarkdown",
new Func<string, string>(FormatMarkdown));
new Func<string?, string>(FormatMarkdown));
scriptObject.Import("GetUserColor", new Func<Guild, User, string>(Guild.GetUserColor));
@ -94,8 +89,8 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
return templateContext;
}
private string FormatMarkdown(string markdown) =>
HtmlMarkdownVisitor.Format(Context, markdown);
private string FormatMarkdown(string? markdown) =>
HtmlMarkdownVisitor.Format(Context, markdown ?? "");
private async Task RenderCurrentMessageGroupAsync()
{
@ -180,7 +175,5 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
ResourcesAssembly
.GetManifestResourceString($"{ResourcesNamespace}.HtmlLayoutTemplate.html")
.SubstringAfter("{{~ %SPLIT% ~}}");
private static string HtmlEncode(string s) => WebUtility.HtmlEncode(s);
}
}

@ -22,6 +22,124 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
});
}
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private void WriteAttachment(Attachment attachment)
{
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id);
_writer.WriteString("url", attachment.Url);
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
_writer.WriteEndObject();
}
private void WriteEmbedAuthor(EmbedAuthor embedAuthor)
{
_writer.WriteStartObject("author");
_writer.WriteString("name", embedAuthor.Name);
_writer.WriteString("url", embedAuthor.Url);
_writer.WriteString("iconUrl", embedAuthor.IconUrl);
_writer.WriteEndObject();
}
private void WriteEmbedThumbnail(EmbedImage embedThumbnail)
{
_writer.WriteStartObject("thumbnail");
_writer.WriteString("url", embedThumbnail.Url);
_writer.WriteNumber("width", embedThumbnail.Width);
_writer.WriteNumber("height", embedThumbnail.Height);
_writer.WriteEndObject();
}
private void WriteEmbedImage(EmbedImage embedImage)
{
_writer.WriteStartObject("image");
_writer.WriteString("url", embedImage.Url);
_writer.WriteNumber("width", embedImage.Width);
_writer.WriteNumber("height", embedImage.Height);
_writer.WriteEndObject();
}
private void WriteEmbedFooter(EmbedFooter embedFooter)
{
_writer.WriteStartObject("footer");
_writer.WriteString("text", embedFooter.Text);
_writer.WriteString("iconUrl", embedFooter.IconUrl);
_writer.WriteEndObject();
}
private void WriteEmbedField(EmbedField embedField)
{
_writer.WriteStartObject();
_writer.WriteString("name", embedField.Name);
_writer.WriteString("value", embedField.Value);
_writer.WriteBoolean("isInline", embedField.IsInline);
_writer.WriteEndObject();
}
private void WriteEmbed(Embed embed)
{
_writer.WriteStartObject();
_writer.WriteString("title", FormatMarkdown(embed.Title));
_writer.WriteString("url", embed.Url);
_writer.WriteString("timestamp", embed.Timestamp);
_writer.WriteString("description", FormatMarkdown(embed.Description));
if (embed.Author != null)
WriteEmbedAuthor(embed.Author);
if (embed.Thumbnail != null)
WriteEmbedThumbnail(embed.Thumbnail);
if (embed.Image != null)
WriteEmbedImage(embed.Image);
if (embed.Footer != null)
WriteEmbedFooter(embed.Footer);
// Fields
_writer.WriteStartArray("fields");
foreach (var field in embed.Fields)
WriteEmbedField(field);
_writer.WriteEndArray();
_writer.WriteEndObject();
}
private void WriteReaction(Reaction reaction)
{
_writer.WriteStartObject();
// Emoji
_writer.WriteStartObject("emoji");
_writer.WriteString("id", reaction.Emoji.Id);
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", reaction.Emoji.ImageUrl);
_writer.WriteEndObject();
_writer.WriteNumber("count", reaction.Count);
_writer.WriteEndObject();
}
public override async Task WritePreambleAsync()
{
// Root object (start)
@ -66,8 +184,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteBoolean("isPinned", message.IsPinned);
// Content
var content = PlainTextMarkdownVisitor.Format(Context, message.Content);
_writer.WriteString("content", content);
_writer.WriteString("content", FormatMarkdown(message.Content));
// Author
_writer.WriteStartObject("author");
@ -82,16 +199,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartArray("attachments");
foreach (var attachment in message.Attachments)
{
_writer.WriteStartObject();
_writer.WriteString("id", attachment.Id);
_writer.WriteString("url", attachment.Url);
_writer.WriteString("fileName", attachment.FileName);
_writer.WriteNumber("fileSizeBytes", attachment.FileSize.TotalBytes);
_writer.WriteEndObject();
}
WriteAttachment(attachment);
_writer.WriteEndArray();
@ -99,71 +207,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartArray("embeds");
foreach (var embed in message.Embeds)
{
_writer.WriteStartObject();
_writer.WriteString("title", embed.Title);
_writer.WriteString("url", embed.Url);
_writer.WriteString("timestamp", embed.Timestamp);
_writer.WriteString("description", embed.Description);
// Author
if (embed.Author != null)
{
_writer.WriteStartObject("author");
_writer.WriteString("name", embed.Author.Name);
_writer.WriteString("url", embed.Author.Url);
_writer.WriteString("iconUrl", embed.Author.IconUrl);
_writer.WriteEndObject();
}
// Thumbnail
if (embed.Thumbnail != null)
{
_writer.WriteStartObject("thumbnail");
_writer.WriteString("url", embed.Thumbnail.Url);
_writer.WriteNumber("width", embed.Thumbnail.Width);
_writer.WriteNumber("height", embed.Thumbnail.Height);
_writer.WriteEndObject();
}
// Image
if (embed.Image != null)
{
_writer.WriteStartObject("image");
_writer.WriteString("url", embed.Image.Url);
_writer.WriteNumber("width", embed.Image.Width);
_writer.WriteNumber("height", embed.Image.Height);
_writer.WriteEndObject();
}
// Footer
if (embed.Footer != null)
{
_writer.WriteStartObject("footer");
_writer.WriteString("text", embed.Footer.Text);
_writer.WriteString("iconUrl", embed.Footer.IconUrl);
_writer.WriteEndObject();
}
// Fields
_writer.WriteStartArray("fields");
foreach (var field in embed.Fields)
{
_writer.WriteStartObject();
_writer.WriteString("name", field.Name);
_writer.WriteString("value", field.Value);
_writer.WriteBoolean("isInline", field.IsInline);
_writer.WriteEndObject();
}
_writer.WriteEndArray();
_writer.WriteEndObject();
}
WriteEmbed(embed);
_writer.WriteEndArray();
@ -171,31 +215,14 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer.WriteStartArray("reactions");
foreach (var reaction in message.Reactions)
{
_writer.WriteStartObject();
// Emoji
_writer.WriteStartObject("emoji");
_writer.WriteString("id", reaction.Emoji.Id);
_writer.WriteString("name", reaction.Emoji.Name);
_writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
_writer.WriteString("imageUrl", reaction.Emoji.ImageUrl);
_writer.WriteEndObject();
// Count
_writer.WriteNumber("count", reaction.Count);
_writer.WriteEndObject();
}
WriteReaction(reaction);
_writer.WriteEndArray();
_writer.WriteEndObject();
_messageCount++;
// Flush every 100 messages
if (_messageCount % 100 == 0)
if (_messageCount++ % 100 == 0)
await _writer.FlushAsync();
}
@ -204,7 +231,6 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Message array (end)
_writer.WriteEndArray();
// Message count
_writer.WriteNumber("messageCount", _messageCount);
// Root object (end)

@ -115,7 +115,7 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
: "";
_buffer
.Append($"<span class=\"mention\" style=\"{style}>\"")
.Append($"<span class=\"mention\" style=\"{style}\">")
.Append("@").Append(HtmlEncode(role.Name))
.Append("</span>");
}

@ -52,9 +52,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers.MarkdownVisitors
public override MarkdownNode VisitEmoji(EmojiNode emoji)
{
_buffer.Append(emoji.IsCustomEmoji
? $":{emoji.Name}:"
: emoji.Name);
_buffer.Append(
emoji.IsCustomEmoji
? $":{emoji.Name}:"
: emoji.Name
);
return base.VisitEmoji(emoji);
}

@ -22,41 +22,8 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
_writer = new StreamWriter(stream);
}
private string FormatPreamble()
{
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Guild: {Context.Guild.Name}");
buffer.AppendLine($"Channel: {Context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(Context.Channel.Topic))
buffer.AppendLine($"Topic: {Context.Channel.Topic}");
if (Context.After != null)
buffer.AppendLine($"After: {Context.After.Value.ToLocalString(Context.DateFormat)}");
if (Context.Before != null)
buffer.AppendLine($"Before: {Context.Before.Value.ToLocalString(Context.DateFormat)}");
buffer.Append('=', 62).AppendLine();
return buffer.ToString();
}
private string FormatPostamble()
{
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Exported {_messageCount:N0} message(s)");
buffer.Append('=', 62).AppendLine();
return buffer.ToString();
}
private string FormatMarkdown(string markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown);
private string FormatMarkdown(string? markdown) =>
PlainTextMarkdownVisitor.Format(Context, markdown ?? "");
private string FormatMessageHeader(Message message)
{
@ -70,21 +37,11 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
// Whether the message is pinned
if (message.IsPinned)
{
buffer.Append(' ').Append("(pinned)");
}
return buffer.ToString();
}
private string FormatMessageContent(Message message)
{
if (string.IsNullOrWhiteSpace(message.Content))
return "";
return FormatMarkdown(message.Content);
}
private string FormatAttachments(IReadOnlyList<Attachment> attachments)
{
if (!attachments.Any())
@ -109,49 +66,25 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
foreach (var embed in embeds)
{
buffer.AppendLine("{Embed}");
// Author name
if (!string.IsNullOrWhiteSpace(embed.Author?.Name))
buffer.AppendLine(embed.Author.Name);
// URL
if (!string.IsNullOrWhiteSpace(embed.Url))
buffer.AppendLine(embed.Url);
buffer
.AppendLine("{Embed}")
.AppendLineIfNotNullOrWhiteSpace(embed.Author?.Name)
.AppendLineIfNotNullOrWhiteSpace(embed.Url)
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(embed.Title))
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(embed.Description));
// Title
if (!string.IsNullOrWhiteSpace(embed.Title))
buffer.AppendLine(FormatMarkdown(embed.Title));
// Description
if (!string.IsNullOrWhiteSpace(embed.Description))
buffer.AppendLine(FormatMarkdown(embed.Description));
// Fields
foreach (var field in embed.Fields)
{
// Name
if (!string.IsNullOrWhiteSpace(field.Name))
buffer.AppendLine(field.Name);
// Value
if (!string.IsNullOrWhiteSpace(field.Value))
buffer.AppendLine(field.Value);
buffer
.AppendLineIfNotNullOrWhiteSpace(field.Name)
.AppendLineIfNotNullOrWhiteSpace(field.Value);
}
// Thumbnail URL
if (!string.IsNullOrWhiteSpace(embed.Thumbnail?.Url))
buffer.AppendLine(embed.Thumbnail?.Url);
// Image URL
if (!string.IsNullOrWhiteSpace(embed.Image?.Url))
buffer.AppendLine(embed.Image?.Url);
// Footer text
if (!string.IsNullOrWhiteSpace(embed.Footer?.Text))
buffer.AppendLine(embed.Footer?.Text);
buffer.AppendLine();
buffer
.AppendLineIfNotNullOrWhiteSpace(embed.Thumbnail?.Url)
.AppendLineIfNotNullOrWhiteSpace(embed.Image?.Url)
.AppendLineIfNotNullOrWhiteSpace(embed.Footer?.Text)
.AppendLine();
}
return buffer.ToString();
@ -187,18 +120,35 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
buffer
.AppendLine(FormatMessageHeader(message))
.AppendLineIfNotEmpty(FormatMessageContent(message))
.AppendLineIfNotNullOrWhiteSpace(FormatMarkdown(message.Content))
.AppendLine()
.AppendLineIfNotEmpty(FormatAttachments(message.Attachments))
.AppendLineIfNotEmpty(FormatEmbeds(message.Embeds))
.AppendLineIfNotEmpty(FormatReactions(message.Reactions));
.AppendLineIfNotNullOrWhiteSpace(FormatAttachments(message.Attachments))
.AppendLineIfNotNullOrWhiteSpace(FormatEmbeds(message.Embeds))
.AppendLineIfNotNullOrWhiteSpace(FormatReactions(message.Reactions));
return buffer.Trim().ToString();
}
public override async Task WritePreambleAsync()
{
await _writer.WriteLineAsync(FormatPreamble());
var buffer = new StringBuilder();
buffer.Append('=', 62).AppendLine();
buffer.AppendLine($"Guild: {Context.Guild.Name}");
buffer.AppendLine($"Channel: {Context.Channel.Name}");
if (!string.IsNullOrWhiteSpace(Context.Channel.Topic))
buffer.AppendLine($"Topic: {Context.Channel.Topic}");
if (Context.After != null)
buffer.AppendLine($"After: {Context.After.Value.ToLocalString(Context.DateFormat)}");
if (Context.Before != null)
buffer.AppendLine($"Before: {Context.Before.Value.ToLocalString(Context.DateFormat)}");
buffer.Append('=', 62).AppendLine();
await _writer.WriteLineAsync(buffer.ToString());
}
public override async Task WriteMessageAsync(Message message)
@ -211,8 +161,15 @@ namespace DiscordChatExporter.Domain.Exporting.Writers
public override async Task WritePostambleAsync()
{
await _writer.WriteLineAsync();
await _writer.WriteLineAsync(FormatPostamble());
var buffer = new StringBuilder();
buffer
.Append('=', 62).AppendLine()
.AppendLine($"Exported {_messageCount:N0} message(s)")
.Append('=', 62).AppendLine()
.AppendLine();
await _writer.WriteLineAsync(buffer.ToString());
}
public override async ValueTask DisposeAsync()

@ -4,7 +4,7 @@ namespace DiscordChatExporter.Domain.Internal
{
internal static class StringExtensions
{
public static StringBuilder AppendLineIfNotEmpty(this StringBuilder builder, string value) =>
public static StringBuilder AppendLineIfNotNullOrWhiteSpace(this StringBuilder builder, string? value) =>
!string.IsNullOrWhiteSpace(value) ? builder.AppendLine(value) : builder;
public static StringBuilder Trim(this StringBuilder builder)

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
@ -8,6 +9,19 @@ namespace DiscordChatExporter.Domain.Utilities
{
public static class AsyncExtensions
{
private static async ValueTask<IReadOnlyList<T>> AggregateAsync<T>(this IAsyncEnumerable<T> asyncEnumerable)
{
var list = new List<T>();
await foreach (var i in asyncEnumerable)
list.Add(i);
return list;
}
public static ValueTaskAwaiter<IReadOnlyList<T>> GetAwaiter<T>(this IAsyncEnumerable<T> asyncEnumerable) =>
asyncEnumerable.AggregateAsync().GetAwaiter();
public static async Task ParallelForEachAsync<T>(this IEnumerable<T> source, Func<T, Task> handleAsync, int degreeOfParallelism)
{
using var semaphore = new SemaphoreSlim(degreeOfParallelism);

Loading…
Cancel
Save