From eb59cbde28d110181d4619c2353bfd9b2f8f64f1 Mon Sep 17 00:00:00 2001 From: Alexey Golub Date: Wed, 12 Jul 2017 20:10:01 +0300 Subject: [PATCH] Make it work --- DiscordChatExporter.sln | 22 +++ .../DiscordChatExporter.csproj | 27 +++ DiscordChatExporter/Models/Attachment.cs | 21 ++ DiscordChatExporter/Models/ChatLog.cs | 22 +++ DiscordChatExporter/Models/Message.cs | 33 ++++ DiscordChatExporter/Models/User.cs | 29 +++ DiscordChatExporter/Options.cs | 32 ++++ DiscordChatExporter/Program.cs | 37 ++++ .../Services/DiscordApiService.cs | 94 +++++++++ DiscordChatExporter/Services/ExportService.cs | 180 ++++++++++++++++++ .../Services/ExportTemplate.html | 84 ++++++++ 11 files changed, 581 insertions(+) create mode 100644 DiscordChatExporter.sln create mode 100644 DiscordChatExporter/DiscordChatExporter.csproj create mode 100644 DiscordChatExporter/Models/Attachment.cs create mode 100644 DiscordChatExporter/Models/ChatLog.cs create mode 100644 DiscordChatExporter/Models/Message.cs create mode 100644 DiscordChatExporter/Models/User.cs create mode 100644 DiscordChatExporter/Options.cs create mode 100644 DiscordChatExporter/Program.cs create mode 100644 DiscordChatExporter/Services/DiscordApiService.cs create mode 100644 DiscordChatExporter/Services/ExportService.cs create mode 100644 DiscordChatExporter/Services/ExportTemplate.html diff --git a/DiscordChatExporter.sln b/DiscordChatExporter.sln new file mode 100644 index 0000000..dff9ab3 --- /dev/null +++ b/DiscordChatExporter.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.26430.13 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DiscordChatExporter", "DiscordChatExporter\DiscordChatExporter.csproj", "{4BE915D1-129C-49E2-860E-62045ACA5EAD}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4BE915D1-129C-49E2-860E-62045ACA5EAD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4BE915D1-129C-49E2-860E-62045ACA5EAD}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4BE915D1-129C-49E2-860E-62045ACA5EAD}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4BE915D1-129C-49E2-860E-62045ACA5EAD}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/DiscordChatExporter/DiscordChatExporter.csproj b/DiscordChatExporter/DiscordChatExporter.csproj new file mode 100644 index 0000000..2119c69 --- /dev/null +++ b/DiscordChatExporter/DiscordChatExporter.csproj @@ -0,0 +1,27 @@ + + + + Exe + net45 + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/DiscordChatExporter/Models/Attachment.cs b/DiscordChatExporter/Models/Attachment.cs new file mode 100644 index 0000000..bbd3dfe --- /dev/null +++ b/DiscordChatExporter/Models/Attachment.cs @@ -0,0 +1,21 @@ +namespace DiscordChatExporter.Models +{ + public class Attachment + { + public string Id { get; } + + public string Url { get; } + + public string FileName { get; } + + public long ContentLength { get; } + + public Attachment(string id, string url, string fileName, long contentLength) + { + Id = id; + Url = url; + FileName = fileName; + ContentLength = contentLength; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter/Models/ChatLog.cs b/DiscordChatExporter/Models/ChatLog.cs new file mode 100644 index 0000000..ca79d9b --- /dev/null +++ b/DiscordChatExporter/Models/ChatLog.cs @@ -0,0 +1,22 @@ +using System.Collections.Generic; +using System.Linq; +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Models +{ + public class ChatLog + { + public string ChannelId { get; } + + public IReadOnlyList Participants { get; } + + public IReadOnlyList Messages { get; } + + public ChatLog(string channelId, IEnumerable messages) + { + ChannelId = channelId; + Messages = messages.ToArray(); + Participants = Messages.Select(m => m.Author).Distinct(a => a.Name).ToArray(); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter/Models/Message.cs b/DiscordChatExporter/Models/Message.cs new file mode 100644 index 0000000..4f1e1f1 --- /dev/null +++ b/DiscordChatExporter/Models/Message.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace DiscordChatExporter.Models +{ + public class Message + { + public string Id { get; } + + public DateTime TimeStamp { get; } + + public User Author { get; } + + public string Content { get; } + + public IReadOnlyList Attachments { get; } + + public Message(string id, DateTime timeStamp, User author, string content, IEnumerable attachments) + { + Id = id; + TimeStamp = timeStamp; + Author = author; + Content = content; + Attachments = attachments.ToArray(); + } + + public override string ToString() + { + return Content; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter/Models/User.cs b/DiscordChatExporter/Models/User.cs new file mode 100644 index 0000000..7de5509 --- /dev/null +++ b/DiscordChatExporter/Models/User.cs @@ -0,0 +1,29 @@ +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Models +{ + public class User + { + public string Id { get; } + + public string Name { get; } + + public string AvatarHash { get; } + + public string AvatarUrl => AvatarHash.IsNotBlank() + ? $"https://cdn.discordapp.com/avatars/{Id}/{AvatarHash}.png?size=256" + : "https://discordapp.com/assets/6debd47ed13483642cf09e832ed0bc1b.png"; + + public User(string id, string name, string avatarHash) + { + Id = id; + Name = name; + AvatarHash = avatarHash; + } + + public override string ToString() + { + return Name; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter/Options.cs b/DiscordChatExporter/Options.cs new file mode 100644 index 0000000..17c4070 --- /dev/null +++ b/DiscordChatExporter/Options.cs @@ -0,0 +1,32 @@ +using CommandLine; +using CommandLine.Text; + +namespace DiscordChatExporter +{ + public class Options + { + [Option('t', "token", Required = true, HelpText = "Discord access token")] + public string Token { get; set; } + + [Option('c', "channel", Required = true, HelpText = "ID of the text channel to export")] + public string ChannelId { get; set; } + + [HelpOption] + public string GetUsage() + { + var help = new HelpText + { + Heading = new HeadingInfo("DiscordChatExporter"), + Copyright = new CopyrightInfo("Alexey 'Tyrrrz' Golub", 2017), + AdditionalNewLineAfterOption = true, + AddDashesToOption = true + }; + help.AddPreOptionsLine("Usage: DiscordChatExporter.exe " + + "-t REkOTVqm9RWOTNOLCdiuMpWd.QiglBz.Lub0E0TZ1xX4ZxCtnwtpBhWt3v1 " + + "-c 459360869055190534"); + help.AddOptions(this); + + return help; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter/Program.cs b/DiscordChatExporter/Program.cs new file mode 100644 index 0000000..b39349d --- /dev/null +++ b/DiscordChatExporter/Program.cs @@ -0,0 +1,37 @@ +using System; +using System.Threading.Tasks; +using DiscordChatExporter.Models; +using DiscordChatExporter.Services; + +namespace DiscordChatExporter +{ + public static class Program + { + private static readonly Options Options = new Options(); + + private static readonly DiscordApiService DiscordApiService = new DiscordApiService(); + private static readonly ExportService ExportService = new ExportService(); + + private static async Task MainAsync(string[] args) + { + // Parse cmd args + CommandLine.Parser.Default.ParseArgumentsStrict(args, Options); + + // Get messages + Console.WriteLine("Getting messages..."); + var messages = await DiscordApiService.GetMessagesAsync(Options.Token, Options.ChannelId); + var chatLog = new ChatLog(Options.ChannelId, messages); + + // Export + Console.WriteLine("Exporting messages..."); + ExportService.Export($"{Options.ChannelId}.html", chatLog); + } + + public static void Main(string[] args) + { + Console.Title = "Discord Chat Exporter"; + + MainAsync(args).GetAwaiter().GetResult(); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter/Services/DiscordApiService.cs b/DiscordChatExporter/Services/DiscordApiService.cs new file mode 100644 index 0000000..db1a606 --- /dev/null +++ b/DiscordChatExporter/Services/DiscordApiService.cs @@ -0,0 +1,94 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading.Tasks; +using DiscordChatExporter.Models; +using Newtonsoft.Json.Linq; +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Services +{ + public class DiscordApiService + { + private const string ApiRoot = "https://discordapp.com/api"; + private readonly HttpClient _httpClient = new HttpClient(); + + private IEnumerable ParseMessages(string json) + { + var messagesJson = JArray.Parse(json); + foreach (var messageJson in messagesJson) + { + // Get basic data + string id = messageJson.Value("id"); + var timeStamp = messageJson.Value("timestamp"); + string content = messageJson.Value("content"); + + // Get author + var authorJson = messageJson["author"]; + string authorId = authorJson.Value("id"); + string authorName = authorJson.Value("username"); + string authorAvatarHash = authorJson.Value("avatar"); + + // Get attachment + var attachmentsJson = messageJson["attachments"]; + var attachments = new List(); + foreach (var attachmentJson in attachmentsJson) + { + string attachmentId = attachmentJson.Value("id"); + string attachmentUrl = attachmentJson.Value("url"); + string attachmentFileName = attachmentJson.Value("filename"); + long attachmentContentLength = attachmentJson.Value("size"); + + var attachment = new Attachment(attachmentId, attachmentUrl, attachmentFileName, attachmentContentLength); + attachments.Add(attachment); + } + + var author = new User(authorId, authorName, authorAvatarHash); + var message = new Message(id, timeStamp, author, content, attachments); + + yield return message; + } + } + + public async Task> GetMessagesAsync(string token, string channelId) + { + var result = new List(); + + // We are going backwards from last message to first + // ...collecting everything between them in batches + string beforeId = null; + while (true) + { + // Form request url + string url = $"{ApiRoot}/channels/{channelId}/messages?token={token}&limit=100"; + if (beforeId.IsNotBlank()) + url += $"&before={beforeId}"; + + // Get response + string response = await _httpClient.GetStringAsync(url); + + // Parse + var messages = ParseMessages(response); + + // Add messages to list + string currentMessageId = null; + foreach (var message in messages) + { + result.Add(message); + currentMessageId = message.Id; + } + + // If no messages - break + if (currentMessageId == null) break; + + // Otherwise offset the next request + beforeId = currentMessageId; + } + + // Messages appear newest first, we need to reverse + result.Reverse(); + + return result; + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter/Services/ExportService.cs b/DiscordChatExporter/Services/ExportService.cs new file mode 100644 index 0000000..9cb38a7 --- /dev/null +++ b/DiscordChatExporter/Services/ExportService.cs @@ -0,0 +1,180 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Text.RegularExpressions; +using DiscordChatExporter.Models; +using HtmlAgilityPack; +using Tyrrrz.Extensions; + +namespace DiscordChatExporter.Services +{ + public class ExportService + { + private class MessageGroup + { + public User Author { get; } + + public DateTime FirstTimeStamp { get; } + + public IReadOnlyList Messages { get; } + + public MessageGroup(User author, DateTime firstTimeStamp, IEnumerable messages) + { + Author = author; + FirstTimeStamp = firstTimeStamp; + Messages = messages.ToArray(); + } + } + + private HtmlDocument GetTemplate() + { + const string templateName = "DiscordChatExporter.Services.ExportTemplate.html"; + var assembly = Assembly.GetExecutingAssembly(); + using (var stream = assembly.GetManifestResourceStream(templateName)) + { + var doc = new HtmlDocument(); + doc.Load(stream); + return doc; + } + } + + private IEnumerable GroupMessages(IEnumerable messages) + { + var result = new List(); + + // Group adjacent messages by timestamp and author + var buffer = new List(); + foreach (var message in messages) + { + var bufferFirst = buffer.FirstOrDefault(); + + // Group break condition + bool breakCondition = + bufferFirst != null && + ( + message.Author.Id != bufferFirst.Author.Id || + (message.TimeStamp - bufferFirst.TimeStamp).TotalHours > 1 || + message.TimeStamp.Hour != bufferFirst.TimeStamp.Hour + ); + + // If condition is true - flush buffer + if (breakCondition) + { + var group = new MessageGroup(bufferFirst.Author, bufferFirst.TimeStamp, buffer); + result.Add(group); + buffer.Clear(); + } + + // Add message to buffer + buffer.Add(message); + } + + // Add what's remaining in buffer + if (buffer.Any()) + { + var bufferFirst = buffer.First(); + var group = new MessageGroup(bufferFirst.Author, bufferFirst.TimeStamp, buffer); + result.Add(group); + } + + return result; + } + + private string FormatMessageContent(string content) + { + // Encode HTML + content = HtmlDocument.HtmlEncode(content); + + // Links from URLs + content = Regex.Replace(content, "((^|\\s)(https?|ftp)://[^\\s/$.?#].[^\\s]*($|\\s))", + "$1"); + + // Preformatted multiline + content = Regex.Replace(content, "```([^`]*?)```", e => "
" + e.Groups[1].Value + "
"); + + // Preformatted + content = Regex.Replace(content, "`([^`]*?)`", e => "
" + e.Groups[1].Value + "
"); + + // Bold + content = Regex.Replace(content, "\\*\\*([^\\*]*?)\\*\\*", "$1"); + + // Italic + content = Regex.Replace(content, "\\*([^\\*]*?)\\*", "$1"); + + // Underline + content = Regex.Replace(content, "__([^_]*?)__", "$1"); + + // Strike through + content = Regex.Replace(content, "~~([^~]*?)~~", "$1"); + + // New lines + content = content.Replace("\n", "
"); + + return content; + } + + public void Export(string filePath, ChatLog chatLog) + { + var doc = GetTemplate(); + + // Info + var infoHtml = doc.GetElementbyId("info"); + infoHtml.AppendChild(HtmlNode.CreateNode($"
Channel ID: {chatLog.ChannelId}
")); + string participants = HtmlDocument.HtmlEncode(chatLog.Participants.Select(u => u.Name).JoinToString(", ")); + infoHtml.AppendChild(HtmlNode.CreateNode($"
Participants: {participants}
")); + infoHtml.AppendChild(HtmlNode.CreateNode($"
Messages: {chatLog.Messages.Count:N0}
")); + + // Messages + var logHtml = doc.GetElementbyId("log"); + var messageGroups = GroupMessages(chatLog.Messages); + foreach (var messageGroup in messageGroups) + { + // Container + var messageHtml = logHtml.AppendChild(HtmlNode.CreateNode("
")); + + // Avatar + messageHtml.AppendChild(HtmlNode.CreateNode("")); + + // Body + var messageBodyHtml = messageHtml.AppendChild(HtmlNode.CreateNode("
")); + + // Author + string authorName = HtmlDocument.HtmlEncode(messageGroup.Author.Name); + messageBodyHtml.AppendChild(HtmlNode.CreateNode($"{authorName}")); + + // Date + string timeStamp = HtmlDocument.HtmlEncode(messageGroup.FirstTimeStamp.ToString("g")); + messageBodyHtml.AppendChild(HtmlNode.CreateNode($"{timeStamp}")); + + // Separate messages + foreach (var message in messageGroup.Messages) + { + // Content + if (message.Content.IsNotBlank()) + { + string content = FormatMessageContent(message.Content); + messageBodyHtml.AppendChild(HtmlNode.CreateNode($"
{content}
")); + } + + // Attachments + if (message.Attachments.Any()) + { + // Attachments + foreach (var attachment in message.Attachments) + { + messageBodyHtml.AppendChild( + HtmlNode.CreateNode("
" + + $"" + + $"" + + "
")); + } + } + } + } + + doc.Save(filePath); + } + } +} \ No newline at end of file diff --git a/DiscordChatExporter/Services/ExportTemplate.html b/DiscordChatExporter/Services/ExportTemplate.html new file mode 100644 index 0000000..3befc29 --- /dev/null +++ b/DiscordChatExporter/Services/ExportTemplate.html @@ -0,0 +1,84 @@ + + + + Discord Chat Log + + + + + + + +
+
+ + \ No newline at end of file