open-source-mirrors
/
DiscordChatExporter
mirror of https://github.com/Tyrrrz/DiscordChatExporter
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use System.Text.Json instead of Newtonsoft.Json

Browse Source
pull/321/head
Alexey Golub 5 years ago
parent f153aad3f1
commit 130c0b6fe2
14 changed files with 201 additions and 185 deletions
Show Stats Download Patch File Download Diff File

1
DiscordChatExporter.Cli/Program.cs
Unescape View File

@ -1,5 +1,4 @@
using System;
using System.Drawing;
using System.Threading.Tasks;
using CliFx;
using DiscordChatExporter.Cli.Commands;

3
DiscordChatExporter.Core.Markdown/MarkdownParser.cs
Unescape View File

@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using DiscordChatExporter.Core.Markdown.Ast;

7
DiscordChatExporter.Core.Models/FileSize.cs
Unescape View File

@ -4,7 +4,7 @@ namespace DiscordChatExporter.Core.Models
{
// Loosely based on https://github.com/omar/ByteSize (MIT license)
public readonly struct FileSize
public readonly partial struct FileSize
{
public long TotalBytes { get; }
@ -58,4 +58,9 @@ namespace DiscordChatExporter.Core.Models
public override string ToString() => $"{GetLargestWholeNumberValue():0.##} {GetLargestWholeNumberSymbol()}";
}
public partial struct FileSize
{
public static FileSize FromBytes(long bytes) => new FileSize(bytes);
}
}

1
DiscordChatExporter.Core.Models/Member.cs
Unescape View File

@ -1,5 +1,4 @@
using System.Collections.Generic;
using System.Linq;
namespace DiscordChatExporter.Core.Models
{

258
DiscordChatExporter.Core.Services/DataService.Parsers.cs
Unescape View File

@ -1,225 +1,195 @@
using System;
using System.Drawing;
using System.Linq;
using System.Text.Json;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services.Internal;
using Newtonsoft.Json.Linq;
using DiscordChatExporter.Core.Services.Internal.Extensions;
using Tyrrrz.Extensions;
namespace DiscordChatExporter.Core.Services
{
public partial class DataService
{
private User ParseUser(JToken json)
private string ParseId(JsonElement json) =>
json.GetProperty("id").GetString();
private User ParseUser(JsonElement json)
{
var id = json["id"]!.Value<string>();
var discriminator = json["discriminator"]!.Value<int>();
var name = json["username"]!.Value<string>();
var avatarHash = json["avatar"]!.Value<string>();
var isBot = json["bot"]?.Value<bool>() ?? false;
var id = ParseId(json);
var discriminator = json.GetProperty("discriminator").GetString().Pipe(int.Parse);
var name = json.GetProperty("username").GetString();
var avatarHash = json.GetProperty("avatar").GetString();
var isBot = json.GetPropertyOrNull("bot")?.GetBoolean() ?? false;
return new User(id, discriminator, name, avatarHash, isBot);
}
private Member ParseMember(JToken json)
private Member ParseMember(JsonElement json)
{
var userId = ParseUser(json["user"]!).Id;
var nick = json["nick"]?.Value<string>();
var roles = (json["roles"] ?? Enumerable.Empty<JToken>()).Select(j => j.Value<string>()).ToArray();
var userId = json.GetProperty("user").Pipe(ParseId);
var nick = json.GetPropertyOrNull("nick")?.GetString();
var roles = json.GetPropertyOrNull("roles")?.EnumerateArray().Select(j => j.GetString()).ToArray() ??
Array.Empty<string>();
return new Member(userId, nick, roles);
}
private Guild ParseGuild(JToken json)
private Guild ParseGuild(JsonElement json)
{
var id = json["id"]!.Value<string>();
var name = json["name"]!.Value<string>();
var iconHash = json["icon"]!.Value<string>();
var roles = (json["roles"] ?? Enumerable.Empty<JToken>()).Select(ParseRole).ToArray();
var id = ParseId(json);
var name = json.GetProperty("name").GetString();
var iconHash = json.GetProperty("icon").GetString();
var roles = json.GetPropertyOrNull("roles")?.EnumerateArray().Select(ParseRole).ToArray() ??
Array.Empty<Role>();
return new Guild(id, name, roles, iconHash);
}
private Channel ParseChannel(JToken json)
private Channel ParseChannel(JsonElement json)
{
// Get basic data
var id = json["id"]!.Value<string>();
var parentId = json["parent_id"]?.Value<string>();
var type = (ChannelType) json["type"]!.Value<int>();
var topic = json["topic"]?.Value<string>();
// Try to extract guild ID
var guildId = json["guild_id"]?.Value<string>();
// If the guild ID is blank, it's direct messages
if (string.IsNullOrWhiteSpace(guildId))
guildId = Guild.DirectMessages.Id;
var id = ParseId(json);
var parentId = json.GetPropertyOrNull("parent_id")?.GetString();
var type = (ChannelType) json.GetProperty("type").GetInt32();
var topic = json.GetPropertyOrNull("topic")?.GetString();
// Try to extract name
var name = json["name"]?.Value<string>();
var guildId = json.GetPropertyOrNull("guild_id")?.GetString() ??
Guild.DirectMessages.Id;
// If the name is blank, it's direct messages
if (string.IsNullOrWhiteSpace(name))
name = json["recipients"]?.Select(ParseUser).Select(u => u.Name).JoinToString(", ");
// If the name is still blank for some reason, fallback to ID
// (blind fix to https://github.com/Tyrrrz/DiscordChatExporter/issues/227)
if (string.IsNullOrWhiteSpace(name))
name = id;
var name =
json.GetPropertyOrNull("name")?.GetString() ??
json.GetPropertyOrNull("recipients")?.EnumerateArray().Select(ParseUser).Select(u => u.Name).JoinToString(", ") ??
id;
return new Channel(id, parentId, guildId, name, topic, type);
}
private Role ParseRole(JToken json)
private Role ParseRole(JsonElement json)
{
var id = json["id"]!.Value<string>();
var name = json["name"]!.Value<string>();
var color = json["color"]!.Value<int>();
var position = json["position"]!.Value<int>();
var id = ParseId(json);
var name = json.GetProperty("name").GetString();
var color = json.GetProperty("color").GetInt32().Pipe(Color.FromArgb);
var position = json.GetProperty("position").GetInt32();
return new Role(id, name, Color.FromArgb(color), position);
return new Role(id, name, color, position);
}
private Attachment ParseAttachment(JToken json)
private Attachment ParseAttachment(JsonElement json)
{
var id = json["id"]!.Value<string>();
var url = json["url"]!.Value<string>();
var width = json["width"]?.Value<int>();
var height = json["height"]?.Value<int>();
var fileName = json["filename"]!.Value<string>();
var fileSizeBytes = json["size"]!.Value<long>();
var fileSize = new FileSize(fileSizeBytes);
var id = ParseId(json);
var url = json.GetProperty("url").GetString();
var width = json.GetPropertyOrNull("width")?.GetInt32();
var height = json.GetPropertyOrNull("height")?.GetInt32();
var fileName = json.GetProperty("filename").GetString();
var fileSize = json.GetProperty("size").GetInt64().Pipe(FileSize.FromBytes);
return new Attachment(id, width, height, url, fileName, fileSize);
}
private EmbedAuthor ParseEmbedAuthor(JToken json)
private EmbedAuthor ParseEmbedAuthor(JsonElement json)
{
var name = json["name"]?.Value<string>();
var url = json["url"]?.Value<string>();
var iconUrl = json["icon_url"]?.Value<string>();
var name = json.GetPropertyOrNull("name")?.GetString();
var url = json.GetPropertyOrNull("url")?.GetString();
var iconUrl = json.GetPropertyOrNull("icon_url")?.GetString();
return new EmbedAuthor(name, url, iconUrl);
}
private EmbedField ParseEmbedField(JToken json)
private EmbedField ParseEmbedField(JsonElement json)
{
var name = json["name"]!.Value<string>();
var value = json["value"]!.Value<string>();
var isInline = json["inline"]?.Value<bool>() ?? false;
var name = json.GetProperty("name").GetString();
var value = json.GetProperty("value").GetString();
var isInline = json.GetPropertyOrNull("inline")?.GetBoolean() ?? false;
return new EmbedField(name, value, isInline);
}
private EmbedImage ParseEmbedImage(JToken json)
private EmbedImage ParseEmbedImage(JsonElement json)
{
var url = json["url"]?.Value<string>();
var width = json["width"]?.Value<int>();
var height = json["height"]?.Value<int>();
var url = json.GetPropertyOrNull("url")?.GetString();
var width = json.GetPropertyOrNull("width")?.GetInt32();
var height = json.GetPropertyOrNull("height")?.GetInt32();
return new EmbedImage(url, width, height);
}
private EmbedFooter ParseEmbedFooter(JToken json)
private EmbedFooter ParseEmbedFooter(JsonElement json)
{
var text = json["text"]!.Value<string>();
var iconUrl = json["icon_url"]?.Value<string>();
var text = json.GetProperty("text").GetString();
var iconUrl = json.GetPropertyOrNull("icon_url")?.GetString();
return new EmbedFooter(text, iconUrl);
}
private Embed ParseEmbed(JToken json)
private Embed ParseEmbed(JsonElement json)
{
// Get basic data
var title = json["title"]?.Value<string>();
var description = json["description"]?.Value<string>();
var url = json["url"]?.Value<string>();
var timestamp = json["timestamp"]?.Value<DateTime>().ToDateTimeOffset();
// Get color
var color = json["color"] != null
? Color.FromArgb(json["color"]!.Value<int>()).ResetAlpha()
: default(Color?);
// Get author
var author = json["author"] != null ? ParseEmbedAuthor(json["author"]!) : null;
// Get fields
var fields = (json["fields"] ?? Enumerable.Empty<JToken>()).Select(ParseEmbedField).ToArray();
// Get thumbnail
var thumbnail = json["thumbnail"] != null ? ParseEmbedImage(json["thumbnail"]!) : null;
var title = json.GetPropertyOrNull("title")?.GetString();
var description = json.GetPropertyOrNull("description")?.GetString();
var url = json.GetPropertyOrNull("url")?.GetString();
var timestamp = json.GetPropertyOrNull("timestamp")?.GetDateTimeOffset();
var color = json.GetPropertyOrNull("color")?.GetInt32().Pipe(Color.FromArgb).ResetAlpha();
// Get image
var image = json["image"] != null ? ParseEmbedImage(json["image"]!) : null;
var author = json.GetPropertyOrNull("author")?.Pipe(ParseEmbedAuthor);
var thumbnail = json.GetPropertyOrNull("thumbnail")?.Pipe(ParseEmbedImage);
var image = json.GetPropertyOrNull("image")?.Pipe(ParseEmbedImage);
var footer = json.GetPropertyOrNull("footer")?.Pipe(ParseEmbedFooter);
// Get footer
var footer = json["footer"] != null ? ParseEmbedFooter(json["footer"]!) : null;
var fields = json.GetPropertyOrNull("fields")?.EnumerateArray().Select(ParseEmbedField).ToArray() ??
Array.Empty<EmbedField>();
return new Embed(title, url, timestamp, color, author, description, fields, thumbnail, image, footer);
}
private Emoji ParseEmoji(JToken json)
private Emoji ParseEmoji(JsonElement json)
{
var id = json["id"]?.Value<string>();
var name = json["name"]!.Value<string>();
var isAnimated = json["animated"]?.Value<bool>() ?? false;
var id = json.GetPropertyOrNull("id")?.GetString();
var name = json.GetProperty("name").GetString();
var isAnimated = json.GetPropertyOrNull("animated")?.GetBoolean() ?? false;
return new Emoji(id, name, isAnimated);
}
private Reaction ParseReaction(JToken json)
private Reaction ParseReaction(JsonElement json)
{
var count = json["count"]!.Value<int>();
var emoji = ParseEmoji(json["emoji"]!);
var count = json.GetProperty("count").GetInt32();
var emoji = json.GetProperty("emoji").Pipe(ParseEmoji);
return new Reaction(count, emoji);
}
private Message ParseMessage(JToken json)
{
// Get basic data
var id = json["id"]!.Value<string>();
var channelId = json["channel_id"]!.Value<string>();
var timestamp = json["timestamp"]!.Value<DateTime>().ToDateTimeOffset();
var editedTimestamp = json["edited_timestamp"]?.Value<DateTime?>()?.ToDateTimeOffset();
var content = json["content"]!.Value<string>();
var type = (MessageType) json["type"]!.Value<int>();
// Workarounds for non-default types
if (type == MessageType.RecipientAdd)
content = "Added a recipient.";
else if (type == MessageType.RecipientRemove)
content = "Removed a recipient.";
else if (type == MessageType.Call)
content = "Started a call.";
else if (type == MessageType.ChannelNameChange)
content = "Changed the channel name.";
else if (type == MessageType.ChannelIconChange)
content = "Changed the channel icon.";
else if (type == MessageType.ChannelPinnedMessage)
content = "Pinned a message.";
else if (type == MessageType.GuildMemberJoin)
content = "Joined the server.";
// Get author
var author = ParseUser(json["author"]!);
// Get attachments
var attachments = (json["attachments"] ?? Enumerable.Empty<JToken>()).Select(ParseAttachment).ToArray();
// Get embeds
var embeds = (json["embeds"] ?? Enumerable.Empty<JToken>()).Select(ParseEmbed).ToArray();
// Get reactions
var reactions = (json["reactions"] ?? Enumerable.Empty<JToken>()).Select(ParseReaction).ToArray();
// Get mentions
var mentionedUsers = (json["mentions"] ?? Enumerable.Empty<JToken>()).Select(ParseUser).ToArray();
// Get whether this message is pinned
var isPinned = json["pinned"]!.Value<bool>();
private Message ParseMessage(JsonElement json)
{
var id = ParseId(json);
var channelId = json.GetProperty("channel_id").GetString();
var timestamp = json.GetProperty("timestamp").GetDateTimeOffset();
var editedTimestamp = json.GetPropertyOrNull("edited_timestamp")?.GetDateTimeOffset();
var type = (MessageType) json.GetProperty("type").GetInt32();
var isPinned = json.GetPropertyOrNull("pinned")?.GetBoolean() ?? false;
var content = type switch
{
MessageType.RecipientAdd => "Added a recipient.",
MessageType.RecipientRemove => "Removed a recipient.",
MessageType.Call => "Started a call.",
MessageType.ChannelNameChange => "Changed the channel name.",
MessageType.ChannelIconChange => "Changed the channel icon.",
MessageType.ChannelPinnedMessage => "Pinned a message.",
MessageType.GuildMemberJoin => "Joined the server.",
_ => json.GetPropertyOrNull("content")?.GetString() ?? ""
};
var author = json.GetProperty("author").Pipe(ParseUser);
var attachments = json.GetPropertyOrNull("attachments")?.EnumerateArray().Select(ParseAttachment).ToArray() ??
Array.Empty<Attachment>();
var embeds = json.GetPropertyOrNull("embeds")?.EnumerateArray().Select(ParseEmbed).ToArray() ??
Array.Empty<Embed>();
var reactions = json.GetPropertyOrNull("reactions")?.EnumerateArray().Select(ParseReaction).ToArray() ??
Array.Empty<Reaction>();
var mentionedUsers = json.GetPropertyOrNull("mentions")?.EnumerateArray().Select(ParseUser).ToArray() ??
Array.Empty<User>();
return new Message(id, channelId, type, author, timestamp, editedTimestamp, isPinned, content, attachments, embeds,
reactions, mentionedUsers);

41
DiscordChatExporter.Core.Services/DataService.cs
Unescape View File

@ -4,11 +4,12 @@ using System.Linq;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text.Json;
using System.Threading.Tasks;
using DiscordChatExporter.Core.Models;
using DiscordChatExporter.Core.Services.Exceptions;
using DiscordChatExporter.Core.Services.Internal;
using Newtonsoft.Json.Linq;
using DiscordChatExporter.Core.Services.Internal.Extensions;
using Polly;
namespace DiscordChatExporter.Core.Services
@ -41,12 +42,12 @@ namespace DiscordChatExporter.Core.Services
(response, timespan, retryCount, context) => Task.CompletedTask);
}
private async Task<JToken> GetApiResponseAsync(AuthToken token, string route)
private async Task<JsonElement> GetApiResponseAsync(AuthToken token, string route)
{
return (await GetApiResponseAsync(token, route, true))!;
return (await GetApiResponseAsync(token, route, true))!.Value;
}
private async Task<JToken?> GetApiResponseAsync(AuthToken token, string route, bool errorOnFail)
private async Task<JsonElement?> GetApiResponseAsync(AuthToken token, string route, bool errorOnFail)
{
using var response = await _httpPolicy.ExecuteAsync(async () =>
{
@ -62,12 +63,14 @@ namespace DiscordChatExporter.Core.Services
// We throw our own exception here because default one doesn't have status code
if (!response.IsSuccessStatusCode)
{
if (errorOnFail) throw new HttpErrorStatusCodeException(response.StatusCode, response.ReasonPhrase);
else return null;
if (errorOnFail)
throw new HttpErrorStatusCodeException(response.StatusCode, response.ReasonPhrase);
return null;
}
var jsonRaw = await response.Content.ReadAsStringAsync();
return JToken.Parse(jsonRaw);
return Json.Parse(jsonRaw);
}
public async Task<Guild> GetGuildAsync(AuthToken token, string guildId)
@ -85,10 +88,7 @@ namespace DiscordChatExporter.Core.Services
public async Task<Member?> GetGuildMemberAsync(AuthToken token, string guildId, string userId)
{
var response = await GetApiResponseAsync(token, $"guilds/{guildId}/members/{userId}", false);
if (response == null) return null;
var member = ParseMember(response);
return member;
return response?.Pipe(ParseMember);
}
public async Task<Channel> GetChannelAsync(AuthToken token, string channelId)
@ -111,22 +111,28 @@ namespace DiscordChatExporter.Core.Services
var response = await GetApiResponseAsync(token, route);
if (!response.HasValues)
yield break;
var isEmpty = true;
// Get full guild object
foreach (var guildId in response.Select(j => j["id"]!.Value<string>()))
foreach (var guildJson in response.EnumerateArray())
{
var guildId = ParseId(guildJson);
yield return await GetGuildAsync(token, guildId);
afterId = guildId;
isEmpty = false;
}
if (isEmpty)
yield break;
}
}
public async Task<IReadOnlyList<Channel>> GetDirectMessageChannelsAsync(AuthToken token)
{
var response = await GetApiResponseAsync(token, "users/@me/channels");
var channels = response.Select(ParseChannel).ToArray();
var channels = response.EnumerateArray().Select(ParseChannel).ToArray();
return channels;
}
@ -138,7 +144,7 @@ namespace DiscordChatExporter.Core.Services
return Array.Empty<Channel>();
var response = await GetApiResponseAsync(token, $"guilds/{guildId}/channels");
var channels = response.Select(ParseChannel).ToArray();
var channels = response.EnumerateArray().Select(ParseChannel).ToArray();
return channels;
}
@ -151,7 +157,7 @@ namespace DiscordChatExporter.Core.Services
var response = await GetApiResponseAsync(token, route);
return response.Select(ParseMessage).FirstOrDefault();
return response.EnumerateArray().Select(ParseMessage).FirstOrDefault();
}
public async IAsyncEnumerable<Message> GetMessagesAsync(AuthToken token, string channelId,
@ -178,6 +184,7 @@ namespace DiscordChatExporter.Core.Services
// Parse
var messages = response
.EnumerateArray()
.Select(ParseMessage)
.Reverse() // reverse because messages appear newest first
.ToArray();

2
DiscordChatExporter.Core.Services/DiscordChatExporter.Core.Services.csproj
Unescape View File

@ -2,8 +2,6 @@
<Import Project="../DiscordChatExporter.props" />
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="Onova" Version="2.5.2" />
<PackageReference Include="Polly" Version="7.2.0" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
<PackageReference Include="Tyrrrz.Settings" Version="1.3.4" />

18
DiscordChatExporter.Core.Services/Internal/Extensions.cs
Unescape View File

@ -1,18 +0,0 @@
using System;
using System.Drawing;
namespace DiscordChatExporter.Core.Services.Internal
{
internal static class Extensions
{
public static DateTimeOffset ToDateTimeOffset(this DateTime dateTime) => new DateTimeOffset(dateTime);
public static string ToSnowflake(this DateTimeOffset dateTime)
{
var value = ((ulong) dateTime.ToUnixTimeMilliseconds() - 1420070400000UL) << 22;
return value.ToString();
}
public static Color ResetAlpha(this Color color) => Color.FromArgb(1, color);
}
}

9
DiscordChatExporter.Core.Services/Internal/Extensions/ColorExtensions.cs
Unescape View File

@ -0,0 +1,9 @@
using System.Drawing;
namespace DiscordChatExporter.Core.Services.Internal.Extensions
{
internal static class ColorExtensions
{
public static Color ResetAlpha(this Color color) => Color.FromArgb(1, color);
}
}

13
DiscordChatExporter.Core.Services/Internal/Extensions/DateExtensions.cs
Unescape View File

@ -0,0 +1,13 @@
using System;
namespace DiscordChatExporter.Core.Services.Internal.Extensions
{
internal static class DateExtensions
{
public static string ToSnowflake(this DateTimeOffset dateTime)
{
var value = ((ulong) dateTime.ToUnixTimeMilliseconds() - 1420070400000UL) << 22;
return value.ToString();
}
}
}

9
DiscordChatExporter.Core.Services/Internal/Extensions/GenericExtensions.cs
Unescape View File

@ -0,0 +1,9 @@
using System;
namespace DiscordChatExporter.Core.Services.Internal.Extensions
{
internal static class GenericExtensions
{
public static TOut Pipe<TIn, TOut>(this TIn input, Func<TIn, TOut> transform) => transform(input);
}
}

12
DiscordChatExporter.Core.Services/Internal/Extensions/JsonElementExtensions.cs
Unescape View File

@ -0,0 +1,12 @@
using System.Text.Json;
namespace DiscordChatExporter.Core.Services.Internal.Extensions
{
internal static class JsonElementExtensions
{
public static JsonElement? GetPropertyOrNull(this JsonElement element, string propertyName) =>
element.TryGetProperty(propertyName, out var result) && result.ValueKind != JsonValueKind.Null
? result
: (JsonElement?) null;
}
}

13
DiscordChatExporter.Core.Services/Internal/Json.cs
Unescape View File

@ -0,0 +1,13 @@
using System.Text.Json;
namespace DiscordChatExporter.Core.Services.Internal
{
internal static class Json
{
public static JsonElement Parse(string json)
{
using var document = JsonDocument.Parse(json);
return document.RootElement.Clone();
}
}
}

1
DiscordChatExporter.Gui/DiscordChatExporter.Gui.csproj
Unescape View File

@ -18,6 +18,7 @@
<PackageReference Include="MaterialDesignThemes" Version="3.0.1" />
<PackageReference Include="Microsoft.Xaml.Behaviors.Wpf" Version="1.1.19" />
<PackageReference Include="Ookii.Dialogs.Wpf" Version="1.1.0" />
<PackageReference Include="Onova" Version="2.5.2" />
<PackageReference Include="Stylet" Version="1.3.0" />
<PackageReference Include="Tyrrrz.Extensions" Version="1.6.5" />
<PackageReference Include="PropertyChanged.Fody" Version="3.2.5" PrivateAssets="all" />

Write Preview
Loading…
Cancel
Save