From 8678043f0db4c69e50042fa8418d3c4c0d62dee0 Mon Sep 17 00:00:00 2001 From: Alexey Golub Date: Mon, 25 Jun 2018 23:11:34 +0300 Subject: [PATCH] Implement progress reporting when downloading messages (#57) --- .../ViewModels/MainViewModel.cs | 4 +- .../Services/DataService.cs | 90 ++++++++++++------- .../Services/IDataService.cs | 2 +- .../ViewModels/IMainViewModel.cs | 3 + .../ViewModels/MainViewModel.cs | 19 +++- DiscordChatExporter.Gui/Views/MainWindow.xaml | 3 +- 6 files changed, 84 insertions(+), 37 deletions(-) diff --git a/DiscordChatExporter.Cli/ViewModels/MainViewModel.cs b/DiscordChatExporter.Cli/ViewModels/MainViewModel.cs index afa073f..ef9db3d 100644 --- a/DiscordChatExporter.Cli/ViewModels/MainViewModel.cs +++ b/DiscordChatExporter.Cli/ViewModels/MainViewModel.cs @@ -21,8 +21,8 @@ namespace DiscordChatExporter.Cli.ViewModels _exportService = exportService; } - public async Task ExportAsync(string token, string channelId, string filePath, ExportFormat format, DateTime? from, - DateTime? to) + public async Task ExportAsync(string token, string channelId, string filePath, ExportFormat format, + DateTime? from, DateTime? to) { // Get channel and guild var channel = await _dataService.GetChannelAsync(token, channelId); diff --git a/DiscordChatExporter.Core/Services/DataService.cs b/DiscordChatExporter.Core/Services/DataService.cs index d20f23d..2f9df87 100644 --- a/DiscordChatExporter.Core/Services/DataService.cs +++ b/DiscordChatExporter.Core/Services/DataService.cs @@ -14,7 +14,8 @@ namespace DiscordChatExporter.Core.Services { private readonly HttpClient _httpClient = new HttpClient(); - private async Task GetApiResponseAsync(string token, string resource, string endpoint, params string[] parameters) + private async Task GetApiResponseAsync(string token, string resource, string endpoint, + params string[] parameters) { // Format URL const string apiRoot = "https://discordapp.com/api/v6"; @@ -89,48 +90,72 @@ namespace DiscordChatExporter.Core.Services } public async Task> GetChannelMessagesAsync(string token, string channelId, - DateTime? from, DateTime? to) + DateTime? from = null, DateTime? to = null, IProgress progress = null) { var result = new List(); - // We are going backwards from last message to first - // collecting everything between them in batches - var beforeId = to?.ToSnowflake() ?? DateTime.MaxValue.ToSnowflake(); + // Report indeterminate progress + progress?.Report(-1); + + // Get the snowflakes for the selected range + var firstId = from != null ? from.Value.ToSnowflake() : "0"; + var lastId = to != null ? to.Value.ToSnowflake() : DateTime.MaxValue.ToSnowflake(); + + // Get the last message + var response = await GetApiResponseAsync(token, "channels", $"{channelId}/messages", + "limit=1", $"before={lastId}"); + var lastMessage = response.Select(ParseMessage).FirstOrDefault(); + + // If the last message doesn't exist or it's outside of range - return + if (lastMessage == null || lastMessage.Timestamp < from) + { + progress?.Report(1); + return result; + } + + // Get other messages + var offsetId = firstId; while (true) { - // Get response - var response = await GetApiResponseAsync(token, "channels", $"{channelId}/messages", - "limit=100", $"before={beforeId}"); + // Get message batch + response = await GetApiResponseAsync(token, "channels", $"{channelId}/messages", + "limit=100", $"after={offsetId}"); // Parse - var messages = response.Select(ParseMessage); - - // Add messages to list - string currentMessageId = null; - foreach (var message in messages) - { - // Break when the message is older than from date - if (from != null && message.Timestamp < from) - { - currentMessageId = null; - break; - } - - // Add message - result.Add(message); - currentMessageId = message.Id; - } - - // If no messages - break - if (currentMessageId == null) + var messages = response + .Select(ParseMessage) + .Reverse() // reverse because messages appear newest first + .ToArray(); + + // Break if there are no messages (can happen if messages are deleted during execution) + if (!messages.Any()) break; - // Otherwise offset the next request - beforeId = currentMessageId; + // Trim messages to range (until last message) + var messagesInRange = messages + .TakeWhile(m => m.Id != lastMessage.Id && m.Timestamp < lastMessage.Timestamp) + .ToArray(); + + // Add to result + result.AddRange(messagesInRange); + + // Break if messages were trimmed (which means the last message was encountered) + if (messagesInRange.Length != messages.Length) + break; + + // Report progress (based on the time range of parsed messages compared to total) + progress?.Report((result.Last().Timestamp - result.First().Timestamp).TotalSeconds / + (lastMessage.Timestamp - result.First().Timestamp).TotalSeconds); + + // Move offset + offsetId = result.Last().Id; } - // Messages appear newest first, we need to reverse - result.Reverse(); + // Add last message + result.Add(lastMessage); + + // Report progress + progress?.Report(1); return result; } @@ -157,6 +182,7 @@ namespace DiscordChatExporter.Core.Services foreach (var mentionedUser in message.MentionedUsers) userMap[mentionedUser.Id] = mentionedUser; } + var users = userMap.Values.ToArray(); return new Mentionables(users, channels, roles); diff --git a/DiscordChatExporter.Core/Services/IDataService.cs b/DiscordChatExporter.Core/Services/IDataService.cs index f0173c8..eb0bcd2 100644 --- a/DiscordChatExporter.Core/Services/IDataService.cs +++ b/DiscordChatExporter.Core/Services/IDataService.cs @@ -20,7 +20,7 @@ namespace DiscordChatExporter.Core.Services Task> GetGuildRolesAsync(string token, string guildId); Task> GetChannelMessagesAsync(string token, string channelId, - DateTime? from, DateTime? to); + DateTime? from = null, DateTime? to = null, IProgress progress = null); Task GetMentionablesAsync(string token, string guildId, IEnumerable messages); diff --git a/DiscordChatExporter.Gui/ViewModels/IMainViewModel.cs b/DiscordChatExporter.Gui/ViewModels/IMainViewModel.cs index ad67e48..aef393d 100644 --- a/DiscordChatExporter.Gui/ViewModels/IMainViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/IMainViewModel.cs @@ -9,6 +9,9 @@ namespace DiscordChatExporter.Gui.ViewModels bool IsBusy { get; } bool IsDataAvailable { get; } + bool IsProgressIndeterminate { get; } + double Progress { get; } + string Token { get; set; } IReadOnlyList AvailableGuilds { get; } diff --git a/DiscordChatExporter.Gui/ViewModels/MainViewModel.cs b/DiscordChatExporter.Gui/ViewModels/MainViewModel.cs index 0b496f0..4bd3241 100644 --- a/DiscordChatExporter.Gui/ViewModels/MainViewModel.cs +++ b/DiscordChatExporter.Gui/ViewModels/MainViewModel.cs @@ -25,6 +25,7 @@ namespace DiscordChatExporter.Gui.ViewModels private readonly Dictionary> _guildChannelsMap; private bool _isBusy; + private double _progress; private string _token; private IReadOnlyList _availableGuilds; private Guild _selectedGuild; @@ -43,6 +44,18 @@ namespace DiscordChatExporter.Gui.ViewModels public bool IsDataAvailable => AvailableGuilds.NotNullAndAny(); + public bool IsProgressIndeterminate => Progress <= 0; + + public double Progress + { + get => _progress; + private set + { + Set(ref _progress, value); + RaisePropertyChanged(() => IsProgressIndeterminate); + } + } + public string Token { get => _token; @@ -225,10 +238,13 @@ namespace DiscordChatExporter.Gui.ViewModels // Get guild var guild = SelectedGuild; + // Create progress handler + var progressHandler = new Progress(p => Progress = p); + try { // Get messages - var messages = await _dataService.GetChannelMessagesAsync(token, channel.Id, from, to); + var messages = await _dataService.GetChannelMessagesAsync(token, channel.Id, from, to, progressHandler); // Group messages var messageGroups = _messageGroupService.GroupMessages(messages); @@ -253,6 +269,7 @@ namespace DiscordChatExporter.Gui.ViewModels MessengerInstance.Send(new ShowNotificationMessage("You don't have access to this channel")); } + Progress = 0; IsBusy = false; } } diff --git a/DiscordChatExporter.Gui/Views/MainWindow.xaml b/DiscordChatExporter.Gui/Views/MainWindow.xaml index 37104af..4eab1c0 100644 --- a/DiscordChatExporter.Gui/Views/MainWindow.xaml +++ b/DiscordChatExporter.Gui/Views/MainWindow.xaml @@ -98,7 +98,8 @@