Implement progress reporting when downloading messages (#57)

DiscordChatExporter.Cli/ViewModels/MainViewModel.cs

@@ -21,8 +21,8 @@ namespace DiscordChatExporter.Cli.ViewModels
             _exportService = exportService;
         }
 
-        public async Task ExportAsync(string token, string channelId, string filePath, ExportFormat format, DateTime? from,
+        public async Task ExportAsync(string token, string channelId, string filePath, ExportFormat format,
-            DateTime? to)
+            DateTime? from, DateTime? to)
         {
             // Get channel and guild
             var channel = await _dataService.GetChannelAsync(token, channelId);

DiscordChatExporter.Core/Services/DataService.cs

@@ -14,7 +14,8 @@ namespace DiscordChatExporter.Core.Services
     {
         private readonly HttpClient _httpClient = new HttpClient();
 
-        private async Task<JToken> GetApiResponseAsync(string token, string resource, string endpoint, params string[] parameters)
+        private async Task<JToken> GetApiResponseAsync(string token, string resource, string endpoint,
+            params string[] parameters)
         {
             // Format URL
             const string apiRoot = "https://discordapp.com/api/v6";
@@ -89,48 +90,72 @@ namespace DiscordChatExporter.Core.Services
         }
 
         public async Task<IReadOnlyList<Message>> GetChannelMessagesAsync(string token, string channelId,
-            DateTime? from, DateTime? to)
+            DateTime? from = null, DateTime? to = null, IProgress<double> progress = null)
         {
             var result = new List<Message>();
 
-            // We are going backwards from last message to first
+            // Report indeterminate progress
-            // collecting everything between them in batches
+            progress?.Report(-1);
-            var beforeId = to?.ToSnowflake() ?? DateTime.MaxValue.ToSnowflake();
+
+            // Get the snowflakes for the selected range
+            var firstId = from != null ? from.Value.ToSnowflake() : "0";
+            var lastId = to != null ? to.Value.ToSnowflake() : DateTime.MaxValue.ToSnowflake();
+
+            // Get the last message
+            var response = await GetApiResponseAsync(token, "channels", $"{channelId}/messages",
+                "limit=1", $"before={lastId}");
+            var lastMessage = response.Select(ParseMessage).FirstOrDefault();
+
+            // If the last message doesn't exist or it's outside of range - return
+            if (lastMessage == null || lastMessage.Timestamp < from)
+            {
+                progress?.Report(1);
+                return result;
+            }
+
+            // Get other messages
+            var offsetId = firstId;
             while (true)
             {
-                // Get response
+                // Get message batch
-                var response = await GetApiResponseAsync(token, "channels", $"{channelId}/messages",
+                response = await GetApiResponseAsync(token, "channels", $"{channelId}/messages",
-                    "limit=100", $"before={beforeId}");
+                    "limit=100", $"after={offsetId}");
 
                 // Parse
-                var messages = response.Select(ParseMessage);
+                var messages = response
-
+                    .Select(ParseMessage)
-                // Add messages to list
+                    .Reverse() // reverse because messages appear newest first
-                string currentMessageId = null;
+                    .ToArray();
-                foreach (var message in messages)
+
-                {
+                // Break if there are no messages (can happen if messages are deleted during execution)
-                    // Break when the message is older than from date
+                if (!messages.Any())
-                    if (from != null && message.Timestamp < from)
-                    {
-                        currentMessageId = null;
-                        break;
-                    }
-
-                    // Add message
-                    result.Add(message);
-                    currentMessageId = message.Id;
-                }
-
-                // If no messages - break
-                if (currentMessageId == null)
                     break;
 
-                // Otherwise offset the next request
+                // Trim messages to range (until last message)
-                beforeId = currentMessageId;
+                var messagesInRange = messages
+                    .TakeWhile(m => m.Id != lastMessage.Id && m.Timestamp < lastMessage.Timestamp)
+                    .ToArray();
+
+                // Add to result
+                result.AddRange(messagesInRange);
+
+                // Break if messages were trimmed (which means the last message was encountered)
+                if (messagesInRange.Length != messages.Length)
+                    break;
+
+                // Report progress (based on the time range of parsed messages compared to total)
+                progress?.Report((result.Last().Timestamp - result.First().Timestamp).TotalSeconds /
+                                 (lastMessage.Timestamp - result.First().Timestamp).TotalSeconds);
+
+                // Move offset
+                offsetId = result.Last().Id;
             }
 
-            // Messages appear newest first, we need to reverse
+            // Add last message
-            result.Reverse();
+            result.Add(lastMessage);
+
+            // Report progress
+            progress?.Report(1);
 
             return result;
         }
@@ -157,6 +182,7 @@ namespace DiscordChatExporter.Core.Services
                 foreach (var mentionedUser in message.MentionedUsers)
                     userMap[mentionedUser.Id] = mentionedUser;
             }
+
             var users = userMap.Values.ToArray();
 
             return new Mentionables(users, channels, roles);

DiscordChatExporter.Core/Services/IDataService.cs

@@ -20,7 +20,7 @@ namespace DiscordChatExporter.Core.Services
         Task<IReadOnlyList<Role>> GetGuildRolesAsync(string token, string guildId);
 
         Task<IReadOnlyList<Message>> GetChannelMessagesAsync(string token, string channelId,
-            DateTime? from, DateTime? to);
+            DateTime? from = null, DateTime? to = null, IProgress<double> progress = null);
 
         Task<Mentionables> GetMentionablesAsync(string token, string guildId,
             IEnumerable<Message> messages);

DiscordChatExporter.Gui/ViewModels/IMainViewModel.cs

@@ -9,6 +9,9 @@ namespace DiscordChatExporter.Gui.ViewModels
         bool IsBusy { get; }
         bool IsDataAvailable { get; }
 
+        bool IsProgressIndeterminate { get; }
+        double Progress { get; }
+
         string Token { get; set; }
 
         IReadOnlyList<Guild> AvailableGuilds { get; }

DiscordChatExporter.Gui/ViewModels/MainViewModel.cs

@@ -25,6 +25,7 @@ namespace DiscordChatExporter.Gui.ViewModels
         private readonly Dictionary<Guild, IReadOnlyList<Channel>> _guildChannelsMap;
 
         private bool _isBusy;
+        private double _progress;
         private string _token;
         private IReadOnlyList<Guild> _availableGuilds;
         private Guild _selectedGuild;
@@ -43,6 +44,18 @@ namespace DiscordChatExporter.Gui.ViewModels
 
         public bool IsDataAvailable => AvailableGuilds.NotNullAndAny();
 
+        public bool IsProgressIndeterminate => Progress <= 0;
+
+        public double Progress
+        {
+            get => _progress;
+            private set
+            {
+                Set(ref _progress, value);
+                RaisePropertyChanged(() => IsProgressIndeterminate);
+            }
+        }
+
         public string Token
         {
             get => _token;
@@ -225,10 +238,13 @@ namespace DiscordChatExporter.Gui.ViewModels
             // Get guild
             var guild = SelectedGuild;
 
+            // Create progress handler
+            var progressHandler = new Progress<double>(p => Progress = p);
+
             try
             {
                 // Get messages
-                var messages = await _dataService.GetChannelMessagesAsync(token, channel.Id, from, to);
+                var messages = await _dataService.GetChannelMessagesAsync(token, channel.Id, from, to, progressHandler);
 
                 // Group messages
                 var messageGroups = _messageGroupService.GroupMessages(messages);
@@ -253,6 +269,7 @@ namespace DiscordChatExporter.Gui.ViewModels
                 MessengerInstance.Send(new ShowNotificationMessage("You don't have access to this channel"));
             }
 
+            Progress = 0;
             IsBusy = false;
         }
     }

DiscordChatExporter.Gui/Views/MainWindow.xaml

@@ -98,7 +98,8 @@
                     <!--  Progress bar  -->
                     <ProgressBar
                         Background="Transparent"
-                        IsIndeterminate="True"
+                        IsIndeterminate="{Binding IsProgressIndeterminate}"
+                        Value="{Binding Progress, Mode=OneWay}"
                         Visibility="{Binding IsBusy, Converter={StaticResource BoolToVisibilityConverter}}" />
                 </StackPanel>
             </Border>