rework refresh queue

Emby.Server.Implementations/Channels/ChannelManager.cs

@@ -1416,7 +1416,7 @@ namespace Emby.Server.Implementations.Channels
             if (!_refreshedItems.ContainsKey(program.Id))
             {
                 _refreshedItems.TryAdd(program.Id, true);
-                _providerManager.QueueRefresh(program.Id, new MetadataRefreshOptions(_fileSystem));
+                _providerManager.QueueRefresh(program.Id, new MetadataRefreshOptions(_fileSystem), RefreshPriority.Low);
             }
 
         }

Emby.Server.Implementations/Collections/CollectionManager.cs

@@ -102,7 +102,7 @@ namespace Emby.Server.Implementations.Collections
                 }
                 else
                 {
-                    _providerManager.QueueRefresh(collection.Id, new MetadataRefreshOptions(_fileSystem));
+                    _providerManager.QueueRefresh(collection.Id, new MetadataRefreshOptions(_fileSystem), RefreshPriority.High);
                 }
 
                 EventHelper.FireEventIfNotNull(CollectionCreated, this, new CollectionCreatedEventArgs
@@ -191,7 +191,7 @@ namespace Emby.Server.Implementations.Collections
 
                 await collection.UpdateToRepository(ItemUpdateType.MetadataEdit, CancellationToken.None).ConfigureAwait(false);
 
-                _providerManager.QueueRefresh(collection.Id, refreshOptions);
+                _providerManager.QueueRefresh(collection.Id, refreshOptions, RefreshPriority.High);
 
                 if (fireEvent)
                 {
@@ -244,7 +244,7 @@ namespace Emby.Server.Implementations.Collections
             collection.UpdateRatingToContent();
 
             await collection.UpdateToRepository(ItemUpdateType.MetadataEdit, CancellationToken.None).ConfigureAwait(false);
-            _providerManager.QueueRefresh(collection.Id, new MetadataRefreshOptions(_fileSystem));
+            _providerManager.QueueRefresh(collection.Id, new MetadataRefreshOptions(_fileSystem), RefreshPriority.High);
 
             EventHelper.FireEventIfNotNull(ItemsRemovedFromCollection, this, new CollectionModifiedEventArgs
             {

Emby.Server.Implementations/Library/LibraryManager.cs

@@ -2126,7 +2126,8 @@ namespace Emby.Server.Implementations.Library
                     // Not sure why this is necessary but need to figure it out
                     // View images are not getting utilized without this
                     ForceSave = true
-                });
+
+                }, RefreshPriority.Normal);
             }
 
             return item;
@@ -2188,7 +2189,8 @@ namespace Emby.Server.Implementations.Library
                 {
                     // Need to force save to increment DateLastSaved
                     ForceSave = true
-                });
+
+                }, RefreshPriority.Normal);
             }
 
             return item;
@@ -2252,7 +2254,8 @@ namespace Emby.Server.Implementations.Library
                 {
                     // Need to force save to increment DateLastSaved
                     ForceSave = true
-                });
+
+                }, RefreshPriority.Normal);
             }
 
             return item;
@@ -2328,7 +2331,7 @@ namespace Emby.Server.Implementations.Library
                 {
                     // Need to force save to increment DateLastSaved
                     ForceSave = true
-                });
+                }, RefreshPriority.Normal);
             }
 
             return item;

Emby.Server.Implementations/LiveTv/LiveTvManager.cs

@@ -857,7 +857,8 @@ namespace Emby.Server.Implementations.LiveTv
                 _providerManager.QueueRefresh(item.Id, new MetadataRefreshOptions(_fileSystem)
                 {
                     MetadataRefreshMode = metadataRefreshMode
-                });
+
+                }, RefreshPriority.Normal);
             }
 
             return item.Id;
@@ -1395,11 +1396,11 @@ namespace Emby.Server.Implementations.LiveTv
 
                     foreach (var program in newPrograms)
                     {
-                        _providerManager.QueueRefresh(program.Id, new MetadataRefreshOptions(_fileSystem));
+                        _providerManager.QueueRefresh(program.Id, new MetadataRefreshOptions(_fileSystem), RefreshPriority.Low);
                     }
                     foreach (var program in updatedPrograms)
                     {
-                        _providerManager.QueueRefresh(program.Id, new MetadataRefreshOptions(_fileSystem));
+                        _providerManager.QueueRefresh(program.Id, new MetadataRefreshOptions(_fileSystem), RefreshPriority.Low);
                     }
 
                     currentChannel.IsMovie = isMovie;

Emby.Server.Implementations/Playlists/PlaylistManager.cs

@@ -201,7 +201,8 @@ namespace Emby.Server.Implementations.Playlists
             _providerManager.QueueRefresh(playlist.Id, new MetadataRefreshOptions(_fileSystem)
             {
                 ForceSave = true
-            });
+
+            }, RefreshPriority.High);
         }
 
         public async Task RemoveFromPlaylist(string playlistId, IEnumerable<string> entryIds)
@@ -228,7 +229,8 @@ namespace Emby.Server.Implementations.Playlists
             _providerManager.QueueRefresh(playlist.Id, new MetadataRefreshOptions(_fileSystem)
             {
                 ForceSave = true
-            });
+
+            }, RefreshPriority.High);
         }
 
         public async Task MoveItem(string playlistId, string entryId, int newIndex)

MediaBrowser.Api/ItemRefreshService.cs

@@ -62,14 +62,7 @@ namespace MediaBrowser.Api
 
             var options = GetRefreshOptions(request);
 
-            if (item is Folder)
-            {
-                _providerManager.QueueRefresh(item.Id, options);
-            }
-            else
-            {
-                _providerManager.RefreshFullItem(item, options, CancellationToken.None);
-            }
+            _providerManager.QueueRefresh(item.Id, options, RefreshPriority.High);
         }
 
         private MetadataRefreshOptions GetRefreshOptions(BaseRefreshRequest request)

MediaBrowser.Api/Subtitles/SubtitleService.cs

@@ -288,7 +288,7 @@ namespace MediaBrowser.Api.Subtitles
                     await _subtitleManager.DownloadSubtitles(video, request.SubtitleId, CancellationToken.None)
                         .ConfigureAwait(false);
 
-                    _providerManager.QueueRefresh(video.Id, new MetadataRefreshOptions(_fileSystem));
+                    _providerManager.QueueRefresh(video.Id, new MetadataRefreshOptions(_fileSystem), RefreshPriority.High);
                 }
                 catch (Exception ex)
                 {

MediaBrowser.Controller/Entities/BaseItem.cs

@@ -1817,7 +1817,7 @@ namespace MediaBrowser.Controller.Entities
         /// <returns>Task.</returns>
         public virtual Task ChangedExternally()
         {
-            ProviderManager.QueueRefresh(Id, new MetadataRefreshOptions(FileSystem));
+            ProviderManager.QueueRefresh(Id, new MetadataRefreshOptions(FileSystem), RefreshPriority.High);
             return Task.FromResult(true);
         }
 

MediaBrowser.Controller/Providers/IProviderManager.cs

@@ -20,9 +20,7 @@ namespace MediaBrowser.Controller.Providers
         /// <summary>
         /// Queues the refresh.
         /// </summary>
-        /// <param name="itemId">The item identifier.</param>
-        /// <param name="options">The options.</param>
-        void QueueRefresh(Guid itemId, MetadataRefreshOptions options);
+        void QueueRefresh(Guid itemId, MetadataRefreshOptions options, RefreshPriority priority);
 
         /// <summary>
         /// Refreshes the full item.
@@ -161,4 +159,11 @@ namespace MediaBrowser.Controller.Providers
         /// <returns>Task{HttpResponseInfo}.</returns>
         Task<HttpResponseInfo> GetSearchImage(string providerName, string url, CancellationToken cancellationToken);
     }
+
+    public enum RefreshPriority
+    {
+        High = 0,
+        Normal = 1,
+        Low = 2
+    }
 }

MediaBrowser.Providers/Manager/GenericPriorityQueue.cs

@@ -0,0 +1,426 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Priority_Queue
+{
+    /// <summary>
+    /// Credit: https://github.com/BlueRaja/High-Speed-Priority-Queue-for-C-Sharp
+    /// A copy of StablePriorityQueue which also has generic priority-type
+    /// </summary>
+    /// <typeparam name="TItem">The values in the queue.  Must extend the GenericPriorityQueue class</typeparam>
+    /// <typeparam name="TPriority">The priority-type.  Must extend IComparable&lt;TPriority&gt;</typeparam>
+    public sealed class GenericPriorityQueue<TItem, TPriority> : IFixedSizePriorityQueue<TItem, TPriority>
+        where TItem : GenericPriorityQueueNode<TPriority>
+        where TPriority : IComparable<TPriority>
+    {
+        private int _numNodes;
+        private TItem[] _nodes;
+        private long _numNodesEverEnqueued;
+
+        /// <summary>
+        /// Instantiate a new Priority Queue
+        /// </summary>
+        /// <param name="maxNodes">The max nodes ever allowed to be enqueued (going over this will cause undefined behavior)</param>
+        public GenericPriorityQueue(int maxNodes)
+        {
+#if DEBUG
+            if (maxNodes <= 0)
+            {
+                throw new InvalidOperationException("New queue size cannot be smaller than 1");
+            }
+#endif
+
+            _numNodes = 0;
+            _nodes = new TItem[maxNodes + 1];
+            _numNodesEverEnqueued = 0;
+        }
+
+        /// <summary>
+        /// Returns the number of nodes in the queue.
+        /// O(1)
+        /// </summary>
+        public int Count
+        {
+            get
+            {
+                return _numNodes;
+            }
+        }
+
+        /// <summary>
+        /// Returns the maximum number of items that can be enqueued at once in this queue.  Once you hit this number (ie. once Count == MaxSize),
+        /// attempting to enqueue another item will cause undefined behavior.  O(1)
+        /// </summary>
+        public int MaxSize
+        {
+            get
+            {
+                return _nodes.Length - 1;
+            }
+        }
+
+        /// <summary>
+        /// Removes every node from the queue.
+        /// O(n) (So, don't do this often!)
+        /// </summary>
+#if NET_VERSION_4_5
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
+        public void Clear()
+        {
+            Array.Clear(_nodes, 1, _numNodes);
+            _numNodes = 0;
+        }
+
+        /// <summary>
+        /// Returns (in O(1)!) whether the given node is in the queue.  O(1)
+        /// </summary>
+#if NET_VERSION_4_5
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
+        public bool Contains(TItem node)
+        {
+#if DEBUG
+            if (node == null)
+            {
+                throw new ArgumentNullException("node");
+            }
+            if (node.QueueIndex < 0 || node.QueueIndex >= _nodes.Length)
+            {
+                throw new InvalidOperationException("node.QueueIndex has been corrupted. Did you change it manually? Or add this node to another queue?");
+            }
+#endif
+
+            return (_nodes[node.QueueIndex] == node);
+        }
+
+        /// <summary>
+        /// Enqueue a node to the priority queue.  Lower values are placed in front. Ties are broken by first-in-first-out.
+        /// If the queue is full, the result is undefined.
+        /// If the node is already enqueued, the result is undefined.
+        /// O(log n)
+        /// </summary>
+#if NET_VERSION_4_5
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
+        public void Enqueue(TItem node, TPriority priority)
+        {
+#if DEBUG
+            if (node == null)
+            {
+                throw new ArgumentNullException("node");
+            }
+            if (_numNodes >= _nodes.Length - 1)
+            {
+                throw new InvalidOperationException("Queue is full - node cannot be added: " + node);
+            }
+            if (Contains(node))
+            {
+                throw new InvalidOperationException("Node is already enqueued: " + node);
+            }
+#endif
+
+            node.Priority = priority;
+            _numNodes++;
+            _nodes[_numNodes] = node;
+            node.QueueIndex = _numNodes;
+            node.InsertionIndex = _numNodesEverEnqueued++;
+            CascadeUp(_nodes[_numNodes]);
+        }
+
+#if NET_VERSION_4_5
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
+        private void Swap(TItem node1, TItem node2)
+        {
+            //Swap the nodes
+            _nodes[node1.QueueIndex] = node2;
+            _nodes[node2.QueueIndex] = node1;
+
+            //Swap their indicies
+            int temp = node1.QueueIndex;
+            node1.QueueIndex = node2.QueueIndex;
+            node2.QueueIndex = temp;
+        }
+
+        //Performance appears to be slightly better when this is NOT inlined o_O
+        private void CascadeUp(TItem node)
+        {
+            //aka Heapify-up
+            int parent = node.QueueIndex / 2;
+            while (parent >= 1)
+            {
+                TItem parentNode = _nodes[parent];
+                if (HasHigherPriority(parentNode, node))
+                    break;
+
+                //Node has lower priority value, so move it up the heap
+                Swap(node, parentNode); //For some reason, this is faster with Swap() rather than (less..?) individual operations, like in CascadeDown()
+
+                parent = node.QueueIndex / 2;
+            }
+        }
+
+#if NET_VERSION_4_5
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
+        private void CascadeDown(TItem node)
+        {
+            //aka Heapify-down
+            TItem newParent;
+            int finalQueueIndex = node.QueueIndex;
+            while (true)
+            {
+                newParent = node;
+                int childLeftIndex = 2 * finalQueueIndex;
+
+                //Check if the left-child is higher-priority than the current node
+                if (childLeftIndex > _numNodes)
+                {
+                    //This could be placed outside the loop, but then we'd have to check newParent != node twice
+                    node.QueueIndex = finalQueueIndex;
+                    _nodes[finalQueueIndex] = node;
+                    break;
+                }
+
+                TItem childLeft = _nodes[childLeftIndex];
+                if (HasHigherPriority(childLeft, newParent))
+                {
+                    newParent = childLeft;
+                }
+
+                //Check if the right-child is higher-priority than either the current node or the left child
+                int childRightIndex = childLeftIndex + 1;
+                if (childRightIndex <= _numNodes)
+                {
+                    TItem childRight = _nodes[childRightIndex];
+                    if (HasHigherPriority(childRight, newParent))
+                    {
+                        newParent = childRight;
+                    }
+                }
+
+                //If either of the children has higher (smaller) priority, swap and continue cascading
+                if (newParent != node)
+                {
+                    //Move new parent to its new index.  node will be moved once, at the end
+                    //Doing it this way is one less assignment operation than calling Swap()
+                    _nodes[finalQueueIndex] = newParent;
+
+                    int temp = newParent.QueueIndex;
+                    newParent.QueueIndex = finalQueueIndex;
+                    finalQueueIndex = temp;
+                }
+                else
+                {
+                    //See note above
+                    node.QueueIndex = finalQueueIndex;
+                    _nodes[finalQueueIndex] = node;
+                    break;
+                }
+            }
+        }
+
+        /// <summary>
+        /// Returns true if 'higher' has higher priority than 'lower', false otherwise.
+        /// Note that calling HasHigherPriority(node, node) (ie. both arguments the same node) will return false
+        /// </summary>
+#if NET_VERSION_4_5
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
+        private bool HasHigherPriority(TItem higher, TItem lower)
+        {
+            var cmp = higher.Priority.CompareTo(lower.Priority);
+            return (cmp < 0 || (cmp == 0 && higher.InsertionIndex < lower.InsertionIndex));
+        }
+
+        /// <summary>
+        /// Removes the head of the queue (node with minimum priority; ties are broken by order of insertion), and returns it.
+        /// If queue is empty, result is undefined
+        /// O(log n)
+        /// </summary>
+        public TItem Dequeue()
+        {
+#if DEBUG
+            if (_numNodes <= 0)
+            {
+                throw new InvalidOperationException("Cannot call Dequeue() on an empty queue");
+            }
+
+            if (!IsValidQueue())
+            {
+                throw new InvalidOperationException("Queue has been corrupted (Did you update a node priority manually instead of calling UpdatePriority()?" +
+                                                    "Or add the same node to two different queues?)");
+            }
+#endif
+
+            TItem returnMe = _nodes[1];
+            Remove(returnMe);
+            return returnMe;
+        }
+
+        /// <summary>
+        /// Resize the queue so it can accept more nodes.  All currently enqueued nodes are remain.
+        /// Attempting to decrease the queue size to a size too small to hold the existing nodes results in undefined behavior
+        /// O(n)
+        /// </summary>
+        public void Resize(int maxNodes)
+        {
+#if DEBUG
+            if (maxNodes <= 0)
+            {
+                throw new InvalidOperationException("Queue size cannot be smaller than 1");
+            }
+
+            if (maxNodes < _numNodes)
+            {
+                throw new InvalidOperationException("Called Resize(" + maxNodes + "), but current queue contains " + _numNodes + " nodes");
+            }
+#endif
+
+            TItem[] newArray = new TItem[maxNodes + 1];
+            int highestIndexToCopy = Math.Min(maxNodes, _numNodes);
+            for (int i = 1; i <= highestIndexToCopy; i++)
+            {
+                newArray[i] = _nodes[i];
+            }
+            _nodes = newArray;
+        }
+
+        /// <summary>
+        /// Returns the head of the queue, without removing it (use Dequeue() for that).
+        /// If the queue is empty, behavior is undefined.
+        /// O(1)
+        /// </summary>
+        public TItem First
+        {
+            get
+            {
+#if DEBUG
+                if (_numNodes <= 0)
+                {
+                    throw new InvalidOperationException("Cannot call .First on an empty queue");
+                }
+#endif
+
+                return _nodes[1];
+            }
+        }
+
+        /// <summary>
+        /// This method must be called on a node every time its priority changes while it is in the queue.  
+        /// <b>Forgetting to call this method will result in a corrupted queue!</b>
+        /// Calling this method on a node not in the queue results in undefined behavior
+        /// O(log n)
+        /// </summary>
+#if NET_VERSION_4_5
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+#endif
+        public void UpdatePriority(TItem node, TPriority priority)
+        {
+#if DEBUG
+            if (node == null)
+            {
+                throw new ArgumentNullException("node");
+            }
+            if (!Contains(node))
+            {
+                throw new InvalidOperationException("Cannot call UpdatePriority() on a node which is not enqueued: " + node);
+            }
+#endif
+
+            node.Priority = priority;
+            OnNodeUpdated(node);
+        }
+
+        private void OnNodeUpdated(TItem node)
+        {
+            //Bubble the updated node up or down as appropriate
+            int parentIndex = node.QueueIndex / 2;
+            TItem parentNode = _nodes[parentIndex];
+
+            if (parentIndex > 0 && HasHigherPriority(node, parentNode))
+            {
+                CascadeUp(node);
+            }
+            else
+            {
+                //Note that CascadeDown will be called if parentNode == node (that is, node is the root)
+                CascadeDown(node);
+            }
+        }
+
+        /// <summary>
+        /// Removes a node from the queue.  The node does not need to be the head of the queue.  
+        /// If the node is not in the queue, the result is undefined.  If unsure, check Contains() first
+        /// O(log n)
+        /// </summary>
+        public void Remove(TItem node)
+        {
+#if DEBUG
+            if (node == null)
+            {
+                throw new ArgumentNullException("node");
+            }
+            if (!Contains(node))
+            {
+                throw new InvalidOperationException("Cannot call Remove() on a node which is not enqueued: " + node);
+            }
+#endif
+
+            //If the node is already the last node, we can remove it immediately
+            if (node.QueueIndex == _numNodes)
+            {
+                _nodes[_numNodes] = null;
+                _numNodes--;
+                return;
+            }
+
+            //Swap the node with the last node
+            TItem formerLastNode = _nodes[_numNodes];
+            Swap(node, formerLastNode);
+            _nodes[_numNodes] = null;
+            _numNodes--;
+
+            //Now bubble formerLastNode (which is no longer the last node) up or down as appropriate
+            OnNodeUpdated(formerLastNode);
+        }
+
+        public IEnumerator<TItem> GetEnumerator()
+        {
+            for (int i = 1; i <= _numNodes; i++)
+                yield return _nodes[i];
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+
+        /// <summary>
+        /// <b>Should not be called in production code.</b>
+        /// Checks to make sure the queue is still in a valid state.  Used for testing/debugging the queue.
+        /// </summary>
+        public bool IsValidQueue()
+        {
+            for (int i = 1; i < _nodes.Length; i++)
+            {
+                if (_nodes[i] != null)
+                {
+                    int childLeftIndex = 2 * i;
+                    if (childLeftIndex < _nodes.Length && _nodes[childLeftIndex] != null && HasHigherPriority(_nodes[childLeftIndex], _nodes[i]))
+                        return false;
+
+                    int childRightIndex = childLeftIndex + 1;
+                    if (childRightIndex < _nodes.Length && _nodes[childRightIndex] != null && HasHigherPriority(_nodes[childRightIndex], _nodes[i]))
+                        return false;
+                }
+            }
+            return true;
+        }
+    }
+}

MediaBrowser.Providers/Manager/GenericPriorityQueueNode.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Priority_Queue
+{
+    /// Credit: https://github.com/BlueRaja/High-Speed-Priority-Queue-for-C-Sharp
+    public class GenericPriorityQueueNode<TPriority>
+    {
+        /// <summary>
+        /// The Priority to insert this node at.  Must be set BEFORE adding a node to the queue (ideally just once, in the node's constructor).
+        /// Should not be manually edited once the node has been enqueued - use queue.UpdatePriority() instead
+        /// </summary>
+        public TPriority Priority { get; protected internal set; }
+
+        /// <summary>
+        /// Represents the current position in the queue
+        /// </summary>
+        public int QueueIndex { get; internal set; }
+
+        /// <summary>
+        /// Represents the order the node was inserted in
+        /// </summary>
+        public long InsertionIndex { get; internal set; }
+    }
+}

MediaBrowser.Providers/Manager/IFixedSizePriorityQueue.cs

@@ -0,0 +1,28 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Priority_Queue
+{
+    /// <summary>
+    /// Credit: https://github.com/BlueRaja/High-Speed-Priority-Queue-for-C-Sharp
+    /// A helper-interface only needed to make writing unit tests a bit easier (hence the 'internal' access modifier)
+    /// </summary>
+    internal interface IFixedSizePriorityQueue<TItem, in TPriority> : IPriorityQueue<TItem, TPriority>
+        where TPriority : IComparable<TPriority>
+    {
+        /// <summary>
+        /// Resize the queue so it can accept more nodes.  All currently enqueued nodes are remain.
+        /// Attempting to decrease the queue size to a size too small to hold the existing nodes results in undefined behavior
+        /// </summary>
+        void Resize(int maxNodes);
+
+        /// <summary>
+        /// Returns the maximum number of items that can be enqueued at once in this queue.  Once you hit this number (ie. once Count == MaxSize),
+        /// attempting to enqueue another item will cause undefined behavior.
+        /// </summary>
+        int MaxSize { get; }
+    }
+}

MediaBrowser.Providers/Manager/IPriorityQueue.cs

@@ -0,0 +1,59 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Priority_Queue
+{
+    /// <summary>
+    /// Credit: https://github.com/BlueRaja/High-Speed-Priority-Queue-for-C-Sharp
+    /// The IPriorityQueue interface.  This is mainly here for purists, and in case I decide to add more implementations later.
+    /// For speed purposes, it is actually recommended that you *don't* access the priority queue through this interface, since the JIT can
+    /// (theoretically?) optimize method calls from concrete-types slightly better.
+    /// </summary>
+    public interface IPriorityQueue<TItem, in TPriority> : IEnumerable<TItem>
+        where TPriority : IComparable<TPriority>
+    {
+        /// <summary>
+        /// Enqueue a node to the priority queue.  Lower values are placed in front. Ties are broken by first-in-first-out.
+        /// See implementation for how duplicates are handled.
+        /// </summary>
+        void Enqueue(TItem node, TPriority priority);
+
+        /// <summary>
+        /// Removes the head of the queue (node with minimum priority; ties are broken by order of insertion), and returns it.
+        /// </summary>
+        TItem Dequeue();
+
+        /// <summary>
+        /// Removes every node from the queue.
+        /// </summary>
+        void Clear();
+
+        /// <summary>
+        /// Returns whether the given node is in the queue.
+        /// </summary>
+        bool Contains(TItem node);
+
+        /// <summary>
+        /// Removes a node from the queue.  The node does not need to be the head of the queue.  
+        /// </summary>
+        void Remove(TItem node);
+
+        /// <summary>
+        /// Call this method to change the priority of a node.  
+        /// </summary>
+        void UpdatePriority(TItem node, TPriority priority);
+
+        /// <summary>
+        /// Returns the head of the queue, without removing it (use Dequeue() for that).
+        /// </summary>
+        TItem First { get; }
+
+        /// <summary>
+        /// Returns the number of nodes in the queue.
+        /// </summary>
+        int Count { get; }
+    }
+}

MediaBrowser.Providers/Manager/ProviderManager.cs

@@ -24,6 +24,7 @@ using MediaBrowser.Common.IO;
 using MediaBrowser.Controller.IO;
 using MediaBrowser.Model.IO;
 using MediaBrowser.Model.Serialization;
+using Priority_Queue;
 
 namespace MediaBrowser.Providers.Manager
 {
@@ -577,7 +578,6 @@ namespace MediaBrowser.Providers.Manager
             return SaveMetadata(item, updateType, _savers.Where(i => savers.Contains(i.Name, StringComparer.OrdinalIgnoreCase)));
         }
 
-        private readonly SemaphoreSlim _saveLock = new SemaphoreSlim(1, 1);
         /// <summary>
         /// Saves the metadata.
         /// </summary>
@@ -607,8 +607,6 @@ namespace MediaBrowser.Providers.Manager
                         continue;
                     }
 
-                    await _saveLock.WaitAsync().ConfigureAwait(false);
-
                     try
                     {
                         _libraryMonitor.ReportFileSystemChangeBeginning(path);
@@ -620,7 +618,6 @@ namespace MediaBrowser.Providers.Manager
                     }
                     finally
                     {
-                        _saveLock.Release();
                         _libraryMonitor.ReportFileSystemChangeComplete(path, false);
                     }
                 }
@@ -851,20 +848,20 @@ namespace MediaBrowser.Providers.Manager
                 });
         }
 
-        private readonly ConcurrentQueue<Tuple<Guid, MetadataRefreshOptions>> _refreshQueue =
-            new ConcurrentQueue<Tuple<Guid, MetadataRefreshOptions>>();
+        private readonly SimplePriorityQueue<Tuple<Guid, MetadataRefreshOptions>> _refreshQueue =
+            new SimplePriorityQueue<Tuple<Guid, MetadataRefreshOptions>>();
 
         private readonly object _refreshQueueLock = new object();
         private bool _isProcessingRefreshQueue;
 
-        public void QueueRefresh(Guid id, MetadataRefreshOptions options)
+        public void QueueRefresh(Guid id, MetadataRefreshOptions options, RefreshPriority priority)
         {
             if (_disposed)
             {
                 return;
             }
 
-            _refreshQueue.Enqueue(new Tuple<Guid, MetadataRefreshOptions>(id, options));
+            _refreshQueue.Enqueue(new Tuple<Guid, MetadataRefreshOptions>(id, options), (int)priority);
 
             lock (_refreshQueueLock)
             {
@@ -876,12 +873,19 @@ namespace MediaBrowser.Providers.Manager
             }
         }
 
+        private bool TryDequeue(out Tuple<Guid, MetadataRefreshOptions> item)
+        {
+            item = _refreshQueue.Dequeue();
+
+            return item != null;
+        }
+
         private async Task StartProcessingRefreshQueue()
         {
             Tuple<Guid, MetadataRefreshOptions> refreshItem;
             var libraryManager = _libraryManagerFactory();
 
-            while (_refreshQueue.TryDequeue(out refreshItem))
+            while (TryDequeue(out refreshItem))
             {
                 if (_disposed)
                 {

MediaBrowser.Providers/Manager/SimplePriorityQueue.cs

@@ -0,0 +1,243 @@
+using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Priority_Queue
+{
+    /// <summary>
+    /// Credit: https://github.com/BlueRaja/High-Speed-Priority-Queue-for-C-Sharp
+    /// A simplified priority queue implementation.  Is stable, auto-resizes, and thread-safe, at the cost of being slightly slower than
+    /// FastPriorityQueue
+    /// </summary>
+    /// <typeparam name="TItem">The type to enqueue</typeparam>
+    /// <typeparam name="TPriority">The priority-type to use for nodes.  Must extend IComparable&lt;TPriority&gt;</typeparam>
+    public class SimplePriorityQueue<TItem, TPriority> : IPriorityQueue<TItem, TPriority>
+        where TPriority : IComparable<TPriority>
+    {
+        private class SimpleNode : GenericPriorityQueueNode<TPriority>
+        {
+            public TItem Data { get; private set; }
+
+            public SimpleNode(TItem data)
+            {
+                Data = data;
+            }
+        }
+
+        private const int INITIAL_QUEUE_SIZE = 10;
+        private readonly GenericPriorityQueue<SimpleNode, TPriority> _queue;
+
+        public SimplePriorityQueue()
+        {
+            _queue = new GenericPriorityQueue<SimpleNode, TPriority>(INITIAL_QUEUE_SIZE);
+        }
+
+        /// <summary>
+        /// Given an item of type T, returns the exist SimpleNode in the queue
+        /// </summary>
+        private SimpleNode GetExistingNode(TItem item)
+        {
+            var comparer = EqualityComparer<TItem>.Default;
+            foreach (var node in _queue)
+            {
+                if (comparer.Equals(node.Data, item))
+                {
+                    return node;
+                }
+            }
+            throw new InvalidOperationException("Item cannot be found in queue: " + item);
+        }
+
+        /// <summary>
+        /// Returns the number of nodes in the queue.
+        /// O(1)
+        /// </summary>
+        public int Count
+        {
+            get
+            {
+                lock (_queue)
+                {
+                    return _queue.Count;
+                }
+            }
+        }
+
+
+        /// <summary>
+        /// Returns the head of the queue, without removing it (use Dequeue() for that).
+        /// Throws an exception when the queue is empty.
+        /// O(1)
+        /// </summary>
+        public TItem First
+        {
+            get
+            {
+                lock (_queue)
+                {
+                    if (_queue.Count <= 0)
+                    {
+                        throw new InvalidOperationException("Cannot call .First on an empty queue");
+                    }
+
+                    SimpleNode first = _queue.First;
+                    return (first != null ? first.Data : default(TItem));
+                }
+            }
+        }
+
+        /// <summary>
+        /// Removes every node from the queue.
+        /// O(n)
+        /// </summary>
+        public void Clear()
+        {
+            lock (_queue)
+            {
+                _queue.Clear();
+            }
+        }
+
+        /// <summary>
+        /// Returns whether the given item is in the queue.
+        /// O(n)
+        /// </summary>
+        public bool Contains(TItem item)
+        {
+            lock (_queue)
+            {
+                var comparer = EqualityComparer<TItem>.Default;
+                foreach (var node in _queue)
+                {
+                    if (comparer.Equals(node.Data, item))
+                    {
+                        return true;
+                    }
+                }
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Removes the head of the queue (node with minimum priority; ties are broken by order of insertion), and returns it.
+        /// If queue is empty, throws an exception
+        /// O(log n)
+        /// </summary>
+        public TItem Dequeue()
+        {
+            lock (_queue)
+            {
+                if (_queue.Count <= 0)
+                {
+                    throw new InvalidOperationException("Cannot call Dequeue() on an empty queue");
+                }
+
+                SimpleNode node = _queue.Dequeue();
+                return node.Data;
+            }
+        }
+
+        /// <summary>
+        /// Enqueue a node to the priority queue.  Lower values are placed in front. Ties are broken by first-in-first-out.
+        /// This queue automatically resizes itself, so there's no concern of the queue becoming 'full'.
+        /// Duplicates are allowed.
+        /// O(log n)
+        /// </summary>
+        public void Enqueue(TItem item, TPriority priority)
+        {
+            lock (_queue)
+            {
+                SimpleNode node = new SimpleNode(item);
+                if (_queue.Count == _queue.MaxSize)
+                {
+                    _queue.Resize(_queue.MaxSize * 2 + 1);
+                }
+                _queue.Enqueue(node, priority);
+            }
+        }
+
+        /// <summary>
+        /// Removes an item from the queue.  The item does not need to be the head of the queue.  
+        /// If the item is not in the queue, an exception is thrown.  If unsure, check Contains() first.
+        /// If multiple copies of the item are enqueued, only the first one is removed. 
+        /// O(n)
+        /// </summary>
+        public void Remove(TItem item)
+        {
+            lock (_queue)
+            {
+                try
+                {
+                    _queue.Remove(GetExistingNode(item));
+                }
+                catch (InvalidOperationException ex)
+                {
+                    throw new InvalidOperationException("Cannot call Remove() on a node which is not enqueued: " + item, ex);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Call this method to change the priority of an item.
+        /// Calling this method on a item not in the queue will throw an exception.
+        /// If the item is enqueued multiple times, only the first one will be updated.
+        /// (If your requirements are complex enough that you need to enqueue the same item multiple times <i>and</i> be able
+        /// to update all of them, please wrap your items in a wrapper class so they can be distinguished).
+        /// O(n)
+        /// </summary>
+        public void UpdatePriority(TItem item, TPriority priority)
+        {
+            lock (_queue)
+            {
+                try
+                {
+                    SimpleNode updateMe = GetExistingNode(item);
+                    _queue.UpdatePriority(updateMe, priority);
+                }
+                catch (InvalidOperationException ex)
+                {
+                    throw new InvalidOperationException("Cannot call UpdatePriority() on a node which is not enqueued: " + item, ex);
+                }
+            }
+        }
+
+        public IEnumerator<TItem> GetEnumerator()
+        {
+            List<TItem> queueData = new List<TItem>();
+            lock (_queue)
+            {
+                //Copy to a separate list because we don't want to 'yield return' inside a lock
+                foreach (var node in _queue)
+                {
+                    queueData.Add(node.Data);
+                }
+            }
+
+            return queueData.GetEnumerator();
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+
+        public bool IsValidQueue()
+        {
+            lock (_queue)
+            {
+                return _queue.IsValidQueue();
+            }
+        }
+    }
+
+    /// <summary>
+    /// A simplified priority queue implementation.  Is stable, auto-resizes, and thread-safe, at the cost of being slightly slower than
+    /// FastPriorityQueue
+    /// This class is kept here for backwards compatibility.  It's recommended you use Simple
+    /// </summary>
+    /// <typeparam name="TItem">The type to enqueue</typeparam>
+    public class SimplePriorityQueue<TItem> : SimplePriorityQueue<TItem, float> { }
+}

MediaBrowser.Providers/MediaBrowser.Providers.csproj

@@ -65,10 +65,15 @@
     <Compile Include="LiveTv\ChannelMetadataService.cs" />
     <Compile Include="LiveTv\ProgramMetadataService.cs" />
     <Compile Include="LiveTv\VideoRecordingService.cs" />
+    <Compile Include="Manager\GenericPriorityQueue.cs" />
+    <Compile Include="Manager\GenericPriorityQueueNode.cs" />
+    <Compile Include="Manager\IFixedSizePriorityQueue.cs" />
     <Compile Include="Manager\ImageSaver.cs" />
+    <Compile Include="Manager\IPriorityQueue.cs" />
     <Compile Include="Manager\ItemImageProvider.cs" />
     <Compile Include="Manager\ProviderManager.cs" />
     <Compile Include="Manager\MetadataService.cs" />
+    <Compile Include="Manager\SimplePriorityQueue.cs" />
     <Compile Include="MediaInfo\FFProbeAudioInfo.cs" />
     <Compile Include="MediaInfo\FFProbeProvider.cs" />
     <Compile Include="MediaInfo\FFProbeVideoInfo.cs" />

MediaBrowser.Providers/Movies/FanartMovieImageProvider.cs

@@ -284,7 +284,6 @@ namespace MediaBrowser.Providers.Movies
                 using (var response = await _httpClient.Get(new HttpRequestOptions
                 {
                     Url = url,
-                    ResourcePool = FanartArtistProvider.Current.FanArtResourcePool,
                     CancellationToken = cancellationToken,
                     BufferContent = true
 

MediaBrowser.Providers/Movies/MovieDbProvider.cs

@@ -30,8 +30,6 @@ namespace MediaBrowser.Providers.Movies
     /// </summary>
     public class MovieDbProvider : IRemoteMetadataProvider<Movie, MovieInfo>, IDisposable, IHasOrder
     {
-        internal readonly SemaphoreSlim MovieDbResourcePool = new SemaphoreSlim(1, 1);
-
         internal static MovieDbProvider Current { get; private set; }
 
         private readonly IJsonSerializer _jsonSerializer;
@@ -137,10 +135,6 @@ namespace MediaBrowser.Providers.Movies
         /// <param name="dispose"><c>true</c> to release both managed and unmanaged resources; <c>false</c> to release only unmanaged resources.</param>
         protected virtual void Dispose(bool dispose)
         {
-            if (dispose)
-            {
-                MovieDbResourcePool.Dispose();
-            }
         }
 
         /// <summary>
@@ -431,7 +425,6 @@ namespace MediaBrowser.Providers.Movies
                 await Task.Delay(Convert.ToInt32(delayMs)).ConfigureAwait(false);
             }
 
-            options.ResourcePool = MovieDbResourcePool;
             _lastRequestTicks = DateTime.UtcNow.Ticks;
 
             options.BufferContent = true;

MediaBrowser.Providers/Music/AudioDbArtistProvider.cs

@@ -27,7 +27,6 @@ namespace MediaBrowser.Providers.Music
 
         public static AudioDbArtistProvider Current;
 
-        public SemaphoreSlim AudioDbResourcePool = new SemaphoreSlim(2, 2);
         private const string ApiKey = "49jhsf8248yfahka89724011";
         public const string BaseUrl = "http://www.theaudiodb.com/api/v1/json/" + ApiKey;
 
@@ -151,7 +150,6 @@ namespace MediaBrowser.Providers.Music
             using (var response = await _httpClient.Get(new HttpRequestOptions
             {
                 Url = url,
-                ResourcePool = AudioDbResourcePool,
                 CancellationToken = cancellationToken,
                 BufferContent = true
 

MediaBrowser.Providers/Music/FanArtArtistProvider.cs

@@ -27,7 +27,6 @@ namespace MediaBrowser.Providers.Music
 {
     public class FanartArtistProvider : IRemoteImageProvider, IHasOrder
     {
-        internal readonly SemaphoreSlim FanArtResourcePool = new SemaphoreSlim(3, 3);
         internal const string ApiKey = "5c6b04c68e904cfed1e6cbc9a9e683d4";
         private const string FanArtBaseUrl = "https://webservice.fanart.tv/v3.1/music/{1}?api_key={0}";
 
@@ -255,7 +254,6 @@ namespace MediaBrowser.Providers.Music
                 using (var response = await _httpClient.Get(new HttpRequestOptions
                 {
                     Url = url,
-                    ResourcePool = FanArtResourcePool,
                     CancellationToken = cancellationToken,
                     BufferContent = true
 

MediaBrowser.Providers/Music/MusicBrainzAlbumProvider.cs

@@ -548,11 +548,6 @@ namespace MediaBrowser.Providers.Music
             return null;
         }
 
-        /// <summary>
-        /// The _music brainz resource pool
-        /// </summary>
-        private readonly SemaphoreSlim _musicBrainzResourcePool = new SemaphoreSlim(1, 1);
-
         private long _lastMbzUrlQueryTicks = 0;
         private List<MbzUrl> _mbzUrls = null;
         private MbzUrl _chosenUrl;
@@ -656,7 +651,6 @@ namespace MediaBrowser.Providers.Music
                 Url = url,
                 CancellationToken = cancellationToken,
                 UserAgent = _appHost.Name + "/" + _appHost.ApplicationVersion,
-                ResourcePool = _musicBrainzResourcePool,
                 BufferContent = throttleMs > 0
             };
 

MediaBrowser.Providers/Omdb/OmdbProvider.cs

@@ -19,7 +19,6 @@ namespace MediaBrowser.Providers.Omdb
 {
     public class OmdbProvider
     {
-        internal static readonly SemaphoreSlim ResourcePool = new SemaphoreSlim(1, 1);
         private readonly IJsonSerializer _jsonSerializer;
         private readonly IFileSystem _fileSystem;
         private readonly IServerConfigurationManager _configurationManager;
@@ -330,7 +329,6 @@ namespace MediaBrowser.Providers.Omdb
             return httpClient.Get(new HttpRequestOptions
             {
                 Url = url,
-                ResourcePool = ResourcePool,
                 CancellationToken = cancellationToken,
                 BufferContent = true,
                 EnableDefaultUserAgent = true

MediaBrowser.Providers/Subtitles/SubtitleManager.cs

@@ -256,12 +256,7 @@ namespace MediaBrowser.Providers.Subtitles
                 _monitor.ReportFileSystemChangeComplete(path, false);
             }
 
-            return _libraryManager.GetItemById(itemId).RefreshMetadata(new MetadataRefreshOptions(new DirectoryService(_logger, _fileSystem))
-            {
-                ImageRefreshMode = ImageRefreshMode.ValidationOnly,
-                MetadataRefreshMode = MetadataRefreshMode.ValidationOnly
-
-            }, CancellationToken.None);
+            return _libraryManager.GetItemById(itemId).ChangedExternally();
         }
 
         public Task<SubtitleResponse> GetRemoteSubtitles(string id, CancellationToken cancellationToken)

MediaBrowser.Providers/TV/FanArt/FanartSeriesProvider.cs

@@ -319,7 +319,6 @@ namespace MediaBrowser.Providers.TV
                 using (var response = await _httpClient.Get(new HttpRequestOptions
                 {
                     Url = url,
-                    ResourcePool = FanartArtistProvider.Current.FanArtResourcePool,
                     CancellationToken = cancellationToken,
                     BufferContent = true
 

MediaBrowser.Providers/TV/TheMovieDb/MovieDbProviderBase.cs

@@ -142,8 +142,7 @@ namespace MediaBrowser.Providers.TV
             return _httpClient.GetResponse(new HttpRequestOptions
             {
                 CancellationToken = cancellationToken,
-                Url = url,
-                ResourcePool = MovieDbProvider.Current.MovieDbResourcePool
+                Url = url
             });
         }
 

MediaBrowser.Providers/TV/TheTVDB/TvdbEpisodeImageProvider.cs

@@ -175,8 +175,7 @@ namespace MediaBrowser.Providers.TV
             return _httpClient.GetResponse(new HttpRequestOptions
             {
                 CancellationToken = cancellationToken,
-                Url = url,
-                ResourcePool = TvdbSeriesProvider.Current.TvDbResourcePool
+                Url = url
             });
         }
     }

MediaBrowser.Providers/TV/TheTVDB/TvdbEpisodeProvider.cs

@@ -919,8 +919,7 @@ namespace MediaBrowser.Providers.TV
             return _httpClient.GetResponse(new HttpRequestOptions
             {
                 CancellationToken = cancellationToken,
-                Url = url,
-                ResourcePool = TvdbSeriesProvider.Current.TvDbResourcePool
+                Url = url
             });
         }
 

MediaBrowser.Providers/TV/TheTVDB/TvdbPrescanTask.cs

@@ -143,7 +143,6 @@ namespace MediaBrowser.Providers.TV
                     Url = ServerTimeUrl,
                     CancellationToken = cancellationToken,
                     EnableHttpCompression = true,
-                    ResourcePool = TvdbSeriesProvider.Current.TvDbResourcePool,
                     BufferContent = false
 
                 }).ConfigureAwait(false))
@@ -240,7 +239,6 @@ namespace MediaBrowser.Providers.TV
                 Url = string.Format(UpdatesUrl, lastUpdateTime),
                 CancellationToken = cancellationToken,
                 EnableHttpCompression = true,
-                ResourcePool = TvdbSeriesProvider.Current.TvDbResourcePool,
                 BufferContent = false
 
             }).ConfigureAwait(false))

MediaBrowser.Providers/TV/TheTVDB/TvdbSeasonImageProvider.cs

@@ -358,8 +358,7 @@ namespace MediaBrowser.Providers.TV
             return _httpClient.GetResponse(new HttpRequestOptions
             {
                 CancellationToken = cancellationToken,
-                Url = url,
-                ResourcePool = TvdbSeriesProvider.Current.TvDbResourcePool
+                Url = url
             });
         }
     }

MediaBrowser.Providers/TV/TheTVDB/TvdbSeriesImageProvider.cs

@@ -347,8 +347,7 @@ namespace MediaBrowser.Providers.TV
             return _httpClient.GetResponse(new HttpRequestOptions
             {
                 CancellationToken = cancellationToken,
-                Url = url,
-                ResourcePool = TvdbSeriesProvider.Current.TvDbResourcePool
+                Url = url
             });
         }
     }

MediaBrowser.Providers/TV/TheTVDB/TvdbSeriesProvider.cs

@@ -27,7 +27,6 @@ namespace MediaBrowser.Providers.TV
 {
     public class TvdbSeriesProvider : IRemoteMetadataProvider<Series, SeriesInfo>, IHasOrder
     {
-        internal readonly SemaphoreSlim TvDbResourcePool = new SemaphoreSlim(2, 2);
         internal static TvdbSeriesProvider Current { get; private set; }
         private readonly IZipClient _zipClient;
         private readonly IHttpClient _httpClient;
@@ -220,7 +219,6 @@ namespace MediaBrowser.Providers.TV
             using (var zipStream = await _httpClient.Get(new HttpRequestOptions
             {
                 Url = url,
-                ResourcePool = TvDbResourcePool,
                 CancellationToken = cancellationToken,
                 BufferContent = false
 
@@ -265,7 +263,6 @@ namespace MediaBrowser.Providers.TV
             using (var result = await _httpClient.Get(new HttpRequestOptions
             {
                 Url = url,
-                ResourcePool = TvDbResourcePool,
                 CancellationToken = cancellationToken,
                 BufferContent = false
 
@@ -520,7 +517,6 @@ namespace MediaBrowser.Providers.TV
             using (var stream = await _httpClient.Get(new HttpRequestOptions
             {
                 Url = url,
-                ResourcePool = TvDbResourcePool,
                 CancellationToken = cancellationToken,
                 BufferContent = false
 
@@ -1651,7 +1647,6 @@ namespace MediaBrowser.Providers.TV
             {
                 CancellationToken = cancellationToken,
                 Url = url,
-                ResourcePool = TvDbResourcePool,
                 BufferContent = false
             });
         }