New: Search for all missing episodes

src/NzbDrone.Common/NzbDrone.Common.csproj

@@ -105,6 +105,7 @@
     <Compile Include="PathEqualityComparer.cs" />
     <Compile Include="Processes\INzbDroneProcessProvider.cs" />
     <Compile Include="Processes\ProcessOutput.cs" />
+    <Compile Include="RateGate.cs" />
     <Compile Include="Serializer\IntConverter.cs" />
     <Compile Include="Services.cs" />
     <Compile Include="Extensions\StreamExtensions.cs" />

src/NzbDrone.Common/RateGate.cs

@@ -0,0 +1,197 @@
+/*
+ * Code from: http://www.jackleitch.net/2010/10/better-rate-limiting-with-dot-net/
+ */
+
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+
+namespace NzbDrone.Common
+{
+    /// <summary>
+    /// Used to control the rate of some occurrence per unit of time.
+    /// </summary>
+    /// <remarks>
+    ///     <para>
+    ///     To control the rate of an action using a <see cref="RateGate"/>, 
+    ///     code should simply call <see cref="WaitToProceed()"/> prior to 
+    ///     performing the action. <see cref="WaitToProceed()"/> will block
+    ///     the current thread until the action is allowed based on the rate 
+    ///     limit.
+    ///     </para>
+    ///     <para>
+    ///     This class is thread safe. A single <see cref="RateGate"/> instance 
+    ///     may be used to control the rate of an occurrence across multiple 
+    ///     threads.
+    ///     </para>
+    /// </remarks>
+    public class RateGate : IDisposable
+    {
+        // Semaphore used to count and limit the number of occurrences per
+        // unit time.
+        private readonly SemaphoreSlim _semaphore;
+
+        // Times (in millisecond ticks) at which the semaphore should be exited.
+        private readonly ConcurrentQueue<int> _exitTimes;
+
+        // Timer used to trigger exiting the semaphore.
+        private readonly Timer _exitTimer;
+
+        // Whether this instance is disposed.
+        private bool _isDisposed;
+
+        /// <summary>
+        /// Number of occurrences allowed per unit of time.
+        /// </summary>
+        public int Occurrences { get; private set; }
+
+        /// <summary>
+        /// The length of the time unit, in milliseconds.
+        /// </summary>
+        public int TimeUnitMilliseconds { get; private set; }
+
+        /// <summary>
+        /// Initializes a <see cref="RateGate"/> with a rate of <paramref name="occurrences"/> 
+        /// per <paramref name="timeUnit"/>.
+        /// </summary>
+        /// <param name="occurrences">Number of occurrences allowed per unit of time.</param>
+        /// <param name="timeUnit">Length of the time unit.</param>
+        /// <exception cref="ArgumentOutOfRangeException">
+        /// If <paramref name="occurrences"/> or <paramref name="timeUnit"/> is negative.
+        /// </exception>
+        public RateGate(int occurrences, TimeSpan timeUnit)
+        {
+            // Check the arguments.
+            if (occurrences <= 0)
+                throw new ArgumentOutOfRangeException("occurrences", "Number of occurrences must be a positive integer");
+            if (timeUnit != timeUnit.Duration())
+                throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be a positive span of time");
+            if (timeUnit >= TimeSpan.FromMilliseconds(UInt32.MaxValue))
+                throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be less than 2^32 milliseconds");
+
+            Occurrences = occurrences;
+            TimeUnitMilliseconds = (int)timeUnit.TotalMilliseconds;
+
+            // Create the semaphore, with the number of occurrences as the maximum count.
+            _semaphore = new SemaphoreSlim(Occurrences, Occurrences);
+
+            // Create a queue to hold the semaphore exit times.
+            _exitTimes = new ConcurrentQueue<int>();
+
+            // Create a timer to exit the semaphore. Use the time unit as the original
+            // interval length because that's the earliest we will need to exit the semaphore.
+            _exitTimer = new Timer(ExitTimerCallback, null, TimeUnitMilliseconds, -1);
+        }
+
+        // Callback for the exit timer that exits the semaphore based on exit times 
+        // in the queue and then sets the timer for the nextexit time.
+        private void ExitTimerCallback(object state)
+        {
+            // While there are exit times that are passed due still in the queue,
+            // exit the semaphore and dequeue the exit time.
+            int exitTime;
+            while (_exitTimes.TryPeek(out exitTime)
+                    && unchecked(exitTime - Environment.TickCount) <= 0)
+            {
+                _semaphore.Release();
+                _exitTimes.TryDequeue(out exitTime);
+            }
+
+            // Try to get the next exit time from the queue and compute
+            // the time until the next check should take place. If the 
+            // queue is empty, then no exit times will occur until at least
+            // one time unit has passed.
+            int timeUntilNextCheck;
+            if (_exitTimes.TryPeek(out exitTime))
+                timeUntilNextCheck = unchecked(exitTime - Environment.TickCount);
+            else
+                timeUntilNextCheck = TimeUnitMilliseconds;
+
+            // Set the timer.
+            _exitTimer.Change(timeUntilNextCheck, -1);
+        }
+
+        /// <summary>
+        /// Blocks the current thread until allowed to proceed or until the
+        /// specified timeout elapses.
+        /// </summary>
+        /// <param name="millisecondsTimeout">Number of milliseconds to wait, or -1 to wait indefinitely.</param>
+        /// <returns>true if the thread is allowed to proceed, or false if timed out</returns>
+        public bool WaitToProceed(int millisecondsTimeout)
+        {
+            // Check the arguments.
+            if (millisecondsTimeout < -1)
+                throw new ArgumentOutOfRangeException("millisecondsTimeout");
+
+            CheckDisposed();
+
+            // Block until we can enter the semaphore or until the timeout expires.
+            var entered = _semaphore.Wait(millisecondsTimeout);
+
+            // If we entered the semaphore, compute the corresponding exit time 
+            // and add it to the queue.
+            if (entered)
+            {
+                var timeToExit = unchecked(Environment.TickCount + TimeUnitMilliseconds);
+                _exitTimes.Enqueue(timeToExit);
+            }
+
+            return entered;
+        }
+
+        /// <summary>
+        /// Blocks the current thread until allowed to proceed or until the
+        /// specified timeout elapses.
+        /// </summary>
+        /// <param name="timeout"></param>
+        /// <returns>true if the thread is allowed to proceed, or false if timed out</returns>
+        public bool WaitToProceed(TimeSpan timeout)
+        {
+            return WaitToProceed((int)timeout.TotalMilliseconds);
+        }
+
+        /// <summary>
+        /// Blocks the current thread indefinitely until allowed to proceed.
+        /// </summary>
+        public void WaitToProceed()
+        {
+            WaitToProceed(Timeout.Infinite);
+        }
+
+        // Throws an ObjectDisposedException if this object is disposed.
+        private void CheckDisposed()
+        {
+            if (_isDisposed)
+                throw new ObjectDisposedException("RateGate is already disposed");
+        }
+
+        /// <summary>
+        /// Releases unmanaged resources held by an instance of this class.
+        /// </summary>
+        public void Dispose()
+        {
+            Dispose(true);
+            GC.SuppressFinalize(this);
+        }
+
+        /// <summary>
+        /// Releases unmanaged resources held by an instance of this class.
+        /// </summary>
+        /// <param name="isDisposing">Whether this object is being disposed.</param>
+        protected virtual void Dispose(bool isDisposing)
+        {
+            if (!_isDisposed)
+            {
+                if (isDisposing)
+                {
+                    // The semaphore and timer both implement IDisposable and 
+                    // therefore must be disposed.
+                    _semaphore.Dispose();
+                    _exitTimer.Dispose();
+
+                    _isDisposed = true;
+                }
+            }
+        }
+    }
+}

src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs

@@ -3,7 +3,7 @@ using NzbDrone.Core.Messaging.Commands;
 
 namespace NzbDrone.Core.IndexerSearch
 {
-    public class EpisodeSearchCommand : Command
+    public class MissingEpisodeSearchCommand : Command
     {
         public List<int> EpisodeIds { get; set; }
 

src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs

@@ -1,22 +1,30 @@
-using NLog;
+using System;
+using System.Linq;
+using NLog;
+using NzbDrone.Common;
+using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Download;
 using NzbDrone.Core.Instrumentation.Extensions;
 using NzbDrone.Core.Messaging.Commands;
+using NzbDrone.Core.Tv;
 
 namespace NzbDrone.Core.IndexerSearch
 {
-    public class EpisodeSearchService : IExecute<EpisodeSearchCommand>
+    public class MissingEpisodeSearchService : IExecute<EpisodeSearchCommand>, IExecute<MissingEpisodeSearchCommand>
     {
         private readonly ISearchForNzb _nzbSearchService;
         private readonly IDownloadApprovedReports _downloadApprovedReports;
+        private readonly IEpisodeService _episodeService;
         private readonly Logger _logger;
 
-        public EpisodeSearchService(ISearchForNzb nzbSearchService,
+        public MissingEpisodeSearchService(ISearchForNzb nzbSearchService,
                                     IDownloadApprovedReports downloadApprovedReports,
+                                    IEpisodeService episodeService,
                                     Logger logger)
         {
             _nzbSearchService = nzbSearchService;
             _downloadApprovedReports = downloadApprovedReports;
+            _episodeService = episodeService;
             _logger = logger;
         }
 
@@ -30,5 +38,38 @@ namespace NzbDrone.Core.IndexerSearch
                 _logger.ProgressInfo("Episode search completed. {0} reports downloaded.", downloaded.Count);
             }
         }
+
+        public void Execute(MissingEpisodeSearchCommand message)
+        {
+            //TODO: Look at ways to make this more efficient (grouping by series/season)
+
+            var episodes =
+                _episodeService.EpisodesWithoutFiles(new PagingSpec<Episode>
+                                                     {
+                                                         Page = 1,
+                                                         PageSize = 100000,
+                                                         SortDirection = SortDirection.Ascending,
+                                                         SortKey = "Id"
+                                                     }).Records.ToList();
+
+            _logger.ProgressInfo("Performing missing search for {0} episodes", episodes.Count);
+            var downloadedCount = 0;
+
+            //Limit requests to indexers at 100 per minute
+            using (var rateGate = new RateGate(100, TimeSpan.FromSeconds(60)))
+            {
+                foreach (var episode in episodes)
+                {
+                    rateGate.WaitToProceed();
+                    var decisions = _nzbSearchService.EpisodeSearch(episode);
+                    var downloaded = _downloadApprovedReports.DownloadApproved(decisions);
+                    downloadedCount += downloaded.Count;
+
+                    _logger.ProgressInfo("Episode search completed. {0} reports downloaded.", downloaded.Count);
+                }
+            }
+
+            _logger.ProgressInfo("Completed missing search for {0} episodes. {1} reports downloaded.", episodes.Count, downloadedCount);
+        }
     }
 }

src/NzbDrone.Core/IndexerSearch/MissingEpisodeSearchCommand.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using NzbDrone.Core.Messaging.Commands;
+
+namespace NzbDrone.Core.IndexerSearch
+{
+    public class EpisodeSearchCommand : Command
+    {
+        public List<int> EpisodeIds { get; set; }
+
+        public override bool SendUpdatesToClient
+        {
+            get
+            {
+                return true;
+            }
+        }
+    }
+}

src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs

@@ -19,6 +19,7 @@ namespace NzbDrone.Core.IndexerSearch
     public interface ISearchForNzb
     {
         List<DownloadDecision> EpisodeSearch(int episodeId);
+        List<DownloadDecision> EpisodeSearch(Episode episode);
         List<DownloadDecision> SeasonSearch(int seriesId, int seasonNumber);
     }
 
@@ -52,6 +53,12 @@ namespace NzbDrone.Core.IndexerSearch
         public List<DownloadDecision> EpisodeSearch(int episodeId)
         {
             var episode = _episodeService.GetEpisode(episodeId);
+
+            return EpisodeSearch(episode);
+        }
+
+        public List<DownloadDecision> EpisodeSearch(Episode episode)
+        {
             var series = _seriesService.GetSeries(episode.SeriesId);
 
             if (series.SeriesType == SeriesTypes.Daily)
@@ -67,7 +74,7 @@ namespace NzbDrone.Core.IndexerSearch
             if (episode.SeasonNumber == 0)
             {
                 // search for special episodes in season 0 
-                return SearchSpecial(series, new List<Episode>{episode});
+                return SearchSpecial(series, new List<Episode> { episode });
             }
 
             return SearchSingle(series, episode);

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -288,6 +288,7 @@
     <Compile Include="Housekeeping\HousekeepingCommand.cs" />
     <Compile Include="Housekeeping\HousekeepingService.cs" />
     <Compile Include="Housekeeping\IHousekeepingTask.cs" />
+    <Compile Include="IndexerSearch\MissingEpisodeSearchCommand.cs" />
     <Compile Include="IndexerSearch\Definitions\SpecialEpisodeSearchCriteria.cs" />
     <Compile Include="IndexerSearch\SeriesSearchService.cs" />
     <Compile Include="IndexerSearch\SeriesSearchCommand.cs" />

src/UI/Wanted/Missing/MissingLayout.js

@@ -67,7 +67,7 @@ define(
                         name    : 'this',
                         label   : 'Episode Title',
                         sortable  : false,
-                        cell    : EpisodeTitleCell,
+                        cell    : EpisodeTitleCell
                     },
                     {
                         name    : 'airDateUtc',
@@ -121,6 +121,12 @@ define(
                             callback: this._searchSelected,
                             ownerContext: this
                         },
+                        {
+                            title: 'Search All Missing',
+                            icon : 'icon-search',
+                            callback: this._searchMissing,
+                            ownerContext: this
+                        },
                         {
                             title: 'Season Pass',
                             icon : 'icon-bookmark',
@@ -201,6 +207,16 @@ define(
                     name    : 'episodeSearch',
                     episodeIds: ids
                 });
+            },
+
+            _searchMissing: function () {
+                if (window.confirm('Are you sure you want to search for {0} missing episodes? '.format(this.collection.state.totalRecords) +
+                                   'One API request to each indexer will be used for each episode. ' +
+                                   'This cannot be stopped once started.')) {
+                    CommandController.Execute('missingEpisodeSearch', {
+                        name    : 'missingEpisodeSearch'
+                    });
+                }
             }
         });
     });