From 89e0b41ebcf06539886b570b9c429065dd84978b Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Thu, 20 Mar 2014 16:41:28 -0700 Subject: [PATCH] New: Search for all missing episodes --- src/NzbDrone.Common/NzbDrone.Common.csproj | 1 + src/NzbDrone.Common/RateGate.cs | 197 ++++++++++++++++++ .../IndexerSearch/EpisodeSearchCommand.cs | 2 +- .../IndexerSearch/EpisodeSearchService.cs | 47 ++++- .../MissingEpisodeSearchCommand.cs | 18 ++ .../IndexerSearch/NzbSearchService.cs | 9 +- src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + src/UI/Wanted/Missing/MissingLayout.js | 18 +- 8 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 src/NzbDrone.Common/RateGate.cs create mode 100644 src/NzbDrone.Core/IndexerSearch/MissingEpisodeSearchCommand.cs diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index 9d1e80e38..b6666bda7 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -105,6 +105,7 @@ + diff --git a/src/NzbDrone.Common/RateGate.cs b/src/NzbDrone.Common/RateGate.cs new file mode 100644 index 000000000..598bf30e1 --- /dev/null +++ b/src/NzbDrone.Common/RateGate.cs @@ -0,0 +1,197 @@ +/* + * Code from: http://www.jackleitch.net/2010/10/better-rate-limiting-with-dot-net/ + */ + +using System; +using System.Collections.Concurrent; +using System.Threading; + +namespace NzbDrone.Common +{ + /// + /// Used to control the rate of some occurrence per unit of time. + /// + /// + /// + /// To control the rate of an action using a , + /// code should simply call prior to + /// performing the action. will block + /// the current thread until the action is allowed based on the rate + /// limit. + /// + /// + /// This class is thread safe. A single instance + /// may be used to control the rate of an occurrence across multiple + /// threads. + /// + /// + public class RateGate : IDisposable + { + // Semaphore used to count and limit the number of occurrences per + // unit time. + private readonly SemaphoreSlim _semaphore; + + // Times (in millisecond ticks) at which the semaphore should be exited. + private readonly ConcurrentQueue _exitTimes; + + // Timer used to trigger exiting the semaphore. + private readonly Timer _exitTimer; + + // Whether this instance is disposed. + private bool _isDisposed; + + /// + /// Number of occurrences allowed per unit of time. + /// + public int Occurrences { get; private set; } + + /// + /// The length of the time unit, in milliseconds. + /// + public int TimeUnitMilliseconds { get; private set; } + + /// + /// Initializes a with a rate of + /// per . + /// + /// Number of occurrences allowed per unit of time. + /// Length of the time unit. + /// + /// If or is negative. + /// + public RateGate(int occurrences, TimeSpan timeUnit) + { + // Check the arguments. + if (occurrences <= 0) + throw new ArgumentOutOfRangeException("occurrences", "Number of occurrences must be a positive integer"); + if (timeUnit != timeUnit.Duration()) + throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be a positive span of time"); + if (timeUnit >= TimeSpan.FromMilliseconds(UInt32.MaxValue)) + throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be less than 2^32 milliseconds"); + + Occurrences = occurrences; + TimeUnitMilliseconds = (int)timeUnit.TotalMilliseconds; + + // Create the semaphore, with the number of occurrences as the maximum count. + _semaphore = new SemaphoreSlim(Occurrences, Occurrences); + + // Create a queue to hold the semaphore exit times. + _exitTimes = new ConcurrentQueue(); + + // Create a timer to exit the semaphore. Use the time unit as the original + // interval length because that's the earliest we will need to exit the semaphore. + _exitTimer = new Timer(ExitTimerCallback, null, TimeUnitMilliseconds, -1); + } + + // Callback for the exit timer that exits the semaphore based on exit times + // in the queue and then sets the timer for the nextexit time. + private void ExitTimerCallback(object state) + { + // While there are exit times that are passed due still in the queue, + // exit the semaphore and dequeue the exit time. + int exitTime; + while (_exitTimes.TryPeek(out exitTime) + && unchecked(exitTime - Environment.TickCount) <= 0) + { + _semaphore.Release(); + _exitTimes.TryDequeue(out exitTime); + } + + // Try to get the next exit time from the queue and compute + // the time until the next check should take place. If the + // queue is empty, then no exit times will occur until at least + // one time unit has passed. + int timeUntilNextCheck; + if (_exitTimes.TryPeek(out exitTime)) + timeUntilNextCheck = unchecked(exitTime - Environment.TickCount); + else + timeUntilNextCheck = TimeUnitMilliseconds; + + // Set the timer. + _exitTimer.Change(timeUntilNextCheck, -1); + } + + /// + /// Blocks the current thread until allowed to proceed or until the + /// specified timeout elapses. + /// + /// Number of milliseconds to wait, or -1 to wait indefinitely. + /// true if the thread is allowed to proceed, or false if timed out + public bool WaitToProceed(int millisecondsTimeout) + { + // Check the arguments. + if (millisecondsTimeout < -1) + throw new ArgumentOutOfRangeException("millisecondsTimeout"); + + CheckDisposed(); + + // Block until we can enter the semaphore or until the timeout expires. + var entered = _semaphore.Wait(millisecondsTimeout); + + // If we entered the semaphore, compute the corresponding exit time + // and add it to the queue. + if (entered) + { + var timeToExit = unchecked(Environment.TickCount + TimeUnitMilliseconds); + _exitTimes.Enqueue(timeToExit); + } + + return entered; + } + + /// + /// Blocks the current thread until allowed to proceed or until the + /// specified timeout elapses. + /// + /// + /// true if the thread is allowed to proceed, or false if timed out + public bool WaitToProceed(TimeSpan timeout) + { + return WaitToProceed((int)timeout.TotalMilliseconds); + } + + /// + /// Blocks the current thread indefinitely until allowed to proceed. + /// + public void WaitToProceed() + { + WaitToProceed(Timeout.Infinite); + } + + // Throws an ObjectDisposedException if this object is disposed. + private void CheckDisposed() + { + if (_isDisposed) + throw new ObjectDisposedException("RateGate is already disposed"); + } + + /// + /// Releases unmanaged resources held by an instance of this class. + /// + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); + } + + /// + /// Releases unmanaged resources held by an instance of this class. + /// + /// Whether this object is being disposed. + protected virtual void Dispose(bool isDisposing) + { + if (!_isDisposed) + { + if (isDisposing) + { + // The semaphore and timer both implement IDisposable and + // therefore must be disposed. + _semaphore.Dispose(); + _exitTimer.Dispose(); + + _isDisposed = true; + } + } + } + } +} diff --git a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs index 27861c869..f595f2a52 100644 --- a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs +++ b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs @@ -3,7 +3,7 @@ using NzbDrone.Core.Messaging.Commands; namespace NzbDrone.Core.IndexerSearch { - public class EpisodeSearchCommand : Command + public class MissingEpisodeSearchCommand : Command { public List EpisodeIds { get; set; } diff --git a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs index 9aec6bbc3..0ae8e05d0 100644 --- a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs @@ -1,22 +1,30 @@ -using NLog; +using System; +using System.Linq; +using NLog; +using NzbDrone.Common; +using NzbDrone.Core.Datastore; using NzbDrone.Core.Download; using NzbDrone.Core.Instrumentation.Extensions; using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.Tv; namespace NzbDrone.Core.IndexerSearch { - public class EpisodeSearchService : IExecute + public class MissingEpisodeSearchService : IExecute, IExecute { private readonly ISearchForNzb _nzbSearchService; private readonly IDownloadApprovedReports _downloadApprovedReports; + private readonly IEpisodeService _episodeService; private readonly Logger _logger; - public EpisodeSearchService(ISearchForNzb nzbSearchService, + public MissingEpisodeSearchService(ISearchForNzb nzbSearchService, IDownloadApprovedReports downloadApprovedReports, + IEpisodeService episodeService, Logger logger) { _nzbSearchService = nzbSearchService; _downloadApprovedReports = downloadApprovedReports; + _episodeService = episodeService; _logger = logger; } @@ -30,5 +38,38 @@ namespace NzbDrone.Core.IndexerSearch _logger.ProgressInfo("Episode search completed. {0} reports downloaded.", downloaded.Count); } } + + public void Execute(MissingEpisodeSearchCommand message) + { + //TODO: Look at ways to make this more efficient (grouping by series/season) + + var episodes = + _episodeService.EpisodesWithoutFiles(new PagingSpec + { + Page = 1, + PageSize = 100000, + SortDirection = SortDirection.Ascending, + SortKey = "Id" + }).Records.ToList(); + + _logger.ProgressInfo("Performing missing search for {0} episodes", episodes.Count); + var downloadedCount = 0; + + //Limit requests to indexers at 100 per minute + using (var rateGate = new RateGate(100, TimeSpan.FromSeconds(60))) + { + foreach (var episode in episodes) + { + rateGate.WaitToProceed(); + var decisions = _nzbSearchService.EpisodeSearch(episode); + var downloaded = _downloadApprovedReports.DownloadApproved(decisions); + downloadedCount += downloaded.Count; + + _logger.ProgressInfo("Episode search completed. {0} reports downloaded.", downloaded.Count); + } + } + + _logger.ProgressInfo("Completed missing search for {0} episodes. {1} reports downloaded.", episodes.Count, downloadedCount); + } } } diff --git a/src/NzbDrone.Core/IndexerSearch/MissingEpisodeSearchCommand.cs b/src/NzbDrone.Core/IndexerSearch/MissingEpisodeSearchCommand.cs new file mode 100644 index 000000000..27861c869 --- /dev/null +++ b/src/NzbDrone.Core/IndexerSearch/MissingEpisodeSearchCommand.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.IndexerSearch +{ + public class EpisodeSearchCommand : Command + { + public List EpisodeIds { get; set; } + + public override bool SendUpdatesToClient + { + get + { + return true; + } + } + } +} \ No newline at end of file diff --git a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs index 9af07cf51..b2b3ddb1d 100644 --- a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs @@ -19,6 +19,7 @@ namespace NzbDrone.Core.IndexerSearch public interface ISearchForNzb { List EpisodeSearch(int episodeId); + List EpisodeSearch(Episode episode); List SeasonSearch(int seriesId, int seasonNumber); } @@ -52,6 +53,12 @@ namespace NzbDrone.Core.IndexerSearch public List EpisodeSearch(int episodeId) { var episode = _episodeService.GetEpisode(episodeId); + + return EpisodeSearch(episode); + } + + public List EpisodeSearch(Episode episode) + { var series = _seriesService.GetSeries(episode.SeriesId); if (series.SeriesType == SeriesTypes.Daily) @@ -67,7 +74,7 @@ namespace NzbDrone.Core.IndexerSearch if (episode.SeasonNumber == 0) { // search for special episodes in season 0 - return SearchSpecial(series, new List{episode}); + return SearchSpecial(series, new List { episode }); } return SearchSingle(series, episode); diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 892cea786..3bf7c995c 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -288,6 +288,7 @@ + diff --git a/src/UI/Wanted/Missing/MissingLayout.js b/src/UI/Wanted/Missing/MissingLayout.js index 5b52251a1..cee0b33b4 100644 --- a/src/UI/Wanted/Missing/MissingLayout.js +++ b/src/UI/Wanted/Missing/MissingLayout.js @@ -67,7 +67,7 @@ define( name : 'this', label : 'Episode Title', sortable : false, - cell : EpisodeTitleCell, + cell : EpisodeTitleCell }, { name : 'airDateUtc', @@ -121,6 +121,12 @@ define( callback: this._searchSelected, ownerContext: this }, + { + title: 'Search All Missing', + icon : 'icon-search', + callback: this._searchMissing, + ownerContext: this + }, { title: 'Season Pass', icon : 'icon-bookmark', @@ -201,6 +207,16 @@ define( name : 'episodeSearch', episodeIds: ids }); + }, + + _searchMissing: function () { + if (window.confirm('Are you sure you want to search for {0} missing episodes? '.format(this.collection.state.totalRecords) + + 'One API request to each indexer will be used for each episode. ' + + 'This cannot be stopped once started.')) { + CommandController.Execute('missingEpisodeSearch', { + name : 'missingEpisodeSearch' + }); + } } }); });