diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj
index 9d1e80e38..b6666bda7 100644
--- a/src/NzbDrone.Common/NzbDrone.Common.csproj
+++ b/src/NzbDrone.Common/NzbDrone.Common.csproj
@@ -105,6 +105,7 @@
+
diff --git a/src/NzbDrone.Common/RateGate.cs b/src/NzbDrone.Common/RateGate.cs
new file mode 100644
index 000000000..598bf30e1
--- /dev/null
+++ b/src/NzbDrone.Common/RateGate.cs
@@ -0,0 +1,197 @@
+/*
+ * Code from: http://www.jackleitch.net/2010/10/better-rate-limiting-with-dot-net/
+ */
+
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+
+namespace NzbDrone.Common
+{
+ ///
+ /// Used to control the rate of some occurrence per unit of time.
+ ///
+ ///
+ ///
+ /// To control the rate of an action using a ,
+ /// code should simply call prior to
+ /// performing the action. will block
+ /// the current thread until the action is allowed based on the rate
+ /// limit.
+ ///
+ ///
+ /// This class is thread safe. A single instance
+ /// may be used to control the rate of an occurrence across multiple
+ /// threads.
+ ///
+ ///
+ public class RateGate : IDisposable
+ {
+ // Semaphore used to count and limit the number of occurrences per
+ // unit time.
+ private readonly SemaphoreSlim _semaphore;
+
+ // Times (in millisecond ticks) at which the semaphore should be exited.
+ private readonly ConcurrentQueue _exitTimes;
+
+ // Timer used to trigger exiting the semaphore.
+ private readonly Timer _exitTimer;
+
+ // Whether this instance is disposed.
+ private bool _isDisposed;
+
+ ///
+ /// Number of occurrences allowed per unit of time.
+ ///
+ public int Occurrences { get; private set; }
+
+ ///
+ /// The length of the time unit, in milliseconds.
+ ///
+ public int TimeUnitMilliseconds { get; private set; }
+
+ ///
+ /// Initializes a with a rate of
+ /// per .
+ ///
+ /// Number of occurrences allowed per unit of time.
+ /// Length of the time unit.
+ ///
+ /// If or is negative.
+ ///
+ public RateGate(int occurrences, TimeSpan timeUnit)
+ {
+ // Check the arguments.
+ if (occurrences <= 0)
+ throw new ArgumentOutOfRangeException("occurrences", "Number of occurrences must be a positive integer");
+ if (timeUnit != timeUnit.Duration())
+ throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be a positive span of time");
+ if (timeUnit >= TimeSpan.FromMilliseconds(UInt32.MaxValue))
+ throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be less than 2^32 milliseconds");
+
+ Occurrences = occurrences;
+ TimeUnitMilliseconds = (int)timeUnit.TotalMilliseconds;
+
+ // Create the semaphore, with the number of occurrences as the maximum count.
+ _semaphore = new SemaphoreSlim(Occurrences, Occurrences);
+
+ // Create a queue to hold the semaphore exit times.
+ _exitTimes = new ConcurrentQueue();
+
+ // Create a timer to exit the semaphore. Use the time unit as the original
+ // interval length because that's the earliest we will need to exit the semaphore.
+ _exitTimer = new Timer(ExitTimerCallback, null, TimeUnitMilliseconds, -1);
+ }
+
+ // Callback for the exit timer that exits the semaphore based on exit times
+ // in the queue and then sets the timer for the nextexit time.
+ private void ExitTimerCallback(object state)
+ {
+ // While there are exit times that are passed due still in the queue,
+ // exit the semaphore and dequeue the exit time.
+ int exitTime;
+ while (_exitTimes.TryPeek(out exitTime)
+ && unchecked(exitTime - Environment.TickCount) <= 0)
+ {
+ _semaphore.Release();
+ _exitTimes.TryDequeue(out exitTime);
+ }
+
+ // Try to get the next exit time from the queue and compute
+ // the time until the next check should take place. If the
+ // queue is empty, then no exit times will occur until at least
+ // one time unit has passed.
+ int timeUntilNextCheck;
+ if (_exitTimes.TryPeek(out exitTime))
+ timeUntilNextCheck = unchecked(exitTime - Environment.TickCount);
+ else
+ timeUntilNextCheck = TimeUnitMilliseconds;
+
+ // Set the timer.
+ _exitTimer.Change(timeUntilNextCheck, -1);
+ }
+
+ ///
+ /// Blocks the current thread until allowed to proceed or until the
+ /// specified timeout elapses.
+ ///
+ /// Number of milliseconds to wait, or -1 to wait indefinitely.
+ /// true if the thread is allowed to proceed, or false if timed out
+ public bool WaitToProceed(int millisecondsTimeout)
+ {
+ // Check the arguments.
+ if (millisecondsTimeout < -1)
+ throw new ArgumentOutOfRangeException("millisecondsTimeout");
+
+ CheckDisposed();
+
+ // Block until we can enter the semaphore or until the timeout expires.
+ var entered = _semaphore.Wait(millisecondsTimeout);
+
+ // If we entered the semaphore, compute the corresponding exit time
+ // and add it to the queue.
+ if (entered)
+ {
+ var timeToExit = unchecked(Environment.TickCount + TimeUnitMilliseconds);
+ _exitTimes.Enqueue(timeToExit);
+ }
+
+ return entered;
+ }
+
+ ///
+ /// Blocks the current thread until allowed to proceed or until the
+ /// specified timeout elapses.
+ ///
+ ///
+ /// true if the thread is allowed to proceed, or false if timed out
+ public bool WaitToProceed(TimeSpan timeout)
+ {
+ return WaitToProceed((int)timeout.TotalMilliseconds);
+ }
+
+ ///
+ /// Blocks the current thread indefinitely until allowed to proceed.
+ ///
+ public void WaitToProceed()
+ {
+ WaitToProceed(Timeout.Infinite);
+ }
+
+ // Throws an ObjectDisposedException if this object is disposed.
+ private void CheckDisposed()
+ {
+ if (_isDisposed)
+ throw new ObjectDisposedException("RateGate is already disposed");
+ }
+
+ ///
+ /// Releases unmanaged resources held by an instance of this class.
+ ///
+ public void Dispose()
+ {
+ Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ ///
+ /// Releases unmanaged resources held by an instance of this class.
+ ///
+ /// Whether this object is being disposed.
+ protected virtual void Dispose(bool isDisposing)
+ {
+ if (!_isDisposed)
+ {
+ if (isDisposing)
+ {
+ // The semaphore and timer both implement IDisposable and
+ // therefore must be disposed.
+ _semaphore.Dispose();
+ _exitTimer.Dispose();
+
+ _isDisposed = true;
+ }
+ }
+ }
+ }
+}
diff --git a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs
index 27861c869..f595f2a52 100644
--- a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs
+++ b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchCommand.cs
@@ -3,7 +3,7 @@ using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.IndexerSearch
{
- public class EpisodeSearchCommand : Command
+ public class MissingEpisodeSearchCommand : Command
{
public List EpisodeIds { get; set; }
diff --git a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs
index 9aec6bbc3..0ae8e05d0 100644
--- a/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs
+++ b/src/NzbDrone.Core/IndexerSearch/EpisodeSearchService.cs
@@ -1,22 +1,30 @@
-using NLog;
+using System;
+using System.Linq;
+using NLog;
+using NzbDrone.Common;
+using NzbDrone.Core.Datastore;
using NzbDrone.Core.Download;
using NzbDrone.Core.Instrumentation.Extensions;
using NzbDrone.Core.Messaging.Commands;
+using NzbDrone.Core.Tv;
namespace NzbDrone.Core.IndexerSearch
{
- public class EpisodeSearchService : IExecute
+ public class MissingEpisodeSearchService : IExecute, IExecute
{
private readonly ISearchForNzb _nzbSearchService;
private readonly IDownloadApprovedReports _downloadApprovedReports;
+ private readonly IEpisodeService _episodeService;
private readonly Logger _logger;
- public EpisodeSearchService(ISearchForNzb nzbSearchService,
+ public MissingEpisodeSearchService(ISearchForNzb nzbSearchService,
IDownloadApprovedReports downloadApprovedReports,
+ IEpisodeService episodeService,
Logger logger)
{
_nzbSearchService = nzbSearchService;
_downloadApprovedReports = downloadApprovedReports;
+ _episodeService = episodeService;
_logger = logger;
}
@@ -30,5 +38,38 @@ namespace NzbDrone.Core.IndexerSearch
_logger.ProgressInfo("Episode search completed. {0} reports downloaded.", downloaded.Count);
}
}
+
+ public void Execute(MissingEpisodeSearchCommand message)
+ {
+ //TODO: Look at ways to make this more efficient (grouping by series/season)
+
+ var episodes =
+ _episodeService.EpisodesWithoutFiles(new PagingSpec
+ {
+ Page = 1,
+ PageSize = 100000,
+ SortDirection = SortDirection.Ascending,
+ SortKey = "Id"
+ }).Records.ToList();
+
+ _logger.ProgressInfo("Performing missing search for {0} episodes", episodes.Count);
+ var downloadedCount = 0;
+
+ //Limit requests to indexers at 100 per minute
+ using (var rateGate = new RateGate(100, TimeSpan.FromSeconds(60)))
+ {
+ foreach (var episode in episodes)
+ {
+ rateGate.WaitToProceed();
+ var decisions = _nzbSearchService.EpisodeSearch(episode);
+ var downloaded = _downloadApprovedReports.DownloadApproved(decisions);
+ downloadedCount += downloaded.Count;
+
+ _logger.ProgressInfo("Episode search completed. {0} reports downloaded.", downloaded.Count);
+ }
+ }
+
+ _logger.ProgressInfo("Completed missing search for {0} episodes. {1} reports downloaded.", episodes.Count, downloadedCount);
+ }
}
}
diff --git a/src/NzbDrone.Core/IndexerSearch/MissingEpisodeSearchCommand.cs b/src/NzbDrone.Core/IndexerSearch/MissingEpisodeSearchCommand.cs
new file mode 100644
index 000000000..27861c869
--- /dev/null
+++ b/src/NzbDrone.Core/IndexerSearch/MissingEpisodeSearchCommand.cs
@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using NzbDrone.Core.Messaging.Commands;
+
+namespace NzbDrone.Core.IndexerSearch
+{
+ public class EpisodeSearchCommand : Command
+ {
+ public List EpisodeIds { get; set; }
+
+ public override bool SendUpdatesToClient
+ {
+ get
+ {
+ return true;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs
index 9af07cf51..b2b3ddb1d 100644
--- a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs
+++ b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs
@@ -19,6 +19,7 @@ namespace NzbDrone.Core.IndexerSearch
public interface ISearchForNzb
{
List EpisodeSearch(int episodeId);
+ List EpisodeSearch(Episode episode);
List SeasonSearch(int seriesId, int seasonNumber);
}
@@ -52,6 +53,12 @@ namespace NzbDrone.Core.IndexerSearch
public List EpisodeSearch(int episodeId)
{
var episode = _episodeService.GetEpisode(episodeId);
+
+ return EpisodeSearch(episode);
+ }
+
+ public List EpisodeSearch(Episode episode)
+ {
var series = _seriesService.GetSeries(episode.SeriesId);
if (series.SeriesType == SeriesTypes.Daily)
@@ -67,7 +74,7 @@ namespace NzbDrone.Core.IndexerSearch
if (episode.SeasonNumber == 0)
{
// search for special episodes in season 0
- return SearchSpecial(series, new List{episode});
+ return SearchSpecial(series, new List { episode });
}
return SearchSingle(series, episode);
diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj
index 892cea786..3bf7c995c 100644
--- a/src/NzbDrone.Core/NzbDrone.Core.csproj
+++ b/src/NzbDrone.Core/NzbDrone.Core.csproj
@@ -288,6 +288,7 @@
+
diff --git a/src/UI/Wanted/Missing/MissingLayout.js b/src/UI/Wanted/Missing/MissingLayout.js
index 5b52251a1..cee0b33b4 100644
--- a/src/UI/Wanted/Missing/MissingLayout.js
+++ b/src/UI/Wanted/Missing/MissingLayout.js
@@ -67,7 +67,7 @@ define(
name : 'this',
label : 'Episode Title',
sortable : false,
- cell : EpisodeTitleCell,
+ cell : EpisodeTitleCell
},
{
name : 'airDateUtc',
@@ -121,6 +121,12 @@ define(
callback: this._searchSelected,
ownerContext: this
},
+ {
+ title: 'Search All Missing',
+ icon : 'icon-search',
+ callback: this._searchMissing,
+ ownerContext: this
+ },
{
title: 'Season Pass',
icon : 'icon-bookmark',
@@ -201,6 +207,16 @@ define(
name : 'episodeSearch',
episodeIds: ids
});
+ },
+
+ _searchMissing: function () {
+ if (window.confirm('Are you sure you want to search for {0} missing episodes? '.format(this.collection.state.totalRecords) +
+ 'One API request to each indexer will be used for each episode. ' +
+ 'This cannot be stopped once started.')) {
+ CommandController.Execute('missingEpisodeSearch', {
+ name : 'missingEpisodeSearch'
+ });
+ }
}
});
});