New: Search for all missing episodes

pull/3113/head
Mark McDowall 11 years ago
parent 1e42bf18e2
commit 89e0b41ebc

@ -105,6 +105,7 @@
<Compile Include="PathEqualityComparer.cs" />
<Compile Include="Processes\INzbDroneProcessProvider.cs" />
<Compile Include="Processes\ProcessOutput.cs" />
<Compile Include="RateGate.cs" />
<Compile Include="Serializer\IntConverter.cs" />
<Compile Include="Services.cs" />
<Compile Include="Extensions\StreamExtensions.cs" />

@ -0,0 +1,197 @@
/*
* Code from: http://www.jackleitch.net/2010/10/better-rate-limiting-with-dot-net/
*/
using System;
using System.Collections.Concurrent;
using System.Threading;
namespace NzbDrone.Common
{
/// <summary>
/// Used to control the rate of some occurrence per unit of time.
/// </summary>
/// <remarks>
/// <para>
/// To control the rate of an action using a <see cref="RateGate"/>,
/// code should simply call <see cref="WaitToProceed()"/> prior to
/// performing the action. <see cref="WaitToProceed()"/> will block
/// the current thread until the action is allowed based on the rate
/// limit.
/// </para>
/// <para>
/// This class is thread safe. A single <see cref="RateGate"/> instance
/// may be used to control the rate of an occurrence across multiple
/// threads.
/// </para>
/// </remarks>
public class RateGate : IDisposable
{
// Semaphore used to count and limit the number of occurrences per
// unit time.
private readonly SemaphoreSlim _semaphore;
// Times (in millisecond ticks) at which the semaphore should be exited.
private readonly ConcurrentQueue<int> _exitTimes;
// Timer used to trigger exiting the semaphore.
private readonly Timer _exitTimer;
// Whether this instance is disposed.
private bool _isDisposed;
/// <summary>
/// Number of occurrences allowed per unit of time.
/// </summary>
public int Occurrences { get; private set; }
/// <summary>
/// The length of the time unit, in milliseconds.
/// </summary>
public int TimeUnitMilliseconds { get; private set; }
/// <summary>
/// Initializes a <see cref="RateGate"/> with a rate of <paramref name="occurrences"/>
/// per <paramref name="timeUnit"/>.
/// </summary>
/// <param name="occurrences">Number of occurrences allowed per unit of time.</param>
/// <param name="timeUnit">Length of the time unit.</param>
/// <exception cref="ArgumentOutOfRangeException">
/// If <paramref name="occurrences"/> or <paramref name="timeUnit"/> is negative.
/// </exception>
public RateGate(int occurrences, TimeSpan timeUnit)
{
// Check the arguments.
if (occurrences <= 0)
throw new ArgumentOutOfRangeException("occurrences", "Number of occurrences must be a positive integer");
if (timeUnit != timeUnit.Duration())
throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be a positive span of time");
if (timeUnit >= TimeSpan.FromMilliseconds(UInt32.MaxValue))
throw new ArgumentOutOfRangeException("timeUnit", "Time unit must be less than 2^32 milliseconds");
Occurrences = occurrences;
TimeUnitMilliseconds = (int)timeUnit.TotalMilliseconds;
// Create the semaphore, with the number of occurrences as the maximum count.
_semaphore = new SemaphoreSlim(Occurrences, Occurrences);
// Create a queue to hold the semaphore exit times.
_exitTimes = new ConcurrentQueue<int>();
// Create a timer to exit the semaphore. Use the time unit as the original
// interval length because that's the earliest we will need to exit the semaphore.
_exitTimer = new Timer(ExitTimerCallback, null, TimeUnitMilliseconds, -1);
}
// Callback for the exit timer that exits the semaphore based on exit times
// in the queue and then sets the timer for the nextexit time.
private void ExitTimerCallback(object state)
{
// While there are exit times that are passed due still in the queue,
// exit the semaphore and dequeue the exit time.
int exitTime;
while (_exitTimes.TryPeek(out exitTime)
&& unchecked(exitTime - Environment.TickCount) <= 0)
{
_semaphore.Release();
_exitTimes.TryDequeue(out exitTime);
}
// Try to get the next exit time from the queue and compute
// the time until the next check should take place. If the
// queue is empty, then no exit times will occur until at least
// one time unit has passed.
int timeUntilNextCheck;
if (_exitTimes.TryPeek(out exitTime))
timeUntilNextCheck = unchecked(exitTime - Environment.TickCount);
else
timeUntilNextCheck = TimeUnitMilliseconds;
// Set the timer.
_exitTimer.Change(timeUntilNextCheck, -1);
}
/// <summary>
/// Blocks the current thread until allowed to proceed or until the
/// specified timeout elapses.
/// </summary>
/// <param name="millisecondsTimeout">Number of milliseconds to wait, or -1 to wait indefinitely.</param>
/// <returns>true if the thread is allowed to proceed, or false if timed out</returns>
public bool WaitToProceed(int millisecondsTimeout)
{
// Check the arguments.
if (millisecondsTimeout < -1)
throw new ArgumentOutOfRangeException("millisecondsTimeout");
CheckDisposed();
// Block until we can enter the semaphore or until the timeout expires.
var entered = _semaphore.Wait(millisecondsTimeout);
// If we entered the semaphore, compute the corresponding exit time
// and add it to the queue.
if (entered)
{
var timeToExit = unchecked(Environment.TickCount + TimeUnitMilliseconds);
_exitTimes.Enqueue(timeToExit);
}
return entered;
}
/// <summary>
/// Blocks the current thread until allowed to proceed or until the
/// specified timeout elapses.
/// </summary>
/// <param name="timeout"></param>
/// <returns>true if the thread is allowed to proceed, or false if timed out</returns>
public bool WaitToProceed(TimeSpan timeout)
{
return WaitToProceed((int)timeout.TotalMilliseconds);
}
/// <summary>
/// Blocks the current thread indefinitely until allowed to proceed.
/// </summary>
public void WaitToProceed()
{
WaitToProceed(Timeout.Infinite);
}
// Throws an ObjectDisposedException if this object is disposed.
private void CheckDisposed()
{
if (_isDisposed)
throw new ObjectDisposedException("RateGate is already disposed");
}
/// <summary>
/// Releases unmanaged resources held by an instance of this class.
/// </summary>
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
/// <summary>
/// Releases unmanaged resources held by an instance of this class.
/// </summary>
/// <param name="isDisposing">Whether this object is being disposed.</param>
protected virtual void Dispose(bool isDisposing)
{
if (!_isDisposed)
{
if (isDisposing)
{
// The semaphore and timer both implement IDisposable and
// therefore must be disposed.
_semaphore.Dispose();
_exitTimer.Dispose();
_isDisposed = true;
}
}
}
}
}

@ -3,7 +3,7 @@ using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.IndexerSearch
{
public class EpisodeSearchCommand : Command
public class MissingEpisodeSearchCommand : Command
{
public List<int> EpisodeIds { get; set; }

@ -1,22 +1,30 @@
using NLog;
using System;
using System.Linq;
using NLog;
using NzbDrone.Common;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.Download;
using NzbDrone.Core.Instrumentation.Extensions;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.IndexerSearch
{
public class EpisodeSearchService : IExecute<EpisodeSearchCommand>
public class MissingEpisodeSearchService : IExecute<EpisodeSearchCommand>, IExecute<MissingEpisodeSearchCommand>
{
private readonly ISearchForNzb _nzbSearchService;
private readonly IDownloadApprovedReports _downloadApprovedReports;
private readonly IEpisodeService _episodeService;
private readonly Logger _logger;
public EpisodeSearchService(ISearchForNzb nzbSearchService,
public MissingEpisodeSearchService(ISearchForNzb nzbSearchService,
IDownloadApprovedReports downloadApprovedReports,
IEpisodeService episodeService,
Logger logger)
{
_nzbSearchService = nzbSearchService;
_downloadApprovedReports = downloadApprovedReports;
_episodeService = episodeService;
_logger = logger;
}
@ -30,5 +38,38 @@ namespace NzbDrone.Core.IndexerSearch
_logger.ProgressInfo("Episode search completed. {0} reports downloaded.", downloaded.Count);
}
}
public void Execute(MissingEpisodeSearchCommand message)
{
//TODO: Look at ways to make this more efficient (grouping by series/season)
var episodes =
_episodeService.EpisodesWithoutFiles(new PagingSpec<Episode>
{
Page = 1,
PageSize = 100000,
SortDirection = SortDirection.Ascending,
SortKey = "Id"
}).Records.ToList();
_logger.ProgressInfo("Performing missing search for {0} episodes", episodes.Count);
var downloadedCount = 0;
//Limit requests to indexers at 100 per minute
using (var rateGate = new RateGate(100, TimeSpan.FromSeconds(60)))
{
foreach (var episode in episodes)
{
rateGate.WaitToProceed();
var decisions = _nzbSearchService.EpisodeSearch(episode);
var downloaded = _downloadApprovedReports.DownloadApproved(decisions);
downloadedCount += downloaded.Count;
_logger.ProgressInfo("Episode search completed. {0} reports downloaded.", downloaded.Count);
}
}
_logger.ProgressInfo("Completed missing search for {0} episodes. {1} reports downloaded.", episodes.Count, downloadedCount);
}
}
}

@ -0,0 +1,18 @@
using System.Collections.Generic;
using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.IndexerSearch
{
public class EpisodeSearchCommand : Command
{
public List<int> EpisodeIds { get; set; }
public override bool SendUpdatesToClient
{
get
{
return true;
}
}
}
}

@ -19,6 +19,7 @@ namespace NzbDrone.Core.IndexerSearch
public interface ISearchForNzb
{
List<DownloadDecision> EpisodeSearch(int episodeId);
List<DownloadDecision> EpisodeSearch(Episode episode);
List<DownloadDecision> SeasonSearch(int seriesId, int seasonNumber);
}
@ -52,6 +53,12 @@ namespace NzbDrone.Core.IndexerSearch
public List<DownloadDecision> EpisodeSearch(int episodeId)
{
var episode = _episodeService.GetEpisode(episodeId);
return EpisodeSearch(episode);
}
public List<DownloadDecision> EpisodeSearch(Episode episode)
{
var series = _seriesService.GetSeries(episode.SeriesId);
if (series.SeriesType == SeriesTypes.Daily)
@ -67,7 +74,7 @@ namespace NzbDrone.Core.IndexerSearch
if (episode.SeasonNumber == 0)
{
// search for special episodes in season 0
return SearchSpecial(series, new List<Episode>{episode});
return SearchSpecial(series, new List<Episode> { episode });
}
return SearchSingle(series, episode);

@ -288,6 +288,7 @@
<Compile Include="Housekeeping\HousekeepingCommand.cs" />
<Compile Include="Housekeeping\HousekeepingService.cs" />
<Compile Include="Housekeeping\IHousekeepingTask.cs" />
<Compile Include="IndexerSearch\MissingEpisodeSearchCommand.cs" />
<Compile Include="IndexerSearch\Definitions\SpecialEpisodeSearchCriteria.cs" />
<Compile Include="IndexerSearch\SeriesSearchService.cs" />
<Compile Include="IndexerSearch\SeriesSearchCommand.cs" />

@ -67,7 +67,7 @@ define(
name : 'this',
label : 'Episode Title',
sortable : false,
cell : EpisodeTitleCell,
cell : EpisodeTitleCell
},
{
name : 'airDateUtc',
@ -121,6 +121,12 @@ define(
callback: this._searchSelected,
ownerContext: this
},
{
title: 'Search All Missing',
icon : 'icon-search',
callback: this._searchMissing,
ownerContext: this
},
{
title: 'Season Pass',
icon : 'icon-bookmark',
@ -201,6 +207,16 @@ define(
name : 'episodeSearch',
episodeIds: ids
});
},
_searchMissing: function () {
if (window.confirm('Are you sure you want to search for {0} missing episodes? '.format(this.collection.state.totalRecords) +
'One API request to each indexer will be used for each episode. ' +
'This cannot be stopped once started.')) {
CommandController.Execute('missingEpisodeSearch', {
name : 'missingEpisodeSearch'
});
}
}
});
});

Loading…
Cancel
Save