Fixed: Don't purge xem scene mapping cache when new series gets added.

pull/1174/head v2.0.0.3953
Taloth Saldono 9 years ago
parent 03e2adc332
commit 7818f0c59b

@ -0,0 +1,102 @@
using System;
using System.Collections.Generic;
using System.Threading;
using FluentAssertions;
using NUnit.Framework;
using NzbDrone.Common.Cache;
namespace NzbDrone.Common.Test.CacheTests
{
[TestFixture]
public class CachedDictionaryFixture
{
private CachedDictionary<string> _cachedString;
private DictionaryWorker _worker;
[SetUp]
public void SetUp()
{
_worker = new DictionaryWorker();
_cachedString = new CachedDictionary<string>(_worker.GetDict, TimeSpan.FromMilliseconds(100));
}
[Test]
public void should_not_fetch_on_create()
{
_worker.HitCount.Should().Be(0);
}
[Test]
public void should_fetch_on_first_call()
{
var result = _cachedString.Get("Hi");
_worker.HitCount.Should().Be(1);
result.Should().Be("Value");
}
[Test]
public void should_fetch_once()
{
var result1 = _cachedString.Get("Hi");
var result2 = _cachedString.Get("HitCount");
_worker.HitCount.Should().Be(1);
}
[Test]
public void should_auto_refresh_after_lifetime()
{
var result1 = _cachedString.Get("Hi");
Thread.Sleep(200);
var result2 = _cachedString.Get("Hi");
_worker.HitCount.Should().Be(2);
}
[Test]
public void should_refresh_early_if_requested()
{
var result1 = _cachedString.Get("Hi");
Thread.Sleep(10);
_cachedString.RefreshIfExpired(TimeSpan.FromMilliseconds(1));
var result2 = _cachedString.Get("Hi");
_worker.HitCount.Should().Be(2);
}
[Test]
public void should_not_refresh_early_if_not_expired()
{
var result1 = _cachedString.Get("Hi");
_cachedString.RefreshIfExpired(TimeSpan.FromMilliseconds(50));
var result2 = _cachedString.Get("Hi");
_worker.HitCount.Should().Be(1);
}
}
public class DictionaryWorker
{
public int HitCount { get; private set; }
public Dictionary<string, string> GetDict()
{
HitCount++;
var result = new Dictionary<string, string>();
result["Hi"] = "Value";
result["HitCount"] = "Hit count is " + HitCount;
return result;
}
}
}

@ -66,6 +66,7 @@
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="CacheTests\CachedDictionaryFixture.cs" />
<Compile Include="CacheTests\CachedFixture.cs" />
<Compile Include="CacheTests\CachedManagerFixture.cs" />
<Compile Include="ConfigFileProviderTest.cs" />

@ -6,8 +6,9 @@ namespace NzbDrone.Common.Cache
{
public interface ICacheManager
{
ICached<T> GetCache<T>(Type host, string name);
ICached<T> GetCache<T>(Type host);
ICached<T> GetCache<T>(Type host, string name);
ICachedDictionary<T> GetCacheDictionary<T>(Type host, string name, Func<IDictionary<string, T>> fetchFunc = null, TimeSpan? lifeTime = null);
void Clear();
ICollection<ICached> Caches { get; }
}
@ -22,12 +23,6 @@ namespace NzbDrone.Common.Cache
}
public ICached<T> GetCache<T>(Type host)
{
Ensure.That(host, () => host).IsNotNull();
return GetCache<T>(host, host.FullName);
}
public void Clear()
{
_cache.Clear();
@ -35,6 +30,12 @@ namespace NzbDrone.Common.Cache
public ICollection<ICached> Caches { get { return _cache.Values; } }
public ICached<T> GetCache<T>(Type host)
{
Ensure.That(host, () => host).IsNotNull();
return GetCache<T>(host, host.FullName);
}
public ICached<T> GetCache<T>(Type host, string name)
{
Ensure.That(host, () => host).IsNotNull();
@ -42,5 +43,13 @@ namespace NzbDrone.Common.Cache
return (ICached<T>)_cache.Get(host.FullName + "_" + name, () => new Cached<T>());
}
public ICachedDictionary<T> GetCacheDictionary<T>(Type host, string name, Func<IDictionary<string, T>> fetchFunc = null, TimeSpan? lifeTime = null)
{
Ensure.That(host, () => host).IsNotNull();
Ensure.That(name, () => name).IsNotNullOrWhiteSpace();
return (ICachedDictionary<T>)_cache.Get("dict_" + host.FullName + "_" + name, () => new CachedDictionary<T>(fetchFunc, lifeTime));
}
}
}

@ -0,0 +1,137 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
namespace NzbDrone.Common.Cache
{
public class CachedDictionary<TValue> : ICachedDictionary<TValue>
{
private readonly Func<IDictionary<string, TValue>> _fetchFunc;
private readonly TimeSpan? _ttl;
private DateTime _lastRefreshed = DateTime.MinValue;
private ConcurrentDictionary<string, TValue> _items = new ConcurrentDictionary<string, TValue>();
public CachedDictionary(Func<IDictionary<string, TValue>> fetchFunc = null, TimeSpan? ttl = null)
{
_fetchFunc = fetchFunc;
_ttl = ttl;
}
public bool IsExpired(TimeSpan ttl)
{
return _lastRefreshed.Add(ttl) < DateTime.UtcNow;
}
public void RefreshIfExpired()
{
if (_ttl.HasValue && _fetchFunc != null)
{
RefreshIfExpired(_ttl.Value);
}
}
public void RefreshIfExpired(TimeSpan ttl)
{
if (IsExpired(ttl))
{
Refresh();
}
}
public void Refresh()
{
if (_fetchFunc == null)
{
throw new InvalidOperationException("Cannot update cache without data source.");
}
Update(_fetchFunc());
ExtendTTL();
}
public void Update(IDictionary<string, TValue> items)
{
_items = new ConcurrentDictionary<string, TValue>(items);
ExtendTTL();
}
public void ExtendTTL()
{
_lastRefreshed = DateTime.UtcNow;
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public ICollection<TValue> Values
{
get
{
RefreshIfExpired();
return _items.Values;
}
}
[DebuggerBrowsable(DebuggerBrowsableState.Never)]
public int Count
{
get
{
RefreshIfExpired();
return _items.Count;
}
}
public TValue Get(string key)
{
RefreshIfExpired();
TValue result;
if (!_items.TryGetValue(key, out result))
{
throw new KeyNotFoundException(string.Format("Item {0} not found in cache.", key));
}
return result;
}
public TValue Find(string key)
{
RefreshIfExpired();
TValue result;
_items.TryGetValue(key, out result);
return result;
}
public void Clear()
{
_items.Clear();
_lastRefreshed = DateTime.MinValue;
}
public void ClearExpired()
{
if (!_ttl.HasValue)
{
throw new InvalidOperationException("Checking expiry without ttl not possible.");
}
if (IsExpired(_ttl.Value))
{
Clear();
}
}
public void Remove(string key)
{
TValue item;
_items.TryRemove(key, out item);
}
}
}

@ -0,0 +1,18 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace NzbDrone.Common.Cache
{
public interface ICachedDictionary<TValue> : ICached
{
void RefreshIfExpired();
void RefreshIfExpired(TimeSpan ttl);
void Refresh();
void Update(IDictionary<string, TValue> items);
void ExtendTTL();
TValue Get(string key);
TValue Find(string key);
bool IsExpired(TimeSpan ttl);
}
}

@ -28,7 +28,6 @@ namespace NzbDrone.Common.Http
private readonly Logger _logger;
private readonly IRateLimitService _rateLimitService;
private readonly ICached<CookieContainer> _cookieContainerCache;
private readonly ICached<bool> _curlTLSFallbackCache;
private readonly List<IHttpRequestInterceptor> _requestInterceptors;
private readonly IHttpDispatcher _httpDispatcher;

@ -64,7 +64,9 @@
<Compile Include="ArchiveService.cs" />
<Compile Include="Cache\Cached.cs" />
<Compile Include="Cache\CacheManager.cs" />
<Compile Include="Cache\CachedDictionary.cs" />
<Compile Include="Cache\ICached.cs" />
<Compile Include="Cache\ICachedDictionary.cs" />
<Compile Include="Cloud\CloudClient.cs" />
<Compile Include="Composition\Container.cs" />
<Compile Include="Composition\ContainerBuilderBase.cs" />

@ -11,6 +11,7 @@ using NzbDrone.Core.DataAugmentation.Xem.Model;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.Tv;
using NzbDrone.Core.Tv.Events;
using NzbDrone.Test.Common;
namespace NzbDrone.Core.Test.DataAugmentation.SceneNumbering
{
@ -144,6 +145,25 @@ namespace NzbDrone.Core.Test.DataAugmentation.SceneNumbering
Mocker.GetMock<ISeriesService>()
.Verify(v => v.UpdateSeries(It.IsAny<Series>()), Times.Never());
ExceptionVerification.ExpectedWarns(1);
}
[Test]
public void should_not_clear_scenenumbering_if_thexem_throws()
{
GivenExistingMapping();
Mocker.GetMock<IXemProxy>()
.Setup(v => v.GetXemSeriesIds())
.Throws(new InvalidOperationException());
Subject.Handle(new SeriesUpdatedEvent(_series));
Mocker.GetMock<ISeriesService>()
.Verify(v => v.UpdateSeries(It.IsAny<Series>()), Times.Never());
ExceptionVerification.ExpectedWarns(1);
}
[Test]

@ -28,8 +28,8 @@ namespace NzbDrone.Core.DataAugmentation.Scene
private readonly IEnumerable<ISceneMappingProvider> _sceneMappingProviders;
private readonly IEventAggregator _eventAggregator;
private readonly Logger _logger;
private readonly ICached<List<SceneMapping>> _getTvdbIdCache;
private readonly ICached<List<SceneMapping>> _findByTvdbIdCache;
private readonly ICachedDictionary<List<SceneMapping>> _getTvdbIdCache;
private readonly ICachedDictionary<List<SceneMapping>> _findByTvdbIdCache;
public SceneMappingService(ISceneMappingRepository repository,
ICacheManager cacheManager,
@ -40,10 +40,10 @@ namespace NzbDrone.Core.DataAugmentation.Scene
_repository = repository;
_sceneMappingProviders = sceneMappingProviders;
_eventAggregator = eventAggregator;
_getTvdbIdCache = cacheManager.GetCache<List<SceneMapping>>(GetType(), "tvdb_id");
_findByTvdbIdCache = cacheManager.GetCache<List<SceneMapping>>(GetType(), "find_tvdb_id");
_logger = logger;
_getTvdbIdCache = cacheManager.GetCacheDictionary<List<SceneMapping>>(GetType(), "tvdb_id");
_findByTvdbIdCache = cacheManager.GetCacheDictionary<List<SceneMapping>>(GetType(), "find_tvdb_id");
}
public List<string> GetSceneNames(int tvdbId, IEnumerable<int> seasonNumbers)
@ -143,6 +143,7 @@ namespace NzbDrone.Core.DataAugmentation.Scene
}
RefreshCache();
_eventAggregator.PublishEvent(new SceneMappingsUpdatedEvent());
}
@ -184,18 +185,8 @@ namespace NzbDrone.Core.DataAugmentation.Scene
{
var mappings = _repository.All().ToList();
_getTvdbIdCache.Clear();
_findByTvdbIdCache.Clear();
foreach (var sceneMapping in mappings.GroupBy(v => v.ParseTerm))
{
_getTvdbIdCache.Set(sceneMapping.Key, sceneMapping.ToList());
}
foreach (var sceneMapping in mappings.GroupBy(x => x.TvdbId))
{
_findByTvdbIdCache.Set(sceneMapping.Key.ToString(), sceneMapping.ToList());
}
_getTvdbIdCache.Update(mappings.GroupBy(v => v.ParseTerm).ToDictionary(v => v.Key, v => v.ToList()));
_findByTvdbIdCache.Update(mappings.GroupBy(v => v.TvdbId).ToDictionary(v => v.Key.ToString(), v => v.ToList()));
}
private List<string> FilterNonEnglish(List<string> titles)
@ -205,7 +196,10 @@ namespace NzbDrone.Core.DataAugmentation.Scene
public void Handle(SeriesRefreshStartingEvent message)
{
UpdateMappings();
if (message.ManualTrigger && _findByTvdbIdCache.IsExpired(TimeSpan.FromMinutes(1)))
{
UpdateMappings();
}
}
public void Execute(UpdateSceneMappingCommand message)

@ -16,7 +16,7 @@ namespace NzbDrone.Core.DataAugmentation.Xem
private readonly IXemProxy _xemProxy;
private readonly ISeriesService _seriesService;
private readonly Logger _logger;
private readonly ICached<bool> _cache;
private readonly ICachedDictionary<bool> _cache;
public XemService(IEpisodeService episodeService,
IXemProxy xemProxy,
@ -26,7 +26,7 @@ namespace NzbDrone.Core.DataAugmentation.Xem
_xemProxy = xemProxy;
_seriesService = seriesService;
_logger = logger;
_cache = cacheManager.GetCache<bool>(GetType());
_cache = cacheManager.GetCacheDictionary<bool>(GetType(), "mappedTvdbid");
}
private void PerformUpdate(Series series)
@ -40,7 +40,6 @@ namespace NzbDrone.Core.DataAugmentation.Xem
if (!mappings.Any() && !series.UseSceneNumbering)
{
_logger.Debug("Mappings for: {0} are empty, skipping", series);
_cache.Remove(series.TvdbId.ToString());
return;
}
@ -171,18 +170,25 @@ namespace NzbDrone.Core.DataAugmentation.Xem
}
}
private void RefreshCache()
private void UpdateXemSeriesIds()
{
var ids = _xemProxy.GetXemSeriesIds();
if (ids.Any())
try
{
_cache.Clear();
}
var ids = _xemProxy.GetXemSeriesIds();
if (ids.Any())
{
_cache.Update(ids.ToDictionary(v => v.ToString(), v => true));
return;
}
foreach (var id in ids)
_cache.ExtendTTL();
_logger.Warn("Failed to update Xem series list.");
}
catch (Exception ex)
{
_cache.Set(id.ToString(), true, TimeSpan.FromHours(1));
_cache.ExtendTTL();
_logger.Warn(ex, "Failed to update Xem series list.");
}
}
@ -206,9 +212,9 @@ namespace NzbDrone.Core.DataAugmentation.Xem
public void Handle(SeriesUpdatedEvent message)
{
if (_cache.Count == 0)
if (_cache.IsExpired(TimeSpan.FromHours(3)))
{
RefreshCache();
UpdateXemSeriesIds();
}
if (_cache.Count == 0)
@ -228,7 +234,10 @@ namespace NzbDrone.Core.DataAugmentation.Xem
public void Handle(SeriesRefreshStartingEvent message)
{
RefreshCache();
if (message.ManualTrigger && _cache.IsExpired(TimeSpan.FromMinutes(1)))
{
UpdateXemSeriesIds();
}
}
}
}

@ -4,5 +4,11 @@ namespace NzbDrone.Core.Tv.Events
{
public class SeriesRefreshStartingEvent : IEvent
{
public bool ManualTrigger { get; set; }
public SeriesRefreshStartingEvent(bool manualTrigger)
{
ManualTrigger = manualTrigger;
}
}
}

@ -144,7 +144,7 @@ namespace NzbDrone.Core.Tv
public void Execute(RefreshSeriesCommand message)
{
_eventAggregator.PublishEvent(new SeriesRefreshStartingEvent());
_eventAggregator.PublishEvent(new SeriesRefreshStartingEvent(message.Trigger == CommandTrigger.Manual));
if (message.SeriesId.HasValue)
{

Loading…
Cancel
Save