diff --git a/src/NzbDrone.Common.Test/ServiceFactoryFixture.cs b/src/NzbDrone.Common.Test/ServiceFactoryFixture.cs index b10f1adee..55c97bc13 100644 --- a/src/NzbDrone.Common.Test/ServiceFactoryFixture.cs +++ b/src/NzbDrone.Common.Test/ServiceFactoryFixture.cs @@ -18,6 +18,7 @@ namespace NzbDrone.Common.Test { var container = MainAppContainerBuilder.BuildContainer(new StartupContext()); container.Register(new MainDatabase(null)); + container.Register(new CacheDatabase(null)); container.Resolve().Register(); Mocker.SetConstant(container); diff --git a/src/NzbDrone.Common/Extensions/PathExtensions.cs b/src/NzbDrone.Common/Extensions/PathExtensions.cs index 0ca739d6c..82dbb49c4 100644 --- a/src/NzbDrone.Common/Extensions/PathExtensions.cs +++ b/src/NzbDrone.Common/Extensions/PathExtensions.cs @@ -14,6 +14,7 @@ namespace NzbDrone.Common.Extensions private const string DB = "readarr.db"; private const string DB_RESTORE = "readarr.restore"; private const string LOG_DB = "logs.db"; + private const string CACHE_DB = "cache.db"; private const string NLOG_CONFIG_FILE = "nlog.config"; private const string UPDATE_CLIENT_EXE_NAME = "Readarr.Update"; @@ -322,6 +323,11 @@ namespace NzbDrone.Common.Extensions return Path.Combine(GetAppDataPath(appFolderInfo), LOG_DB); } + public static string GetCacheDatabase(this IAppFolderInfo appFolderInfo) + { + return Path.Combine(GetAppDataPath(appFolderInfo), CACHE_DB); + } + public static string GetNlogConfigPath(this IAppFolderInfo appFolderInfo) { return Path.Combine(appFolderInfo.StartUpFolder, NLOG_CONFIG_FILE); diff --git a/src/NzbDrone.Core.Test/Framework/CoreTest.cs b/src/NzbDrone.Core.Test/Framework/CoreTest.cs index 84fe17d12..bfd934e7b 100644 --- a/src/NzbDrone.Core.Test/Framework/CoreTest.cs +++ b/src/NzbDrone.Core.Test/Framework/CoreTest.cs @@ -1,4 +1,5 @@ using System; +using Moq; using NUnit.Framework; using NzbDrone.Common.Cache; using NzbDrone.Common.Cloud; @@ -28,6 +29,11 @@ namespace NzbDrone.Core.Test.Framework Mocker.SetConstant(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); Mocker.SetConstant(new ReadarrCloudRequestBuilder()); Mocker.SetConstant(Mocker.Resolve()); + + var httpClient = Mocker.Resolve(); + Mocker.GetMock() + .Setup(x => x.Get(It.IsAny(), It.IsAny())) + .Returns((HttpRequest request, TimeSpan ttl) => httpClient.Get(request)); } } diff --git a/src/NzbDrone.Core/Datastore/CacheDatabase.cs b/src/NzbDrone.Core/Datastore/CacheDatabase.cs new file mode 100644 index 000000000..50c1be8c1 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/CacheDatabase.cs @@ -0,0 +1,33 @@ +using System; +using System.Data; + +namespace NzbDrone.Core.Datastore +{ + public interface ICacheDatabase : IDatabase + { + } + + public class CacheDatabase : ICacheDatabase + { + private readonly IDatabase _database; + + public CacheDatabase(IDatabase database) + { + _database = database; + } + + public IDbConnection OpenConnection() + { + return _database.OpenConnection(); + } + + public Version Version => _database.Version; + + public int Migration => _database.Migration; + + public void Vacuum() + { + _database.Vacuum(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs b/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs index cc087b450..1e6c0cc2d 100644 --- a/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs +++ b/src/NzbDrone.Core/Datastore/ConnectionStringFactory.cs @@ -9,6 +9,7 @@ namespace NzbDrone.Core.Datastore { string MainDbConnectionString { get; } string LogDbConnectionString { get; } + string CacheDbConnectionString { get; } string GetDatabasePath(string connectionString); } @@ -18,10 +19,12 @@ namespace NzbDrone.Core.Datastore { MainDbConnectionString = GetConnectionString(appFolderInfo.GetDatabase()); LogDbConnectionString = GetConnectionString(appFolderInfo.GetLogDatabase()); + CacheDbConnectionString = GetConnectionString(appFolderInfo.GetCacheDatabase()); } public string MainDbConnectionString { get; private set; } public string LogDbConnectionString { get; private set; } + public string CacheDbConnectionString { get; private set; } public string GetDatabasePath(string connectionString) { diff --git a/src/NzbDrone.Core/Datastore/DbFactory.cs b/src/NzbDrone.Core/Datastore/DbFactory.cs index 68042286b..64b967499 100644 --- a/src/NzbDrone.Core/Datastore/DbFactory.cs +++ b/src/NzbDrone.Core/Datastore/DbFactory.cs @@ -48,6 +48,10 @@ namespace NzbDrone.Core.Datastore var logDb = new LogDatabase(container.Resolve().Create(MigrationType.Log)); container.Register(logDb); + + var cacheDb = new CacheDatabase(container.Resolve().Create(MigrationType.Cache)); + + container.Register(cacheDb); } public DbFactory(IMigrationController migrationController, @@ -88,6 +92,14 @@ namespace NzbDrone.Core.Datastore break; } + case MigrationType.Cache: + { + connectionString = _connectionStringFactory.CacheDbConnectionString; + CreateLog(connectionString, migrationContext); + + break; + } + default: { throw new ArgumentException("Invalid MigrationType"); diff --git a/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs b/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs index 886662e0c..eb9c15be1 100644 --- a/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs +++ b/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs @@ -364,5 +364,15 @@ namespace NzbDrone.Core.Datastore.Migration .WithColumn("ExceptionType").AsString().Nullable() .WithColumn("Level").AsString(); } + + protected override void CacheDbUpgrade() + { + Create.TableForModel("HttpResponse") + .WithColumn("Url").AsString().Indexed() + .WithColumn("LastRefresh").AsDateTime() + .WithColumn("Expiry").AsDateTime().Indexed() + .WithColumn("Value").AsString() + .WithColumn("StatusCode").AsInt32(); + } } } diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationType.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationType.cs index eb72b996b..ec646124e 100644 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationType.cs +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/MigrationType.cs @@ -3,6 +3,7 @@ public enum MigrationType { Main, - Log + Log, + Cache } } diff --git a/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs index c437e2349..860bba839 100644 --- a/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs +++ b/src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs @@ -22,6 +22,10 @@ namespace NzbDrone.Core.Datastore.Migration.Framework { } + protected virtual void CacheDbUpgrade() + { + } + public int Version { get @@ -48,6 +52,11 @@ namespace NzbDrone.Core.Datastore.Migration.Framework _logger.Info("Starting migration to " + Version); LogDbUpgrade(); return; + case MigrationType.Cache: + _logger.Info("Starting migration to " + Version); + CacheDbUpgrade(); + return; + default: LogDbUpgrade(); MainDbUpgrade(); diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 78370c63f..154daa300 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -14,6 +14,7 @@ using NzbDrone.Core.Download.Pending; using NzbDrone.Core.Extras.Metadata; using NzbDrone.Core.Extras.Metadata.Files; using NzbDrone.Core.Extras.Others; +using NzbDrone.Core.Http; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.Exclusions; using NzbDrone.Core.Indexers; @@ -186,6 +187,8 @@ namespace NzbDrone.Core.Datastore Mapper.Entity("CustomFilters").RegisterModel(); Mapper.Entity("ImportListExclusions").RegisterModel(); + + Mapper.Entity("HttpResponse").RegisterModel(); } private static void RegisterMappers() diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs new file mode 100644 index 000000000..e6c89baf8 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/TrimHttpCache.cs @@ -0,0 +1,23 @@ +using Dapper; +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class TrimHttpCache : IHousekeepingTask + { + private readonly ICacheDatabase _database; + + public TrimHttpCache(ICacheDatabase database) + { + _database = database; + } + + public void Clean() + { + using (var mapper = _database.OpenConnection()) + { + mapper.Execute(@"DELETE FROM HttpResponse WHERE Expiry < date('now')"); + } + } + } +} diff --git a/src/NzbDrone.Core/Http/CachedHttpResponse.cs b/src/NzbDrone.Core/Http/CachedHttpResponse.cs new file mode 100644 index 000000000..2cb1e3060 --- /dev/null +++ b/src/NzbDrone.Core/Http/CachedHttpResponse.cs @@ -0,0 +1,14 @@ +using System; +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.Http +{ + public class CachedHttpResponse : ModelBase + { + public string Url { get; set; } + public DateTime LastRefresh { get; set; } + public DateTime Expiry { get; set; } + public string Value { get; set; } + public int StatusCode { get; set; } + } +} diff --git a/src/NzbDrone.Core/Http/CachedHttpResponseRepository.cs b/src/NzbDrone.Core/Http/CachedHttpResponseRepository.cs new file mode 100644 index 000000000..5a358cbda --- /dev/null +++ b/src/NzbDrone.Core/Http/CachedHttpResponseRepository.cs @@ -0,0 +1,27 @@ +using System.Linq; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; + +namespace NzbDrone.Core.Http +{ + public interface ICachedHttpResponseRepository : IBasicRepository + { + CachedHttpResponse FindByUrl(string url); + } + + public class CachedHttpResponseRepository : BasicRepository, ICachedHttpResponseRepository + { + public CachedHttpResponseRepository(ICacheDatabase database, + IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + + public CachedHttpResponse FindByUrl(string url) + { + var edition = Query(x => x.Url == url).SingleOrDefault(); + + return edition; + } + } +} diff --git a/src/NzbDrone.Core/Http/CachedHttpResponseService.cs b/src/NzbDrone.Core/Http/CachedHttpResponseService.cs new file mode 100644 index 000000000..74a53a086 --- /dev/null +++ b/src/NzbDrone.Core/Http/CachedHttpResponseService.cs @@ -0,0 +1,58 @@ +using System; +using System.Net; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.Http +{ + public interface ICachedHttpResponseService + { + HttpResponse Get(HttpRequest request, TimeSpan ttl); + } + + public class CachedHttpResponseService : ICachedHttpResponseService + { + private readonly ICachedHttpResponseRepository _repo; + private readonly IHttpClient _httpClient; + + public CachedHttpResponseService(ICachedHttpResponseRepository httpResponseRepository, + IHttpClient httpClient) + { + _repo = httpResponseRepository; + _httpClient = httpClient; + } + + public HttpResponse Get(HttpRequest request, TimeSpan ttl) + { + var cached = _repo.FindByUrl(request.Url.ToString()); + + if (cached != null && cached.Expiry > DateTime.UtcNow) + { + return new HttpResponse(request, new HttpHeader(), cached.Value, (HttpStatusCode)cached.StatusCode); + } + + var result = _httpClient.Get(request); + + if (!result.HasHttpError) + { + if (cached == null) + { + cached = new CachedHttpResponse + { + Url = request.Url.ToString(), + }; + } + + var now = DateTime.UtcNow; + + cached.LastRefresh = now; + cached.Expiry = now.Add(ttl); + cached.Value = result.Content; + cached.StatusCode = (int)result.StatusCode; + + _repo.Upsert(cached); + } + + return result; + } + } +} diff --git a/src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs b/src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs index 96408a6db..000948247 100644 --- a/src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs @@ -10,6 +10,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Books; using NzbDrone.Core.Exceptions; +using NzbDrone.Core.Http; using NzbDrone.Core.MediaCover; using NzbDrone.Core.Parser; @@ -27,6 +28,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads RegexOptions.IgnoreCase | RegexOptions.Compiled); private readonly IHttpClient _httpClient; + private readonly ICachedHttpResponseService _cachedHttpClient; private readonly Logger _logger; private readonly IAuthorService _authorService; private readonly IBookService _bookService; @@ -36,6 +38,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads private readonly ICached> _cache; public GoodreadsProxy(IHttpClient httpClient, + ICachedHttpResponseService cachedHttpClient, IAuthorService authorService, IBookService bookService, IEditionService editionService, @@ -43,6 +46,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads ICacheManager cacheManager) { _httpClient = httpClient; + _cachedHttpClient = cachedHttpClient; _authorService = authorService; _bookService = bookService; _editionService = editionService; @@ -80,7 +84,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads httpRequest.AllowAutoRedirect = true; httpRequest.SuppressHttpError = true; - var httpResponse = _httpClient.Get(httpRequest); + var httpResponse = _cachedHttpClient.Get(httpRequest, TimeSpan.FromDays(30)); if (httpResponse.HasHttpError) { @@ -208,7 +212,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads httpRequest.AllowAutoRedirect = true; httpRequest.SuppressHttpError = true; - var httpResponse = _httpClient.Get(httpRequest); + var httpResponse = _cachedHttpClient.Get(httpRequest, TimeSpan.FromDays(7)); if (httpResponse.HasHttpError) { @@ -240,7 +244,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads httpRequest.AllowAutoRedirect = true; httpRequest.SuppressHttpError = true; - var httpResponse = _httpClient.Get(httpRequest); + var httpResponse = _cachedHttpClient.Get(httpRequest, TimeSpan.FromDays(90)); if (httpResponse.HasHttpError) { @@ -314,7 +318,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads httpRequest.AllowAutoRedirect = true; httpRequest.SuppressHttpError = true; - var httpResponse = _httpClient.Get(httpRequest); + var httpResponse = _cachedHttpClient.Get(httpRequest, TimeSpan.FromDays(90)); if (httpResponse.HasHttpError) { diff --git a/src/NzbDrone.Host.Test/ContainerFixture.cs b/src/NzbDrone.Host.Test/ContainerFixture.cs index d42c5c31f..dde0715e2 100644 --- a/src/NzbDrone.Host.Test/ContainerFixture.cs +++ b/src/NzbDrone.Host.Test/ContainerFixture.cs @@ -31,6 +31,7 @@ namespace NzbDrone.App.Test _container = MainAppContainerBuilder.BuildContainer(args); _container.Register(new MainDatabase(null)); + _container.Register(new CacheDatabase(null)); // set up a dummy broadcaster to allow tests to resolve var mockBroadcaster = new Mock();