From 8edadbbfa25521ebd3ee0a0de25837b7fe687bba Mon Sep 17 00:00:00 2001 From: ta264 Date: Thu, 2 Sep 2021 21:12:13 +0100 Subject: [PATCH] Separate out Goodreads search from Author/Book data --- .../Goodreads/GoodreadsProxySearchFixture.cs | 5 +- .../Goodreads/GoodreadsProxy.cs | 335 +----------------- .../GoodreadsSearchProxy.cs | 318 +++++++++++++++++ .../Resources/SearchJsonResource.cs | 0 4 files changed, 323 insertions(+), 335 deletions(-) create mode 100644 src/NzbDrone.Core/MetadataSource/GoodreadsSearchProxy/GoodreadsSearchProxy.cs rename src/NzbDrone.Core/MetadataSource/{Goodreads => GoodreadsSearchProxy}/Resources/SearchJsonResource.cs (100%) diff --git a/src/NzbDrone.Core.Test/MetadataSource/Goodreads/GoodreadsProxySearchFixture.cs b/src/NzbDrone.Core.Test/MetadataSource/Goodreads/GoodreadsProxySearchFixture.cs index c2db0c911..3932e0d04 100644 --- a/src/NzbDrone.Core.Test/MetadataSource/Goodreads/GoodreadsProxySearchFixture.cs +++ b/src/NzbDrone.Core.Test/MetadataSource/Goodreads/GoodreadsProxySearchFixture.cs @@ -7,6 +7,7 @@ using NUnit.Framework; using NzbDrone.Common.Http; using NzbDrone.Core.Books; using NzbDrone.Core.Http; +using NzbDrone.Core.MetadataSource; using NzbDrone.Core.MetadataSource.Goodreads; using NzbDrone.Core.Profiles.Metadata; using NzbDrone.Core.Test.Framework; @@ -15,13 +16,15 @@ using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.MetadataSource.Goodreads { [TestFixture] - public class GoodreadsProxySearchFixture : CoreTest + public class GoodreadsProxySearchFixture : CoreTest { [SetUp] public void Setup() { UseRealHttp(); + Mocker.SetConstant(Mocker.Resolve()); + var httpClient = Mocker.Resolve(); Mocker.GetMock() .Setup(x => x.Get>(It.IsAny(), It.IsAny(), It.IsAny())) diff --git a/src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs b/src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs index 11ebff51b..2663d7a71 100644 --- a/src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/Goodreads/GoodreadsProxy.cs @@ -17,7 +17,7 @@ using NzbDrone.Core.Parser; namespace NzbDrone.Core.MetadataSource.Goodreads { - public class GoodreadsProxy : IProvideAuthorInfo, ISearchForNewAuthor, IProvideBookInfo, ISearchForNewBook, ISearchForNewEntity + public class GoodreadsProxy : IProvideAuthorInfo, IProvideBookInfo { private static readonly RegexReplace FullSizeImageRegex = new RegexReplace(@"\._[SU][XY]\d+_.jpg$", ".jpg", @@ -37,22 +37,18 @@ namespace NzbDrone.Core.MetadataSource.Goodreads private readonly ICachedHttpResponseService _cachedHttpClient; private readonly Logger _logger; private readonly IAuthorService _authorService; - private readonly IBookService _bookService; private readonly IEditionService _editionService; private readonly IHttpRequestBuilderFactory _requestBuilder; - private readonly IHttpRequestBuilderFactory _searchBuilder; private readonly ICached> _cache; public GoodreadsProxy(ICachedHttpResponseService cachedHttpClient, IAuthorService authorService, - IBookService bookService, IEditionService editionService, Logger logger, ICacheManager cacheManager) { _cachedHttpClient = cachedHttpClient; _authorService = authorService; - _bookService = bookService; _editionService = editionService; _cache = cacheManager.GetCache>(GetType()); _logger = logger; @@ -63,12 +59,6 @@ namespace NzbDrone.Core.MetadataSource.Goodreads .SetHeader("User-Agent", "Dalvik/1.6.0 (Linux; U; Android 4.1.2; GT-I9100 Build/JZO54K)") .KeepAlive() .CreateFactory(); - - _searchBuilder = new HttpRequestBuilder("https://www.goodreads.com/book/auto_complete") - .AddQueryParam("format", "json") - .SetHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36") - .KeepAlive() - .CreateFactory(); } public HashSet GetChangedAuthors(DateTime startTime) @@ -383,168 +373,6 @@ namespace NzbDrone.Core.MetadataSource.Goodreads return new Tuple>(resource.Authors.First().Id.ToString(), book, authors); } - public List SearchForNewAuthor(string title) - { - var books = SearchForNewBook(title, null); - - return books.Select(x => x.Author.Value).ToList(); - } - - public List SearchForNewBook(string title, string author) - { - try - { - var lowerTitle = title.ToLowerInvariant(); - - var split = lowerTitle.Split(':'); - var prefix = split[0]; - - if (split.Length == 2 && new[] { "readarr", "readarrid", "goodreads", "isbn", "asin" }.Contains(prefix)) - { - var slug = split[1].Trim(); - - if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace)) - { - return new List(); - } - - if (prefix == "goodreads" || prefix == "readarr" || prefix == "readarrid") - { - var isValid = int.TryParse(slug, out var searchId); - if (!isValid) - { - return new List(); - } - - return SearchByGoodreadsId(searchId); - } - else if (prefix == "isbn") - { - return SearchByIsbn(slug); - } - else if (prefix == "asin") - { - return SearchByAsin(slug); - } - } - - var q = title.ToLower().Trim(); - if (author != null) - { - q += " " + author; - } - - return SearchByField("all", q); - } - catch (HttpException) - { - throw new GoodreadsException("Search for '{0}' failed. Unable to communicate with Goodreads.", title); - } - catch (Exception ex) - { - _logger.Warn(ex, ex.Message); - throw new GoodreadsException("Search for '{0}' failed. Invalid response received from Goodreads.", title); - } - } - - public List SearchByIsbn(string isbn) - { - return SearchByField("isbn", isbn, e => e.Isbn13 = isbn); - } - - public List SearchByAsin(string asin) - { - return SearchByField("asin", asin, e => e.Asin = asin); - } - - public List SearchByGoodreadsId(int id) - { - try - { - var remote = GetBookInfo(id.ToString()); - - var book = _bookService.FindById(remote.Item2.ForeignBookId); - var result = book ?? remote.Item2; - - // at this point, book could have the wrong edition. - // Check if we already have the correct edition. - var remoteEdition = remote.Item2.Editions.Value.Single(x => x.Monitored); - var localEdition = _editionService.GetEditionByForeignEditionId(remoteEdition.ForeignEditionId); - if (localEdition != null) - { - result.Editions = new List { localEdition }; - } - - // If we don't have the correct edition in the response, add it in. - if (!result.Editions.Value.Any(x => x.ForeignEditionId == remoteEdition.ForeignEditionId)) - { - result.Editions.Value.ForEach(x => x.Monitored = false); - result.Editions.Value.Add(remoteEdition); - } - - var author = _authorService.FindById(remote.Item1); - if (author == null) - { - author = new Author - { - CleanName = Parser.Parser.CleanAuthorName(remote.Item2.AuthorMetadata.Value.Name), - Metadata = remote.Item2.AuthorMetadata.Value - }; - } - - result.Author = author; - - return new List { result }; - } - catch (BookNotFoundException) - { - return new List(); - } - } - - public List SearchByField(string field, string query, Action applyData = null) - { - try - { - var httpRequest = _searchBuilder.Create() - .AddQueryParam("q", query) - .Build(); - - var response = _cachedHttpClient.Get>(httpRequest, true, TimeSpan.FromDays(5)); - - return response.Resource.SelectList(x => MapJsonSearchResult(x, response.Resource.Count == 1 ? applyData : null)); - } - catch (HttpException) - { - throw new GoodreadsException("Search for {0} '{1}' failed. Unable to communicate with Goodreads.", field, query); - } - catch (Exception ex) - { - _logger.Warn(ex, ex.Message); - throw new GoodreadsException("Search for {0} '{1}' failed. Invalid response received from Goodreads.", field, query); - } - } - - public List SearchForNewEntity(string title) - { - var books = SearchForNewBook(title, null); - - var result = new List(); - foreach (var book in books) - { - var author = book.Author.Value; - - if (!result.Contains(author)) - { - result.Add(author); - } - - result.Add(book); - } - - return result; - } - private static AuthorMetadata MapAuthor(AuthorResource resource) { var author = new AuthorMetadata @@ -683,167 +511,6 @@ namespace NzbDrone.Core.MetadataSource.Goodreads return book; } - private Book MapSearchResult(WorkResource resource) - { - var book = _bookService.FindById(resource.Id.ToString()); - if (resource.BestBook != null) - { - var edition = _editionService.GetEditionByForeignEditionId(resource.BestBook.Id.ToString()); - - if (edition == null) - { - edition = new Edition - { - ForeignEditionId = resource.BestBook.Id.ToString(), - Title = resource.BestBook.Title, - TitleSlug = resource.BestBook.Id.ToString(), - Ratings = new Ratings { Votes = resource.RatingsCount, Value = resource.AverageRating }, - }; - } - - edition.Monitored = true; - edition.ManualAdd = true; - - if (resource.BestBook.ImageUrl.IsNotNullOrWhiteSpace() && !NoPhotoRegex.IsMatch(resource.BestBook.ImageUrl)) - { - edition.Images.Add(new MediaCover.MediaCover - { - Url = FullSizeImageRegex.Replace(resource.BestBook.ImageUrl), - CoverType = MediaCoverTypes.Cover - }); - } - - if (book == null) - { - book = new Book - { - ForeignBookId = resource.Id.ToString(), - Title = resource.BestBook.Title, - TitleSlug = resource.Id.ToString(), - ReleaseDate = resource.OriginalPublicationDate, - Ratings = new Ratings { Votes = resource.RatingsCount, Value = resource.AverageRating }, - AnyEditionOk = true - }; - } - - book.Editions = new List { edition }; - - var authorId = resource.BestBook.AuthorId.ToString(); - var author = _authorService.FindById(authorId); - - if (author == null) - { - author = new Author - { - CleanName = Parser.Parser.CleanAuthorName(resource.BestBook.AuthorName), - Metadata = new AuthorMetadata() - { - ForeignAuthorId = resource.BestBook.AuthorId.ToString(), - Name = resource.BestBook.AuthorName, - NameLastFirst = resource.BestBook.AuthorName.ToLastFirst(), - SortName = resource.BestBook.AuthorName.ToLower(), - SortNameLastFirst = resource.BestBook.AuthorName.ToLastFirst().ToLower(), - TitleSlug = resource.BestBook.AuthorId.ToString() - } - }; - } - - book.Author = author; - book.AuthorMetadata = book.Author.Value.Metadata.Value; - book.AuthorMetadataId = author.AuthorMetadataId; - book.CleanTitle = book.Title.CleanAuthorName(); - } - - return book; - } - - private Book MapJsonSearchResult(SearchJsonResource resource, Action applyData = null) - { - var book = _bookService.FindById(resource.WorkId.ToString()); - var edition = _editionService.GetEditionByForeignEditionId(resource.BookId.ToString()); - - if (edition == null) - { - edition = new Edition - { - ForeignEditionId = resource.BookId.ToString(), - Title = resource.BookTitleBare, - TitleSlug = resource.BookId.ToString(), - Ratings = new Ratings { Votes = resource.RatingsCount, Value = resource.AverageRating }, - PageCount = resource.PageCount, - Overview = resource.Description?.Html ?? string.Empty - }; - - if (applyData != null) - { - applyData(edition); - } - } - - edition.Monitored = true; - edition.ManualAdd = true; - - if (resource.ImageUrl.IsNotNullOrWhiteSpace() && !NoPhotoRegex.IsMatch(resource.ImageUrl)) - { - edition.Images.Add(new MediaCover.MediaCover - { - Url = FullSizeImageRegex.Replace(resource.ImageUrl), - CoverType = MediaCoverTypes.Cover - }); - } - - if (book == null) - { - book = new Book - { - ForeignBookId = resource.WorkId.ToString(), - Title = resource.BookTitleBare, - TitleSlug = resource.WorkId.ToString(), - Ratings = new Ratings { Votes = resource.RatingsCount, Value = resource.AverageRating }, - AnyEditionOk = true - }; - } - - if (book.Editions != null) - { - if (book.Editions.Value.Any()) - { - edition.Monitored = false; - } - - book.Editions.Value.Add(edition); - } - else - { - book.Editions = new List { edition }; - } - - var authorId = resource.Author.Id.ToString(); - var author = _authorService.FindById(authorId); - - if (author == null) - { - author = new Author - { - CleanName = Parser.Parser.CleanAuthorName(resource.Author.Name), - Metadata = new AuthorMetadata() - { - ForeignAuthorId = resource.Author.Id.ToString(), - Name = DuplicateSpacesRegex.Replace(resource.Author.Name, " "), - TitleSlug = resource.Author.Id.ToString() - } - }; - } - - book.Author = author; - book.AuthorMetadata = book.Author.Value.Metadata.Value; - book.AuthorMetadataId = author.AuthorMetadataId; - book.CleanTitle = book.Title.CleanAuthorName(); - book.SeriesLinks = MapSearchSeries(resource.Title, resource.BookTitleBare); - - return book; - } - public static List MapSearchSeries(string title, string titleWithoutSeries) { if (title != titleWithoutSeries && diff --git a/src/NzbDrone.Core/MetadataSource/GoodreadsSearchProxy/GoodreadsSearchProxy.cs b/src/NzbDrone.Core/MetadataSource/GoodreadsSearchProxy/GoodreadsSearchProxy.cs new file mode 100644 index 000000000..6383f2a4b --- /dev/null +++ b/src/NzbDrone.Core/MetadataSource/GoodreadsSearchProxy/GoodreadsSearchProxy.cs @@ -0,0 +1,318 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.Books; +using NzbDrone.Core.Exceptions; +using NzbDrone.Core.Http; +using NzbDrone.Core.MediaCover; +using NzbDrone.Core.Parser; + +namespace NzbDrone.Core.MetadataSource.Goodreads +{ + public class GoodreadsSearchProxy : ISearchForNewAuthor, ISearchForNewBook, ISearchForNewEntity + { + private static readonly RegexReplace FullSizeImageRegex = new RegexReplace(@"\._[SU][XY]\d+_.jpg$", + ".jpg", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly Regex DuplicateSpacesRegex = new Regex(@"\s{2,}", RegexOptions.Compiled); + + private static readonly Regex NoPhotoRegex = new Regex(@"/nophoto/(book|user)/", + RegexOptions.IgnoreCase | RegexOptions.Compiled); + + private static readonly List SeriesRegex = new List + { + new Regex(@"\((?[^,]+),\s+#(?[\w\d\.]+)\)$", RegexOptions.Compiled), + new Regex(@"(The\s+(?.+)\s+Series\s+Book\s+(?[\w\d\.]+)\)$)", RegexOptions.Compiled) + }; + + private readonly ICachedHttpResponseService _cachedHttpClient; + private readonly Logger _logger; + private readonly IProvideBookInfo _bookInfo; + private readonly IAuthorService _authorService; + private readonly IBookService _bookService; + private readonly IEditionService _editionService; + private readonly IHttpRequestBuilderFactory _searchBuilder; + private readonly ICached> _cache; + + public GoodreadsSearchProxy(ICachedHttpResponseService cachedHttpClient, + IProvideBookInfo bookInfo, + IAuthorService authorService, + IBookService bookService, + IEditionService editionService, + Logger logger, + ICacheManager cacheManager) + { + _cachedHttpClient = cachedHttpClient; + _bookInfo = bookInfo; + _authorService = authorService; + _bookService = bookService; + _editionService = editionService; + _cache = cacheManager.GetCache>(GetType()); + _logger = logger; + + _searchBuilder = new HttpRequestBuilder("https://www.goodreads.com/book/auto_complete") + .AddQueryParam("format", "json") + .SetHeader("User-Agent", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36") + .KeepAlive() + .CreateFactory(); + } + + public List SearchForNewAuthor(string title) + { + var books = SearchForNewBook(title, null); + + return books.Select(x => x.Author.Value).ToList(); + } + + public List SearchForNewBook(string title, string author) + { + try + { + var lowerTitle = title.ToLowerInvariant(); + + var split = lowerTitle.Split(':'); + var prefix = split[0]; + + if (split.Length == 2 && new[] { "readarr", "readarrid", "goodreads", "isbn", "asin" }.Contains(prefix)) + { + var slug = split[1].Trim(); + + if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace)) + { + return new List(); + } + + if (prefix == "goodreads" || prefix == "readarr" || prefix == "readarrid") + { + var isValid = int.TryParse(slug, out var searchId); + if (!isValid) + { + return new List(); + } + + return SearchByGoodreadsId(searchId); + } + else if (prefix == "isbn") + { + return SearchByIsbn(slug); + } + else if (prefix == "asin") + { + return SearchByAsin(slug); + } + } + + var q = title.ToLower().Trim(); + if (author != null) + { + q += " " + author; + } + + return SearchByField("all", q); + } + catch (HttpException) + { + throw new GoodreadsException("Search for '{0}' failed. Unable to communicate with Goodreads.", title); + } + catch (Exception ex) + { + _logger.Warn(ex, ex.Message); + throw new GoodreadsException("Search for '{0}' failed. Invalid response received from Goodreads.", + title); + } + } + + public List SearchByIsbn(string isbn) + { + return SearchByField("isbn", isbn, e => e.Isbn13 = isbn); + } + + public List SearchByAsin(string asin) + { + return SearchByField("asin", asin, e => e.Asin = asin); + } + + public List SearchByGoodreadsId(int id) + { + try + { + var remote = _bookInfo.GetBookInfo(id.ToString()); + + var book = _bookService.FindById(remote.Item2.ForeignBookId); + var result = book ?? remote.Item2; + + // at this point, book could have the wrong edition. + // Check if we already have the correct edition. + var remoteEdition = remote.Item2.Editions.Value.Single(x => x.Monitored); + var localEdition = _editionService.GetEditionByForeignEditionId(remoteEdition.ForeignEditionId); + if (localEdition != null) + { + result.Editions = new List { localEdition }; + } + + // If we don't have the correct edition in the response, add it in. + if (!result.Editions.Value.Any(x => x.ForeignEditionId == remoteEdition.ForeignEditionId)) + { + result.Editions.Value.ForEach(x => x.Monitored = false); + result.Editions.Value.Add(remoteEdition); + } + + var author = _authorService.FindById(remote.Item1); + if (author == null) + { + author = new Author + { + CleanName = Parser.Parser.CleanAuthorName(remote.Item2.AuthorMetadata.Value.Name), + Metadata = remote.Item2.AuthorMetadata.Value + }; + } + + result.Author = author; + + return new List { result }; + } + catch (BookNotFoundException) + { + return new List(); + } + } + + public List SearchByField(string field, string query, Action applyData = null) + { + try + { + var httpRequest = _searchBuilder.Create() + .AddQueryParam("q", query) + .Build(); + + var response = _cachedHttpClient.Get>(httpRequest, true, TimeSpan.FromDays(5)); + + return response.Resource.SelectList(x => + MapJsonSearchResult(x, response.Resource.Count == 1 ? applyData : null)); + } + catch (HttpException) + { + throw new GoodreadsException("Search for {0} '{1}' failed. Unable to communicate with Goodreads.", field, query); + } + catch (Exception ex) + { + _logger.Warn(ex, ex.Message); + throw new GoodreadsException("Search for {0} '{1}' failed. Invalid response received from Goodreads.", field, query); + } + } + + public List SearchForNewEntity(string title) + { + var books = SearchForNewBook(title, null); + + var result = new List(); + foreach (var book in books) + { + var author = book.Author.Value; + + if (!result.Contains(author)) + { + result.Add(author); + } + + result.Add(book); + } + + return result; + } + + private Book MapJsonSearchResult(SearchJsonResource resource, Action applyData = null) + { + var book = _bookService.FindById(resource.WorkId.ToString()); + var edition = _editionService.GetEditionByForeignEditionId(resource.BookId.ToString()); + + if (edition == null) + { + edition = new Edition + { + ForeignEditionId = resource.BookId.ToString(), + Title = resource.BookTitleBare, + TitleSlug = resource.BookId.ToString(), + Ratings = new Ratings { Votes = resource.RatingsCount, Value = resource.AverageRating }, + PageCount = resource.PageCount, + Overview = resource.Description?.Html ?? string.Empty + }; + + if (applyData != null) + { + applyData(edition); + } + } + + edition.Monitored = true; + edition.ManualAdd = true; + + if (resource.ImageUrl.IsNotNullOrWhiteSpace() && !NoPhotoRegex.IsMatch(resource.ImageUrl)) + { + edition.Images.Add(new MediaCover.MediaCover + { + Url = FullSizeImageRegex.Replace(resource.ImageUrl), + CoverType = MediaCoverTypes.Cover + }); + } + + if (book == null) + { + book = new Book + { + ForeignBookId = resource.WorkId.ToString(), + Title = resource.BookTitleBare, + TitleSlug = resource.WorkId.ToString(), + Ratings = new Ratings { Votes = resource.RatingsCount, Value = resource.AverageRating }, + AnyEditionOk = true + }; + } + + if (book.Editions != null) + { + if (book.Editions.Value.Any()) + { + edition.Monitored = false; + } + + book.Editions.Value.Add(edition); + } + else + { + book.Editions = new List { edition }; + } + + var authorId = resource.Author.Id.ToString(); + var author = _authorService.FindById(authorId); + + if (author == null) + { + author = new Author + { + CleanName = Parser.Parser.CleanAuthorName(resource.Author.Name), + Metadata = new AuthorMetadata() + { + ForeignAuthorId = resource.Author.Id.ToString(), + Name = DuplicateSpacesRegex.Replace(resource.Author.Name, " "), + TitleSlug = resource.Author.Id.ToString() + } + }; + } + + book.Author = author; + book.AuthorMetadata = book.Author.Value.Metadata.Value; + book.AuthorMetadataId = author.AuthorMetadataId; + book.CleanTitle = book.Title.CleanAuthorName(); + book.SeriesLinks = GoodreadsProxy.MapSearchSeries(resource.Title, resource.BookTitleBare); + + return book; + } + } +} diff --git a/src/NzbDrone.Core/MetadataSource/Goodreads/Resources/SearchJsonResource.cs b/src/NzbDrone.Core/MetadataSource/GoodreadsSearchProxy/Resources/SearchJsonResource.cs similarity index 100% rename from src/NzbDrone.Core/MetadataSource/Goodreads/Resources/SearchJsonResource.cs rename to src/NzbDrone.Core/MetadataSource/GoodreadsSearchProxy/Resources/SearchJsonResource.cs