Alternative metadata source

pull/1319/head
BookInfo 3 years ago committed by ta264
parent 225f0b310a
commit 6bdfe01fbc

@ -157,17 +157,28 @@ namespace NzbDrone.Common.Test.Http
response.Resource.Data.Should().Be(message);
}
[TestCase("gzip")]
public void should_execute_get_using_gzip(string compression)
[Test]
public void should_execute_get_using_gzip()
{
var request = new HttpRequest($"https://{_httpBinHost}/{compression}");
var request = new HttpRequest($"https://{_httpBinHost}/gzip");
var response = Subject.Get<HttpBinResource>(request);
response.Resource.Headers["Accept-Encoding"].ToString().Should().Be(compression);
response.Resource.Headers["Accept-Encoding"].ToString().Should().Contain("gzip");
response.Resource.Gzipped.Should().BeTrue();
}
[Test]
public void should_execute_get_using_brotli()
{
var request = new HttpRequest($"https://{_httpBinHost}/brotli");
var response = Subject.Get<HttpBinResource>(request);
response.Resource.Headers["Accept-Encoding"].ToString().Should().Contain("br");
response.Resource.Brotli.Should().BeTrue();
}
[TestCase(HttpStatusCode.Unauthorized)]
[TestCase(HttpStatusCode.Forbidden)]
[TestCase(HttpStatusCode.NotFound)]
@ -783,6 +794,7 @@ namespace NzbDrone.Common.Test.Http
public string Url { get; set; }
public string Data { get; set; }
public bool Gzipped { get; set; }
public bool Brotli { get; set; }
}
public class HttpCookieResource

@ -5,8 +5,7 @@ namespace NzbDrone.Common.Cloud
public interface IReadarrCloudRequestBuilder
{
IHttpRequestBuilderFactory Services { get; }
IHttpRequestBuilderFactory Search { get; }
IHttpRequestBuilderFactory InternalSearch { get; }
IHttpRequestBuilderFactory Metadata { get; }
}
public class ReadarrCloudRequestBuilder : IReadarrCloudRequestBuilder
@ -17,15 +16,12 @@ namespace NzbDrone.Common.Cloud
Services = new HttpRequestBuilder("https://readarr.servarr.com/v1/")
.CreateFactory();
Search = new HttpRequestBuilder("https://api.readarr.com/v0.2/{route}")
.KeepAlive()
Metadata = new HttpRequestBuilder("https://api.bookinfo.club/v1/{route}")
.CreateFactory();
}
public IHttpRequestBuilderFactory Services { get; }
public IHttpRequestBuilderFactory Search { get; }
public IHttpRequestBuilderFactory InternalSearch { get; }
public IHttpRequestBuilderFactory Metadata { get; }
}
}

@ -1,16 +1,12 @@
using System;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Net;
using System.Reflection;
using System.Text;
using NLog;
using NLog.Fluent;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http.Proxy;
using NzbDrone.Common.Instrumentation.Extensions;
namespace NzbDrone.Common.Http.Dispatchers
{
@ -38,7 +34,7 @@ namespace NzbDrone.Common.Http.Dispatchers
// Deflate is not a standard and could break depending on implementation.
// we should just stick with the more compatible Gzip
//http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net
webRequest.AutomaticDecompression = DecompressionMethods.GZip;
webRequest.AutomaticDecompression = DecompressionMethods.Brotli | DecompressionMethods.GZip;
webRequest.Method = request.Method.ToString();
webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent);

@ -7,7 +7,6 @@ using FluentValidation.Results;
using Moq;
using Newtonsoft.Json;
using NUnit.Framework;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Books;
using NzbDrone.Core.Books.Commands;
using NzbDrone.Core.Configuration;
@ -19,6 +18,7 @@ using NzbDrone.Core.MediaFiles.BookImport.Aggregation.Aggregators;
using NzbDrone.Core.MediaFiles.BookImport.Identification;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.MetadataSource.BookInfo;
using NzbDrone.Core.MetadataSource.Goodreads;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
@ -59,7 +59,7 @@ namespace NzbDrone.Core.Test.MediaFiles.BookImport.Identification
Mocker.SetConstant<IMediaFileService>(Mocker.Resolve<MediaFileService>());
Mocker.SetConstant<IConfigService>(Mocker.Resolve<IConfigService>());
Mocker.SetConstant<IProvideAuthorInfo>(Mocker.Resolve<GoodreadsProxy>());
Mocker.SetConstant<IProvideAuthorInfo>(Mocker.Resolve<BookInfoProxy>());
Mocker.SetConstant<IProvideBookInfo>(Mocker.Resolve<GoodreadsProxy>());
_addAuthorService = Mocker.Resolve<AddAuthorService>();

@ -19,8 +19,8 @@ namespace NzbDrone.Core.Test.MetadataSource
.Returns("");
Mocker.GetMock<IReadarrCloudRequestBuilder>()
.Setup(s => s.Search)
.Returns(new HttpRequestBuilder("https://api.readarr.com/api/v0.4/{route}").CreateFactory());
.Setup(s => s.Metadata)
.Returns(new HttpRequestBuilder("https://api.bookinfo.club/v1/{route}").CreateFactory());
}
private void WithCustomProvider()
@ -45,7 +45,7 @@ namespace NzbDrone.Core.Test.MetadataSource
{
var details = Subject.GetRequestBuilder().Create();
details.BaseUrl.ToString().Should().Contain("v0.4");
details.BaseUrl.ToString().Should().Contain("bookinfo.club/v1");
}
}
}

@ -33,7 +33,7 @@ namespace NzbDrone.Core.Test.MusicTests
private void GivenValidAuthor(string readarrId)
{
Mocker.GetMock<IProvideAuthorInfo>()
.Setup(s => s.GetAuthorInfo(readarrId, true))
.Setup(s => s.GetAuthorInfo(readarrId, true, false))
.Returns(_fakeAuthor);
}
@ -113,7 +113,7 @@ namespace NzbDrone.Core.Test.MusicTests
};
Mocker.GetMock<IProvideAuthorInfo>()
.Setup(s => s.GetAuthorInfo(newAuthor.ForeignAuthorId, true))
.Setup(s => s.GetAuthorInfo(newAuthor.ForeignAuthorId, true, false))
.Throws(new AuthorNotFoundException(newAuthor.ForeignAuthorId));
Mocker.GetMock<IAddAuthorValidator>()

@ -14,6 +14,7 @@ namespace NzbDrone.Core.Books
{
Links = new List<Links>();
Genres = new List<string>();
RelatedBooks = new List<int>();
Ratings = new Ratings();
Author = new Author();
AddOptions = new AddBookOptions();
@ -28,6 +29,7 @@ namespace NzbDrone.Core.Books
public DateTime? ReleaseDate { get; set; }
public List<Links> Links { get; set; }
public List<string> Genres { get; set; }
public List<int> RelatedBooks { get; set; }
public Ratings Ratings { get; set; }
// These are Readarr generated/config
@ -72,6 +74,7 @@ namespace NzbDrone.Core.Books
ReleaseDate = other.ReleaseDate;
Links = other.Links;
Genres = other.Genres;
RelatedBooks = other.RelatedBooks;
Ratings = other.Ratings;
CleanTitle = other.CleanTitle;
}

@ -97,7 +97,7 @@ namespace NzbDrone.Core.Books
try
{
author = _authorInfo.GetAuthorInfo(newAuthor.Metadata.Value.ForeignAuthorId);
author = _authorInfo.GetAuthorInfo(newAuthor.Metadata.Value.ForeignAuthorId, includeBooks: false);
}
catch (AuthorNotFoundException)
{

@ -385,6 +385,7 @@ namespace NzbDrone.Core.Books
{
try
{
LogProgress(author);
var data = GetSkyhookData(author.ForeignAuthorId, author.MetadataProfile.Value.MinPopularity);
updated |= RefreshEntityInfo(author, null, data, manualTrigger, false, message.LastStartTime);
}

@ -339,18 +339,9 @@ namespace NzbDrone.Core.Books
{
var updated = false;
HashSet<string> updatedGoodreadsBooks = null;
if (lastUpdate.HasValue && lastUpdate.Value.AddDays(14) > DateTime.UtcNow)
{
updatedGoodreadsBooks = _bookInfo.GetChangedBooks(lastUpdate.Value);
}
foreach (var book in books)
{
if (forceBookRefresh ||
(updatedGoodreadsBooks == null && _checkIfBookShouldBeRefreshed.ShouldRefresh(book)) ||
(updatedGoodreadsBooks != null && updatedGoodreadsBooks.Contains(book.ForeignBookId)))
if (forceBookRefresh || _checkIfBookShouldBeRefreshed.ShouldRefresh(book))
{
updated |= RefreshBookInfo(book, remoteBooks, remoteData, forceUpdateFileTags);
}

@ -0,0 +1,14 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(16)]
public class AddRelatedBooks : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Alter.Table("Books").AddColumn("RelatedBooks").AsString().WithDefaultValue("[]");
}
}
}

@ -0,0 +1,18 @@
using System.Net;
using NzbDrone.Core.Exceptions;
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class BookInfoException : NzbDroneClientException
{
public BookInfoException(string message)
: base(HttpStatusCode.ServiceUnavailable, message)
{
}
public BookInfoException(string message, params object[] args)
: base(HttpStatusCode.ServiceUnavailable, message, args)
{
}
}
}

@ -0,0 +1,315 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using System.Threading;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Books;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.MediaCover;
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class BookInfoProxy : IProvideAuthorInfo
{
private readonly IHttpClient _httpClient;
private readonly Logger _logger;
private readonly IMetadataRequestBuilder _requestBuilder;
private readonly ICached<HashSet<string>> _cache;
public BookInfoProxy(IHttpClient httpClient,
IMetadataRequestBuilder requestBuilder,
Logger logger,
ICacheManager cacheManager)
{
_httpClient = httpClient;
_requestBuilder = requestBuilder;
_cache = cacheManager.GetCache<HashSet<string>>(GetType());
_logger = logger;
}
public HashSet<string> GetChangedAuthors(DateTime startTime)
{
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", "author/changed")
.AddQueryParam("since", startTime.ToString("o"))
.Build();
httpRequest.SuppressHttpError = true;
var httpResponse = _httpClient.Get<RecentUpdatesResource>(httpRequest);
if (httpResponse.Resource.Limited)
{
return null;
}
return new HashSet<string>(httpResponse.Resource.Ids.Select(x => x.ToString()));
}
public Author GetAuthorInfo(string foreignAuthorId, bool useCache = false, bool includeBooks = true)
{
_logger.Debug("Getting Author details GoodreadsId of {0}", foreignAuthorId);
return PollAuthor(foreignAuthorId, includeBooks);
}
private Author PollAuthor(string foreignAuthorId, bool includeBooks)
{
AuthorResource resource = null;
for (var i = 0; i < 60; i++)
{
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", $"author/{foreignAuthorId}")
.Build();
httpRequest.AllowAutoRedirect = true;
httpRequest.SuppressHttpError = true;
var httpResponse = _httpClient.Get<AuthorResource>(httpRequest);
if (httpResponse.HasHttpError)
{
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
throw new AuthorNotFoundException(foreignAuthorId);
}
else if (httpResponse.StatusCode == HttpStatusCode.BadRequest)
{
throw new BadRequestException(foreignAuthorId);
}
else
{
throw new HttpException(httpRequest, httpResponse);
}
}
resource = httpResponse.Resource;
if (resource.Works != null || !includeBooks)
{
resource.Works ??= new List<WorkResource>();
resource.Series ??= new List<SeriesResource>();
break;
}
Thread.Sleep(2000);
}
if (resource?.Works == null)
{
throw new BookInfoException($"Failed to get works for {foreignAuthorId}");
}
return MapAuthor(resource);
}
public Author GetAuthorAndBooks(string foreignAuthorId, double minPopularity = 0)
{
return GetAuthorInfo(foreignAuthorId);
}
public HashSet<string> GetChangedBooks(DateTime startTime)
{
return _cache.Get("ChangedBooks", () => GetChangedBooksUncached(startTime), TimeSpan.FromMinutes(30));
}
private HashSet<string> GetChangedBooksUncached(DateTime startTime)
{
return null;
}
public Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string foreignBookId)
{
return null;
}
private Author MapAuthor(AuthorResource resource)
{
var metadata = new AuthorMetadata
{
ForeignAuthorId = resource.ForeignId.ToString(),
TitleSlug = resource.ForeignId.ToString(),
Name = resource.Name.CleanSpaces(),
Overview = resource.Description,
Ratings = new Ratings { Votes = resource.RatingCount, Value = (decimal)resource.AverageRating },
Status = AuthorStatusType.Continuing
};
metadata.SortName = metadata.Name.ToLower();
metadata.NameLastFirst = metadata.Name.ToLastFirst();
metadata.SortNameLastFirst = metadata.NameLastFirst.ToLower();
if (resource.ImageUrl.IsNotNullOrWhiteSpace())
{
metadata.Images.Add(new MediaCover.MediaCover
{
Url = resource.ImageUrl,
CoverType = MediaCoverTypes.Poster
});
}
if (resource.Url.IsNotNullOrWhiteSpace())
{
metadata.Links.Add(new Links { Url = resource.Url, Name = "Goodreads" });
}
var books = resource.Works
.Where(x => x.ForeignId > 0 && GetAuthorId(x) == resource.ForeignId)
.Select(MapBook)
.ToList();
books.ForEach(x => x.AuthorMetadata = metadata);
var series = resource.Series.Select(MapSeries).ToList();
MapSeriesLinks(series, books, resource);
var result = new Author
{
Metadata = metadata,
CleanName = Parser.Parser.CleanAuthorName(metadata.Name),
Books = books,
Series = series
};
return result;
}
private static void MapSeriesLinks(List<Series> series, List<Book> books, AuthorResource resource)
{
var bookDict = books.ToDictionary(x => x.ForeignBookId);
var seriesDict = series.ToDictionary(x => x.ForeignSeriesId);
// only take series where there are some works
foreach (var s in resource.Series.Where(x => x.LinkItems.Any()))
{
if (seriesDict.TryGetValue(s.ForeignId.ToString(), out var curr))
{
curr.LinkItems = s.LinkItems.Where(x => x.ForeignWorkId.IsNotNullOrWhiteSpace() && bookDict.ContainsKey(x.ForeignWorkId.ToString())).Select(l => new SeriesBookLink
{
Book = bookDict[l.ForeignWorkId.ToString()],
Series = curr,
IsPrimary = l.Primary,
Position = l.PositionInSeries
}).ToList();
}
}
}
private static Series MapSeries(SeriesResource resource)
{
var series = new Series
{
ForeignSeriesId = resource.ForeignId.ToString(),
Title = resource.Title,
Description = resource.Description
};
return series;
}
private static Book MapBook(WorkResource resource)
{
var book = new Book
{
ForeignBookId = resource.ForeignId.ToString(),
Title = resource.Title,
TitleSlug = resource.ForeignId.ToString(),
CleanTitle = Parser.Parser.CleanAuthorName(resource.Title),
ReleaseDate = resource.ReleaseDate,
Genres = resource.Genres,
RelatedBooks = resource.RelatedWorks
};
book.Links.Add(new Links { Url = resource.Url, Name = "Goodreads Editions" });
if (resource.Books != null)
{
book.Editions = resource.Books.Select(x => MapEdition(x)).ToList();
// monitor the most rated release
var mostPopular = book.Editions.Value.OrderByDescending(x => x.Ratings.Votes).FirstOrDefault();
if (mostPopular != null)
{
mostPopular.Monitored = true;
// fix work title if missing
if (book.Title.IsNullOrWhiteSpace())
{
book.Title = mostPopular.Title;
}
}
}
else
{
book.Editions = new List<Edition>();
}
Debug.Assert(!book.Editions.Value.Any() || book.Editions.Value.Count(x => x.Monitored) == 1, "one edition monitored");
book.AnyEditionOk = true;
var ratingCount = book.Editions.Value.Sum(x => x.Ratings.Votes);
if (ratingCount > 0)
{
book.Ratings = new Ratings
{
Votes = ratingCount,
Value = book.Editions.Value.Sum(x => x.Ratings.Votes * x.Ratings.Value) / ratingCount
};
}
else
{
book.Ratings = new Ratings { Votes = 0, Value = 0 };
}
return book;
}
private static Edition MapEdition(BookResource resource)
{
var edition = new Edition
{
ForeignEditionId = resource.ForeignId.ToString(),
TitleSlug = resource.ForeignId.ToString(),
Isbn13 = resource.Isbn13,
Asin = resource.Asin,
Title = resource.Title.CleanSpaces(),
Language = resource.Language,
Overview = resource.Description,
Format = resource.Format,
IsEbook = resource.IsEbook,
Disambiguation = resource.EditionInformation,
Publisher = resource.Publisher,
PageCount = resource.NumPages ?? 0,
ReleaseDate = resource.ReleaseDate,
Ratings = new Ratings { Votes = resource.RatingCount, Value = (decimal)resource.AverageRating }
};
if (resource.ImageUrl.IsNotNullOrWhiteSpace())
{
edition.Images.Add(new MediaCover.MediaCover
{
Url = resource.ImageUrl,
CoverType = MediaCoverTypes.Cover
});
}
edition.Links.Add(new Links { Url = resource.Url, Name = "Goodreads Book" });
return edition;
}
private int GetAuthorId(WorkResource b)
{
return b.Books.First().Contributors.FirstOrDefault()?.ForeignId ?? 0;
}
}
}

@ -0,0 +1,28 @@
using System;
using System.Collections.Generic;
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class AuthorResource
{
public int ForeignId { get; set; }
public string Name { get; set; }
public string TitleSlug { get; set; }
public string Description { get; set; }
public string ImageUrl { get; set; }
public string Url { get; set; }
public int ReviewCount { get; set; }
public int RatingCount { get; set; }
public double AverageRating { get; set; }
public DateTime LastChange { get; set; }
public DateTime LastRefresh { get; set; }
public List<WorkResource> Works { get; set; }
public List<SeriesResource> Series { get; set; }
}
}

@ -1,11 +1,11 @@
using System;
using System.Collections.Generic;
namespace NzbDrone.Core.MetadataSource.SkyHook
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class BookResource
{
public int GoodreadsId { get; set; }
public int ForeignId { get; set; }
public string TitleSlug { get; set; }
public string Asin { get; set; }
public string Description { get; set; }

@ -1,8 +1,8 @@
namespace NzbDrone.Core.MetadataSource.SkyHook
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class ContributorResource
{
public int GoodreadsId { get; set; }
public int ForeignId { get; set; }
public string Role { get; set; }
}
}

@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class RecentUpdatesResource
{
public bool Limited { get; set; }
public DateTime Since { get; set; }
public List<int> Ids { get; set; }
}
}

@ -0,0 +1,22 @@
using System.Collections.Generic;
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class SeriesResource
{
public int ForeignId { get; set; }
public string Title { get; set; }
public string Description { get; set; }
public List<SeriesWorkLinkResource> LinkItems { get; set; }
}
public class SeriesWorkLinkResource
{
public string ForeignSeriesId { get; set; }
public string ForeignWorkId { get; set; }
public string PositionInSeries { get; set; }
public int SeriesPosition { get; set; }
public bool Primary { get; set; }
}
}

@ -1,15 +1,17 @@
using System;
using System.Collections.Generic;
namespace NzbDrone.Core.MetadataSource.SkyHook
namespace NzbDrone.Core.MetadataSource.BookInfo
{
public class WorkResource
{
public int GoodreadsId { get; set; }
public int ForeignId { get; set; }
public string Title { get; set; }
public string TitleSlug { get; set; }
public string Url { get; set; }
public DateTime? ReleaseDate { get; set; }
public List<string> Genres { get; set; }
public List<int> RelatedWorks { get; set; }
public List<BookResource> Books { get; set; } = new List<BookResource>();
}
}

@ -4,7 +4,7 @@ using System.Xml;
using System.Xml.Linq;
using System.Xml.XPath;
using NzbDrone.Common.Http;
using NzbDrone.Core.MetadataSource.SkyHook;
using NzbDrone.Core.MetadataSource.BookInfo;
namespace NzbDrone.Core.MetadataSource.Goodreads
{
@ -110,7 +110,7 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
// If we found any error at all above, throw an exception
if (!string.IsNullOrWhiteSpace(error))
{
throw new SkyHookException("Received an error from Goodreads " + error);
throw new BookInfoException("Received an error from Goodreads " + error);
}
}
}

@ -17,7 +17,7 @@ using NzbDrone.Core.Parser;
namespace NzbDrone.Core.MetadataSource.Goodreads
{
public class GoodreadsProxy : IProvideAuthorInfo, IProvideBookInfo
public class GoodreadsProxy : IProvideBookInfo
{
private static readonly RegexReplace FullSizeImageRegex = new RegexReplace(@"\._[SU][XY]\d+_.jpg$",
".jpg",
@ -307,14 +307,8 @@ namespace NzbDrone.Core.MetadataSource.Goodreads
return result;
}
public HashSet<string> GetChangedBooks(DateTime startTime)
{
return _cache.Get("ChangedBooks", () => GetChangedBooksUncached(startTime), TimeSpan.FromMinutes(30));
}
private HashSet<string> GetChangedBooksUncached(DateTime startTime)
{
return null;
}
private bool TryGetBookInfo(string foreignEditionId, bool useCache, out Tuple<string, Book, List<AuthorMetadata>> result)

@ -30,7 +30,7 @@ namespace NzbDrone.Core.MetadataSource
}
else
{
return _defaultRequestFactory.Search;
return _defaultRequestFactory.Metadata;
}
}
}

@ -6,7 +6,7 @@ namespace NzbDrone.Core.MetadataSource
{
public interface IProvideAuthorInfo
{
Author GetAuthorInfo(string readarrId, bool useCache = true);
Author GetAuthorInfo(string readarrId, bool useCache = true, bool includeBooks = true);
Author GetAuthorAndBooks(string readarrId, double minPopularity = 0);
HashSet<string> GetChangedAuthors(DateTime startTime);
}

@ -7,6 +7,5 @@ namespace NzbDrone.Core.MetadataSource
public interface IProvideBookInfo
{
Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string id, bool useCache = true);
HashSet<string> GetChangedBooks(DateTime startTime);
}
}

@ -1,18 +0,0 @@
using System.Net;
using NzbDrone.Core.Exceptions;
namespace NzbDrone.Core.MetadataSource.SkyHook
{
public class SkyHookException : NzbDroneClientException
{
public SkyHookException(string message)
: base(HttpStatusCode.ServiceUnavailable, message)
{
}
public SkyHookException(string message, params object[] args)
: base(HttpStatusCode.ServiceUnavailable, message, args)
{
}
}
}

@ -1,491 +0,0 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Net;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Books;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.MediaCover;
namespace NzbDrone.Core.MetadataSource.SkyHook
{
public class SkyHookProxy
{
private readonly IHttpClient _httpClient;
private readonly Logger _logger;
private readonly IAuthorService _authorService;
private readonly IBookService _bookService;
private readonly IMetadataRequestBuilder _requestBuilder;
private readonly ICached<HashSet<string>> _cache;
public SkyHookProxy(IHttpClient httpClient,
IMetadataRequestBuilder requestBuilder,
IAuthorService authorService,
IBookService bookService,
Logger logger,
ICacheManager cacheManager)
{
_httpClient = httpClient;
_requestBuilder = requestBuilder;
_authorService = authorService;
_bookService = bookService;
_cache = cacheManager.GetCache<HashSet<string>>(GetType());
_logger = logger;
}
public HashSet<string> GetChangedAuthors(DateTime startTime)
{
return null;
}
public Author GetAuthorInfo(string foreignAuthorId)
{
_logger.Debug("Getting Author details ReadarrAPI.MetadataID of {0}", foreignAuthorId);
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", $"author/{foreignAuthorId}")
.Build();
httpRequest.AllowAutoRedirect = true;
httpRequest.SuppressHttpError = true;
var httpResponse = _httpClient.Get<AuthorResource>(httpRequest);
if (httpResponse.HasHttpError)
{
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
throw new AuthorNotFoundException(foreignAuthorId);
}
else if (httpResponse.StatusCode == HttpStatusCode.BadRequest)
{
throw new BadRequestException(foreignAuthorId);
}
else
{
throw new HttpException(httpRequest, httpResponse);
}
}
return MapAuthor(httpResponse.Resource);
}
public HashSet<string> GetChangedBooks(DateTime startTime)
{
return _cache.Get("ChangedBooks", () => GetChangedBooksUncached(startTime), TimeSpan.FromMinutes(30));
}
private HashSet<string> GetChangedBooksUncached(DateTime startTime)
{
return null;
}
public Tuple<string, Book, List<AuthorMetadata>> GetBookInfo(string foreignBookId)
{
return null;
/*
_logger.Debug("Getting Book with ReadarrAPI.MetadataID of {0}", foreignBookId);
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", $"book/{foreignBookId}")
.Build();
httpRequest.AllowAutoRedirect = true;
httpRequest.SuppressHttpError = true;
var httpResponse = _httpClient.Get<BookResource>(httpRequest);
if (httpResponse.HasHttpError)
{
if (httpResponse.StatusCode == HttpStatusCode.NotFound)
{
throw new BookNotFoundException(foreignBookId);
}
else if (httpResponse.StatusCode == HttpStatusCode.BadRequest)
{
throw new BadRequestException(foreignBookId);
}
else
{
throw new HttpException(httpRequest, httpResponse);
}
}
var b = httpResponse.Resource;
var book = MapBook(b);
// var authors = httpResponse.Resource.AuthorMetadata.SelectList(MapAuthor);
var authorid = GetAuthorId(b).ToString();
// book.AuthorMetadata = authors.First(x => x.ForeignAuthorId == authorid);
return new Tuple<string, Book, List<AuthorMetadata>>(authorid, book, null);
*/
}
public List<Author> SearchForNewAuthor(string title)
{
var books = SearchForNewBook(title, null);
return books.Select(x => x.Author.Value).ToList();
}
public List<Book> SearchForNewBook(string title, string author)
{
try
{
var lowerTitle = title.ToLowerInvariant();
var split = lowerTitle.Split(':');
var prefix = split[0];
if (split.Length == 2 && new[] { "readarr", "readarrid", "goodreads", "isbn", "asin" }.Contains(prefix))
{
var slug = split[1].Trim();
if (slug.IsNullOrWhiteSpace() || slug.Any(char.IsWhiteSpace))
{
return new List<Book>();
}
if (prefix == "goodreads" || prefix == "readarr" || prefix == "readarrid")
{
var isValid = int.TryParse(slug, out var searchId);
if (!isValid)
{
return new List<Book>();
}
return SearchByGoodreadsId(searchId);
}
else if (prefix == "isbn")
{
return SearchByIsbn(slug);
}
else if (prefix == "asin")
{
return SearchByAsin(slug);
}
}
var q = title.ToLower().Trim();
if (author != null)
{
q += " " + author;
}
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", "search")
.AddQueryParam("q", q)
.Build();
var result = _httpClient.Get<BookSearchResource>(httpRequest);
return MapSearchResult(result.Resource);
}
catch (HttpException)
{
throw new SkyHookException("Search for '{0}' failed. Unable to communicate with ReadarrAPI.", title);
}
catch (Exception ex)
{
_logger.Warn(ex, ex.Message);
throw new SkyHookException("Search for '{0}' failed. Invalid response received from ReadarrAPI.", title);
}
}
public List<Book> SearchByIsbn(string isbn)
{
return SearchByAlternateId("isbn", isbn);
}
public List<Book> SearchByAsin(string asin)
{
return SearchByAlternateId("asin", asin.ToUpper());
}
public List<Book> SearchByGoodreadsId(int goodreadsId)
{
return SearchByAlternateId("goodreads", goodreadsId.ToString());
}
private List<Book> SearchByAlternateId(string type, string id)
{
try
{
var httpRequest = _requestBuilder.GetRequestBuilder().Create()
.SetSegment("route", $"book/{type}/{id}")
.Build();
var httpResponse = _httpClient.Get<BookSearchResource>(httpRequest);
var result = _httpClient.Get<BookSearchResource>(httpRequest);
return MapSearchResult(result.Resource);
}
catch (HttpException)
{
throw new SkyHookException("Search for {0} '{1}' failed. Unable to communicate with ReadarrAPI.", type, id);
}
catch (Exception ex)
{
_logger.Warn(ex, ex.Message);
throw new SkyHookException("Search for {0 }'{1}' failed. Invalid response received from ReadarrAPI.", type, id);
}
}
public List<object> SearchForNewEntity(string title)
{
var books = SearchForNewBook(title, null);
var result = new List<object>();
foreach (var book in books)
{
var author = book.Author.Value;
if (!result.Contains(author))
{
result.Add(author);
}
result.Add(book);
}
return result;
}
private Author MapAuthor(AuthorResource resource)
{
var metadata = MapAuthor(resource.AuthorMetadata.First(x => x.GoodreadsId == resource.GoodreadsId));
var books = resource.Works
.Where(x => GetAuthorId(x) == resource.GoodreadsId)
.Select(MapBook)
.ToList();
books.ForEach(x => x.AuthorMetadata = metadata);
var series = resource.Series.Select(MapSeries).ToList();
MapSeriesLinks(series, books, resource);
var result = new Author
{
Metadata = metadata,
CleanName = Parser.Parser.CleanAuthorName(metadata.Name),
Books = books,
Series = series
};
return result;
}
private void MapSeriesLinks(List<Series> series, List<Book> books, BulkResource resource)
{
var bookDict = books.ToDictionary(x => x.ForeignBookId);
var seriesDict = series.ToDictionary(x => x.ForeignSeriesId);
// only take series where there are some works
foreach (var s in resource.Series.Where(x => x.Works.Any()))
{
if (seriesDict.TryGetValue(s.GoodreadsId.ToString(), out var curr))
{
curr.LinkItems = s.Works.Where(x => bookDict.ContainsKey(x.GoodreadsId.ToString())).Select(l => new SeriesBookLink
{
Book = bookDict[l.GoodreadsId.ToString()],
Series = curr,
IsPrimary = l.Primary,
Position = l.Position
}).ToList();
}
}
}
private static AuthorMetadata MapAuthor(AuthorSummaryResource resource)
{
var author = new AuthorMetadata
{
ForeignAuthorId = resource.GoodreadsId.ToString(),
TitleSlug = resource.TitleSlug,
Name = resource.Name.CleanSpaces(),
Overview = resource.Description,
Ratings = new Ratings { Votes = resource.RatingsCount, Value = (decimal)resource.AverageRating }
};
author.NameLastFirst = author.Name.ToLastFirst();
author.SortName = author.Name.ToLower();
author.SortNameLastFirst = author.Name.ToLastFirst().ToLower();
if (resource.ImageUrl.IsNotNullOrWhiteSpace())
{
author.Images.Add(new MediaCover.MediaCover
{
Url = resource.ImageUrl,
CoverType = MediaCoverTypes.Poster
});
}
author.Links.Add(new Links { Url = resource.Url, Name = "Goodreads" });
return author;
}
private static Series MapSeries(SeriesResource resource)
{
var series = new Series
{
ForeignSeriesId = resource.GoodreadsId.ToString(),
Title = resource.Title,
Description = resource.Description
};
return series;
}
private static Book MapBook(WorkResource resource)
{
var book = new Book
{
ForeignBookId = resource.GoodreadsId.ToString(),
Title = resource.Title,
TitleSlug = resource.TitleSlug,
CleanTitle = Parser.Parser.CleanAuthorName(resource.Title),
ReleaseDate = resource.ReleaseDate,
};
book.Links.Add(new Links { Url = resource.Url, Name = "Goodreads Editions" });
if (resource.Books != null)
{
book.Editions = resource.Books.Select(x => MapEdition(x)).ToList();
// monitor the most rated release
var mostPopular = book.Editions.Value.OrderByDescending(x => x.Ratings.Votes).FirstOrDefault();
if (mostPopular != null)
{
mostPopular.Monitored = true;
// fix work title if missing
if (book.Title.IsNullOrWhiteSpace())
{
book.Title = mostPopular.Title;
}
}
}
else
{
book.Editions = new List<Edition>();
}
Debug.Assert(!book.Editions.Value.Any() || book.Editions.Value.Count(x => x.Monitored) == 1, "one edition monitored");
book.AnyEditionOk = true;
var ratingCount = book.Editions.Value.Sum(x => x.Ratings.Votes);
if (ratingCount > 0)
{
book.Ratings = new Ratings
{
Votes = ratingCount,
Value = book.Editions.Value.Sum(x => x.Ratings.Votes * x.Ratings.Value) / ratingCount
};
}
else
{
book.Ratings = new Ratings { Votes = 0, Value = 0 };
}
return book;
}
private static Edition MapEdition(BookResource resource)
{
var edition = new Edition
{
ForeignEditionId = resource.GoodreadsId.ToString(),
TitleSlug = resource.TitleSlug,
Isbn13 = resource.Isbn13,
Asin = resource.Asin,
Title = resource.Title.CleanSpaces(),
Language = resource.Language,
Overview = resource.Description,
Format = resource.Format,
IsEbook = resource.IsEbook,
Disambiguation = resource.EditionInformation,
Publisher = resource.Publisher,
PageCount = resource.NumPages ?? 0,
ReleaseDate = resource.ReleaseDate,
Ratings = new Ratings { Votes = resource.RatingCount, Value = (decimal)resource.AverageRating }
};
if (resource.ImageUrl.IsNotNullOrWhiteSpace())
{
edition.Images.Add(new MediaCover.MediaCover
{
Url = resource.ImageUrl,
CoverType = MediaCoverTypes.Cover
});
}
edition.Links.Add(new Links { Url = resource.Url, Name = "Goodreads Book" });
return edition;
}
private List<Book> MapSearchResult(BookSearchResource resource)
{
var metadata = resource.AuthorMetadata.SelectList(MapAuthor).ToDictionary(x => x.ForeignAuthorId);
var result = new List<Book>();
foreach (var b in resource.Works)
{
var book = _bookService.FindById(b.GoodreadsId.ToString());
if (book == null)
{
book = MapBook(b);
var authorid = GetAuthorId(b);
if (authorid == 0)
{
continue;
}
var author = _authorService.FindById(authorid.ToString());
if (author == null)
{
var authorMetadata = metadata[authorid.ToString()];
author = new Author
{
CleanName = Parser.Parser.CleanAuthorName(authorMetadata.Name),
Metadata = authorMetadata
};
}
book.Author = author;
book.AuthorMetadata = author.Metadata.Value;
}
result.Add(book);
}
var seriesList = resource.Series.Select(MapSeries).ToList();
MapSeriesLinks(seriesList, result, resource);
return result;
}
private int GetAuthorId(WorkResource b)
{
return b.Books.First().Contributors.FirstOrDefault()?.GoodreadsId ?? 0;
}
}
}

@ -1,7 +0,0 @@
namespace NzbDrone.Core.MetadataSource.SkyHook
{
public class AuthorResource : BulkResource
{
public int GoodreadsId { get; set; }
}
}

@ -1,17 +0,0 @@
namespace NzbDrone.Core.MetadataSource.SkyHook
{
public class AuthorSummaryResource
{
public int GoodreadsId { get; set; }
public string TitleSlug { get; set; }
public string Name { get; set; }
public string Description { get; set; }
public string ImageUrl { get; set; }
public string Url { get; set; }
public int ReviewCount { get; set; }
public int RatingsCount { get; set; }
public double AverageRating { get; set; }
}
}

@ -1,6 +0,0 @@
namespace NzbDrone.Core.MetadataSource.SkyHook
{
public class BookSearchResource : BulkResource
{
}
}

@ -1,11 +0,0 @@
using System.Collections.Generic;
namespace NzbDrone.Core.MetadataSource.SkyHook
{
public class BulkResource
{
public List<AuthorSummaryResource> AuthorMetadata { get; set; } = new List<AuthorSummaryResource>();
public List<WorkResource> Works { get; set; }
public List<SeriesResource> Series { get; set; }
}
}

@ -1,20 +0,0 @@
using System.Collections.Generic;
namespace NzbDrone.Core.MetadataSource.SkyHook
{
public class SeriesResource
{
public int GoodreadsId { get; set; }
public string Title { get; set; }
public string Description { get; set; }
public List<SeriesWorkLinkResource> Works { get; set; }
}
public class SeriesWorkLinkResource
{
public int GoodreadsId { get; set; }
public string Position { get; set; }
public bool Primary { get; set; }
}
}
Loading…
Cancel
Save