using System ;
using System.Collections.Generic ;
using System.Linq ;
using System.Text.RegularExpressions ;
using NLog ;
using NzbDrone.Common.Extensions ;
using NzbDrone.Core.Books ;
using NzbDrone.Core.Books.Calibre ;
using NzbDrone.Core.ImportLists ;
using NzbDrone.Core.Lifecycle ;
using NzbDrone.Core.MediaFiles ;
using NzbDrone.Core.Messaging.Events ;
using NzbDrone.Core.Profiles.Releases ;
using NzbDrone.Core.RootFolders ;
namespace NzbDrone.Core.Profiles.Metadata
{
public interface IMetadataProfileService
{
MetadataProfile Add ( MetadataProfile profile ) ;
void Update ( MetadataProfile profile ) ;
void Delete ( int id ) ;
List < MetadataProfile > All ( ) ;
MetadataProfile Get ( int id ) ;
bool Exists ( int id ) ;
List < Book > FilterBooks ( Author input , int profileId ) ;
}
public class MetadataProfileService : IMetadataProfileService , IHandle < ApplicationStartedEvent >
{
public const string NONE_PROFILE_NAME = "None" ;
public const double NONE_PROFILE_MIN_POPULARITY = 1e10 ;
private static readonly Regex PartOrSetRegex = new Regex ( @"(?<from>\d+) of (?<to>\d+)|(?<from>\d+)\s?/\s?(?<to>\d+)|(?<from>\d+)\s?-\s?(?<to>\d+)" ,
RegexOptions . Compiled | RegexOptions . IgnoreCase ) ;
private readonly IMetadataProfileRepository _profileRepository ;
private readonly IAuthorService _authorService ;
private readonly IBookService _bookService ;
private readonly IEditionService _editionService ;
private readonly IMediaFileService _mediaFileService ;
private readonly IImportListFactory _importListFactory ;
private readonly IRootFolderService _rootFolderService ;
private readonly ITermMatcherService _termMatcherService ;
private readonly Logger _logger ;
public MetadataProfileService ( IMetadataProfileRepository profileRepository ,
IAuthorService authorService ,
IBookService bookService ,
IEditionService editionService ,
IMediaFileService mediaFileService ,
IImportListFactory importListFactory ,
IRootFolderService rootFolderService ,
ITermMatcherService termMatcherService ,
Logger logger )
{
_profileRepository = profileRepository ;
_authorService = authorService ;
_bookService = bookService ;
_editionService = editionService ;
_mediaFileService = mediaFileService ;
_importListFactory = importListFactory ;
_rootFolderService = rootFolderService ;
_termMatcherService = termMatcherService ;
_logger = logger ;
}
public MetadataProfile Add ( MetadataProfile profile )
{
return _profileRepository . Insert ( profile ) ;
}
public void Update ( MetadataProfile profile )
{
if ( profile . Name = = NONE_PROFILE_NAME )
{
throw new InvalidOperationException ( "Not permitted to alter None metadata profile" ) ;
}
_profileRepository . Update ( profile ) ;
}
public void Delete ( int id )
{
var profile = _profileRepository . Get ( id ) ;
if ( profile . Name = = NONE_PROFILE_NAME | |
_authorService . GetAllAuthors ( ) . Any ( c = > c . MetadataProfileId = = id ) | |
_importListFactory . All ( ) . Any ( c = > c . MetadataProfileId = = id ) | |
_rootFolderService . All ( ) . Any ( c = > c . DefaultMetadataProfileId = = id ) )
{
throw new MetadataProfileInUseException ( profile . Name ) ;
}
_profileRepository . Delete ( id ) ;
}
public List < MetadataProfile > All ( )
{
return _profileRepository . All ( ) . ToList ( ) ;
}
public MetadataProfile Get ( int id )
{
return _profileRepository . Get ( id ) ;
}
public bool Exists ( int id )
{
return _profileRepository . Exists ( id ) ;
}
public List < Book > FilterBooks ( Author input , int profileId )
{
var seriesLinks = input . Series . Value . SelectMany ( x = > x . LinkItems . Value )
. GroupBy ( x = > x . Book . Value )
. ToDictionary ( x = > x . Key , y = > y . ToList ( ) ) ;
var dbAuthor = _authorService . FindById ( input . ForeignAuthorId ) ;
var localBooks = new List < Book > ( ) ;
if ( dbAuthor ! = null )
{
localBooks = _bookService . GetBooksByAuthorMetadataId ( dbAuthor . AuthorMetadataId ) ;
var editions = _editionService . GetEditionsByAuthor ( dbAuthor . Id ) . GroupBy ( x = > x . BookId ) . ToDictionary ( x = > x . Key , y = > y . ToList ( ) ) ;
foreach ( var book in localBooks )
{
if ( editions . TryGetValue ( book . Id , out var bookEditions ) )
{
book . Editions = bookEditions ;
}
else
{
book . Editions = new List < Edition > ( ) ;
}
}
}
var localFiles = _mediaFileService . GetFilesByAuthor ( dbAuthor ? . Id ? ? 0 ) ;
return FilterBooks ( input . Books . Value , localBooks , localFiles , seriesLinks , profileId ) ;
}
private List < Book > FilterBooks ( IEnumerable < Book > remoteBooks , List < Book > localBooks , List < BookFile > localFiles , Dictionary < Book , List < SeriesBookLink > > seriesLinks , int metadataProfileId )
{
var profile = Get ( metadataProfileId ) ;
_logger . Trace ( $"Filtering:\n{remoteBooks.Select(x => x.ToString()).Join(" \ n ")}" ) ;
var hash = new HashSet < Book > ( remoteBooks ) ;
var titles = new HashSet < string > ( remoteBooks . Select ( x = > x . Title ) ) ;
var localHash = new HashSet < string > ( localBooks . Where ( x = > x . AddOptions . AddType = = BookAddType . Manual ) . Select ( x = > x . ForeignBookId ) ) ;
localHash . UnionWith ( localFiles . Select ( x = > x . Edition . Value . Book . Value . ForeignBookId ) ) ;
FilterByPredicate ( hash , x = > x . ForeignBookId , localHash , profile , BookAllowedByRating , "rating criteria not met" ) ;
FilterByPredicate ( hash , x = > x . ForeignBookId , localHash , profile , ( x , p ) = > ! p . SkipMissingDate | | x . ReleaseDate . HasValue , "release date is missing" ) ;
FilterByPredicate ( hash , x = > x . ForeignBookId , localHash , profile , ( x , p ) = > ! p . SkipPartsAndSets | | ! IsPartOrSet ( x , seriesLinks . GetValueOrDefault ( x ) , titles ) , "book is part of set" ) ;
FilterByPredicate ( hash , x = > x . ForeignBookId , localHash , profile , ( x , p ) = > ! p . SkipSeriesSecondary | | ! seriesLinks . ContainsKey ( x ) | | seriesLinks [ x ] . Any ( y = > y . IsPrimary ) , "book is a secondary series item" ) ;
FilterByPredicate ( hash , x = > x . ForeignBookId , localHash , profile , ( x , p ) = > ! MatchesTerms ( x . Title , p . Ignored ) , "contains ignored terms" ) ;
foreach ( var book in hash )
{
var localEditions = localBooks . SingleOrDefault ( x = > x . ForeignBookId = = book . ForeignBookId ) ? . Editions . Value ? ? new List < Edition > ( ) ;
book . Editions = FilterEditions ( book . Editions . Value , localEditions , localFiles , profile ) ;
}
FilterByPredicate ( hash , x = > x . ForeignBookId , localHash , profile , ( x , p ) = > x . Editions . Value . Any ( e = > e . PageCount > p . MinPages ) | | x . Editions . Value . All ( e = > e . PageCount = = 0 ) , "minimum page count not met" ) ;
FilterByPredicate ( hash , x = > x . ForeignBookId , localHash , profile , ( x , p ) = > x . Editions . Value . Any ( ) , "all editions filtered out" ) ;
return hash . ToList ( ) ;
}
private List < Edition > FilterEditions ( IEnumerable < Edition > editions , List < Edition > localEditions , List < BookFile > localFiles , MetadataProfile profile )
{
var allowedLanguages = profile . AllowedLanguages . IsNotNullOrWhiteSpace ( ) ? new HashSet < string > ( profile . AllowedLanguages . Trim ( ',' ) . Split ( ',' ) . Select ( x = > x . CanonicalizeLanguage ( ) ) ) : new HashSet < string > ( ) ;
var hash = new HashSet < Edition > ( editions ) ;
var localHash = new HashSet < string > ( localEditions . Where ( x = > x . ManualAdd ) . Select ( x = > x . ForeignEditionId ) ) ;
localHash . UnionWith ( localFiles . Select ( x = > x . Edition . Value . ForeignEditionId ) ) ;
FilterByPredicate ( hash , x = > x . ForeignEditionId , localHash , profile , ( x , p ) = > ! allowedLanguages . Any ( ) | | allowedLanguages . Contains ( x . Language ? . CanonicalizeLanguage ( ) ) , "edition language not allowed" ) ;
FilterByPredicate ( hash , x = > x . ForeignEditionId , localHash , profile , ( x , p ) = > ! p . SkipMissingIsbn | | x . Isbn13 . IsNotNullOrWhiteSpace ( ) | | x . Asin . IsNotNullOrWhiteSpace ( ) , "isbn and asin is missing" ) ;
FilterByPredicate ( hash , x = > x . ForeignEditionId , localHash , profile , ( x , p ) = > ! MatchesTerms ( x . Title , p . Ignored ) , "contains ignored terms" ) ;
return hash . ToList ( ) ;
}
private void FilterByPredicate < T > ( HashSet < T > remoteItems , Func < T , string > getId , HashSet < string > localItems , MetadataProfile profile , Func < T , MetadataProfile , bool > bookAllowed , string message )
{
var filtered = new HashSet < T > ( remoteItems . Where ( x = > ! bookAllowed ( x , profile ) & & ! localItems . Contains ( getId ( x ) ) ) ) ;
if ( filtered . Any ( ) )
{
_logger . Trace ( $"Skipping {filtered.Count} {typeof(T).Name} because {message}:\n{filtered.ConcatToString(x => x.ToString(), " \ n ")}" ) ;
remoteItems . RemoveWhere ( x = > filtered . Contains ( x ) ) ;
}
}
private bool BookAllowedByRating ( Book b , MetadataProfile p )
{
// hack for the 'none' metadata profile
if ( p . MinPopularity = = NONE_PROFILE_MIN_POPULARITY )
{
return false ;
}
return ( b . Ratings . Popularity > = p . MinPopularity ) | | b . ReleaseDate > DateTime . UtcNow ;
}
private bool IsPartOrSet ( Book book , List < SeriesBookLink > seriesLinks , HashSet < string > titles )
{
if ( seriesLinks ! = null & &
seriesLinks . Any ( x = > x . Position . IsNotNullOrWhiteSpace ( ) ) & &
! seriesLinks . Any ( s = > double . TryParse ( s . Position , out _ ) ) )
{
// No non-empty series entries parse to a number, so all like 1-3 etc.
return true ;
}
// Skip things of form Title1 / Title2 when Title1 and Title2 are already in the list
var bookTitles = new [ ] { book . Title } . Concat ( book . Editions . Value . Select ( x = > x . Title ) ) . ToList ( ) ;
foreach ( var title in bookTitles )
{
var split = title . Split ( '/' ) . Select ( x = > x . Trim ( ) ) . ToList ( ) ;
if ( split . Count > 1 & & split . All ( x = > titles . Contains ( x ) ) )
{
return true ;
}
}
var match = PartOrSetRegex . Match ( book . Title ) ;
if ( match . Groups [ "from" ] . Success )
{
var from = int . Parse ( match . Groups [ "from" ] . Value ) ;
return from < = 1800 | | from > DateTime . UtcNow . Year ;
}
return false ;
}
private bool MatchesTerms ( string value , string terms )
{
if ( terms . IsNullOrWhiteSpace ( ) | | value . IsNullOrWhiteSpace ( ) )
{
return false ;
}
var split = terms . Split ( new [ ] { ',' } , StringSplitOptions . RemoveEmptyEntries ) . ToList ( ) ;
var foundTerms = ContainsAny ( split , value ) ;
return foundTerms . Any ( ) ;
}
private List < string > ContainsAny ( List < string > terms , string title )
{
return terms . Where ( t = > _termMatcherService . IsMatch ( t , title ) ) . ToList ( ) ;
}
public void Handle ( ApplicationStartedEvent message )
{
var profiles = All ( ) ;
// Name is a unique property
var emptyProfile = profiles . FirstOrDefault ( x = > x . Name = = NONE_PROFILE_NAME ) ;
// make sure empty profile exists and is actually empty
// TODO: reinstate
if ( emptyProfile ! = null & &
emptyProfile . MinPopularity = = NONE_PROFILE_MIN_POPULARITY )
{
return ;
}
if ( ! profiles . Any ( ) )
{
_logger . Info ( "Setting up standard metadata profile" ) ;
Add ( new MetadataProfile
{
Name = "Standard" ,
MinPopularity = 350 ,
SkipMissingDate = true ,
SkipPartsAndSets = true ,
AllowedLanguages = "eng, null"
} ) ;
}
if ( emptyProfile ! = null )
{
// emptyProfile is not the correct empty profile - move it out of the way
_logger . Info ( $"Renaming non-empty metadata profile {emptyProfile.Name}" ) ;
var names = profiles . Select ( x = > x . Name ) . ToList ( ) ;
var i = 1 ;
emptyProfile . Name = $"{NONE_PROFILE_NAME}.{i}" ;
while ( names . Contains ( emptyProfile . Name ) )
{
i + + ;
}
_profileRepository . Update ( emptyProfile ) ;
}
_logger . Info ( "Setting up empty metadata profile" ) ;
Add ( new MetadataProfile
{
Name = NONE_PROFILE_NAME ,
MinPopularity = NONE_PROFILE_MIN_POPULARITY
} ) ;
}
}
}