From 3abda061ba14debdd4d8082f7576e93ccbc11b0c Mon Sep 17 00:00:00 2001 From: ta264 Date: Wed, 26 May 2021 22:09:31 +0100 Subject: [PATCH] New: Detect audio vs text from newznab categories --- .../DownloadDecisionMakerFixture.cs | 4 +++- .../Profiles/ProfileServiceFixture.cs | 2 +- .../DecisionEngine/DownloadDecisionMaker.cs | 11 +++++++++-- .../Indexers/Newznab/NewznabRssParser.cs | 19 +++++++++++++++++++ src/NzbDrone.Core/Indexers/RssParser.cs | 6 ++++++ .../Indexers/Torznab/TorznabRssParser.cs | 18 ++++++++++++++++-- src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs | 3 ++- src/NzbDrone.Core/Parser/QualityParser.cs | 14 +++++++++++++- .../Qualities/QualityDetectionSource.cs | 5 +++-- 9 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/DownloadDecisionMakerFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/DownloadDecisionMakerFixture.cs index 2af1fa54a..12f34e453 100644 --- a/src/NzbDrone.Core.Test/DecisionEngineTests/DownloadDecisionMakerFixture.cs +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/DownloadDecisionMakerFixture.cs @@ -11,6 +11,7 @@ using NzbDrone.Core.DecisionEngine.Specifications; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Qualities; using NzbDrone.Core.Test.Framework; using NzbDrone.Test.Common; @@ -60,7 +61,8 @@ namespace NzbDrone.Core.Test.DecisionEngineTests _remoteBook = new RemoteBook { Author = new Author(), - Books = new List { new Book() } + Books = new List { new Book() }, + ParsedBookInfo = Builder.CreateNew().With(x => x.Quality = new QualityModel(Quality.FLAC)).Build() }; Mocker.GetMock() diff --git a/src/NzbDrone.Core.Test/Profiles/ProfileServiceFixture.cs b/src/NzbDrone.Core.Test/Profiles/ProfileServiceFixture.cs index d61174dc4..6ea394673 100644 --- a/src/NzbDrone.Core.Test/Profiles/ProfileServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Profiles/ProfileServiceFixture.cs @@ -21,7 +21,7 @@ namespace NzbDrone.Core.Test.Profiles Subject.Handle(new ApplicationStartedEvent()); Mocker.GetMock() - .Verify(v => v.Insert(It.IsAny()), Times.Exactly(4)); + .Verify(v => v.Insert(It.IsAny()), Times.Exactly(3)); } [Test] diff --git a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs index cc2f99956..0d6b2a95f 100644 --- a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs +++ b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs @@ -10,6 +10,7 @@ using NzbDrone.Core.Download.Aggregation; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Qualities; namespace NzbDrone.Core.DecisionEngine { @@ -105,6 +106,12 @@ namespace NzbDrone.Core.DecisionEngine remoteBook.Release = report; + // parse quality again with title and category if unknown + if (remoteBook.ParsedBookInfo.Quality.Quality == Quality.Unknown) + { + remoteBook.ParsedBookInfo.Quality = QualityParser.ParseQuality(report.Title, null, report.Categories); + } + if (remoteBook.Author == null) { decision = new DownloadDecision(remoteBook, new Rejection("Unknown Author")); @@ -138,7 +145,7 @@ namespace NzbDrone.Core.DecisionEngine { parsedBookInfo = new ParsedBookInfo { - Quality = QualityParser.ParseQuality(report.Title) + Quality = QualityParser.ParseQuality(report.Title, null, report.Categories) }; } @@ -160,7 +167,7 @@ namespace NzbDrone.Core.DecisionEngine { parsedBookInfo = new ParsedBookInfo { - Quality = QualityParser.ParseQuality(report.Title, null) + Quality = QualityParser.ParseQuality(report.Title, null, report.Categories) }; } diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs index b79b098c3..662d78e0f 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs @@ -121,6 +121,25 @@ namespace NzbDrone.Core.Indexers.Newznab return base.GetPublishDate(item); } + protected override List GetCategories(XElement item) + { + var values = item.Elements(ns + "attr") + .Where(e => e.Attribute("name").Value.Equals("category", StringComparison.OrdinalIgnoreCase) && + e.Attribute("value")?.Value != null) + .Select(e => e.Attribute("value").Value); + + var cats = new List(); + foreach (var value in values) + { + if (int.TryParse(value, out var cat)) + { + cats.Add(cat); + } + } + + return cats; + } + protected virtual string GetAuthor(XElement item) { var authorString = TryGetNewznabAttribute(item, "author"); diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index 096045a7b..086d2caeb 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -159,6 +159,7 @@ namespace NzbDrone.Core.Indexers releaseInfo.BasicAuthString = GetBasicAuth(); releaseInfo.InfoUrl = GetInfoUrl(item); releaseInfo.CommentUrl = GetCommentUrl(item); + releaseInfo.Categories = GetCategories(item); try { @@ -230,6 +231,11 @@ namespace NzbDrone.Core.Indexers return ParseUrl((string)item.Element("comments")); } + protected virtual List GetCategories(XElement item) + { + return new List(); + } + protected virtual long GetSize(XElement item) { if (UseEnclosureLength) diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs index 26ddd8110..9fc458e97 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs @@ -91,9 +91,23 @@ namespace NzbDrone.Core.Indexers.Torznab return size; } - protected override DateTime GetPublishDate(XElement item) + protected override List GetCategories(XElement item) { - return base.GetPublishDate(item); + var values = item.Elements(ns + "attr") + .Where(e => e.Attribute("name").Value.Equals("category", StringComparison.OrdinalIgnoreCase) && + e.Attribute("value")?.Value != null) + .Select(e => e.Attribute("value").Value); + + var cats = new List(); + foreach (var value in values) + { + if (int.TryParse(value, out var cat)) + { + cats.Add(cat); + } + } + + return cats; } protected override string GetDownloadUrl(XElement item) diff --git a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs index f2521fe1d..6b3b8b3b7 100644 --- a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Text; using NzbDrone.Core.Indexers; @@ -25,7 +26,7 @@ namespace NzbDrone.Core.Parser.Model public string Source { get; set; } public string Container { get; set; } public string Codec { get; set; } - public string Resolution { get; set; } + public List Categories { get; set; } public int Age { diff --git a/src/NzbDrone.Core/Parser/QualityParser.cs b/src/NzbDrone.Core/Parser/QualityParser.cs index 5f743416e..1375bf082 100644 --- a/src/NzbDrone.Core/Parser/QualityParser.cs +++ b/src/NzbDrone.Core/Parser/QualityParser.cs @@ -1,5 +1,7 @@ using System; +using System.Collections.Generic; using System.IO; +using System.Linq; using System.Text.RegularExpressions; using NLog; using NzbDrone.Common.Disk; @@ -29,7 +31,7 @@ namespace NzbDrone.Core.Parser private static readonly Regex CodecRegex = new Regex(@"\b(?:(?PDF)|(?MOBI)|(?EPUB)|(?AZW3?)|(?MPEG Version \d(.5)? Audio, Layer 1|MP1)|(?MPEG Version \d(.5)? Audio, Layer 2|MP2)|(?MP3.*VBR|MPEG Version \d(.5)? Audio, Layer 3 vbr)|(?MP3|MPEG Version \d(.5)? Audio, Layer 3)|(?flac)|(?wavpack|wv)|(?alac)|(?WMA\d?)|(?WAV|PCM)|(?M4A|M4P|M4B|AAC|mp4a|MPEG-4 Audio(?!.*alac))|(?OGG|OGA|Vorbis))\b|(?monkey's audio|[\[|\(].*\bape\b.*[\]|\)])|(?Opus Version \d(.5)? Audio|[\[|\(].*\bopus\b.*[\]|\)])", RegexOptions.Compiled | RegexOptions.IgnoreCase); - public static QualityModel ParseQuality(string name, string desc = null) + public static QualityModel ParseQuality(string name, string desc = null, List categories = null) { Logger.Debug("Trying to parse quality for {0}", name); @@ -107,6 +109,16 @@ namespace NzbDrone.Core.Parser } } + //Based on category + if (result.Quality == Quality.Unknown && categories != null) + { + if (categories.Any(x => x >= 3000 && x < 4000)) + { + result.Quality = Quality.UnknownAudio; + result.QualityDetectionSource = QualityDetectionSource.Category; + } + } + return result; } diff --git a/src/NzbDrone.Core/Qualities/QualityDetectionSource.cs b/src/NzbDrone.Core/Qualities/QualityDetectionSource.cs index 9260c14bc..97272f004 100644 --- a/src/NzbDrone.Core/Qualities/QualityDetectionSource.cs +++ b/src/NzbDrone.Core/Qualities/QualityDetectionSource.cs @@ -1,9 +1,10 @@ -namespace NzbDrone.Core.Qualities +namespace NzbDrone.Core.Qualities { public enum QualityDetectionSource { Name, Extension, - TagLib + TagLib, + Category } }