From f6d1b77b450aa762c5927711bf17498ed88352ad Mon Sep 17 00:00:00 2001 From: Qstick Date: Wed, 25 Oct 2017 23:08:37 -0400 Subject: [PATCH] Misc Newznab/Torznab Updates --- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 12 ++- src/NzbDrone.Core/Indexers/IndexerBase.cs | 1 + .../Newznab/NewznabCapabilitiesProvider.cs | 20 ++++- .../Newznab/NewznabRequestGenerator.cs | 2 +- .../Indexers/Newznab/NewznabRssParser.cs | 58 +++++++-------- .../Indexers/Newznab/NewznabSettings.cs | 9 ++- src/NzbDrone.Core/Indexers/RssEnclosure.cs | 14 ++++ src/NzbDrone.Core/Indexers/RssParser.cs | 73 ++++++++++++++----- .../Indexers/TorrentRssParser.cs | 4 +- .../Indexers/Torznab/TorznabRssParser.cs | 36 ++++++--- .../Indexers/Torznab/TorznabSettings.cs | 1 + src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 12 files changed, 165 insertions(+), 66 deletions(-) create mode 100644 src/NzbDrone.Core/Indexers/RssEnclosure.cs diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index c2d5618e4..df6877aca 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -137,7 +137,7 @@ namespace NzbDrone.Core.Indexers } } - releases.AddRange(pagedReleases); + releases.AddRange(pagedReleases.Where(IsValidRelease)); } if (releases.Any()) @@ -239,6 +239,16 @@ namespace NzbDrone.Core.Indexers return CleanupReleases(releases); } + protected virtual bool IsValidRelease(ReleaseInfo release) + { + if (release.DownloadUrl.IsNullOrWhiteSpace()) + { + return false; + } + + return true; + } + protected virtual bool IsFullPage(IList page) { return PageSize != 0 && page.Count >= PageSize; diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 1ce6fe0e3..fd8a278fc 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -72,6 +72,7 @@ namespace NzbDrone.Core.Indexers result.ForEach(c => { + c.Guid = string.Concat(Definition.Id, "_", c.Guid); c.IndexerId = Definition.Id; c.Indexer = Definition.Name; c.DownloadProtocol = Protocol; diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs index bf82a18de..a9c8fd9c0 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs @@ -41,7 +41,7 @@ namespace NzbDrone.Core.Indexers.Newznab { var capabilities = new NewznabCapabilities(); - var url = string.Format("{0}/api?t=caps", indexerSettings.BaseUrl.TrimEnd('/')); + var url = string.Format("{0}{1}?t=caps", indexerSettings.BaseUrl.TrimEnd('/'), indexerSettings.ApiPath.TrimEnd('/')); if (indexerSettings.ApiKey.IsNotNullOrWhiteSpace()) { @@ -68,13 +68,13 @@ namespace NzbDrone.Core.Indexers.Newznab } catch (XmlException ex) { - _logger.Debug(ex, "Failed to parse newznab api capabilities for {0}.", indexerSettings.BaseUrl); + _logger.Debug(ex, "Failed to parse newznab api capabilities for {0}", indexerSettings.BaseUrl); ex.WithData(response); throw; } catch (Exception ex) { - _logger.Error(ex, "Failed to determine newznab api capabilities for {0}, using the defaults instead till Lidarr restarts.", indexerSettings.BaseUrl); + _logger.Error(ex, "Failed to determine newznab api capabilities for {0}, using the defaults instead till Lidarr restarts", indexerSettings.BaseUrl); } return capabilities; @@ -84,7 +84,19 @@ namespace NzbDrone.Core.Indexers.Newznab { var capabilities = new NewznabCapabilities(); - var xmlRoot = XDocument.Parse(response.Content).Element("caps"); + var xDoc = XDocument.Parse(response.Content); + + if (xDoc == null) + { + throw new XmlException("Invalid XML"); + } + + var xmlRoot = xDoc.Element("caps"); + + if (xmlRoot == null) + { + throw new XmlException("Unexpected XML"); + } var xmlLimits = xmlRoot.Element("limits"); if (xmlLimits != null) diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs index 6fa21a990..ad3676fc5 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs @@ -134,7 +134,7 @@ namespace NzbDrone.Core.Indexers.Newznab var categoriesQuery = string.Join(",", categories.Distinct()); - var baseUrl = string.Format("{0}/api?t={1}&cat={2}&extended=1{3}", Settings.BaseUrl.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters); + var baseUrl = string.Format("{0}{1}?t={2}&cat={3}&extended=1{4}", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters); if (Settings.ApiKey.IsNotNullOrWhiteSpace()) { diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs index a0d647fcc..b9f1efa54 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Collections.Generic; using System.Linq; using System.Xml.Linq; using NzbDrone.Common.Extensions; @@ -13,7 +14,8 @@ namespace NzbDrone.Core.Indexers.Newznab public NewznabRssParser() { - PreferredEnclosureMimeType = "application/x-nzb"; + PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes; + UseEnclosureUrl = true; } protected override bool PreProcess(IndexerResponse indexerResponse) @@ -45,6 +47,24 @@ namespace NzbDrone.Core.Indexers.Newznab throw new NewznabException(indexerResponse, errorMessage); } + protected override bool PostProcess(IndexerResponse indexerResponse, List items, List releases) + { + var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray(); + if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty()) + { + if (enclosureTypes.Intersect(TorrentEnclosureMimeTypes).Any()) + { + _logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Torznab indexer?", NzbEnclosureMimeType, enclosureTypes[0]); + } + else + { + _logger.Warn("Feed does not contain {0}, found {1}.", NzbEnclosureMimeType, enclosureTypes[0]); + } + } + + return true; + } + protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo) { releaseInfo = base.ProcessItem(item, releaseInfo); @@ -55,17 +75,6 @@ namespace NzbDrone.Core.Indexers.Newznab return releaseInfo; } - protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) - { - var enclosureType = GetEnclosure(item).Attribute("type").Value; - if (enclosureType.Contains("application/x-bittorrent")) - { - throw new UnsupportedFeedException("Feed contains {0}, did you intend to add a Torznab indexer?", enclosureType); - } - - return base.PostProcess(item, releaseInfo); - } - protected override string GetInfoUrl(XElement item) { return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments")); @@ -102,18 +111,6 @@ namespace NzbDrone.Core.Indexers.Newznab return base.GetPublishDate(item); } - protected override string GetDownloadUrl(XElement item) - { - var url = base.GetDownloadUrl(item); - - if (!Uri.IsWellFormedUriString(url, UriKind.Absolute)) - { - url = ParseUrl((string)item.Element("enclosure").Attribute("url")); - } - - return url; - } - protected virtual string GetArtist(XElement item) { var artistString = TryGetNewznabAttribute(item, "artist"); @@ -140,11 +137,14 @@ namespace NzbDrone.Core.Indexers.Newznab protected string TryGetNewznabAttribute(XElement item, string key, string defaultValue = "") { - var attr = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.CurrentCultureIgnoreCase)); - - if (attr != null) + var attrElement = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase)); + if (attrElement != null) { - return attr.Attribute("value").Value; + var attrValue = attrElement.Attribute("value"); + if (attrValue != null) + { + return attrValue.Value; + } } return defaultValue; diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs index ccedd42b6..0412d0c64 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs @@ -47,6 +47,7 @@ namespace NzbDrone.Core.Indexers.Newznab }); RuleFor(c => c.BaseUrl).ValidRootUrl(); + RuleFor(c => c.ApiPath).ValidUrlBase("/api"); RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey); RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex) .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace()); @@ -59,16 +60,20 @@ namespace NzbDrone.Core.Indexers.Newznab public NewznabSettings() { + ApiPath = "/api"; Categories = new[] { 3000, 3010, 3020, 3030, 3040 }; } [FieldDefinition(0, Label = "URL")] public string BaseUrl { get; set; } - [FieldDefinition(1, Label = "API Key")] + [FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)] + public string ApiPath { get; set; } + + [FieldDefinition(2, Label = "API Key")] public string ApiKey { get; set; } - [FieldDefinition(2, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows", Advanced = true)] + [FieldDefinition(3, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows", Advanced = true)] public IEnumerable Categories { get; set; } [FieldDefinition(4, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)] diff --git a/src/NzbDrone.Core/Indexers/RssEnclosure.cs b/src/NzbDrone.Core/Indexers/RssEnclosure.cs new file mode 100644 index 000000000..6c0a59c37 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/RssEnclosure.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.Indexers +{ + public class RssEnclosure + { + public string Url { get; set; } + public string Type { get; set; } + public long Length { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index fb7e60d0e..4cab5b6c7 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -19,6 +19,11 @@ namespace NzbDrone.Core.Indexers public class RssParser : IParseIndexerResponse { private static readonly Regex ReplaceEntities = new Regex("&[a-z]+;", RegexOptions.Compiled | RegexOptions.IgnoreCase); + public const string NzbEnclosureMimeType = "application/x-nzb"; + public const string TorrentEnclosureMimeType = "application/x-bittorrent"; + public const string MagnetEnclosureMimeType = "application/x-bittorrent;x-scheme-handler/magnet"; + public static readonly string[] UsenetEnclosureMimeTypes = new[] { NzbEnclosureMimeType }; + public static readonly string[] TorrentEnclosureMimeTypes = new[] { TorrentEnclosureMimeType, MagnetEnclosureMimeType }; protected readonly Logger _logger; @@ -32,7 +37,7 @@ namespace NzbDrone.Core.Indexers // Parse "Size: 1.3 GB" or "1.3 GB" parts in the description element and use that as Size. public bool ParseSizeInDescription { get; set; } - public string PreferredEnclosureMimeType { get; set; } + public string[] PreferredEnclosureMimeTypes { get; set; } private IndexerResponse _indexerResponse; @@ -53,7 +58,7 @@ namespace NzbDrone.Core.Indexers } var document = LoadXmlDocument(indexerResponse); - var items = GetItems(document); + var items = GetItems(document).ToList(); foreach (var item in items) { @@ -77,6 +82,11 @@ namespace NzbDrone.Core.Indexers } } + if (!PostProcess(indexerResponse, items, releases)) + { + return new List(); + } + return releases; } @@ -124,6 +134,11 @@ namespace NzbDrone.Core.Indexers return true; } + protected virtual bool PostProcess(IndexerResponse indexerResponse, List elements, List releases) + { + return true; + } + protected ReleaseInfo ProcessItem(XElement item) { var releaseInfo = CreateNewReleaseInfo(); @@ -132,7 +147,7 @@ namespace NzbDrone.Core.Indexers _logger.Trace("Parsed: {0}", releaseInfo.Title); - return PostProcess(item, releaseInfo); + return PostProcessItem(item, releaseInfo); } protected virtual ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo) @@ -156,7 +171,7 @@ namespace NzbDrone.Core.Indexers return releaseInfo; } - protected virtual ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) + protected virtual ReleaseInfo PostProcessItem(XElement item, ReleaseInfo releaseInfo) { return releaseInfo; } @@ -187,7 +202,8 @@ namespace NzbDrone.Core.Indexers { if (UseEnclosureUrl) { - return ParseUrl((string)GetEnclosure(item).Attribute("url")); + var enclosure = GetEnclosure(item); + return enclosure != null ? ParseUrl(enclosure.Url) : null; } return ParseUrl((string)item.Element("link")); @@ -228,37 +244,60 @@ namespace NzbDrone.Core.Indexers if (enclosure != null) { - return (long)enclosure.Attribute("length"); + return enclosure.Length; } return 0; } - protected virtual XElement GetEnclosure(XElement item) + protected virtual RssEnclosure[] GetEnclosures(XElement item) + { + var enclosures = item.Elements("enclosure") + .Select(v => new RssEnclosure + { + Url = v.Attribute("url").Value, + Type = v.Attribute("type").Value, + Length = (long)v.Attribute("length") + }) + .ToArray(); + + return enclosures; + } + + protected RssEnclosure GetEnclosure(XElement item, bool enforceMimeType = true) + { + var enclosures = GetEnclosures(item); + + return GetEnclosure(enclosures, enforceMimeType); + } + + protected virtual RssEnclosure GetEnclosure(RssEnclosure[] enclosures, bool enforceMimeType = true) { - var enclosures = item.Elements("enclosure").ToArray(); if (enclosures.Length == 0) { return null; } - if (enclosures.Length == 1) + if (PreferredEnclosureMimeTypes != null) { - return enclosures.First(); - } + foreach (var preferredEnclosureType in PreferredEnclosureMimeTypes) + { + var preferredEnclosure = enclosures.FirstOrDefault(v => v.Type == preferredEnclosureType); - if (PreferredEnclosureMimeType != null) - { - var preferredEnclosure = enclosures.FirstOrDefault(v => v.Attribute("type").Value == PreferredEnclosureMimeType); + if (preferredEnclosure != null) + { + return preferredEnclosure; + } + } - if (preferredEnclosure != null) + if (enforceMimeType) { - return preferredEnclosure; + return null; } } - return item.Elements("enclosure").SingleOrDefault(); + return enclosures.SingleOrDefault(); } protected IEnumerable GetItems(XDocument document) diff --git a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs index b77022540..df55ae6a8 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs @@ -1,4 +1,4 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Text.RegularExpressions; using System.Xml.Linq; using NzbDrone.Common.Extensions; @@ -16,7 +16,7 @@ namespace NzbDrone.Core.Indexers public TorrentRssParser() { - PreferredEnclosureMimeType = "application/x-bittorrent"; + PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes; } public IEnumerable GetItems(IndexerResponse indexerResponse) diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs index 038785214..46bd46533 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Collections.Generic; using System.Linq; using System.Xml.Linq; using NzbDrone.Common.Extensions; @@ -11,6 +12,11 @@ namespace NzbDrone.Core.Indexers.Torznab { public const string ns = "{http://torznab.com/schemas/2015/feed}"; + public TorznabRssParser() + { + UseEnclosureUrl = true; + } + protected override bool PreProcess(IndexerResponse indexerResponse) { var xdoc = LoadXmlDocument(indexerResponse); @@ -36,15 +42,22 @@ namespace NzbDrone.Core.Indexers.Torznab throw new TorznabException("Torznab error detected: {0}", errorMessage); } - protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) + protected override bool PostProcess(IndexerResponse indexerResponse, List items, List releases) { - var enclosureType = item.Element("enclosure").Attribute("type").Value; - if (!enclosureType.Contains("application/x-bittorrent")) + var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray(); + if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty()) { - throw new UnsupportedFeedException("Feed contains {0} instead of application/x-bittorrent", enclosureType); + if (enclosureTypes.Intersect(UsenetEnclosureMimeTypes).Any()) + { + _logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Newznab indexer?", TorrentEnclosureMimeType, enclosureTypes[0]); + } + else + { + _logger.Warn("Feed does not contain {0}, found {1}.", TorrentEnclosureMimeType, enclosureTypes[0]); + } } - return base.PostProcess(item, releaseInfo); + return true; } @@ -134,11 +147,14 @@ namespace NzbDrone.Core.Indexers.Torznab protected string TryGetTorznabAttribute(XElement item, string key, string defaultValue = "") { - var attr = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.CurrentCultureIgnoreCase)); - - if (attr != null) + var attrElement = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase)); + if (attrElement != null) { - return attr.Attribute("value").Value; + var attrValue = attrElement.Attribute("value"); + if (attrValue != null) + { + return attrValue.Value; + } } return defaultValue; diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs index 1c1aea8b8..0c2cda826 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs @@ -41,6 +41,7 @@ namespace NzbDrone.Core.Indexers.Torznab }); RuleFor(c => c.BaseUrl).ValidRootUrl(); + RuleFor(c => c.ApiPath).ValidUrlBase("/api"); RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey); RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex) .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace()); diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 0b80490ab..7eda4001b 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -497,6 +497,7 @@ +