Misc Newznab/Torznab Updates

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -137,7 +137,7 @@ namespace NzbDrone.Core.Indexers
                             }
                         }
 
-                        releases.AddRange(pagedReleases);
+                        releases.AddRange(pagedReleases.Where(IsValidRelease));
                     }
 
                     if (releases.Any())
@@ -239,6 +239,16 @@ namespace NzbDrone.Core.Indexers
             return CleanupReleases(releases);
         }
 
+        protected virtual bool IsValidRelease(ReleaseInfo release)
+        {
+            if (release.DownloadUrl.IsNullOrWhiteSpace())
+            {
+                return false;
+            }
+
+            return true;
+        }
+
         protected virtual bool IsFullPage(IList<ReleaseInfo> page)
         {
             return PageSize != 0 && page.Count >= PageSize;

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -72,6 +72,7 @@ namespace NzbDrone.Core.Indexers
 
             result.ForEach(c =>
             {
+                c.Guid = string.Concat(Definition.Id, "_", c.Guid);
                 c.IndexerId = Definition.Id;
                 c.Indexer = Definition.Name;
                 c.DownloadProtocol = Protocol;

src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs

@@ -41,7 +41,7 @@ namespace NzbDrone.Core.Indexers.Newznab
         {
             var capabilities = new NewznabCapabilities();
 
-            var url = string.Format("{0}/api?t=caps", indexerSettings.BaseUrl.TrimEnd('/'));
+            var url = string.Format("{0}{1}?t=caps", indexerSettings.BaseUrl.TrimEnd('/'), indexerSettings.ApiPath.TrimEnd('/'));
 
             if (indexerSettings.ApiKey.IsNotNullOrWhiteSpace())
             {
@@ -68,13 +68,13 @@ namespace NzbDrone.Core.Indexers.Newznab
             }
             catch (XmlException ex)
             {
-                _logger.Debug(ex, "Failed to parse newznab api capabilities for {0}.", indexerSettings.BaseUrl);
+                _logger.Debug(ex, "Failed to parse newznab api capabilities for {0}", indexerSettings.BaseUrl);
                 ex.WithData(response);
                 throw;
             }
             catch (Exception ex)
             {
-                _logger.Error(ex, "Failed to determine newznab api capabilities for {0}, using the defaults instead till Lidarr restarts.", indexerSettings.BaseUrl);
+                _logger.Error(ex, "Failed to determine newznab api capabilities for {0}, using the defaults instead till Lidarr restarts", indexerSettings.BaseUrl);
             }
 
             return capabilities;
@@ -84,7 +84,19 @@ namespace NzbDrone.Core.Indexers.Newznab
         {
             var capabilities = new NewznabCapabilities();
 
-            var xmlRoot = XDocument.Parse(response.Content).Element("caps");
+            var xDoc = XDocument.Parse(response.Content);
+
+            if (xDoc == null)
+            {
+                throw new XmlException("Invalid XML");
+            }
+
+            var xmlRoot = xDoc.Element("caps");
+
+            if (xmlRoot == null)
+            {
+                throw new XmlException("Unexpected XML");
+            }
 
             var xmlLimits = xmlRoot.Element("limits");
             if (xmlLimits != null)

src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs

@@ -134,7 +134,7 @@ namespace NzbDrone.Core.Indexers.Newznab
 
             var categoriesQuery = string.Join(",", categories.Distinct());
 
-            var baseUrl = string.Format("{0}/api?t={1}&cat={2}&extended=1{3}", Settings.BaseUrl.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters);
+            var baseUrl = string.Format("{0}{1}?t={2}&cat={3}&extended=1{4}", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters);
 
             if (Settings.ApiKey.IsNotNullOrWhiteSpace())
             {

src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs

@@ -1,4 +1,5 @@
-using System;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Xml.Linq;
 using NzbDrone.Common.Extensions;
@@ -13,7 +14,8 @@ namespace NzbDrone.Core.Indexers.Newznab
 
         public NewznabRssParser()
         {
-            PreferredEnclosureMimeType = "application/x-nzb";
+            PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes;
+            UseEnclosureUrl = true;
         }
 
         protected override bool PreProcess(IndexerResponse indexerResponse)
@@ -45,6 +47,24 @@ namespace NzbDrone.Core.Indexers.Newznab
             throw new NewznabException(indexerResponse, errorMessage);
         }
 
+        protected override bool PostProcess(IndexerResponse indexerResponse, List<XElement> items, List<ReleaseInfo> releases)
+        {
+            var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray();
+            if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty())
+            {
+                if (enclosureTypes.Intersect(TorrentEnclosureMimeTypes).Any())
+                {
+                    _logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Torznab indexer?", NzbEnclosureMimeType, enclosureTypes[0]);
+                }
+                else
+                {
+                    _logger.Warn("Feed does not contain {0}, found {1}.", NzbEnclosureMimeType, enclosureTypes[0]);
+                }
+            }
+
+            return true;
+        }
+
         protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo)
         {
             releaseInfo = base.ProcessItem(item, releaseInfo);
@@ -55,17 +75,6 @@ namespace NzbDrone.Core.Indexers.Newznab
             return releaseInfo;
         }
 
-        protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo)
-        {
-            var enclosureType = GetEnclosure(item).Attribute("type").Value;
-            if (enclosureType.Contains("application/x-bittorrent"))
-            {
-                throw new UnsupportedFeedException("Feed contains {0}, did you intend to add a Torznab indexer?", enclosureType);
-            }
-
-            return base.PostProcess(item, releaseInfo);
-        }
-
         protected override string GetInfoUrl(XElement item)
         {
             return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments"));
@@ -102,18 +111,6 @@ namespace NzbDrone.Core.Indexers.Newznab
             return base.GetPublishDate(item);
         }
 
-        protected override string GetDownloadUrl(XElement item)
-        {
-            var url = base.GetDownloadUrl(item);
-
-            if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
-            {
-                url = ParseUrl((string)item.Element("enclosure").Attribute("url"));
-            }
-
-            return url;
-        }
-
         protected virtual string GetArtist(XElement item)
         {
             var artistString = TryGetNewznabAttribute(item, "artist");
@@ -140,11 +137,14 @@ namespace NzbDrone.Core.Indexers.Newznab
 
         protected string TryGetNewznabAttribute(XElement item, string key, string defaultValue = "")
         {
-            var attr = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.CurrentCultureIgnoreCase));
+            var attrElement = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
-
+            if (attrElement != null)
-            if (attr != null)
+            {
+                var attrValue = attrElement.Attribute("value");
+                if (attrValue != null)
                 {
-                return attr.Attribute("value").Value;
+                    return attrValue.Value;
+                }
             }
 
             return defaultValue;

src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs

@@ -47,6 +47,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             });
 
             RuleFor(c => c.BaseUrl).ValidRootUrl();
+            RuleFor(c => c.ApiPath).ValidUrlBase("/api");
             RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
             RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
                                                 .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());
@@ -59,16 +60,20 @@ namespace NzbDrone.Core.Indexers.Newznab
 
         public NewznabSettings()
         {
+            ApiPath = "/api";
             Categories = new[] { 3000, 3010, 3020, 3030, 3040 };
         }
 
         [FieldDefinition(0, Label = "URL")]
         public string BaseUrl { get; set; }
 
-        [FieldDefinition(1, Label = "API Key")]
+        [FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)]
+        public string ApiPath { get; set; }
+
+        [FieldDefinition(2, Label = "API Key")]
         public string ApiKey { get; set; }
 
-        [FieldDefinition(2, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows", Advanced = true)]
+        [FieldDefinition(3, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows", Advanced = true)]
         public IEnumerable<int> Categories { get; set; }
 
         [FieldDefinition(4, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)]

src/NzbDrone.Core/Indexers/RssEnclosure.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace NzbDrone.Core.Indexers
+{
+    public class RssEnclosure
+    {
+        public string Url { get; set; }
+        public string Type { get; set; }
+        public long Length { get; set; }
+    }
+}

src/NzbDrone.Core/Indexers/RssParser.cs

@@ -19,6 +19,11 @@ namespace NzbDrone.Core.Indexers
     public class RssParser : IParseIndexerResponse
     {
         private static readonly Regex ReplaceEntities = new Regex("&[a-z]+;", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+        public const string NzbEnclosureMimeType = "application/x-nzb";
+        public const string TorrentEnclosureMimeType = "application/x-bittorrent";
+        public const string MagnetEnclosureMimeType = "application/x-bittorrent;x-scheme-handler/magnet";
+        public static readonly string[] UsenetEnclosureMimeTypes = new[] { NzbEnclosureMimeType };
+        public static readonly string[] TorrentEnclosureMimeTypes = new[] { TorrentEnclosureMimeType, MagnetEnclosureMimeType };
 
         protected readonly Logger _logger;
 
@@ -32,7 +37,7 @@ namespace NzbDrone.Core.Indexers
         // Parse "Size: 1.3 GB" or "1.3 GB" parts in the description element and use that as Size.
         public bool ParseSizeInDescription { get; set; }
 
-        public string PreferredEnclosureMimeType { get; set; }
+        public string[] PreferredEnclosureMimeTypes { get; set; }
 
         private IndexerResponse _indexerResponse;
 
@@ -53,7 +58,7 @@ namespace NzbDrone.Core.Indexers
             }
 
             var document = LoadXmlDocument(indexerResponse);
-            var items = GetItems(document);
+            var items = GetItems(document).ToList();
 
             foreach (var item in items)
             {
@@ -77,6 +82,11 @@ namespace NzbDrone.Core.Indexers
                 }
             }
 
+            if (!PostProcess(indexerResponse, items, releases))
+            {
+                return new List<ReleaseInfo>();
+            }
+
             return releases;
         }
 
@@ -124,6 +134,11 @@ namespace NzbDrone.Core.Indexers
             return true;
         }
 
+        protected virtual bool PostProcess(IndexerResponse indexerResponse, List<XElement> elements, List<ReleaseInfo> releases)
+        {
+            return true;
+        }
+
         protected ReleaseInfo ProcessItem(XElement item)
         {
             var releaseInfo = CreateNewReleaseInfo();
@@ -132,7 +147,7 @@ namespace NzbDrone.Core.Indexers
 
             _logger.Trace("Parsed: {0}", releaseInfo.Title);
 
-            return PostProcess(item, releaseInfo);
+            return PostProcessItem(item, releaseInfo);
         }
 
         protected virtual ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo)
@@ -156,7 +171,7 @@ namespace NzbDrone.Core.Indexers
             return releaseInfo;
         }
 
-        protected virtual ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo)
+        protected virtual ReleaseInfo PostProcessItem(XElement item, ReleaseInfo releaseInfo)
         {
             return releaseInfo;
         }
@@ -187,7 +202,8 @@ namespace NzbDrone.Core.Indexers
         {
             if (UseEnclosureUrl)
             {
-                return ParseUrl((string)GetEnclosure(item).Attribute("url"));
+                var enclosure = GetEnclosure(item);
+                return enclosure != null ? ParseUrl(enclosure.Url) : null;
             }
 
             return ParseUrl((string)item.Element("link"));
@@ -228,29 +244,46 @@ namespace NzbDrone.Core.Indexers
 
             if (enclosure != null)
             {
-                return (long)enclosure.Attribute("length");
+                return enclosure.Length;
             }
 
             return 0;
         }
 
-        protected virtual XElement GetEnclosure(XElement item)
+        protected virtual RssEnclosure[] GetEnclosures(XElement item)
+        {
+            var enclosures = item.Elements("enclosure")
+                                 .Select(v => new RssEnclosure
                                  {
-            var enclosures = item.Elements("enclosure").ToArray();
+                                    Url = v.Attribute("url").Value,
+                                    Type = v.Attribute("type").Value,
+                                    Length = (long)v.Attribute("length")
+                                 })
+                                 .ToArray();
 
-            if (enclosures.Length == 0)
+            return enclosures;
+        }
+
+        protected RssEnclosure GetEnclosure(XElement item, bool enforceMimeType = true)
         {
-                return null;
+            var enclosures = GetEnclosures(item);
+
+            return GetEnclosure(enclosures, enforceMimeType);
         }
 
-            if (enclosures.Length == 1)
+        protected virtual RssEnclosure GetEnclosure(RssEnclosure[] enclosures, bool enforceMimeType = true)
+        {
+
+            if (enclosures.Length == 0)
             {
-                return enclosures.First();
+                return null;
             }
 
-            if (PreferredEnclosureMimeType != null)
+            if (PreferredEnclosureMimeTypes != null)
             {
-                var preferredEnclosure = enclosures.FirstOrDefault(v => v.Attribute("type").Value == PreferredEnclosureMimeType);
+                foreach (var preferredEnclosureType in PreferredEnclosureMimeTypes)
+                {
+                    var preferredEnclosure = enclosures.FirstOrDefault(v => v.Type == preferredEnclosureType);
 
                     if (preferredEnclosure != null)
                     {
@@ -258,7 +291,13 @@ namespace NzbDrone.Core.Indexers
                     }
                 }
 
-            return item.Elements("enclosure").SingleOrDefault();
+                if (enforceMimeType)
+                {
+                    return null;
+                }
+            }
+
+            return enclosures.SingleOrDefault();
         }
 
         protected IEnumerable<XElement> GetItems(XDocument document)

src/NzbDrone.Core/Indexers/TorrentRssParser.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Text.RegularExpressions;
 using System.Xml.Linq;
 using NzbDrone.Common.Extensions;
@@ -16,7 +16,7 @@ namespace NzbDrone.Core.Indexers
 
         public TorrentRssParser()
         {
-            PreferredEnclosureMimeType = "application/x-bittorrent";
+            PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes;
         }
 
         public IEnumerable<XElement> GetItems(IndexerResponse indexerResponse)

src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs

@@ -1,4 +1,5 @@
-using System;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 using System.Xml.Linq;
 using NzbDrone.Common.Extensions;
@@ -11,6 +12,11 @@ namespace NzbDrone.Core.Indexers.Torznab
     {
         public const string ns = "{http://torznab.com/schemas/2015/feed}";
 
+        public TorznabRssParser()
+        {
+            UseEnclosureUrl = true;
+        }
+
         protected override bool PreProcess(IndexerResponse indexerResponse)
         {
             var xdoc = LoadXmlDocument(indexerResponse);
@@ -36,15 +42,22 @@ namespace NzbDrone.Core.Indexers.Torznab
             throw new TorznabException("Torznab error detected: {0}", errorMessage);
         }
 
-        protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo)
+        protected override bool PostProcess(IndexerResponse indexerResponse, List<XElement> items, List<ReleaseInfo> releases)
+        {
+            var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray();
+            if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty())
+            {
+                if (enclosureTypes.Intersect(UsenetEnclosureMimeTypes).Any())
                 {
-            var enclosureType = item.Element("enclosure").Attribute("type").Value;
+                    _logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Newznab indexer?", TorrentEnclosureMimeType, enclosureTypes[0]);
-            if (!enclosureType.Contains("application/x-bittorrent"))
+                }
+                else
                 {
-                throw new UnsupportedFeedException("Feed contains {0} instead of application/x-bittorrent", enclosureType);
+                    _logger.Warn("Feed does not contain {0}, found {1}.", TorrentEnclosureMimeType, enclosureTypes[0]);
+                }
             }
 
-            return base.PostProcess(item, releaseInfo);
+            return true;
         }
 
 
@@ -134,11 +147,14 @@ namespace NzbDrone.Core.Indexers.Torznab
 
         protected string TryGetTorznabAttribute(XElement item, string key, string defaultValue = "")
         {
-            var attr = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.CurrentCultureIgnoreCase));
+            var attrElement = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
-
+            if (attrElement != null)
-            if (attr != null)
             {
-                return attr.Attribute("value").Value;
+                var attrValue = attrElement.Attribute("value");
+                if (attrValue != null)
+                {
+                    return attrValue.Value;
+                }
             }
 
             return defaultValue;

src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs

@@ -41,6 +41,7 @@ namespace NzbDrone.Core.Indexers.Torznab
             });
 
             RuleFor(c => c.BaseUrl).ValidRootUrl();
+            RuleFor(c => c.ApiPath).ValidUrlBase("/api");
             RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
             RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
                                                 .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -497,6 +497,7 @@
     <Compile Include="Indexers\IIndexerSettings.cs" />
     <Compile Include="Indexers\IndexerDefaults.cs" />
     <Compile Include="Indexers\ITorrentIndexerSettings.cs" />
+    <Compile Include="Indexers\RssEnclosure.cs" />
     <Compile Include="Indexers\Waffles\WafflesRssParser.cs" />
     <Compile Include="Indexers\Waffles\Waffles.cs" />
     <Compile Include="Indexers\Waffles\WafflesRequestGenerator.cs" />