open-source-mirrors
/
Readarr
mirror of https://github.com/Readarr/Readarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Misc Newznab/Torznab Updates

Browse Source
pull/6/head
Qstick 7 years ago
parent 5bee842b26
commit f6d1b77b45
12 changed files with 165 additions and 66 deletions
Show Stats Download Patch File Download Diff File

12
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
Unescape View File

@ -137,7 +137,7 @@ namespace NzbDrone.Core.Indexers
}
}
releases.AddRange(pagedReleases);
releases.AddRange(pagedReleases.Where(IsValidRelease));
}
if (releases.Any())
@ -239,6 +239,16 @@ namespace NzbDrone.Core.Indexers
return CleanupReleases(releases);
}
protected virtual bool IsValidRelease(ReleaseInfo release)
{
if (release.DownloadUrl.IsNullOrWhiteSpace())
{
return false;
}
return true;
}
protected virtual bool IsFullPage(IList<ReleaseInfo> page)
{
return PageSize != 0 && page.Count >= PageSize;

1
src/NzbDrone.Core/Indexers/IndexerBase.cs
Unescape View File

@ -72,6 +72,7 @@ namespace NzbDrone.Core.Indexers
result.ForEach(c =>
{
c.Guid = string.Concat(Definition.Id, "_", c.Guid);
c.IndexerId = Definition.Id;
c.Indexer = Definition.Name;
c.DownloadProtocol = Protocol;

20
src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs
Unescape View File

@ -41,7 +41,7 @@ namespace NzbDrone.Core.Indexers.Newznab
{
var capabilities = new NewznabCapabilities();
var url = string.Format("{0}/api?t=caps", indexerSettings.BaseUrl.TrimEnd('/'));
var url = string.Format("{0}{1}?t=caps", indexerSettings.BaseUrl.TrimEnd('/'), indexerSettings.ApiPath.TrimEnd('/'));
if (indexerSettings.ApiKey.IsNotNullOrWhiteSpace())
{
@ -68,13 +68,13 @@ namespace NzbDrone.Core.Indexers.Newznab
}
catch (XmlException ex)
{
_logger.Debug(ex, "Failed to parse newznab api capabilities for {0}.", indexerSettings.BaseUrl);
_logger.Debug(ex, "Failed to parse newznab api capabilities for {0}", indexerSettings.BaseUrl);
ex.WithData(response);
throw;
}
catch (Exception ex)
{
_logger.Error(ex, "Failed to determine newznab api capabilities for {0}, using the defaults instead till Lidarr restarts.", indexerSettings.BaseUrl);
_logger.Error(ex, "Failed to determine newznab api capabilities for {0}, using the defaults instead till Lidarr restarts", indexerSettings.BaseUrl);
}
return capabilities;
@ -84,7 +84,19 @@ namespace NzbDrone.Core.Indexers.Newznab
{
var capabilities = new NewznabCapabilities();
var xmlRoot = XDocument.Parse(response.Content).Element("caps");
var xDoc = XDocument.Parse(response.Content);
if (xDoc == null)
{
throw new XmlException("Invalid XML");
}
var xmlRoot = xDoc.Element("caps");
if (xmlRoot == null)
{
throw new XmlException("Unexpected XML");
}
var xmlLimits = xmlRoot.Element("limits");
if (xmlLimits != null)

2
src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs
Unescape View File

@ -134,7 +134,7 @@ namespace NzbDrone.Core.Indexers.Newznab
var categoriesQuery = string.Join(",", categories.Distinct());
var baseUrl = string.Format("{0}/api?t={1}&cat={2}&extended=1{3}", Settings.BaseUrl.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters);
var baseUrl = string.Format("{0}{1}?t={2}&cat={3}&extended=1{4}", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters);
if (Settings.ApiKey.IsNotNullOrWhiteSpace())
{

58
src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs
Unescape View File

@ -1,4 +1,5 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
@ -13,7 +14,8 @@ namespace NzbDrone.Core.Indexers.Newznab
public NewznabRssParser()
{
PreferredEnclosureMimeType = "application/x-nzb";
PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes;
UseEnclosureUrl = true;
}
protected override bool PreProcess(IndexerResponse indexerResponse)
@ -45,6 +47,24 @@ namespace NzbDrone.Core.Indexers.Newznab
throw new NewznabException(indexerResponse, errorMessage);
}
protected override bool PostProcess(IndexerResponse indexerResponse, List<XElement> items, List<ReleaseInfo> releases)
{
var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray();
if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty())
{
if (enclosureTypes.Intersect(TorrentEnclosureMimeTypes).Any())
{
_logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Torznab indexer?", NzbEnclosureMimeType, enclosureTypes[0]);
}
else
{
_logger.Warn("Feed does not contain {0}, found {1}.", NzbEnclosureMimeType, enclosureTypes[0]);
}
}
return true;
}
protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo)
{
releaseInfo = base.ProcessItem(item, releaseInfo);
@ -55,17 +75,6 @@ namespace NzbDrone.Core.Indexers.Newznab
return releaseInfo;
}
protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo)
{
var enclosureType = GetEnclosure(item).Attribute("type").Value;
if (enclosureType.Contains("application/x-bittorrent"))
{
throw new UnsupportedFeedException("Feed contains {0}, did you intend to add a Torznab indexer?", enclosureType);
}
return base.PostProcess(item, releaseInfo);
}
protected override string GetInfoUrl(XElement item)
{
return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments"));
@ -102,18 +111,6 @@ namespace NzbDrone.Core.Indexers.Newznab
return base.GetPublishDate(item);
}
protected override string GetDownloadUrl(XElement item)
{
var url = base.GetDownloadUrl(item);
if (!Uri.IsWellFormedUriString(url, UriKind.Absolute))
{
url = ParseUrl((string)item.Element("enclosure").Attribute("url"));
}
return url;
}
protected virtual string GetArtist(XElement item)
{
var artistString = TryGetNewznabAttribute(item, "artist");
@ -140,11 +137,14 @@ namespace NzbDrone.Core.Indexers.Newznab
protected string TryGetNewznabAttribute(XElement item, string key, string defaultValue = "")
{
var attr = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.CurrentCultureIgnoreCase));
if (attr != null)
var attrElement = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
if (attrElement != null)
{
return attr.Attribute("value").Value;
var attrValue = attrElement.Attribute("value");
if (attrValue != null)
{
return attrValue.Value;
}
}
return defaultValue;

9
src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs
Unescape View File

@ -47,6 +47,7 @@ namespace NzbDrone.Core.Indexers.Newznab
});
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.ApiPath).ValidUrlBase("/api");
RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
.When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());
@ -59,16 +60,20 @@ namespace NzbDrone.Core.Indexers.Newznab
public NewznabSettings()
{
ApiPath = "/api";
Categories = new[] { 3000, 3010, 3020, 3030, 3040 };
}
[FieldDefinition(0, Label = "URL")]
public string BaseUrl { get; set; }
[FieldDefinition(1, Label = "API Key")]
[FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)]
public string ApiPath { get; set; }
[FieldDefinition(2, Label = "API Key")]
public string ApiKey { get; set; }
[FieldDefinition(2, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows", Advanced = true)]
[FieldDefinition(3, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows", Advanced = true)]
public IEnumerable<int> Categories { get; set; }
[FieldDefinition(4, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)]

14
src/NzbDrone.Core/Indexers/RssEnclosure.cs
Unescape View File

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
namespace NzbDrone.Core.Indexers
{
public class RssEnclosure
{
public string Url { get; set; }
public string Type { get; set; }
public long Length { get; set; }
}
}

73
src/NzbDrone.Core/Indexers/RssParser.cs
Unescape View File

@ -19,6 +19,11 @@ namespace NzbDrone.Core.Indexers
public class RssParser : IParseIndexerResponse
{
private static readonly Regex ReplaceEntities = new Regex("&[a-z]+;", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public const string NzbEnclosureMimeType = "application/x-nzb";
public const string TorrentEnclosureMimeType = "application/x-bittorrent";
public const string MagnetEnclosureMimeType = "application/x-bittorrent;x-scheme-handler/magnet";
public static readonly string[] UsenetEnclosureMimeTypes = new[] { NzbEnclosureMimeType };
public static readonly string[] TorrentEnclosureMimeTypes = new[] { TorrentEnclosureMimeType, MagnetEnclosureMimeType };
protected readonly Logger _logger;
@ -32,7 +37,7 @@ namespace NzbDrone.Core.Indexers
// Parse "Size: 1.3 GB" or "1.3 GB" parts in the description element and use that as Size.
public bool ParseSizeInDescription { get; set; }
public string PreferredEnclosureMimeType { get; set; }
public string[] PreferredEnclosureMimeTypes { get; set; }
private IndexerResponse _indexerResponse;
@ -53,7 +58,7 @@ namespace NzbDrone.Core.Indexers
}
var document = LoadXmlDocument(indexerResponse);
var items = GetItems(document);
var items = GetItems(document).ToList();
foreach (var item in items)
{
@ -77,6 +82,11 @@ namespace NzbDrone.Core.Indexers
}
}
if (!PostProcess(indexerResponse, items, releases))
{
return new List<ReleaseInfo>();
}
return releases;
}
@ -124,6 +134,11 @@ namespace NzbDrone.Core.Indexers
return true;
}
protected virtual bool PostProcess(IndexerResponse indexerResponse, List<XElement> elements, List<ReleaseInfo> releases)
{
return true;
}
protected ReleaseInfo ProcessItem(XElement item)
{
var releaseInfo = CreateNewReleaseInfo();
@ -132,7 +147,7 @@ namespace NzbDrone.Core.Indexers
_logger.Trace("Parsed: {0}", releaseInfo.Title);
return PostProcess(item, releaseInfo);
return PostProcessItem(item, releaseInfo);
}
protected virtual ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo)
@ -156,7 +171,7 @@ namespace NzbDrone.Core.Indexers
return releaseInfo;
}
protected virtual ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo)
protected virtual ReleaseInfo PostProcessItem(XElement item, ReleaseInfo releaseInfo)
{
return releaseInfo;
}
@ -187,7 +202,8 @@ namespace NzbDrone.Core.Indexers
{
if (UseEnclosureUrl)
{
return ParseUrl((string)GetEnclosure(item).Attribute("url"));
var enclosure = GetEnclosure(item);
return enclosure != null ? ParseUrl(enclosure.Url) : null;
}
return ParseUrl((string)item.Element("link"));
@ -228,37 +244,60 @@ namespace NzbDrone.Core.Indexers
if (enclosure != null)
{
return (long)enclosure.Attribute("length");
return enclosure.Length;
}
return 0;
}
protected virtual XElement GetEnclosure(XElement item)
protected virtual RssEnclosure[] GetEnclosures(XElement item)
{
var enclosures = item.Elements("enclosure")
.Select(v => new RssEnclosure
{
Url = v.Attribute("url").Value,
Type = v.Attribute("type").Value,
Length = (long)v.Attribute("length")
})
.ToArray();
return enclosures;
}
protected RssEnclosure GetEnclosure(XElement item, bool enforceMimeType = true)
{
var enclosures = GetEnclosures(item);
return GetEnclosure(enclosures, enforceMimeType);
}
protected virtual RssEnclosure GetEnclosure(RssEnclosure[] enclosures, bool enforceMimeType = true)
{
var enclosures = item.Elements("enclosure").ToArray();
if (enclosures.Length == 0)
{
return null;
}
if (enclosures.Length == 1)
if (PreferredEnclosureMimeTypes != null)
{
return enclosures.First();
}
foreach (var preferredEnclosureType in PreferredEnclosureMimeTypes)
{
var preferredEnclosure = enclosures.FirstOrDefault(v => v.Type == preferredEnclosureType);
if (PreferredEnclosureMimeType != null)
{
var preferredEnclosure = enclosures.FirstOrDefault(v => v.Attribute("type").Value == PreferredEnclosureMimeType);
if (preferredEnclosure != null)
{
return preferredEnclosure;
}
}
if (preferredEnclosure != null)
if (enforceMimeType)
{
return preferredEnclosure;
return null;
}
}
return item.Elements("enclosure").SingleOrDefault();
return enclosures.SingleOrDefault();
}
protected IEnumerable<XElement> GetItems(XDocument document)

4
src/NzbDrone.Core/Indexers/TorrentRssParser.cs
Unescape View File

@ -1,4 +1,4 @@
using System.Collections.Generic;
using System.Collections.Generic;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
@ -16,7 +16,7 @@ namespace NzbDrone.Core.Indexers
public TorrentRssParser()
{
PreferredEnclosureMimeType = "application/x-bittorrent";
PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes;
}
public IEnumerable<XElement> GetItems(IndexerResponse indexerResponse)

36
src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs
Unescape View File

@ -1,4 +1,5 @@
using System;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
@ -11,6 +12,11 @@ namespace NzbDrone.Core.Indexers.Torznab
{
public const string ns = "{http://torznab.com/schemas/2015/feed}";
public TorznabRssParser()
{
UseEnclosureUrl = true;
}
protected override bool PreProcess(IndexerResponse indexerResponse)
{
var xdoc = LoadXmlDocument(indexerResponse);
@ -36,15 +42,22 @@ namespace NzbDrone.Core.Indexers.Torznab
throw new TorznabException("Torznab error detected: {0}", errorMessage);
}
protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo)
protected override bool PostProcess(IndexerResponse indexerResponse, List<XElement> items, List<ReleaseInfo> releases)
{
var enclosureType = item.Element("enclosure").Attribute("type").Value;
if (!enclosureType.Contains("application/x-bittorrent"))
var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray();
if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty())
{
throw new UnsupportedFeedException("Feed contains {0} instead of application/x-bittorrent", enclosureType);
if (enclosureTypes.Intersect(UsenetEnclosureMimeTypes).Any())
{
_logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Newznab indexer?", TorrentEnclosureMimeType, enclosureTypes[0]);
}
else
{
_logger.Warn("Feed does not contain {0}, found {1}.", TorrentEnclosureMimeType, enclosureTypes[0]);
}
}
return base.PostProcess(item, releaseInfo);
return true;
}
@ -134,11 +147,14 @@ namespace NzbDrone.Core.Indexers.Torznab
protected string TryGetTorznabAttribute(XElement item, string key, string defaultValue = "")
{
var attr = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.CurrentCultureIgnoreCase));
if (attr != null)
var attrElement = item.Elements(ns + "attr").FirstOrDefault(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase));
if (attrElement != null)
{
return attr.Attribute("value").Value;
var attrValue = attrElement.Attribute("value");
if (attrValue != null)
{
return attrValue.Value;
}
}
return defaultValue;

1
src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs
Unescape View File

@ -41,6 +41,7 @@ namespace NzbDrone.Core.Indexers.Torznab
});
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.ApiPath).ValidUrlBase("/api");
RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
.When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());

1
src/NzbDrone.Core/NzbDrone.Core.csproj
Unescape View File

@ -497,6 +497,7 @@
<Compile Include="Indexers\IIndexerSettings.cs" />
<Compile Include="Indexers\IndexerDefaults.cs" />
<Compile Include="Indexers\ITorrentIndexerSettings.cs" />
<Compile Include="Indexers\RssEnclosure.cs" />
<Compile Include="Indexers\Waffles\WafflesRssParser.cs" />
<Compile Include="Indexers\Waffles\Waffles.cs" />
<Compile Include="Indexers\Waffles\WafflesRequestGenerator.cs" />

Write Preview
Loading…
Cancel
Save