Merge branch 'linq-xml' into vnext

pull/4/head
Mark McDowall 11 years ago
commit ff573f358f

@ -2,8 +2,8 @@ using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Globalization; using System.Globalization;
using System.IO; using System.IO;
using System.ServiceModel.Syndication;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Xml.Linq;
using NLog; using NLog;
using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Parser.Model;
@ -25,27 +25,27 @@ namespace NzbDrone.Core.Indexers
public IEnumerable<ReportInfo> Process(Stream source) public IEnumerable<ReportInfo> Process(Stream source)
{ {
//TODO: replace this BS with plain Linq to XML var xdoc = XDocument.Load(source);
var reader = new SyndicationFeedXmlReader(source); var items = xdoc.Descendants("item");
var feed = SyndicationFeed.Load(reader).Items;
var result = new List<ReportInfo>(); var result = new List<ReportInfo>();
foreach (var syndicationItem in feed) foreach (var item in items)
{ {
try try
{ {
var parsedEpisode = ParseFeed(syndicationItem); var reportInfo = ParseFeedItem(item);
if (parsedEpisode != null) if (reportInfo != null)
{ {
parsedEpisode.NzbUrl = GetNzbUrl(syndicationItem); reportInfo.NzbUrl = GetNzbUrl(item);
parsedEpisode.NzbInfoUrl = GetNzbInfoUrl(syndicationItem); reportInfo.NzbInfoUrl = GetNzbInfoUrl(item);
result.Add(parsedEpisode);
result.Add(reportInfo);
} }
} }
catch (Exception itemEx) catch (Exception itemEx)
{ {
itemEx.Data.Add("Item", syndicationItem.Title); itemEx.Data.Add("Item", item.Title());
_logger.ErrorException("An error occurred while processing feed item", itemEx); _logger.ErrorException("An error occurred while processing feed item", itemEx);
} }
} }
@ -54,37 +54,37 @@ namespace NzbDrone.Core.Indexers
} }
protected virtual string GetTitle(SyndicationItem syndicationItem) protected virtual string GetTitle(XElement item)
{ {
return syndicationItem.Title.Text; return item.Title();
} }
protected virtual string GetNzbUrl(SyndicationItem item) protected virtual string GetNzbUrl(XElement item)
{ {
return item.Links[0].Uri.ToString(); return item.Links()[0];
} }
protected virtual string GetNzbInfoUrl(SyndicationItem item) protected virtual string GetNzbInfoUrl(XElement item)
{ {
return String.Empty; return String.Empty;
} }
protected virtual ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) protected virtual ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{ {
return currentResult; return currentResult;
} }
private ReportInfo ParseFeed(SyndicationItem item) private ReportInfo ParseFeedItem(XElement item)
{ {
var title = GetTitle(item); var title = GetTitle(item);
var reportInfo = new ReportInfo(); var reportInfo = new ReportInfo();
reportInfo.Title = title; reportInfo.Title = title;
reportInfo.Age = DateTime.Now.Date.Subtract(item.PublishDate.Date).Days; reportInfo.Age = DateTime.Now.Date.Subtract(item.PublishDate().Date).Days;
reportInfo.ReleaseGroup = ParseReleaseGroup(title); reportInfo.ReleaseGroup = ParseReleaseGroup(title);
_logger.Trace("Parsed: {0} from: {1}", reportInfo, item.Title.Text); _logger.Trace("Parsed: {0} from: {1}", reportInfo, item.Title());
return PostProcessor(item, reportInfo); return PostProcessor(item, reportInfo);
} }

@ -109,7 +109,7 @@ namespace NzbDrone.Core.Indexers
{ {
if (webException.Message.Contains("503") || webException.Message.Contains("timed out")) if (webException.Message.Contains("503") || webException.Message.Contains("timed out"))
{ {
_logger.Warn("{0} server is currently unavailable.{1} {2}", indexer.Name, url, webException.Message); _logger.Warn("{0} server is currently unavailable. {1} {2}", indexer.Name, url, webException.Message);
} }
else else
{ {

@ -1,11 +1,15 @@
using System; using System;
using System.ServiceModel.Syndication; using System.Drawing;
using System.Linq;
using System.Xml.Linq;
using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Newznab namespace NzbDrone.Core.Indexers.Newznab
{ {
public class NewznabParser : BasicRssParser public class NewznabParser : BasicRssParser
{ {
private static XNamespace NEWZNAB = "http://www.newznab.com/DTD/2010/feeds/attributes/";
private readonly Newznab _newznabIndexer; private readonly Newznab _newznabIndexer;
public NewznabParser(Newznab newznabIndexer) public NewznabParser(Newznab newznabIndexer)
@ -13,31 +17,22 @@ namespace NzbDrone.Core.Indexers.Newznab
_newznabIndexer = newznabIndexer; _newznabIndexer = newznabIndexer;
} }
protected override string GetNzbInfoUrl(SyndicationItem item) protected override string GetNzbInfoUrl(XElement item)
{ {
return item.Id; return item.Comments().Replace("#comments", "");
} }
protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{ {
if (currentResult != null) if (currentResult != null)
{ {
if (item.Links.Count > 1) var attributes = item.Elements(NEWZNAB + "attr");
{ var sizeElement = attributes.Single(e => e.Attribute("name").Value == "size");
currentResult.Size = item.Links[1].Length;
}
currentResult.Indexer = GetName(item); currentResult.Size = Convert.ToInt64(sizeElement.Attribute("value").Value);
} }
return currentResult; return currentResult;
} }
private string GetName(SyndicationItem item)
{
var hostname = item.Links[0].Uri.DnsSafeHost.ToLower();
return String.Format("{0}_{1}", _newznabIndexer.Name, hostname);
}
} }
} }

@ -1,6 +1,6 @@
using System; using System;
using System.ServiceModel.Syndication;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Xml.Linq;
using NLog; using NLog;
using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Parser.Model;
@ -18,11 +18,11 @@ namespace NzbDrone.Core.Indexers.NzbClub
} }
protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{ {
if (currentResult != null) if (currentResult != null)
{ {
var match = SizeRegex.Match(item.Summary.Text); var match = SizeRegex.Match(item.Description());
if (match.Success && match.Groups["size"].Success) if (match.Success && match.Groups["size"].Success)
{ {
@ -30,26 +30,33 @@ namespace NzbDrone.Core.Indexers.NzbClub
} }
else else
{ {
logger.Warn("Couldn't parse size from {0}", item.Summary.Text); logger.Warn("Couldn't parse size from {0}", item.Description());
} }
} }
return currentResult; return currentResult;
} }
protected override string GetTitle(SyndicationItem syndicationItem) protected override string GetTitle(XElement item)
{ {
var title = ParseHeader(syndicationItem.Title.Text); var title = ParseHeader(item.Title());
if (String.IsNullOrWhiteSpace(title)) if (String.IsNullOrWhiteSpace(title))
return syndicationItem.Title.Text; return item.Title();
return title; return title;
} }
protected override string GetNzbInfoUrl(SyndicationItem item) protected override string GetNzbInfoUrl(XElement item)
{ {
return item.Links[1].Uri.ToString(); return item.Links()[0];
}
protected override string GetNzbUrl(XElement item)
{
var enclosure = item.Element("enclosure");
return enclosure.Attribute("url").Value;
} }
} }
} }

@ -10,6 +10,14 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs
get { return "omgwtfnzbs"; } get { return "omgwtfnzbs"; }
} }
public override IParseFeed Parser
{
get
{
return new OmgwtfnzbsParser();
}
}
public override IEnumerable<string> RecentFeed public override IEnumerable<string> RecentFeed
{ {
get get

@ -1,21 +1,16 @@
using System; using System;
using System.ServiceModel.Syndication;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using System.Xml.Linq;
using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Omgwtfnzbs namespace NzbDrone.Core.Indexers.Omgwtfnzbs
{ {
public class OmgwtfnzbsParser : BasicRssParser public class OmgwtfnzbsParser : BasicRssParser
{ {
protected override string GetNzbUrl(SyndicationItem item) protected override string GetNzbInfoUrl(XElement item)
{
return item.Links[0].Uri.ToString();
}
protected override string GetNzbInfoUrl(SyndicationItem item)
{ {
//Todo: Me thinks I need to parse details to get this... //Todo: Me thinks I need to parse details to get this...
var match = Regex.Match(item.Summary.Text, @"(?:\<b\>View NZB\:\<\/b\>\s\<a\shref\=\"")(?<URL>.+)(?:\""\starget)", var match = Regex.Match(item.Description(), @"(?:\<b\>View NZB\:\<\/b\>\s\<a\shref\=\"")(?<URL>.+)(?:\""\starget)",
RegexOptions.IgnoreCase | RegexOptions.Compiled); RegexOptions.IgnoreCase | RegexOptions.Compiled);
if (match.Success) if (match.Success)
@ -26,11 +21,11 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs
return String.Empty; return String.Empty;
} }
protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{ {
if (currentResult != null) if (currentResult != null)
{ {
var sizeString = Regex.Match(item.Summary.Text, @"Size:\<\/b\>\s\d+\.\d{1,2}\s\w{2}\<br \/\>", RegexOptions.IgnoreCase | RegexOptions.Compiled).Value; var sizeString = Regex.Match(item.Description(), @"(?:Size:\<\/b\>\s\d+\.)\d{1,2}\s\w{2}(?:\<br \/\>)", RegexOptions.IgnoreCase | RegexOptions.Compiled).Value;
currentResult.Size = GetReportSize(sizeString); currentResult.Size = GetReportSize(sizeString);
} }

@ -1,98 +0,0 @@
//http://stackoverflow.com/questions/210375/problems-reading-rss-with-c-and-net-3-5
//https://connect.microsoft.com/VisualStudio/feedback/details/325421/syndicationfeed-load-fails-to-parse-datetime-against-a-real-world-feeds-ie7-can-read
using System;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Reflection;
using System.ServiceModel.Syndication;
using System.Threading;
using System.Xml;
using NLog;
namespace NzbDrone.Core.Indexers
{
public class SyndicationFeedXmlReader : XmlTextReader
{
private static readonly Logger logger = LogManager.GetCurrentClassLogger();
private static readonly string[] rss20DateTimeHints = { "pubDate" };
private static readonly string[] atom10DateTimeHints = { "updated", "published", "lastBuildDate" };
private bool _isRss2DateTime;
private bool _isAtomDateTime;
private static readonly MethodInfo rss20FeedFormatterMethodInfo = typeof(Rss20FeedFormatter).GetMethod("DateFromString", BindingFlags.NonPublic | BindingFlags.Static);
private static readonly MethodInfo atom10FeedFormatterMethodInfo = typeof(Atom10FeedFormatter).GetMethod("DateFromString", BindingFlags.NonPublic | BindingFlags.Instance);
public SyndicationFeedXmlReader(Stream stream) : base(stream) { }
public override bool IsStartElement(string localname, string ns)
{
_isRss2DateTime = rss20DateTimeHints.Contains(localname);
_isAtomDateTime = atom10DateTimeHints.Contains(localname);
CheckForError();
return base.IsStartElement(localname, ns);
}
public override string ReadString()
{
var dateVal = base.ReadString();
try
{
if (_isRss2DateTime)
{
rss20FeedFormatterMethodInfo.Invoke(null, new object[] { dateVal, this });
}
if (_isAtomDateTime)
{
atom10FeedFormatterMethodInfo.Invoke(new Atom10FeedFormatter(), new object[] { dateVal, this });
}
}
catch (TargetInvocationException e)
{
DateTime parsedDate;
if (!DateTime.TryParse(dateVal, new CultureInfo("en-US"), DateTimeStyles.None, out parsedDate))
{
parsedDate = DateTime.UtcNow;
logger.WarnException("Unable to parse Feed date " + dateVal, e);
}
var currentCulture = Thread.CurrentThread.CurrentCulture;
Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US");
dateVal = parsedDate.ToString("ddd, dd MMM yyyy HH:mm:ss zzz");
dateVal = dateVal.Remove(dateVal.LastIndexOf(':'), 1);
Thread.CurrentThread.CurrentCulture = currentCulture;
}
return dateVal;
}
private void CheckForError()
{
if (MoveToContent() == XmlNodeType.Element)
{
if (Name != "error")
return;
var message = "Error: ";
if (HasAttributes)
{
while (MoveToNextAttribute())
{
message += String.Format(" [{0}:{1}]", Name, Value);
}
}
logger.Error("Error in RSS feed: {0}", message);
throw new Exception(message);
}
}
}
}

@ -1,21 +1,16 @@
using System.ServiceModel.Syndication; using System.Xml.Linq;
using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Wombles namespace NzbDrone.Core.Indexers.Wombles
{ {
public class WomblesParser : BasicRssParser public class WomblesParser : BasicRssParser
{ {
protected override string GetNzbUrl(SyndicationItem item) protected override string GetNzbInfoUrl(XElement item)
{
return item.Links[0].Uri.ToString();
}
protected override string GetNzbInfoUrl(SyndicationItem item)
{ {
return null; return null;
} }
protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult)
{ {
if (currentResult != null) if (currentResult != null)
{ {

@ -0,0 +1,52 @@
using System;
using System.Collections.Generic;
using System.Data.Odbc;
using System.Linq;
using System.Text;
using System.Xml.Linq;
namespace NzbDrone.Core.Indexers
{
public static class XElementExtensions
{
public static string Title(this XElement item)
{
return TryGetValue(item, "title", "Unknown");
}
public static DateTime PublishDate(this XElement item)
{
return DateTime.Parse(TryGetValue(item, "pubDate"));
}
public static List<String> Links(this XElement item)
{
var result = new List<String>();
var elements = item.Elements("link");
foreach (var link in elements)
{
result.Add(link.Value);
}
return result;
}
public static string Description(this XElement item)
{
return TryGetValue(item, "description");
}
public static string Comments(this XElement item)
{
return TryGetValue(item, "comments");
}
private static string TryGetValue(XElement item, string elementName, string defaultValue = "")
{
var element = item.Element(elementName);
return element != null ? element.Value : defaultValue;
}
}
}

@ -207,6 +207,7 @@
<Compile Include="Indexers\IndexerSettingUpdatedEvent.cs" /> <Compile Include="Indexers\IndexerSettingUpdatedEvent.cs" />
<Compile Include="Indexers\IndexerWithSetting.cs" /> <Compile Include="Indexers\IndexerWithSetting.cs" />
<Compile Include="Indexers\RssSyncCommand.cs" /> <Compile Include="Indexers\RssSyncCommand.cs" />
<Compile Include="Indexers\XElementExtensions.cs" />
<Compile Include="Instrumentation\Commands\TrimLogCommand.cs" /> <Compile Include="Instrumentation\Commands\TrimLogCommand.cs" />
<Compile Include="Instrumentation\SetLoggingLevel.cs" /> <Compile Include="Instrumentation\SetLoggingLevel.cs" />
<Compile Include="Jobs\TaskManager.cs" /> <Compile Include="Jobs\TaskManager.cs" />
@ -435,9 +436,6 @@
<Compile Include="Indexers\IndexerFetchService.cs"> <Compile Include="Indexers\IndexerFetchService.cs">
<SubType>Code</SubType> <SubType>Code</SubType>
</Compile> </Compile>
<Compile Include="Indexers\SyndicationFeedXmlReader.cs">
<SubType>Code</SubType>
</Compile>
<Compile Include="MediaFiles\MediaFileService.cs"> <Compile Include="MediaFiles\MediaFileService.cs">
<SubType>Code</SubType> <SubType>Code</SubType>
</Compile> </Compile>

Loading…
Cancel
Save