From 7d43afd7b9e55779d03fd728fbedcf1c7f49ff3c Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Mon, 5 Aug 2013 19:45:57 -0700 Subject: [PATCH] Good riddance SyndicationFeed --- NzbDrone.Core/Indexers/BasicRssParser.cs | 40 ++++---- NzbDrone.Core/Indexers/IndexerFetchService.cs | 2 +- .../Indexers/Newznab/NewznabParser.cs | 27 +++-- .../Indexers/NzbClub/NzbClubParser.cs | 25 +++-- .../Indexers/Omgwtfnzbs/Omgwtfnzbs.cs | 8 ++ .../Indexers/Omgwtfnzbs/OmgwtfnzbsParser.cs | 15 +-- .../Indexers/SyndicationFeedXmlReader.cs | 98 ------------------- .../Indexers/Wombles/WomblesParser.cs | 11 +-- NzbDrone.Core/Indexers/XElementExtensions.cs | 52 ++++++++++ NzbDrone.Core/NzbDrone.Core.csproj | 4 +- 10 files changed, 117 insertions(+), 165 deletions(-) delete mode 100644 NzbDrone.Core/Indexers/SyndicationFeedXmlReader.cs create mode 100644 NzbDrone.Core/Indexers/XElementExtensions.cs diff --git a/NzbDrone.Core/Indexers/BasicRssParser.cs b/NzbDrone.Core/Indexers/BasicRssParser.cs index 1b18c7a25..d8013bd1f 100644 --- a/NzbDrone.Core/Indexers/BasicRssParser.cs +++ b/NzbDrone.Core/Indexers/BasicRssParser.cs @@ -2,8 +2,8 @@ using System; using System.Collections.Generic; using System.Globalization; using System.IO; -using System.ServiceModel.Syndication; using System.Text.RegularExpressions; +using System.Xml.Linq; using NLog; using NzbDrone.Core.Parser.Model; @@ -25,27 +25,27 @@ namespace NzbDrone.Core.Indexers public IEnumerable Process(Stream source) { - //TODO: replace this BS with plain Linq to XML - var reader = new SyndicationFeedXmlReader(source); - var feed = SyndicationFeed.Load(reader).Items; + var xdoc = XDocument.Load(source); + var items = xdoc.Descendants("item"); var result = new List(); - foreach (var syndicationItem in feed) + foreach (var item in items) { try { - var parsedEpisode = ParseFeed(syndicationItem); - if (parsedEpisode != null) + var reportInfo = ParseFeedItem(item); + if (reportInfo != null) { - parsedEpisode.NzbUrl = GetNzbUrl(syndicationItem); - parsedEpisode.NzbInfoUrl = GetNzbInfoUrl(syndicationItem); - result.Add(parsedEpisode); + reportInfo.NzbUrl = GetNzbUrl(item); + reportInfo.NzbInfoUrl = GetNzbInfoUrl(item); + + result.Add(reportInfo); } } catch (Exception itemEx) { - itemEx.Data.Add("Item", syndicationItem.Title); + itemEx.Data.Add("Item", item.Title()); _logger.ErrorException("An error occurred while processing feed item", itemEx); } } @@ -54,37 +54,37 @@ namespace NzbDrone.Core.Indexers } - protected virtual string GetTitle(SyndicationItem syndicationItem) + protected virtual string GetTitle(XElement item) { - return syndicationItem.Title.Text; + return item.Title(); } - protected virtual string GetNzbUrl(SyndicationItem item) + protected virtual string GetNzbUrl(XElement item) { - return item.Links[0].Uri.ToString(); + return item.Links()[0]; } - protected virtual string GetNzbInfoUrl(SyndicationItem item) + protected virtual string GetNzbInfoUrl(XElement item) { return String.Empty; } - protected virtual ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) + protected virtual ReportInfo PostProcessor(XElement item, ReportInfo currentResult) { return currentResult; } - private ReportInfo ParseFeed(SyndicationItem item) + private ReportInfo ParseFeedItem(XElement item) { var title = GetTitle(item); var reportInfo = new ReportInfo(); reportInfo.Title = title; - reportInfo.Age = DateTime.Now.Date.Subtract(item.PublishDate.Date).Days; + reportInfo.Age = DateTime.Now.Date.Subtract(item.PublishDate().Date).Days; reportInfo.ReleaseGroup = ParseReleaseGroup(title); - _logger.Trace("Parsed: {0} from: {1}", reportInfo, item.Title.Text); + _logger.Trace("Parsed: {0} from: {1}", reportInfo, item.Title()); return PostProcessor(item, reportInfo); } diff --git a/NzbDrone.Core/Indexers/IndexerFetchService.cs b/NzbDrone.Core/Indexers/IndexerFetchService.cs index 29a634661..4d020002d 100644 --- a/NzbDrone.Core/Indexers/IndexerFetchService.cs +++ b/NzbDrone.Core/Indexers/IndexerFetchService.cs @@ -106,7 +106,7 @@ namespace NzbDrone.Core.Indexers { if (webException.Message.Contains("503") || webException.Message.Contains("timed out")) { - _logger.Warn("{0} server is currently unavailable.{1} {2}", indexer.Name, url, webException.Message); + _logger.Warn("{0} server is currently unavailable. {1} {2}", indexer.Name, url, webException.Message); } else { diff --git a/NzbDrone.Core/Indexers/Newznab/NewznabParser.cs b/NzbDrone.Core/Indexers/Newznab/NewznabParser.cs index db71d631b..10a2fc036 100644 --- a/NzbDrone.Core/Indexers/Newznab/NewznabParser.cs +++ b/NzbDrone.Core/Indexers/Newznab/NewznabParser.cs @@ -1,11 +1,15 @@ using System; -using System.ServiceModel.Syndication; +using System.Drawing; +using System.Linq; +using System.Xml.Linq; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers.Newznab { public class NewznabParser : BasicRssParser { + private static XNamespace NEWZNAB = "http://www.newznab.com/DTD/2010/feeds/attributes/"; + private readonly Newznab _newznabIndexer; public NewznabParser(Newznab newznabIndexer) @@ -13,31 +17,22 @@ namespace NzbDrone.Core.Indexers.Newznab _newznabIndexer = newznabIndexer; } - protected override string GetNzbInfoUrl(SyndicationItem item) + protected override string GetNzbInfoUrl(XElement item) { - return item.Id; + return item.Comments().Replace("#comments", ""); } - protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) + protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult) { if (currentResult != null) { - if (item.Links.Count > 1) - { - currentResult.Size = item.Links[1].Length; - } + var attributes = item.Elements(NEWZNAB + "attr"); + var sizeElement = attributes.Single(e => e.Attribute("name").Value == "size"); - currentResult.Indexer = GetName(item); + currentResult.Size = Convert.ToInt64(sizeElement.Attribute("value").Value); } return currentResult; } - - - private string GetName(SyndicationItem item) - { - var hostname = item.Links[0].Uri.DnsSafeHost.ToLower(); - return String.Format("{0}_{1}", _newznabIndexer.Name, hostname); - } } } \ No newline at end of file diff --git a/NzbDrone.Core/Indexers/NzbClub/NzbClubParser.cs b/NzbDrone.Core/Indexers/NzbClub/NzbClubParser.cs index 0355aaf82..738846020 100644 --- a/NzbDrone.Core/Indexers/NzbClub/NzbClubParser.cs +++ b/NzbDrone.Core/Indexers/NzbClub/NzbClubParser.cs @@ -1,6 +1,6 @@ using System; -using System.ServiceModel.Syndication; using System.Text.RegularExpressions; +using System.Xml.Linq; using NLog; using NzbDrone.Core.Parser.Model; @@ -18,11 +18,11 @@ namespace NzbDrone.Core.Indexers.NzbClub } - protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) + protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult) { if (currentResult != null) { - var match = SizeRegex.Match(item.Summary.Text); + var match = SizeRegex.Match(item.Description()); if (match.Success && match.Groups["size"].Success) { @@ -30,26 +30,33 @@ namespace NzbDrone.Core.Indexers.NzbClub } else { - logger.Warn("Couldn't parse size from {0}", item.Summary.Text); + logger.Warn("Couldn't parse size from {0}", item.Description()); } } return currentResult; } - protected override string GetTitle(SyndicationItem syndicationItem) + protected override string GetTitle(XElement item) { - var title = ParseHeader(syndicationItem.Title.Text); + var title = ParseHeader(item.Title()); if (String.IsNullOrWhiteSpace(title)) - return syndicationItem.Title.Text; + return item.Title(); return title; } - protected override string GetNzbInfoUrl(SyndicationItem item) + protected override string GetNzbInfoUrl(XElement item) { - return item.Links[1].Uri.ToString(); + return item.Links()[0]; + } + + protected override string GetNzbUrl(XElement item) + { + var enclosure = item.Element("enclosure"); + + return enclosure.Attribute("url").Value; } } } \ No newline at end of file diff --git a/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs b/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs index a25478659..e756ee527 100644 --- a/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs +++ b/NzbDrone.Core/Indexers/Omgwtfnzbs/Omgwtfnzbs.cs @@ -10,6 +10,14 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs get { return "omgwtfnzbs"; } } + public override IParseFeed Parser + { + get + { + return new OmgwtfnzbsParser(); + } + } + public override IEnumerable RecentFeed { get diff --git a/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsParser.cs b/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsParser.cs index 9f4bf3aa5..a2369f81a 100644 --- a/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsParser.cs +++ b/NzbDrone.Core/Indexers/Omgwtfnzbs/OmgwtfnzbsParser.cs @@ -1,21 +1,16 @@ using System; -using System.ServiceModel.Syndication; using System.Text.RegularExpressions; +using System.Xml.Linq; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers.Omgwtfnzbs { public class OmgwtfnzbsParser : BasicRssParser { - protected override string GetNzbUrl(SyndicationItem item) - { - return item.Links[0].Uri.ToString(); - } - - protected override string GetNzbInfoUrl(SyndicationItem item) + protected override string GetNzbInfoUrl(XElement item) { //Todo: Me thinks I need to parse details to get this... - var match = Regex.Match(item.Summary.Text, @"(?:\View NZB\:\<\/b\>\s\.+)(?:\""\starget)", + var match = Regex.Match(item.Description(), @"(?:\View NZB\:\<\/b\>\s\.+)(?:\""\starget)", RegexOptions.IgnoreCase | RegexOptions.Compiled); if (match.Success) @@ -26,11 +21,11 @@ namespace NzbDrone.Core.Indexers.Omgwtfnzbs return String.Empty; } - protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) + protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult) { if (currentResult != null) { - var sizeString = Regex.Match(item.Summary.Text, @"Size:\<\/b\>\s\d+\.\d{1,2}\s\w{2}\
", RegexOptions.IgnoreCase | RegexOptions.Compiled).Value; + var sizeString = Regex.Match(item.Description(), @"(?:Size:\<\/b\>\s\d+\.)\d{1,2}\s\w{2}(?:\
)", RegexOptions.IgnoreCase | RegexOptions.Compiled).Value; currentResult.Size = GetReportSize(sizeString); } diff --git a/NzbDrone.Core/Indexers/SyndicationFeedXmlReader.cs b/NzbDrone.Core/Indexers/SyndicationFeedXmlReader.cs deleted file mode 100644 index 30642a721..000000000 --- a/NzbDrone.Core/Indexers/SyndicationFeedXmlReader.cs +++ /dev/null @@ -1,98 +0,0 @@ -//http://stackoverflow.com/questions/210375/problems-reading-rss-with-c-and-net-3-5 -//https://connect.microsoft.com/VisualStudio/feedback/details/325421/syndicationfeed-load-fails-to-parse-datetime-against-a-real-world-feeds-ie7-can-read - -using System; -using System.Globalization; -using System.IO; -using System.Linq; -using System.Reflection; -using System.ServiceModel.Syndication; -using System.Threading; -using System.Xml; -using NLog; - -namespace NzbDrone.Core.Indexers -{ - public class SyndicationFeedXmlReader : XmlTextReader - { - private static readonly Logger logger = LogManager.GetCurrentClassLogger(); - - private static readonly string[] rss20DateTimeHints = { "pubDate" }; - private static readonly string[] atom10DateTimeHints = { "updated", "published", "lastBuildDate" }; - private bool _isRss2DateTime; - private bool _isAtomDateTime; - - private static readonly MethodInfo rss20FeedFormatterMethodInfo = typeof(Rss20FeedFormatter).GetMethod("DateFromString", BindingFlags.NonPublic | BindingFlags.Static); - private static readonly MethodInfo atom10FeedFormatterMethodInfo = typeof(Atom10FeedFormatter).GetMethod("DateFromString", BindingFlags.NonPublic | BindingFlags.Instance); - - public SyndicationFeedXmlReader(Stream stream) : base(stream) { } - - public override bool IsStartElement(string localname, string ns) - { - _isRss2DateTime = rss20DateTimeHints.Contains(localname); - _isAtomDateTime = atom10DateTimeHints.Contains(localname); - - CheckForError(); - - return base.IsStartElement(localname, ns); - } - - public override string ReadString() - { - var dateVal = base.ReadString(); - - try - { - if (_isRss2DateTime) - { - rss20FeedFormatterMethodInfo.Invoke(null, new object[] { dateVal, this }); - } - if (_isAtomDateTime) - { - atom10FeedFormatterMethodInfo.Invoke(new Atom10FeedFormatter(), new object[] { dateVal, this }); - } - } - catch (TargetInvocationException e) - { - DateTime parsedDate; - - if (!DateTime.TryParse(dateVal, new CultureInfo("en-US"), DateTimeStyles.None, out parsedDate)) - { - parsedDate = DateTime.UtcNow; - logger.WarnException("Unable to parse Feed date " + dateVal, e); - } - - var currentCulture = Thread.CurrentThread.CurrentCulture; - Thread.CurrentThread.CurrentCulture = new CultureInfo("en-US"); - dateVal = parsedDate.ToString("ddd, dd MMM yyyy HH:mm:ss zzz"); - dateVal = dateVal.Remove(dateVal.LastIndexOf(':'), 1); - Thread.CurrentThread.CurrentCulture = currentCulture; - } - - return dateVal; - } - - private void CheckForError() - { - if (MoveToContent() == XmlNodeType.Element) - { - if (Name != "error") - return; - - var message = "Error: "; - - if (HasAttributes) - { - while (MoveToNextAttribute()) - { - message += String.Format(" [{0}:{1}]", Name, Value); - } - } - - logger.Error("Error in RSS feed: {0}", message); - throw new Exception(message); - } - - } - } -} diff --git a/NzbDrone.Core/Indexers/Wombles/WomblesParser.cs b/NzbDrone.Core/Indexers/Wombles/WomblesParser.cs index 1a980a9fb..de2aa0a99 100644 --- a/NzbDrone.Core/Indexers/Wombles/WomblesParser.cs +++ b/NzbDrone.Core/Indexers/Wombles/WomblesParser.cs @@ -1,21 +1,16 @@ -using System.ServiceModel.Syndication; +using System.Xml.Linq; using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers.Wombles { public class WomblesParser : BasicRssParser { - protected override string GetNzbUrl(SyndicationItem item) - { - return item.Links[0].Uri.ToString(); - } - - protected override string GetNzbInfoUrl(SyndicationItem item) + protected override string GetNzbInfoUrl(XElement item) { return null; } - protected override ReportInfo PostProcessor(SyndicationItem item, ReportInfo currentResult) + protected override ReportInfo PostProcessor(XElement item, ReportInfo currentResult) { if (currentResult != null) { diff --git a/NzbDrone.Core/Indexers/XElementExtensions.cs b/NzbDrone.Core/Indexers/XElementExtensions.cs new file mode 100644 index 000000000..b04239900 --- /dev/null +++ b/NzbDrone.Core/Indexers/XElementExtensions.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Data.Odbc; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace NzbDrone.Core.Indexers +{ + public static class XElementExtensions + { + public static string Title(this XElement item) + { + return TryGetValue(item, "title", "Unknown"); + } + + public static DateTime PublishDate(this XElement item) + { + return DateTime.Parse(TryGetValue(item, "pubDate")); + } + + public static List Links(this XElement item) + { + var result = new List(); + var elements = item.Elements("link"); + + foreach (var link in elements) + { + result.Add(link.Value); + } + + return result; + } + + public static string Description(this XElement item) + { + return TryGetValue(item, "description"); + } + + public static string Comments(this XElement item) + { + return TryGetValue(item, "comments"); + } + + private static string TryGetValue(XElement item, string elementName, string defaultValue = "") + { + var element = item.Element(elementName); + + return element != null ? element.Value : defaultValue; + } + } +} diff --git a/NzbDrone.Core/NzbDrone.Core.csproj b/NzbDrone.Core/NzbDrone.Core.csproj index 12a437e7e..2cff2bcd6 100644 --- a/NzbDrone.Core/NzbDrone.Core.csproj +++ b/NzbDrone.Core/NzbDrone.Core.csproj @@ -207,6 +207,7 @@ + @@ -435,9 +436,6 @@ Code - - Code - Code