Big Abstraction for IMDBWatchlist -> RSSImport (With a test)

src/NzbDrone.Core.Test/IndexerTests/BasicRssParserFixture.cs

@@ -45,6 +45,7 @@ namespace NzbDrone.Core.Test.IndexerTests
             return new IndexerResponse(new IndexerRequest(httpRequest), httpResponse);
         }
 
+
         [Test]
         public void should_handle_relative_url()
         {

src/NzbDrone.Core.Test/NetImport/RSSImportFixture.cs

@@ -0,0 +1,34 @@
+using System.Linq;
+using System.Text;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.NetImport;
+using NzbDrone.Core.NetImport.RSSImport;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.NetImport
+{
+    public class RSSImportTest : CoreTest<RSSImportParser>
+    {
+        private NetImportResponse CreateResponse(string url, string content)
+        {
+            var httpRequest = new HttpRequest(url);
+            var httpResponse = new HttpResponse(httpRequest, new HttpHeader(), Encoding.UTF8.GetBytes(content));
+
+            return new NetImportResponse(new NetImportRequest(httpRequest), httpResponse);
+        }
+
+
+        [Test]
+        public void should_handle_relative_url()
+        {
+            var xml = ReadAllText("Files/imdb_watchlist.xml");
+
+            var result = Subject.ParseResponse(CreateResponse("http://my.indexer.com/api?q=My+Favourite+Show", xml));
+
+            result.First().Title.Should().Be("Think Like a Man Too");
+        }
+    }
+}

src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj

@@ -284,6 +284,7 @@
     <Compile Include="MetadataSource\SkyHook\SkyHookProxySearchFixture.cs" />
     <Compile Include="MetadataSource\SearchSeriesComparerFixture.cs" />
     <Compile Include="MetadataSource\SkyHook\SkyHookProxyFixture.cs" />
+    <Compile Include="NetImport\RSSImportFixture.cs" />
     <Compile Include="NotificationTests\SynologyIndexerFixture.cs" />
     <Compile Include="OrganizerTests\FileNameBuilderTests\CleanTitleFixture.cs" />
     <Compile Include="OrganizerTests\FileNameBuilderTests\EpisodeTitleCollapseFixture.cs" />
@@ -409,6 +410,9 @@
       <Link>sqlite3.dll</Link>
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
+    <Content Include="Files\imdb_watchlist.xml">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </Content>
     <Content Include="License.txt" />
     <None Include="Files\Indexers\BroadcastheNet\RecentFeed.json">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>

src/NzbDrone.Core/Datastore/Migration/119_create_netimport_table.cs

@@ -10,7 +10,6 @@ namespace NzbDrone.Core.Datastore.Migration
         {
             Create.TableForModel("NetImport")
                 .WithColumn("Enabled").AsBoolean()
-                .WithColumn("ProfileId").AsInt32()
                 .WithColumn("Name").AsString().Unique()
                 .WithColumn("Implementation").AsString()
                 .WithColumn("Settings").AsString().Nullable();

src/NzbDrone.Core/NetImport/IMDbWatchList/IMDbWatchList.cs

@@ -1,36 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Xml.Serialization;
-using FluentValidation.Results;
-using NLog;
-using NzbDrone.Common.Http;
-using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Indexers;
-using NzbDrone.Core.Indexers.PassThePopcorn;
-using NzbDrone.Core.Parser;
-using NzbDrone.Core.Tv;
-
-namespace NzbDrone.Core.NetImport.IMDbWatchList
-{
-    public class IMDbWatchList : HttpNetImportBase<IMDbWatchListSettings>
-    {
-        public override string Name => "IMDbWatchList";
-        public override string Link => "http://rss.imdb.com/list/";
-        public override int ProfileId => 1;
-        public override bool Enabled => true;
-
-        public IMDbWatchList(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
-            : base(httpClient, configService, parsingService, logger)
-        { }
-
-        public override INetImportRequestGenerator GetRequestGenerator()
-        {
-            return new IMDbWatchListRequestGenerator() { Settings = Settings };
-        }
-
-        public override IParseNetImportResponse GetParser()
-        {
-            return new IMDbWatchListParser(Settings);
-        }
-    }
-}

src/NzbDrone.Core/NetImport/IMDbWatchList/IMDbWatchListParser.cs

@@ -1,57 +0,0 @@
-using Newtonsoft.Json;
-using NzbDrone.Core.NetImport.Exceptions;
-using NzbDrone.Core.Tv;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net;
-using System.Text;
-
-namespace NzbDrone.Core.NetImport.IMDbWatchList
-{
-    public class IMDbWatchListParser : IParseNetImportResponse
-    {
-        private readonly IMDbWatchListSettings _settings;
-
-        public IMDbWatchListParser(IMDbWatchListSettings settings)
-        {
-            _settings = settings;
-        }
-
-        public IList<Movie> ParseResponse(NetImportResponse netImportResponse)
-        {
-            var torrentInfos = new List<Movie>();
-
-            if (netImportResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
-            {
-                throw new NetImportException(netImportResponse,
-                    "Unexpected response status {0} code from API request",
-                    netImportResponse.HttpResponse.StatusCode);
-            }
-
-            var jsonResponse = JsonConvert.DeserializeObject<IMDbWatchListAPI.Channel>(netImportResponse.Content);
-
-            var responseData = jsonResponse.Movie;
-            if (responseData == null)
-            {
-                throw new NetImportException(netImportResponse,
-                    "This list has no movies");
-            }
-
-            foreach (var result in responseData)
-            {
-                var title = Parser.Parser.ParseMovieTitle(result.Title, false);
-
-                torrentInfos.Add(new Movie()
-                {
-                    Title = title.MovieTitle,
-                    Year = title.Year,
-                    ProfileId = _settings.ProfileId,
-                    ImdbId = Parser.Parser.ParseImdbId(result.Link)
-                });
-            }
-
-            return torrentInfos.ToArray();
-        }
-    }
-}

src/NzbDrone.Core/NetImport/INetImport.cs

@@ -8,7 +8,6 @@ namespace NzbDrone.Core.NetImport
 {
     public interface INetImport : IProvider
     {
-        string Link { get; }
         bool Enabled { get; }
         
         IList<Movie> Fetch();

src/NzbDrone.Core/NetImport/NetImportBase.cs

@@ -21,8 +21,6 @@ namespace NzbDrone.Core.NetImport
         protected readonly Logger _logger;
 
         public abstract string Name { get; }
-        public abstract string Link { get; }
-        public abstract int ProfileId { get; }
 
         public abstract bool Enabled { get; }
 
@@ -46,8 +44,6 @@ namespace NzbDrone.Core.NetImport
                 yield return new NetImportDefinition
                 {
                     Name = GetType().Name,
-                    Link = Link,
-                    ProfileId = ProfileId,
                     Enabled = config.Validate().IsValid && Enabled,
                     Implementation = GetType().Name,
                     Settings = config

src/NzbDrone.Core/NetImport/NetImportDefinition.cs

@@ -4,8 +4,6 @@ namespace NzbDrone.Core.NetImport
 {
     public class NetImportDefinition : ProviderDefinition
     {
-        public string Link { get; set; }
-        public int ProfileId { get; set; }
         public bool Enabled { get; set; }
         public override bool Enable => Enabled;
     }

src/NzbDrone.Core/NetImport/RSSImport/IMDbWatchListAPI.cs

@@ -1,7 +1,7 @@
 using System.Collections.Generic;
 using System.Xml.Serialization;
 
-namespace NzbDrone.Core.NetImport.IMDbWatchList
+namespace NzbDrone.Core.NetImport.RSSImport
 {
     class IMDbWatchListAPI
     {

src/NzbDrone.Core/NetImport/RSSImport/IMDbWatchListProxy.cs

@@ -8,12 +8,12 @@ using NzbDrone.Core.Exceptions;
 using RestSharp;
 using NzbDrone.Core.Rest;
 
-namespace NzbDrone.Core.NetImport.IMDbWatchList
+namespace NzbDrone.Core.NetImport.RSSImport
 {
     public interface IIMDbWatchListProxy
     {
         void ImportMovies(string url);
-        ValidationFailure Test(IMDbWatchListSettings settings);
+        ValidationFailure Test(RSSImportSettings settings);
     }
 
     public class IMDbWatchListProxy : IIMDbWatchListProxy
@@ -60,7 +60,7 @@ namespace NzbDrone.Core.NetImport.IMDbWatchList
             }
         }
 
-        public ValidationFailure Test(IMDbWatchListSettings settings)
+        public ValidationFailure Test(RSSImportSettings settings)
         {
             try
             {

src/NzbDrone.Core/NetImport/RSSImport/RSSImport.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Collections.Generic;
+using System.Xml.Serialization;
+using FluentValidation.Results;
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.PassThePopcorn;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Tv;
+
+namespace NzbDrone.Core.NetImport.RSSImport
+{
+    public class RSSImport : HttpNetImportBase<RSSImportSettings>
+    {
+        public override string Name => "RSSList";
+        public override bool Enabled => true;
+
+        public RSSImport(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger)
+            : base(httpClient, configService, parsingService, logger)
+        { }
+
+        public new virtual IEnumerable<ProviderDefinition> DefaultDefinitions
+        {
+            get
+            {
+                var config = (RSSImportSettings)new RSSImportSettings();
+                config.Link = "https://rss.imdb.com/list/YOURLISTID";
+
+                yield return new NetImportDefinition
+                {
+                    Name = GetType().Name,
+                    Enabled = config.Validate().IsValid && Enabled,
+                    Implementation = GetType().Name,
+                    Settings = config
+                };
+            }
+        }
+
+        public override INetImportRequestGenerator GetRequestGenerator()
+        {
+            return new RSSImportRequestGenerator() { Settings = Settings };
+        }
+
+        public override IParseNetImportResponse GetParser()
+        {
+            return new RSSImportParser(Settings);
+        }
+    }
+}

src/NzbDrone.Core/NetImport/RSSImport/RSSImportParser.cs

@@ -0,0 +1,236 @@
+using Newtonsoft.Json;
+using NzbDrone.Core.NetImport.Exceptions;
+using NzbDrone.Core.Tv;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Xml;
+using System.Xml.Linq;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.NetImport.RSSImport
+{
+    public class RSSImportParser : IParseNetImportResponse
+    {
+        private readonly RSSImportSettings _settings;
+        private NetImportResponse _importResponse;
+        private readonly Logger _logger;
+
+        private static readonly Regex ReplaceEntities = new Regex("&[a-z]+;", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+
+        public RSSImportParser(RSSImportSettings settings)
+        {
+            _settings = settings;
+        }
+
+        public virtual IList<Movie> ParseResponse(NetImportResponse importResponse)
+        {
+            _importResponse = importResponse;
+
+            var movies = new List<Movie>();
+
+            if (!PreProcess(importResponse))
+            {
+                return movies;
+            }
+
+            var document = LoadXmlDocument(importResponse);
+            var items = GetItems(document);
+
+            foreach (var item in items)
+            {
+                try
+                {
+                    var reportInfo = ProcessItem(item);
+
+                    movies.AddIfNotNull(reportInfo);
+                }
+                catch (Exception itemEx)
+                {
+                    //itemEx.Data.Add("Item", item.Title());
+                    _logger.Error(itemEx, "An error occurred while processing feed item from " + importResponse.Request.Url);
+                }
+            }
+
+            return movies;
+        }
+
+        protected virtual XDocument LoadXmlDocument(NetImportResponse indexerResponse)
+        {
+            try
+            {
+                var content = indexerResponse.Content;
+                content = ReplaceEntities.Replace(content, ReplaceEntity);
+
+                using (var xmlTextReader = XmlReader.Create(new StringReader(content), new XmlReaderSettings { DtdProcessing = DtdProcessing.Ignore, IgnoreComments = true }))
+                {
+                    return XDocument.Load(xmlTextReader);
+                }
+            }
+            catch (XmlException ex)
+            {
+                var contentSample = indexerResponse.Content.Substring(0, Math.Min(indexerResponse.Content.Length, 512));
+                _logger.Debug("Truncated response content (originally {0} characters): {1}", indexerResponse.Content.Length, contentSample);
+
+                ex.Data.Add("ContentLength", indexerResponse.Content.Length);
+                ex.Data.Add("ContentSample", contentSample);
+
+                throw;
+            }
+        }
+
+        protected virtual string ReplaceEntity(Match match)
+        {
+            try
+            {
+                var character = WebUtility.HtmlDecode(match.Value);
+                return string.Concat("&#", (int)character[0], ";");
+            }
+            catch
+            {
+                return match.Value;
+            }
+        }
+
+        protected virtual Movie CreateNewMovie()
+        {
+            return new Movie();
+        }
+
+        protected virtual bool PreProcess(NetImportResponse indexerResponse)
+        {
+            if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+            {
+                throw new NetImportException(indexerResponse, "Indexer API call resulted in an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
+            }
+
+            if (indexerResponse.HttpResponse.Headers.ContentType != null && indexerResponse.HttpResponse.Headers.ContentType.Contains("text/html") &&
+                indexerResponse.HttpRequest.Headers.Accept != null && !indexerResponse.HttpRequest.Headers.Accept.Contains("text/html"))
+            {
+                throw new NetImportException(indexerResponse, "Indexer responded with html content. Site is likely blocked or unavailable.");
+            }
+
+            return true;
+        }
+
+        protected Movie ProcessItem(XElement item)
+        {
+            var releaseInfo = CreateNewMovie();
+
+            releaseInfo = ProcessItem(item, releaseInfo);
+
+            //_logger.Trace("Parsed: {0}", releaseInfo.Title);
+
+            return PostProcess(item, releaseInfo);
+        }
+
+        protected virtual Movie ProcessItem(XElement item, Movie releaseInfo)
+        {
+            var result = Parser.Parser.ParseMovieTitle(GetTitle(item));
+
+            releaseInfo.Title = GetTitle(item);
+
+            if (result != null)
+            {
+                releaseInfo.Title = result.MovieTitle;
+                releaseInfo.Year = result.Year;
+                releaseInfo.ImdbId = result.ImdbId;
+            }
+
+            try
+            {
+                if (releaseInfo.ImdbId.IsNullOrWhiteSpace())
+                {
+                    releaseInfo.ImdbId = GetImdbId(item);
+                }
+
+            }
+            catch (Exception)
+            {
+                _logger.Debug("Unable to extract Imdb Id :(.");
+            }
+
+            return releaseInfo;
+        }
+
+        protected virtual Movie PostProcess(XElement item, Movie releaseInfo)
+        {
+            return releaseInfo;
+        }
+
+        protected virtual string GetTitle(XElement item)
+        {
+            return item.TryGetValue("title", "Unknown");
+        }
+
+        protected virtual DateTime GetPublishDate(XElement item)
+        {
+            var dateString = item.TryGetValue("pubDate");
+
+            if (dateString.IsNullOrWhiteSpace())
+            {
+                throw new UnsupportedFeedException("Rss feed must have a pubDate element with a valid publish date.");
+            }
+
+            return XElementExtensions.ParseDate(dateString);
+        }
+
+        protected virtual string GetImdbId(XElement item)
+        {
+            var url = item.TryGetValue("link");
+            if (url.IsNullOrWhiteSpace())
+            {
+                return "";
+            }
+            return Parser.Parser.ParseImdbId(url);
+        }
+
+        protected IEnumerable<XElement> GetItems(XDocument document)
+        {
+            var root = document.Root;
+
+            if (root == null)
+            {
+                return Enumerable.Empty<XElement>();
+            }
+
+            var channel = root.Element("channel");
+
+            if (channel == null)
+            {
+                return Enumerable.Empty<XElement>();
+            }
+
+            return channel.Elements("item");
+        }
+
+        protected virtual string ParseUrl(string value)
+        {
+            if (value.IsNullOrWhiteSpace())
+            {
+                return null;
+            }
+
+            try
+            {
+                var url = _importResponse.HttpRequest.Url + new HttpUri(value);
+
+                return url.FullUri;
+            }
+            catch (Exception ex)
+            {
+                _logger.Debug(ex, string.Format("Failed to parse Url {0}, ignoring.", value));
+                return null;
+            }
+        }
+    }
+}

src/NzbDrone.Core/NetImport/RSSImport/RSSImportRequestGenerator.cs

@@ -5,11 +5,11 @@ using NzbDrone.Common.Http;
 using NzbDrone.Common.Serializer;
 using NzbDrone.Core.IndexerSearch.Definitions;
 
-namespace NzbDrone.Core.NetImport.IMDbWatchList
+namespace NzbDrone.Core.NetImport.RSSImport
 {
-    public class IMDbWatchListRequestGenerator : INetImportRequestGenerator
+    public class RSSImportRequestGenerator : INetImportRequestGenerator
     {
-        public IMDbWatchListSettings Settings { get; set; }
+        public RSSImportSettings Settings { get; set; }
 
         public virtual NetImportPageableRequestChain GetMovies()
         {

src/NzbDrone.Core/NetImport/RSSImport/RSSImportSettings.cs

@@ -4,16 +4,16 @@ using NzbDrone.Core.Profiles;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 
-namespace NzbDrone.Core.NetImport.IMDbWatchList
+namespace NzbDrone.Core.NetImport.RSSImport
 {
 
-    public class IMDbWatchListSettings : NetImportBaseSettings
+    public class RSSImportSettings : NetImportBaseSettings
     {
         //private const string helpLink = "https://imdb.com";
 
-        public IMDbWatchListSettings()
+        public RSSImportSettings()
         {
-            Link = "http://rss.imdb.com/list/";
+            Link = "http://rss.yoursite.com";
             ProfileId = 1;
         }
 

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -130,18 +130,18 @@
     <Compile Include="NetImport\NetImportPageableRequest.cs" />
     <Compile Include="NetImport\NetImportPageableRequestChain.cs" />
     <Compile Include="NetImport\INetImportRequestGenerator.cs" />
-    <Compile Include="NetImport\IMDbWatchList\IMDbWatchListAPI.cs" />
-    <Compile Include="NetImport\IMDbWatchList\IMDbWatchListParser.cs" />
-    <Compile Include="NetImport\IMDbWatchList\IMDbWatchListRequestGenerator.cs" />
+    <Compile Include="NetImport\RSSImport\IMDbWatchListAPI.cs" />
+    <Compile Include="NetImport\RSSImport\RSSImportParser.cs" />
+    <Compile Include="NetImport\RSSImport\RSSImportRequestGenerator.cs" />
     <Compile Include="NetImport\NetImportRequest.cs" />
     <Compile Include="NetImport\NetImportResponse.cs" />
     <Compile Include="NetImport\NetImportBase.cs" />
     <Compile Include="NetImport\NetImportRepository.cs" />
     <Compile Include="NetImport\INetImport.cs" />
     <Compile Include="NetImport\NetImportDefinition.cs" />
-    <Compile Include="NetImport\IMDbWatchList\IMDbWatchList.cs" />
-    <Compile Include="NetImport\IMDbWatchList\IMDbWatchListProxy.cs" />
-    <Compile Include="NetImport\IMDbWatchList\IMDbWatchListSettings.cs" />
+    <Compile Include="NetImport\RSSImport\RSSImport.cs" />
+    <Compile Include="NetImport\RSSImport\IMDbWatchListProxy.cs" />
+    <Compile Include="NetImport\RSSImport\RSSImportSettings.cs" />
     <Compile Include="Backup\Backup.cs" />
     <Compile Include="Backup\BackupCommand.cs" />
     <Compile Include="Backup\BackupService.cs" />