New: IMDb List Support

src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs

@@ -25,17 +25,21 @@ namespace NzbDrone.Core.Test.ImportListTests
 
             _importListReports = new List<ImportListItemInfo> { importListItem1 };
 
-            Mocker.GetMock<IFetchAndParseImportList>()
+            Mocker.GetMock<ISeriesService>()
-                  .Setup(v => v.Fetch())
+                  .Setup(v => v.AllSeriesTvdbIds())
-                  .Returns(_importListReports);
+                  .Returns(new List<int>());
 
             Mocker.GetMock<ISearchForNewSeries>()
                   .Setup(v => v.SearchForNewSeries(It.IsAny<string>()))
                   .Returns(new List<Series>());
 
+            Mocker.GetMock<ISearchForNewSeries>()
+                  .Setup(v => v.SearchForNewSeriesByImdbId(It.IsAny<string>()))
+                  .Returns(new List<Series>());
+
             Mocker.GetMock<IImportListFactory>()
-                  .Setup(v => v.Get(It.IsAny<int>()))
+                  .Setup(v => v.All())
-                  .Returns(new ImportListDefinition { ShouldMonitor = MonitorTypes.All });
+                  .Returns(new List<ImportListDefinition> { new ImportListDefinition { ShouldMonitor = MonitorTypes.All } });
 
             Mocker.GetMock<IFetchAndParseImportList>()
                   .Setup(v => v.Fetch())
@@ -51,11 +55,16 @@ namespace NzbDrone.Core.Test.ImportListTests
             _importListReports.First().TvdbId = 81189;
         }
 
+        private void WithImdbId()
+        {
+            _importListReports.First().ImdbId = "tt0496424";
+        }
+
         private void WithExistingSeries()
         {
             Mocker.GetMock<ISeriesService>()
-                  .Setup(v => v.FindByTvdbId(_importListReports.First().TvdbId))
+                  .Setup(v => v.AllSeriesTvdbIds())
-                  .Returns(new Series { TvdbId = _importListReports.First().TvdbId });
+                  .Returns(new List<int> { _importListReports.First().TvdbId });
         }
 
         private void WithExcludedSeries()
@@ -74,8 +83,8 @@ namespace NzbDrone.Core.Test.ImportListTests
         private void WithMonitorType(MonitorTypes monitor)
         {
             Mocker.GetMock<IImportListFactory>()
-                  .Setup(v => v.Get(It.IsAny<int>()))
+                  .Setup(v => v.All())
-                  .Returns(new ImportListDefinition { ShouldMonitor = monitor });
+                  .Returns(new List<ImportListDefinition> { new ImportListDefinition { ShouldMonitor = monitor } });
         }
 
         [Test]
@@ -97,6 +106,16 @@ namespace NzbDrone.Core.Test.ImportListTests
                   .Verify(v => v.SearchForNewSeries(It.IsAny<string>()), Times.Never());
         }
 
+        [Test]
+        public void should_search_by_imdb_if_series_title_and_series_imdb()
+        {
+            WithImdbId();
+            Subject.Execute(new ImportListSyncCommand());
+
+            Mocker.GetMock<ISearchForNewSeries>()
+                  .Verify(v => v.SearchForNewSeriesByImdbId(It.IsAny<string>()), Times.Once());
+        }
+
         [Test]
         public void should_not_add_if_existing_series()
         {

src/NzbDrone.Core.Test/MetadataSource/SkyHook/SkyHookProxySearchFixture.cs

@@ -1,6 +1,7 @@
-using FluentAssertions;
+using FluentAssertions;
 using Moq;
 using NUnit.Framework;
+using NzbDrone.Core.MetadataSource;
 using NzbDrone.Core.MetadataSource.SkyHook;
 using NzbDrone.Core.Test.Framework;
 using NzbDrone.Core.Tv;
@@ -42,6 +43,30 @@ namespace NzbDrone.Core.Test.MetadataSource.SkyHook
             ExceptionVerification.IgnoreWarns();
         }
 
+        [TestCase("tt0496424", "30 Rock")]
+        public void should_search_by_imdb(string title, string expected)
+        {
+            var result = Subject.SearchForNewSeriesByImdbId(title);
+
+            result.Should().NotBeEmpty();
+
+            result[0].Title.Should().Be(expected);
+
+            ExceptionVerification.IgnoreWarns();
+        }
+
+        [TestCase("4565se")]
+        public void should_not_search_by_imdb_if_invalid(string title)
+        {
+            var result = Subject.SearchForNewSeriesByImdbId(title);
+            result.Should().BeEmpty();
+
+            Mocker.GetMock<ISearchForNewSeries>()
+                  .Verify(v => v.SearchForNewSeries(It.IsAny<string>()), Times.Never());
+
+            ExceptionVerification.IgnoreWarns();
+        }
+
         [TestCase("tvdbid:")]
         [TestCase("tvdbid: 99999999999999999999")]
         [TestCase("tvdbid: 0")]

src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs

@@ -71,7 +71,7 @@ namespace NzbDrone.Core.ImportLists
 
             Task.WaitAll(taskList.ToArray());
 
-            result = result.DistinctBy(r => new { r.TvdbId, r.Title }).ToList();
+            result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList();
 
             _logger.Debug("Found {0} reports", result.Count);
 
@@ -118,7 +118,7 @@ namespace NzbDrone.Core.ImportLists
 
             Task.WaitAll(taskList.ToArray());
 
-            result = result.DistinctBy(r => new { r.TvdbId, r.Title }).ToList();
+            result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList();
 
             return result;
         }

src/NzbDrone.Core/ImportLists/HttpImportListBase.cs

@@ -165,9 +165,9 @@ namespace NzbDrone.Core.ImportLists
             return CleanupListItems(releases);
         }
 
-        protected virtual bool IsValidItem(ImportListItemInfo release)
+        protected virtual bool IsValidItem(ImportListItemInfo listItem)
         {
-            if (release.Title.IsNullOrWhiteSpace())
+            if (listItem.Title.IsNullOrWhiteSpace() && listItem.ImdbId.IsNullOrWhiteSpace() && listItem.TmdbId == 0)
             {
                 return false;
             }

src/NzbDrone.Core/ImportLists/Imdb/ImdbListImport.cs

@@ -0,0 +1,49 @@
+using System.Collections.Generic;
+using NLog;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.ThingiProvider;
+
+namespace NzbDrone.Core.ImportLists.Imdb
+{
+    public class ImdbListImport : HttpImportListBase<ImdbListSettings>
+    {
+        public override string Name => "IMDb Lists";
+
+        public override ImportListType ListType => ImportListType.Other;
+
+        public ImdbListImport(IHttpClient httpClient,
+                              IImportListStatusService importListStatusService,
+                              IConfigService configService,
+                              IParsingService parsingService,
+                              Logger logger)
+        : base(httpClient, importListStatusService, configService, parsingService, logger)
+        {
+        }
+
+        public override IEnumerable<ProviderDefinition> DefaultDefinitions
+        {
+            get
+            {
+                foreach (var def in base.DefaultDefinitions)
+                {
+                    yield return def;
+                }
+            }
+        }
+
+        public override IImportListRequestGenerator GetRequestGenerator()
+        {
+            return new ImdbListRequestGenerator()
+            {
+                Settings = Settings
+            };
+        }
+
+        public override IParseImportListResponse GetParser()
+        {
+            return new ImdbListParser();
+        }
+    }
+}

src/NzbDrone.Core/ImportLists/Imdb/ImdbListParser.cs

@@ -0,0 +1,53 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.ImportLists.Exceptions;
+using NzbDrone.Core.Parser.Model;
+
+namespace NzbDrone.Core.ImportLists.Imdb
+{
+    public class ImdbListParser : IParseImportListResponse
+    {
+        public IList<ImportListItemInfo> ParseResponse(ImportListResponse importListResponse)
+        {
+            var importResponse = importListResponse;
+
+            var series = new List<ImportListItemInfo>();
+
+            if (!PreProcess(importResponse))
+            {
+                return series;
+            }
+
+            // Parse TSV response from IMDB export
+            var rows = importResponse.Content.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
+
+            series = rows.Skip(1).SelectList(m => m.Split(',')).Where(m => m.Length > 1).SelectList(i => new ImportListItemInfo { ImdbId = i[1] });
+
+            return series;
+        }
+
+        protected virtual bool PreProcess(ImportListResponse listResponse)
+        {
+            if (listResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
+            {
+                throw new ImportListException(listResponse,
+                    "Imdb call resulted in an unexpected StatusCode [{0}]",
+                    listResponse.HttpResponse.StatusCode);
+            }
+
+            if (listResponse.HttpResponse.Headers.ContentType != null &&
+                listResponse.HttpResponse.Headers.ContentType.Contains("text/json") &&
+                listResponse.HttpRequest.Headers.Accept != null &&
+                !listResponse.HttpRequest.Headers.Accept.Contains("text/json"))
+            {
+                throw new ImportListException(listResponse,
+                    "Imdb responded with html content. Site is likely blocked or unavailable.");
+            }
+
+            return true;
+        }
+    }
+}

src/NzbDrone.Core/ImportLists/Imdb/ImdbListRequestGenerator.cs

@@ -0,0 +1,23 @@
+using System.Collections.Generic;
+using NzbDrone.Common.Http;
+
+namespace NzbDrone.Core.ImportLists.Imdb
+{
+    public class ImdbListRequestGenerator : IImportListRequestGenerator
+    {
+        public ImdbListSettings Settings { get; set; }
+
+        public virtual ImportListPageableRequestChain GetListItems()
+        {
+            var pageableRequests = new ImportListPageableRequestChain();
+            var httpRequest = new HttpRequest($"https://www.imdb.com/list/{Settings.ListId}/export", new HttpAccept("*/*"));
+            var request = new ImportListRequest(httpRequest.Url.ToString(), new HttpAccept(httpRequest.Headers.Accept));
+
+            request.HttpRequest.SuppressHttpError = true;
+
+            pageableRequests.Add(new List<ImportListRequest> { request });
+
+            return pageableRequests;
+        }
+    }
+}

src/NzbDrone.Core/ImportLists/Imdb/ImdbListSettings.cs

@@ -0,0 +1,35 @@
+using FluentValidation;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.ImportLists.Imdb
+{
+    public class ImdbSettingsValidator : AbstractValidator<ImdbListSettings>
+    {
+        public ImdbSettingsValidator()
+        {
+            RuleFor(c => c.ListId)
+                .Matches(@"^ls\d+$")
+                .WithMessage("List ID mist be an IMDb List ID of the form 'ls12345678'");
+        }
+    }
+
+    public class ImdbListSettings : IImportListSettings
+    {
+        private static readonly ImdbSettingsValidator Validator = new ImdbSettingsValidator();
+
+        public ImdbListSettings()
+        {
+        }
+
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(1, Label = "List ID", HelpText = "IMDb list ID (e.g ls12345678)")]
+        public string ListId { get; set; }
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}

src/NzbDrone.Core/ImportLists/ImportListSyncService.cs

@@ -69,6 +69,8 @@ namespace NzbDrone.Core.ImportLists
             var reportNumber = 1;
 
             var listExclusions = _importListExclusionService.All();
+            var importLists = _importListFactory.All();
+            var existingTvdbIds = _seriesService.AllSeriesTvdbIds();
 
             foreach (var report in reports)
             {
@@ -76,12 +78,12 @@ namespace NzbDrone.Core.ImportLists
 
                 reportNumber++;
 
-                var importList = _importListFactory.Get(report.ImportListId);
+                var importList = importLists.Single(x => x.Id == report.ImportListId);
 
-                // Map TVDb if we only have a series name
+                // Map by IMDbId if we have it
-                if (report.TvdbId <= 0 && report.Title.IsNotNullOrWhiteSpace())
+                if (report.TvdbId <= 0 && report.ImdbId.IsNotNullOrWhiteSpace())
                 {
-                    var mappedSeries = _seriesSearchService.SearchForNewSeries(report.Title)
+                    var mappedSeries = _seriesSearchService.SearchForNewSeriesByImdbId(report.ImdbId)
                         .FirstOrDefault();
 
                     if (mappedSeries != null)
@@ -91,14 +93,17 @@ namespace NzbDrone.Core.ImportLists
                     }
                 }
 
-                // Check to see if series in DB
+                // Map TVDb if we only have a series name
-                var existingSeries = _seriesService.FindByTvdbId(report.TvdbId);
+                if (report.TvdbId <= 0 && report.Title.IsNotNullOrWhiteSpace())
-
-                // Break if Series Exists in DB
-                if (existingSeries != null)
                 {
-                    _logger.Debug("{0} [{1}] Rejected, Series Exists in DB", report.TvdbId, report.Title);
+                    var mappedSeries = _seriesSearchService.SearchForNewSeries(report.Title)
-                    continue;
+                        .FirstOrDefault();
+
+                    if (mappedSeries != null)
+                    {
+                        report.TvdbId = mappedSeries.TvdbId;
+                        report.Title = mappedSeries?.Title;
+                    }
                 }
 
                 // Check to see if series excluded
@@ -110,6 +115,13 @@ namespace NzbDrone.Core.ImportLists
                     continue;
                 }
 
+                // Break if Series Exists in DB
+                if (existingTvdbIds.Any(x => x == report.TvdbId))
+                {
+                    _logger.Debug("{0} [{1}] Rejected, Series Exists in DB", report.TvdbId, report.Title);
+                    continue;
+                }
+
                 // Append Series if not already in DB or already on add list
                 if (seriesToAdd.All(s => s.TvdbId != report.TvdbId))
                 {

src/NzbDrone.Core/MetadataSource/ISearchForNewSeries.cs

@@ -1,4 +1,4 @@
-using System.Collections.Generic;
+using System.Collections.Generic;
 using NzbDrone.Core.Tv;
 
 namespace NzbDrone.Core.MetadataSource
@@ -6,5 +6,6 @@ namespace NzbDrone.Core.MetadataSource
     public interface ISearchForNewSeries
     {
         List<Series> SearchForNewSeries(string title);
+        List<Series> SearchForNewSeriesByImdbId(string imdbId);
     }
 }

src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs

@@ -69,6 +69,20 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
             return new Tuple<Series, List<Episode>>(series, episodes.ToList());
         }
 
+        public List<Series> SearchForNewSeriesByImdbId(string imdbId)
+        {
+            imdbId = Parser.Parser.NormalizeImdbId(imdbId);
+
+            if (imdbId == null)
+            {
+                return new List<Series>();
+            }
+
+            var results = SearchForNewSeries($"imdb:{imdbId}");
+
+            return results;
+        }
+
         public List<Series> SearchForNewSeries(string title)
         {
             try

src/NzbDrone.Core/Parser/Parser.cs

@@ -733,6 +733,24 @@ namespace NzbDrone.Core.Parser
             return title.Trim().ToLower();
         }
 
+        public static string NormalizeImdbId(string imdbId)
+        {
+            var imdbRegex = new Regex(@"^(\d{1,10}|(tt)\d{1,10})$");
+
+            if (!imdbRegex.IsMatch(imdbId))
+            {
+                return null;
+            }
+
+            if (imdbId.Length > 2)
+            {
+                imdbId = imdbId.Replace("tt", "").PadLeft(7, '0');
+                return $"tt{imdbId}";
+            }
+
+            return null;
+        }
+
         public static string ParseReleaseGroup(string title)
         {
             title = title.Trim();

src/NzbDrone.Core/Tv/SeriesRepository.cs

@@ -16,6 +16,7 @@ namespace NzbDrone.Core.Tv
         Series FindByTvdbId(int tvdbId);
         Series FindByTvRageId(int tvRageId);
         Series FindByPath(string path);
+        List<int> AllSeriesTvdbIds();
         Dictionary<int, string> AllSeriesPaths();
     }
 
@@ -73,6 +74,14 @@ namespace NzbDrone.Core.Tv
                         .FirstOrDefault();
         }
 
+        public List<int> AllSeriesTvdbIds()
+        {
+            using (var conn = _database.OpenConnection())
+            {
+                return conn.Query<int>("SELECT TvdbId FROM Series").ToList();
+            }
+        }
+
         public Dictionary<int, string> AllSeriesPaths()
         {
             using (var conn = _database.OpenConnection())

src/NzbDrone.Core/Tv/SeriesService.cs

@@ -24,6 +24,7 @@ namespace NzbDrone.Core.Tv
         Series FindByPath(string path);
         void DeleteSeries(int seriesId, bool deleteFiles, bool addImportListExclusion);
         List<Series> GetAllSeries();
+        List<int> AllSeriesTvdbIds();
         Dictionary<int, string> GetAllSeriesPaths();
         List<Series> AllForTag(int tagId);
         Series UpdateSeries(Series series, bool updateEpisodesToMatchSeason = true, bool publishUpdatedEvent = true);
@@ -160,6 +161,11 @@ namespace NzbDrone.Core.Tv
             return _seriesRepository.All().ToList();
         }
 
+        public List<int> AllSeriesTvdbIds()
+        {
+            return _seriesRepository.AllSeriesTvdbIds().ToList();
+        }
+
         public Dictionary<int, string> GetAllSeriesPaths()
         {
             return _seriesRepository.AllSeriesPaths();