New: IMDb List Support

pull/5244/head
Qstick 2 years ago
parent ea7af03d69
commit 381834edce

@ -25,17 +25,21 @@ namespace NzbDrone.Core.Test.ImportListTests
_importListReports = new List<ImportListItemInfo> { importListItem1 };
Mocker.GetMock<IFetchAndParseImportList>()
.Setup(v => v.Fetch())
.Returns(_importListReports);
Mocker.GetMock<ISeriesService>()
.Setup(v => v.AllSeriesTvdbIds())
.Returns(new List<int>());
Mocker.GetMock<ISearchForNewSeries>()
.Setup(v => v.SearchForNewSeries(It.IsAny<string>()))
.Returns(new List<Series>());
Mocker.GetMock<ISearchForNewSeries>()
.Setup(v => v.SearchForNewSeriesByImdbId(It.IsAny<string>()))
.Returns(new List<Series>());
Mocker.GetMock<IImportListFactory>()
.Setup(v => v.Get(It.IsAny<int>()))
.Returns(new ImportListDefinition { ShouldMonitor = MonitorTypes.All });
.Setup(v => v.All())
.Returns(new List<ImportListDefinition> { new ImportListDefinition { ShouldMonitor = MonitorTypes.All } });
Mocker.GetMock<IFetchAndParseImportList>()
.Setup(v => v.Fetch())
@ -51,11 +55,16 @@ namespace NzbDrone.Core.Test.ImportListTests
_importListReports.First().TvdbId = 81189;
}
private void WithImdbId()
{
_importListReports.First().ImdbId = "tt0496424";
}
private void WithExistingSeries()
{
Mocker.GetMock<ISeriesService>()
.Setup(v => v.FindByTvdbId(_importListReports.First().TvdbId))
.Returns(new Series { TvdbId = _importListReports.First().TvdbId });
.Setup(v => v.AllSeriesTvdbIds())
.Returns(new List<int> { _importListReports.First().TvdbId });
}
private void WithExcludedSeries()
@ -74,8 +83,8 @@ namespace NzbDrone.Core.Test.ImportListTests
private void WithMonitorType(MonitorTypes monitor)
{
Mocker.GetMock<IImportListFactory>()
.Setup(v => v.Get(It.IsAny<int>()))
.Returns(new ImportListDefinition { ShouldMonitor = monitor });
.Setup(v => v.All())
.Returns(new List<ImportListDefinition> { new ImportListDefinition { ShouldMonitor = monitor } });
}
[Test]
@ -97,6 +106,16 @@ namespace NzbDrone.Core.Test.ImportListTests
.Verify(v => v.SearchForNewSeries(It.IsAny<string>()), Times.Never());
}
[Test]
public void should_search_by_imdb_if_series_title_and_series_imdb()
{
WithImdbId();
Subject.Execute(new ImportListSyncCommand());
Mocker.GetMock<ISearchForNewSeries>()
.Verify(v => v.SearchForNewSeriesByImdbId(It.IsAny<string>()), Times.Once());
}
[Test]
public void should_not_add_if_existing_series()
{

@ -1,6 +1,7 @@
using FluentAssertions;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.MetadataSource;
using NzbDrone.Core.MetadataSource.SkyHook;
using NzbDrone.Core.Test.Framework;
using NzbDrone.Core.Tv;
@ -42,6 +43,30 @@ namespace NzbDrone.Core.Test.MetadataSource.SkyHook
ExceptionVerification.IgnoreWarns();
}
[TestCase("tt0496424", "30 Rock")]
public void should_search_by_imdb(string title, string expected)
{
var result = Subject.SearchForNewSeriesByImdbId(title);
result.Should().NotBeEmpty();
result[0].Title.Should().Be(expected);
ExceptionVerification.IgnoreWarns();
}
[TestCase("4565se")]
public void should_not_search_by_imdb_if_invalid(string title)
{
var result = Subject.SearchForNewSeriesByImdbId(title);
result.Should().BeEmpty();
Mocker.GetMock<ISearchForNewSeries>()
.Verify(v => v.SearchForNewSeries(It.IsAny<string>()), Times.Never());
ExceptionVerification.IgnoreWarns();
}
[TestCase("tvdbid:")]
[TestCase("tvdbid: 99999999999999999999")]
[TestCase("tvdbid: 0")]

@ -71,7 +71,7 @@ namespace NzbDrone.Core.ImportLists
Task.WaitAll(taskList.ToArray());
result = result.DistinctBy(r => new { r.TvdbId, r.Title }).ToList();
result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList();
_logger.Debug("Found {0} reports", result.Count);
@ -118,7 +118,7 @@ namespace NzbDrone.Core.ImportLists
Task.WaitAll(taskList.ToArray());
result = result.DistinctBy(r => new { r.TvdbId, r.Title }).ToList();
result = result.DistinctBy(r => new { r.TvdbId, r.ImdbId, r.Title }).ToList();
return result;
}

@ -165,9 +165,9 @@ namespace NzbDrone.Core.ImportLists
return CleanupListItems(releases);
}
protected virtual bool IsValidItem(ImportListItemInfo release)
protected virtual bool IsValidItem(ImportListItemInfo listItem)
{
if (release.Title.IsNullOrWhiteSpace())
if (listItem.Title.IsNullOrWhiteSpace() && listItem.ImdbId.IsNullOrWhiteSpace() && listItem.TmdbId == 0)
{
return false;
}

@ -0,0 +1,49 @@
using System.Collections.Generic;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Parser;
using NzbDrone.Core.ThingiProvider;
namespace NzbDrone.Core.ImportLists.Imdb
{
public class ImdbListImport : HttpImportListBase<ImdbListSettings>
{
public override string Name => "IMDb Lists";
public override ImportListType ListType => ImportListType.Other;
public ImdbListImport(IHttpClient httpClient,
IImportListStatusService importListStatusService,
IConfigService configService,
IParsingService parsingService,
Logger logger)
: base(httpClient, importListStatusService, configService, parsingService, logger)
{
}
public override IEnumerable<ProviderDefinition> DefaultDefinitions
{
get
{
foreach (var def in base.DefaultDefinitions)
{
yield return def;
}
}
}
public override IImportListRequestGenerator GetRequestGenerator()
{
return new ImdbListRequestGenerator()
{
Settings = Settings
};
}
public override IParseImportListResponse GetParser()
{
return new ImdbListParser();
}
}
}

@ -0,0 +1,53 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.ImportLists.Exceptions;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.ImportLists.Imdb
{
public class ImdbListParser : IParseImportListResponse
{
public IList<ImportListItemInfo> ParseResponse(ImportListResponse importListResponse)
{
var importResponse = importListResponse;
var series = new List<ImportListItemInfo>();
if (!PreProcess(importResponse))
{
return series;
}
// Parse TSV response from IMDB export
var rows = importResponse.Content.Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
series = rows.Skip(1).SelectList(m => m.Split(',')).Where(m => m.Length > 1).SelectList(i => new ImportListItemInfo { ImdbId = i[1] });
return series;
}
protected virtual bool PreProcess(ImportListResponse listResponse)
{
if (listResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new ImportListException(listResponse,
"Imdb call resulted in an unexpected StatusCode [{0}]",
listResponse.HttpResponse.StatusCode);
}
if (listResponse.HttpResponse.Headers.ContentType != null &&
listResponse.HttpResponse.Headers.ContentType.Contains("text/json") &&
listResponse.HttpRequest.Headers.Accept != null &&
!listResponse.HttpRequest.Headers.Accept.Contains("text/json"))
{
throw new ImportListException(listResponse,
"Imdb responded with html content. Site is likely blocked or unavailable.");
}
return true;
}
}
}

@ -0,0 +1,23 @@
using System.Collections.Generic;
using NzbDrone.Common.Http;
namespace NzbDrone.Core.ImportLists.Imdb
{
public class ImdbListRequestGenerator : IImportListRequestGenerator
{
public ImdbListSettings Settings { get; set; }
public virtual ImportListPageableRequestChain GetListItems()
{
var pageableRequests = new ImportListPageableRequestChain();
var httpRequest = new HttpRequest($"https://www.imdb.com/list/{Settings.ListId}/export", new HttpAccept("*/*"));
var request = new ImportListRequest(httpRequest.Url.ToString(), new HttpAccept(httpRequest.Headers.Accept));
request.HttpRequest.SuppressHttpError = true;
pageableRequests.Add(new List<ImportListRequest> { request });
return pageableRequests;
}
}
}

@ -0,0 +1,35 @@
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.ImportLists.Imdb
{
public class ImdbSettingsValidator : AbstractValidator<ImdbListSettings>
{
public ImdbSettingsValidator()
{
RuleFor(c => c.ListId)
.Matches(@"^ls\d+$")
.WithMessage("List ID mist be an IMDb List ID of the form 'ls12345678'");
}
}
public class ImdbListSettings : IImportListSettings
{
private static readonly ImdbSettingsValidator Validator = new ImdbSettingsValidator();
public ImdbListSettings()
{
}
public string BaseUrl { get; set; }
[FieldDefinition(1, Label = "List ID", HelpText = "IMDb list ID (e.g ls12345678)")]
public string ListId { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}

@ -69,6 +69,8 @@ namespace NzbDrone.Core.ImportLists
var reportNumber = 1;
var listExclusions = _importListExclusionService.All();
var importLists = _importListFactory.All();
var existingTvdbIds = _seriesService.AllSeriesTvdbIds();
foreach (var report in reports)
{
@ -76,12 +78,12 @@ namespace NzbDrone.Core.ImportLists
reportNumber++;
var importList = _importListFactory.Get(report.ImportListId);
var importList = importLists.Single(x => x.Id == report.ImportListId);
// Map TVDb if we only have a series name
if (report.TvdbId <= 0 && report.Title.IsNotNullOrWhiteSpace())
// Map by IMDbId if we have it
if (report.TvdbId <= 0 && report.ImdbId.IsNotNullOrWhiteSpace())
{
var mappedSeries = _seriesSearchService.SearchForNewSeries(report.Title)
var mappedSeries = _seriesSearchService.SearchForNewSeriesByImdbId(report.ImdbId)
.FirstOrDefault();
if (mappedSeries != null)
@ -91,14 +93,17 @@ namespace NzbDrone.Core.ImportLists
}
}
// Check to see if series in DB
var existingSeries = _seriesService.FindByTvdbId(report.TvdbId);
// Break if Series Exists in DB
if (existingSeries != null)
// Map TVDb if we only have a series name
if (report.TvdbId <= 0 && report.Title.IsNotNullOrWhiteSpace())
{
_logger.Debug("{0} [{1}] Rejected, Series Exists in DB", report.TvdbId, report.Title);
continue;
var mappedSeries = _seriesSearchService.SearchForNewSeries(report.Title)
.FirstOrDefault();
if (mappedSeries != null)
{
report.TvdbId = mappedSeries.TvdbId;
report.Title = mappedSeries?.Title;
}
}
// Check to see if series excluded
@ -110,6 +115,13 @@ namespace NzbDrone.Core.ImportLists
continue;
}
// Break if Series Exists in DB
if (existingTvdbIds.Any(x => x == report.TvdbId))
{
_logger.Debug("{0} [{1}] Rejected, Series Exists in DB", report.TvdbId, report.Title);
continue;
}
// Append Series if not already in DB or already on add list
if (seriesToAdd.All(s => s.TvdbId != report.TvdbId))
{

@ -1,4 +1,4 @@
using System.Collections.Generic;
using System.Collections.Generic;
using NzbDrone.Core.Tv;
namespace NzbDrone.Core.MetadataSource
@ -6,5 +6,6 @@ namespace NzbDrone.Core.MetadataSource
public interface ISearchForNewSeries
{
List<Series> SearchForNewSeries(string title);
List<Series> SearchForNewSeriesByImdbId(string imdbId);
}
}

@ -69,6 +69,20 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
return new Tuple<Series, List<Episode>>(series, episodes.ToList());
}
public List<Series> SearchForNewSeriesByImdbId(string imdbId)
{
imdbId = Parser.Parser.NormalizeImdbId(imdbId);
if (imdbId == null)
{
return new List<Series>();
}
var results = SearchForNewSeries($"imdb:{imdbId}");
return results;
}
public List<Series> SearchForNewSeries(string title)
{
try

@ -733,6 +733,24 @@ namespace NzbDrone.Core.Parser
return title.Trim().ToLower();
}
public static string NormalizeImdbId(string imdbId)
{
var imdbRegex = new Regex(@"^(\d{1,10}|(tt)\d{1,10})$");
if (!imdbRegex.IsMatch(imdbId))
{
return null;
}
if (imdbId.Length > 2)
{
imdbId = imdbId.Replace("tt", "").PadLeft(7, '0');
return $"tt{imdbId}";
}
return null;
}
public static string ParseReleaseGroup(string title)
{
title = title.Trim();

@ -16,6 +16,7 @@ namespace NzbDrone.Core.Tv
Series FindByTvdbId(int tvdbId);
Series FindByTvRageId(int tvRageId);
Series FindByPath(string path);
List<int> AllSeriesTvdbIds();
Dictionary<int, string> AllSeriesPaths();
}
@ -73,6 +74,14 @@ namespace NzbDrone.Core.Tv
.FirstOrDefault();
}
public List<int> AllSeriesTvdbIds()
{
using (var conn = _database.OpenConnection())
{
return conn.Query<int>("SELECT TvdbId FROM Series").ToList();
}
}
public Dictionary<int, string> AllSeriesPaths()
{
using (var conn = _database.OpenConnection())

@ -24,6 +24,7 @@ namespace NzbDrone.Core.Tv
Series FindByPath(string path);
void DeleteSeries(int seriesId, bool deleteFiles, bool addImportListExclusion);
List<Series> GetAllSeries();
List<int> AllSeriesTvdbIds();
Dictionary<int, string> GetAllSeriesPaths();
List<Series> AllForTag(int tagId);
Series UpdateSeries(Series series, bool updateEpisodesToMatchSeason = true, bool publishUpdatedEvent = true);
@ -160,6 +161,11 @@ namespace NzbDrone.Core.Tv
return _seriesRepository.All().ToList();
}
public List<int> AllSeriesTvdbIds()
{
return _seriesRepository.AllSeriesTvdbIds().ToList();
}
public Dictionary<int, string> GetAllSeriesPaths()
{
return _seriesRepository.AllSeriesPaths();

Loading…
Cancel
Save