From 3ccc30638f31ab787fb96e20a36d1134d76e66a6 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Tue, 22 Sep 2020 22:32:57 +0200 Subject: [PATCH] New: Added FileList.io indexer support --- .../Files/Indexers/FileList/RecentFeed.json | 38 ++++++++++ .../FileListTests/FileListFixture.cs | 57 +++++++++++++++ .../Annotations/FieldDefinitionAttribute.cs | 15 ++++ .../Indexers/FileList/FileList.cs | 30 ++++++++ .../Indexers/FileList/FileListParser.cs | 73 +++++++++++++++++++ .../FileList/FileListRequestGenerator.cs | 52 +++++++++++++ .../Indexers/FileList/FileListSettings.cs | 70 ++++++++++++++++++ .../Indexers/FileList/FileListTorrent.cs | 24 ++++++ .../Instrumentation/DatabaseTarget.cs | 2 +- src/Sonarr.Http/ClientSchema/SelectOption.cs | 10 +++ 10 files changed, 370 insertions(+), 1 deletion(-) create mode 100644 src/NzbDrone.Core.Test/Files/Indexers/FileList/RecentFeed.json create mode 100644 src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs create mode 100644 src/NzbDrone.Core/Indexers/FileList/FileList.cs create mode 100644 src/NzbDrone.Core/Indexers/FileList/FileListParser.cs create mode 100644 src/NzbDrone.Core/Indexers/FileList/FileListRequestGenerator.cs create mode 100644 src/NzbDrone.Core/Indexers/FileList/FileListSettings.cs create mode 100644 src/NzbDrone.Core/Indexers/FileList/FileListTorrent.cs create mode 100644 src/Sonarr.Http/ClientSchema/SelectOption.cs diff --git a/src/NzbDrone.Core.Test/Files/Indexers/FileList/RecentFeed.json b/src/NzbDrone.Core.Test/Files/Indexers/FileList/RecentFeed.json new file mode 100644 index 000000000..0ae0f3a7d --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/FileList/RecentFeed.json @@ -0,0 +1,38 @@ +[ + { + "id": 1234, + "name": "Mankind.Divided.2019.S01E01.1080p.WEB-DL", + "imdb": "tt1232322", + "freeleech": 0, + "upload_date": "2019-01-22 22:20:19", + "download_link": "https://filelist.io/download.php?id=1234&passkey=somepass", + "size": 830512414, + "internal": 0, + "moderated": 1, + "category": "Seriale HD", + "seeders": 12, + "leechers": 2, + "times_completed": 11, + "comments": 0, + "files": 3, + "small_description": "Much anticipated show about (redacted)" + }, + { + "id": 1235, + "name": "Mankind.Divided.2019.S01E02.1080p.WEB-DL", + "imdb": "tt9999999", + "freeleech": 0, + "upload_date": "2019-01-22 22:19:37", + "download_link": "https://filelist.io/download.php?id=1235&passkey=somepass", + "size": 473149881, + "internal": 0, + "moderated": 1, + "category": "Seriale HD", + "seeders": 9, + "leechers": 1, + "times_completed": 8, + "comments": 0, + "files": 3, + "small_description": "(redacted) finds a way to unify two of the most insignificant factions" + } +] \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs new file mode 100644 index 000000000..cd4da25ee --- /dev/null +++ b/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs @@ -0,0 +1,57 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.FileList; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.IndexerTests.FileListTests +{ + [TestFixture] + public class FileListFixture : CoreTest + { + [SetUp] + public void Setup() + { + Subject.Definition = new IndexerDefinition() + { + Name = "FileList", + Settings = new FileListSettings() { Username = "someuser", Passkey = "somepass" } + }; + } + + [Test] + public void should_parse_recent_feed_from_FileList() + { + var recentFeed = ReadAllText(@"Files/Indexers/FileList/RecentFeed.json"); + + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) + .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(2); + releases.First().Should().BeOfType(); + + var torrentInfo = releases.First() as TorrentInfo; + + torrentInfo.Title.Should().Be("Mankind.Divided.2019.S01E01.1080p.WEB-DL"); + torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + torrentInfo.DownloadUrl.Should().Be("https://filelist.io/download.php?id=1234&passkey=somepass"); + torrentInfo.InfoUrl.Should().Be("https://filelist.io/details.php?id=1234"); + torrentInfo.CommentUrl.Should().BeNullOrEmpty(); + torrentInfo.Indexer.Should().Be(Subject.Definition.Name); + torrentInfo.PublishDate.Should().Be(DateTime.Parse("2019-01-22 22:20:19").ToUniversalTime()); + torrentInfo.Size.Should().Be(830512414); + torrentInfo.InfoHash.Should().Be(null); + torrentInfo.MagnetUrl.Should().Be(null); + torrentInfo.Peers.Should().Be(2 + 12); + torrentInfo.Seeders.Should().Be(12); + } + } +} diff --git a/src/NzbDrone.Core/Annotations/FieldDefinitionAttribute.cs b/src/NzbDrone.Core/Annotations/FieldDefinitionAttribute.cs index e02acd67d..3742c8124 100644 --- a/src/NzbDrone.Core/Annotations/FieldDefinitionAttribute.cs +++ b/src/NzbDrone.Core/Annotations/FieldDefinitionAttribute.cs @@ -1,4 +1,5 @@ using System; +using System.Runtime.CompilerServices; namespace NzbDrone.Core.Annotations { @@ -22,6 +23,20 @@ namespace NzbDrone.Core.Annotations public HiddenType Hidden { get; set; } } + [AttributeUsage(AttributeTargets.Property | AttributeTargets.Field, AllowMultiple = false)] + public class FieldOptionAttribute : Attribute + { + public FieldOptionAttribute(string label = null, [CallerLineNumber] int order = 0) + { + Order = order; + Label = label; + } + + public int Order { get; private set; } + public string Label { get; set; } + public string Hint { get; set; } + } + public enum FieldType { Textbox, diff --git a/src/NzbDrone.Core/Indexers/FileList/FileList.cs b/src/NzbDrone.Core/Indexers/FileList/FileList.cs new file mode 100644 index 000000000..bb0da183d --- /dev/null +++ b/src/NzbDrone.Core/Indexers/FileList/FileList.cs @@ -0,0 +1,30 @@ +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; + +namespace NzbDrone.Core.Indexers.FileList +{ + public class FileList : HttpIndexerBase + { + public override string Name => "FileList"; + public override DownloadProtocol Protocol => DownloadProtocol.Torrent; + public override bool SupportsRss => true; + public override bool SupportsSearch => true; + + public FileList(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) + { + } + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new FileListRequestGenerator() { Settings = Settings }; + } + + public override IParseIndexerResponse GetParser() + { + return new FileListParser(Settings); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/FileList/FileListParser.cs b/src/NzbDrone.Core/Indexers/FileList/FileListParser.cs new file mode 100644 index 000000000..5b3425480 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/FileList/FileListParser.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Net; +using Newtonsoft.Json; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Indexers.FileList +{ + public class FileListParser : IParseIndexerResponse + { + private readonly FileListSettings _settings; + + public FileListParser(FileListSettings settings) + { + _settings = settings; + } + + public IList ParseResponse(IndexerResponse indexerResponse) + { + var torrentInfos = new List(); + + if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new IndexerException(indexerResponse, + "Unexpected response status {0} code from API request", + indexerResponse.HttpResponse.StatusCode); + } + + var queryResults = JsonConvert.DeserializeObject>(indexerResponse.Content); + + foreach (var result in queryResults) + { + var id = result.Id; + + //if (result.FreeLeech) + torrentInfos.Add(new TorrentInfo() + { + Guid = $"FileList-{id}", + Title = result.Name, + Size = result.Size, + DownloadUrl = GetDownloadUrl(id), + InfoUrl = GetInfoUrl(id), + Seeders = result.Seeders, + Peers = result.Leechers + result.Seeders, + PublishDate = result.UploadDate.ToUniversalTime() + }); + } + + return torrentInfos.ToArray(); + } + + private string GetDownloadUrl(string torrentId) + { + var url = new HttpUri(_settings.BaseUrl) + .CombinePath("download.php") + .AddQueryParam("id", torrentId) + .AddQueryParam("passkey", _settings.Passkey); + + return url.FullUri; + } + + private string GetInfoUrl(string torrentId) + { + var url = new HttpUri(_settings.BaseUrl) + .CombinePath("details.php") + .AddQueryParam("id", torrentId); + + return url.FullUri; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/FileList/FileListRequestGenerator.cs b/src/NzbDrone.Core/Indexers/FileList/FileListRequestGenerator.cs new file mode 100644 index 000000000..ce140bb37 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/FileList/FileListRequestGenerator.cs @@ -0,0 +1,52 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NzbDrone.Common.Http; +using NzbDrone.Core.IndexerSearch.Definitions; + +namespace NzbDrone.Core.Indexers.FileList +{ + public class FileListRequestGenerator : IIndexerRequestGenerator + { + public FileListSettings Settings { get; set; } + + public virtual IndexerPageableRequestChain GetRecentRequests() + { + var pageableRequests = new IndexerPageableRequestChain(); + + pageableRequests.Add(GetRequest("latest-torrents", Settings.Categories, "")); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + + pageableRequests.Add(GetRequest("search-torrents", Settings.Categories, string.Format(" & type=name&query={0}+{1}", Uri.EscapeDataString(searchCriteria.ArtistQuery.Trim()), Uri.EscapeDataString(searchCriteria.AlbumQuery.Trim())))); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + + pageableRequests.Add(GetRequest("search-torrents", Settings.Categories, string.Format(" & type=name&query={0}", Uri.EscapeDataString(searchCriteria.ArtistQuery.Trim())))); + + return pageableRequests; + } + + private IEnumerable GetRequest(string searchType, IEnumerable categories, string parameters) + { + var categoriesQuery = string.Join(",", categories.Distinct()); + + var baseUrl = string.Format("{0}/api.php?action={1}&category={2}{3}", Settings.BaseUrl.TrimEnd('/'), searchType, categoriesQuery, parameters); + + var request = new IndexerRequest(baseUrl, HttpAccept.Json); + request.HttpRequest.AddBasicAuthentication(Settings.Username.Trim(), Settings.Passkey.Trim()); + + yield return request; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/FileList/FileListSettings.cs b/src/NzbDrone.Core/Indexers/FileList/FileListSettings.cs new file mode 100644 index 000000000..0160f4731 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/FileList/FileListSettings.cs @@ -0,0 +1,70 @@ +using System.Collections.Generic; +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.FileList +{ + public class FileListSettingsValidator : AbstractValidator + { + public FileListSettingsValidator() + { + RuleFor(c => c.BaseUrl).ValidRootUrl(); + RuleFor(c => c.Username).NotEmpty(); + RuleFor(c => c.Passkey).NotEmpty(); + + RuleFor(c => c.SeedCriteria).SetValidator(_ => new SeedCriteriaSettingsValidator()); + } + } + + public class FileListSettings : ITorrentIndexerSettings + { + private static readonly FileListSettingsValidator Validator = new FileListSettingsValidator(); + + public FileListSettings() + { + BaseUrl = "https://filelist.io"; + MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS; + + Categories = new int[] + { + (int)FileListCategories.AUDIO, + (int)FileListCategories.FLAC + }; + } + + [FieldDefinition(0, Label = "Username")] + public string Username { get; set; } + + [FieldDefinition(1, Label = "Passkey")] + public string Passkey { get; set; } + + [FieldDefinition(3, Label = "API URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since your API key will be sent to that host.")] + public string BaseUrl { get; set; } + + [FieldDefinition(4, Label = "Categories", Type = FieldType.Select, SelectOptions = typeof(FileListCategories), HelpText = "Categories for use in search and feeds, leave blank to disable standard/daily shows")] + public IEnumerable Categories { get; set; } + + [FieldDefinition(5, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Lidarr will download from this indexer, empty is no limit", Advanced = true)] + public int? EarlyReleaseLimit { get; set; } + + [FieldDefinition(6, Type = FieldType.Number, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)] + public int MinimumSeeders { get; set; } + + [FieldDefinition(7)] + public SeedCriteriaSettings SeedCriteria { get; set; } = new SeedCriteriaSettings(); + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } + + public enum FileListCategories + { + [FieldOption] + AUDIO = 5, + [FieldOption] + FLAC = 11 + } +} diff --git a/src/NzbDrone.Core/Indexers/FileList/FileListTorrent.cs b/src/NzbDrone.Core/Indexers/FileList/FileListTorrent.cs new file mode 100644 index 000000000..01ea834ed --- /dev/null +++ b/src/NzbDrone.Core/Indexers/FileList/FileListTorrent.cs @@ -0,0 +1,24 @@ +using System; +using Newtonsoft.Json; + +namespace NzbDrone.Core.Indexers.FileList +{ + public class FileListTorrent + { + public string Id { get; set; } + public string Name { get; set; } + public long Size { get; set; } + public int Leechers { get; set; } + public int Seeders { get; set; } + [JsonProperty(PropertyName = "times_completed")] + public uint TimesCompleted { get; set; } + public uint Comments { get; set; } + public uint Files { get; set; } + [JsonProperty(PropertyName = "imdb")] + public string ImdbId { get; set; } + [JsonProperty(PropertyName = "freeleech")] + public bool FreeLeech { get; set; } + [JsonProperty(PropertyName = "upload_date")] + public DateTime UploadDate { get; set; } + } +} diff --git a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs index 2a74549b9..1e62b6580 100644 --- a/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs +++ b/src/NzbDrone.Core/Instrumentation/DatabaseTarget.cs @@ -104,7 +104,7 @@ namespace NzbDrone.Core.Instrumentation public void Handle(ApplicationShutdownRequested message) { - if (LogManager.Configuration.LoggingRules.Contains(Rule)) + if (LogManager.Configuration != null && LogManager.Configuration.LoggingRules.Contains(Rule)) { UnRegister(); } diff --git a/src/Sonarr.Http/ClientSchema/SelectOption.cs b/src/Sonarr.Http/ClientSchema/SelectOption.cs new file mode 100644 index 000000000..130453095 --- /dev/null +++ b/src/Sonarr.Http/ClientSchema/SelectOption.cs @@ -0,0 +1,10 @@ +namespace Sonarr.Http.ClientSchema +{ + public class SelectOption + { + public int Value { get; set; } + public string Name { get; set; } + public int Order { get; set; } + public string Hint { get; set; } + } +}