From 1b78fd38db8bd566a2abcadc040e9f96c5d3bce7 Mon Sep 17 00:00:00 2001 From: Bogdan Date: Thu, 12 Jan 2023 21:05:11 +0200 Subject: [PATCH] Fixed: (FileList) RequestGenerator refactoring, append slash to IndexerUrl --- .../FileListTests/FileListFixture.cs | 17 ++- .../FileListRequestGeneratorFixture.cs | 41 +++--- .../Indexers/Definitions/FileList/FileList.cs | 36 ++--- .../Definitions/FileList/FileListApi.cs | 3 +- .../Definitions/FileList/FileListParser.cs | 23 ++-- .../FileList/FileListRequestGenerator.cs | 128 +++++++++++------- .../Definitions/FileList/FileListSettings.cs | 7 +- 7 files changed, 152 insertions(+), 103 deletions(-) diff --git a/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs index 208980085..6996b6ffc 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListFixture.cs @@ -21,10 +21,15 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests [SetUp] public void Setup() { - Subject.Definition = new IndexerDefinition() + Subject.Definition = new IndexerDefinition { Name = "FileList", - Settings = new FileListSettings() { Username = "someuser", Passkey = "somepass" } + Settings = new FileListSettings + { + BaseUrl = "https://filelist.io/", + Username = "someuser", + Passkey = "somepass" + } }; } @@ -35,9 +40,9 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests Mocker.GetMock() .Setup(o => o.ExecuteProxiedAsync(It.Is(v => v.Method == HttpMethod.Get), Subject.Definition)) - .Returns((r, d) => Task.FromResult(new HttpResponse(r, new HttpHeader(), new CookieCollection(), recentFeed))); + .Returns((r, d) => Task.FromResult(new HttpResponse(r, new HttpHeader { { "Content-Type", "application/json" } }, new CookieCollection(), recentFeed))); - var releases = (await Subject.Fetch(new MovieSearchCriteria { Categories = new int[] { 2000 } })).Releases; + var releases = (await Subject.Fetch(new MovieSearchCriteria { Categories = new[] { 2000 } })).Releases; releases.Should().HaveCount(4); releases.First().Should().BeOfType(); @@ -50,12 +55,14 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests torrentInfo.InfoUrl.Should().Be("https://filelist.io/details.php?id=665873"); torrentInfo.CommentUrl.Should().BeNullOrEmpty(); torrentInfo.Indexer.Should().Be(Subject.Definition.Name); - torrentInfo.PublishDate.Should().Be(DateTime.Parse("2020-01-25 20:20:19").ToUniversalTime()); + torrentInfo.PublishDate.Should().Be(DateTime.Parse("2020-01-25 20:20:19")); torrentInfo.Size.Should().Be(8300512414); torrentInfo.InfoHash.Should().Be(null); torrentInfo.MagnetUrl.Should().Be(null); torrentInfo.Peers.Should().Be(2 + 12); torrentInfo.Seeders.Should().Be(12); + + releases.Any(t => t.IndexerFlags.Contains(IndexerFlag.Internal)).Should().Be(true); } } } diff --git a/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListRequestGeneratorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListRequestGeneratorFixture.cs index fafc9c04b..6b2655fe1 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListRequestGeneratorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/FileListTests/FileListRequestGeneratorFixture.cs @@ -16,34 +16,35 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests [SetUp] public void Setup() { - Subject.Settings = new FileListSettings() + Subject.Settings = new FileListSettings { + BaseUrl = "https://filelist.io/", Passkey = "abcd", - Username = "somename", - BaseUrl = "https://filelist.io" + Username = "somename" }; Subject.Capabilities = new IndexerCapabilities { TvSearchParams = new List - { - TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep - }, + { + TvSearchParam.Q, TvSearchParam.ImdbId, TvSearchParam.Season, TvSearchParam.Ep + }, MovieSearchParams = new List - { - MovieSearchParam.Q, MovieSearchParam.ImdbId - }, + { + MovieSearchParam.Q, MovieSearchParam.ImdbId + }, MusicSearchParams = new List - { - MusicSearchParam.Q - }, + { + MusicSearchParam.Q + }, BookSearchParams = new List - { - BookSearchParam.Q - }, + { + BookSearchParam.Q + }, Flags = new List { - IndexerFlag.FreeLeech + IndexerFlag.FreeLeech, + IndexerFlag.Internal, } }; @@ -53,7 +54,7 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests _movieSearchCriteria = new MovieSearchCriteria { SearchTerm = "Star Wars", - Categories = new int[] { 2000 } + Categories = new[] { 2000 } }; } @@ -65,13 +66,13 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests [Test] public void should_use_categories_for_feed() { - var results = Subject.GetSearchRequests(new MovieSearchCriteria { Categories = new int[] { NewznabStandardCategory.MoviesSD.Id, NewznabStandardCategory.MoviesDVD.Id } }); + var results = Subject.GetSearchRequests(new MovieSearchCriteria { Categories = new[] { NewznabStandardCategory.MoviesSD.Id, NewznabStandardCategory.MoviesDVD.Id } }); results.GetAllTiers().Should().HaveCount(1); var page = results.GetAllTiers().First().First(); - page.Url.Query.Should().Contain("&category=1,2&"); + page.Url.Query.Should().Contain("&category=1%2C2"); } [Test] @@ -100,7 +101,7 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests var page = results.GetAllTiers().First().First(); page.Url.Query.Should().Contain("type=name"); - page.Url.Query.Should().Contain("query=Star Wars"); + page.Url.Query.Should().Contain("query=Star+Wars"); } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs index 7d725a9ca..2b7aa2e9b 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileList.cs @@ -1,6 +1,5 @@ using System.Collections.Generic; using NLog; -using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Messaging.Events; @@ -9,7 +8,8 @@ namespace NzbDrone.Core.Indexers.FileList public class FileList : TorrentIndexerBase { public override string Name => "FileList.io"; - public override string[] IndexerUrls => new string[] { "https://filelist.io" }; + public override string[] IndexerUrls => new[] { "https://filelist.io/" }; + public override string[] LegacyUrls => new[] { "https://filelist.io" }; public override string Description => "FileList (FL) is a ROMANIAN Private Torrent Tracker for 0DAY / GENERAL"; public override DownloadProtocol Protocol => DownloadProtocol.Torrent; public override IndexerPrivacy Privacy => IndexerPrivacy.Private; @@ -18,14 +18,18 @@ namespace NzbDrone.Core.Indexers.FileList public override bool SupportsRedirect => true; public override IndexerCapabilities Capabilities => SetCapabilities(); - public FileList(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger) + public FileList(IIndexerHttpClient httpClient, + IEventAggregator eventAggregator, + IIndexerStatusService indexerStatusService, + IConfigService configService, + Logger logger) : base(httpClient, eventAggregator, indexerStatusService, configService, logger) { } public override IIndexerRequestGenerator GetRequestGenerator() { - return new FileListRequestGenerator() { Settings = Settings, Capabilities = Capabilities }; + return new FileListRequestGenerator { Settings = Settings, Capabilities = Capabilities }; } public override IParseIndexerResponse GetParser() @@ -38,21 +42,21 @@ namespace NzbDrone.Core.Indexers.FileList var caps = new IndexerCapabilities { TvSearchParams = new List - { - TvSearchParam.Q, TvSearchParam.ImdbId, TvSearchParam.Season, TvSearchParam.Ep - }, + { + TvSearchParam.Q, TvSearchParam.ImdbId, TvSearchParam.Season, TvSearchParam.Ep + }, MovieSearchParams = new List - { - MovieSearchParam.Q, MovieSearchParam.ImdbId - }, + { + MovieSearchParam.Q, MovieSearchParam.ImdbId + }, MusicSearchParams = new List - { - MusicSearchParam.Q - }, + { + MusicSearchParam.Q + }, BookSearchParams = new List - { - BookSearchParam.Q - }, + { + BookSearchParam.Q + }, Flags = new List { IndexerFlag.Internal, diff --git a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListApi.cs b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListApi.cs index 2f4bb734b..9dbbb9a4e 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListApi.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListApi.cs @@ -1,4 +1,3 @@ -using System; using Newtonsoft.Json; namespace NzbDrone.Core.Indexers.FileList @@ -24,5 +23,7 @@ namespace NzbDrone.Core.Indexers.FileList [JsonProperty(PropertyName = "upload_date")] public string UploadDate { get; set; } public string Category { get; set; } + [JsonProperty(PropertyName = "small_description")] + public string SmallDescription { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListParser.cs b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListParser.cs index 9cf925174..4d6257923 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListParser.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.Globalization; +using System.Linq; using System.Net; using Newtonsoft.Json; using NzbDrone.Common.Http; @@ -25,9 +27,12 @@ namespace NzbDrone.Core.Indexers.FileList if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) { - throw new IndexerException(indexerResponse, - "Unexpected response status {0} code from API request", - indexerResponse.HttpResponse.StatusCode); + throw new IndexerException(indexerResponse, "Unexpected response status {0} code from API request", indexerResponse.HttpResponse.StatusCode); + } + + if (!indexerResponse.HttpResponse.Headers.ContentType.Contains(HttpAccept.Json.Value)) + { + throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}"); } var queryResults = JsonConvert.DeserializeObject>(indexerResponse.Content); @@ -49,10 +54,10 @@ namespace NzbDrone.Core.Indexers.FileList imdbId = int.Parse(result.ImdbId.Substring(2)); } - var downloadVolumeFactor = result.FreeLeech == true ? 0 : 1; - var uploadVolumeFactor = result.DoubleUp == true ? 2 : 1; + var downloadVolumeFactor = result.FreeLeech ? 0 : 1; + var uploadVolumeFactor = result.DoubleUp ? 2 : 1; - torrentInfos.Add(new TorrentInfo() + torrentInfos.Add(new TorrentInfo { Guid = string.Format("FileList-{0}", id), Title = result.Name, @@ -62,7 +67,9 @@ namespace NzbDrone.Core.Indexers.FileList InfoUrl = GetInfoUrl(id), Seeders = result.Seeders, Peers = result.Leechers + result.Seeders, - PublishDate = DateTime.Parse(result.UploadDate + " +0200"), + PublishDate = DateTime.Parse(result.UploadDate + " +0200", CultureInfo.InvariantCulture, DateTimeStyles.AdjustToUniversal), + Description = result.SmallDescription, + Genres = result.SmallDescription.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).ToList(), ImdbId = imdbId, IndexerFlags = flags, Files = (int)result.Files, @@ -70,7 +77,7 @@ namespace NzbDrone.Core.Indexers.FileList DownloadVolumeFactor = downloadVolumeFactor, UploadVolumeFactor = uploadVolumeFactor, MinimumRatio = 1, - MinimumSeedTime = 172800, //48 hours + MinimumSeedTime = 172800, // 48 hours }); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListRequestGenerator.cs index 803b7be39..0bbc71bdb 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListRequestGenerator.cs @@ -1,8 +1,10 @@ using System; using System.Collections.Generic; +using System.Collections.Specialized; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Parser; namespace NzbDrone.Core.Indexers.FileList { @@ -13,108 +15,140 @@ namespace NzbDrone.Core.Indexers.FileList public Func> GetCookies { get; set; } public Action, DateTime?> CookiesUpdater { get; set; } - public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) + public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria) { var pageableRequests = new IndexerPageableRequestChain(); + var parameters = GetDefaultParameters(); - if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace()) - { - pageableRequests.Add(GetRequest("search-torrents", searchCriteria.Categories, string.Format("&type=imdb&query={0}", searchCriteria.FullImdbId))); - } - else if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) + if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace() || searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) { - var titleYearSearchQuery = string.Format("{0}", searchCriteria.SanitizedSearchTerm); - pageableRequests.Add(GetRequest("search-torrents", searchCriteria.Categories, string.Format("&type=name&query={0}", titleYearSearchQuery.Trim()))); - } - else - { - pageableRequests.Add(GetRequest("latest-torrents", searchCriteria.Categories, "")); + parameters.Add("action", "search-torrents"); + + if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace()) + { + parameters.Add("type", "imdb"); + parameters.Add("query", searchCriteria.FullImdbId); + } + else if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) + { + parameters.Add("type", "name"); + parameters.Add("query", searchCriteria.SanitizedSearchTerm.Trim()); + } + + if (searchCriteria.Season.HasValue) + { + parameters.Add("season", searchCriteria.Season.ToString()); + parameters.Add("episode", searchCriteria.Episode); + } } + pageableRequests.Add(GetRequest(searchCriteria, parameters)); + return pageableRequests; } - public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria) + public virtual IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) { var pageableRequests = new IndexerPageableRequestChain(); - if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) + var parameters = GetDefaultParameters(); + + if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace()) { - var titleYearSearchQuery = string.Format("{0}", searchCriteria.SanitizedSearchTerm); - pageableRequests.Add(GetRequest("search-torrents", searchCriteria.Categories, string.Format("&type=name&query={0}", titleYearSearchQuery.Trim()))); + parameters.Add("action", "search-torrents"); + parameters.Add("type", "imdb"); + parameters.Add("query", searchCriteria.FullImdbId); } - else + else if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) { - pageableRequests.Add(GetRequest("latest-torrents", searchCriteria.Categories, "")); + parameters.Add("action", "search-torrents"); + parameters.Add("type", "name"); + parameters.Add("query", searchCriteria.SanitizedSearchTerm.Trim()); } + pageableRequests.Add(GetRequest(searchCriteria, parameters)); + return pageableRequests; } - public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria) + public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria) { var pageableRequests = new IndexerPageableRequestChain(); + var parameters = GetDefaultParameters(); - if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace()) - { - pageableRequests.Add(GetRequest("search-torrents", searchCriteria.Categories, string.Format("&type=imdb&query={0}&season={1}&episode={2}", searchCriteria.FullImdbId, searchCriteria.Season, searchCriteria.Episode))); - } - else if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) - { - var titleYearSearchQuery = string.Format("{0}", searchCriteria.SanitizedSearchTerm); - pageableRequests.Add(GetRequest("search-torrents", searchCriteria.Categories, string.Format("&type=name&query={0}&season={1}&episode={2}", titleYearSearchQuery.Trim(), searchCriteria.Season, searchCriteria.Episode))); - } - else + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) { - pageableRequests.Add(GetRequest("latest-torrents", searchCriteria.Categories, "")); + parameters.Add("action", "search-torrents"); + parameters.Add("type", "name"); + parameters.Add("query", searchCriteria.SanitizedSearchTerm.Trim()); } + pageableRequests.Add(GetRequest(searchCriteria, parameters)); + return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria) { var pageableRequests = new IndexerPageableRequestChain(); + var parameters = GetDefaultParameters(); + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) { - var titleYearSearchQuery = string.Format("{0}", searchCriteria.SanitizedSearchTerm); - pageableRequests.Add(GetRequest("search-torrents", searchCriteria.Categories, string.Format("&type=name&query={0}", titleYearSearchQuery.Trim()))); - } - else - { - pageableRequests.Add(GetRequest("latest-torrents", searchCriteria.Categories, "")); + parameters.Add("action", "search-torrents"); + parameters.Add("type", "name"); + parameters.Add("query", searchCriteria.SanitizedSearchTerm.Trim()); } + pageableRequests.Add(GetRequest(searchCriteria, parameters)); + return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria) { var pageableRequests = new IndexerPageableRequestChain(); + var parameters = GetDefaultParameters(); + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) { - var titleYearSearchQuery = string.Format("{0}", searchCriteria.SanitizedSearchTerm); - pageableRequests.Add(GetRequest("search-torrents", searchCriteria.Categories, string.Format("&type=name&query={0}", titleYearSearchQuery.Trim()))); - } - else - { - pageableRequests.Add(GetRequest("latest-torrents", searchCriteria.Categories, "")); + parameters.Add("action", "search-torrents"); + parameters.Add("type", "name"); + parameters.Add("query", searchCriteria.SanitizedSearchTerm.Trim()); } + pageableRequests.Add(GetRequest(searchCriteria, parameters)); + return pageableRequests; } - private IEnumerable GetRequest(string searchType, int[] categories, string parameters) + private IEnumerable GetRequest(SearchCriteriaBase searchCriteria, NameValueCollection parameters) { - var categoriesQuery = string.Join(",", Capabilities.Categories.MapTorznabCapsToTrackers(categories)); + if (parameters.Get("action") is null) + { + parameters.Add("action", "latest-torrents"); + } + + parameters.Add("category", string.Join(",", Capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories))); - var baseUrl = string.Format("{0}/api.php?action={1}&category={2}&username={3}&passkey={4}{5}", Settings.BaseUrl.TrimEnd('/'), searchType, categoriesQuery, Settings.Username.Trim(), Settings.Passkey.Trim(), parameters); + var searchUrl = $"{Settings.BaseUrl.TrimEnd('/')}/api.php?{parameters.GetQueryString()}"; + + yield return new IndexerRequest(searchUrl, HttpAccept.Json); + } + + private NameValueCollection GetDefaultParameters() + { + var parameters = new NameValueCollection + { + { "username", Settings.Username.Trim() }, + { "passkey", Settings.Passkey.Trim() } + }; if (Settings.FreeleechOnly) { - baseUrl += "&freeleech=1"; + parameters.Add("freeleech", "1"); } - yield return new IndexerRequest(baseUrl, HttpAccept.Json); + return parameters; } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListSettings.cs index 89934c9ea..29a8626c7 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/FileList/FileListSettings.cs @@ -16,12 +16,7 @@ namespace NzbDrone.Core.Indexers.FileList public class FileListSettings : NoAuthTorrentBaseSettings { - private static readonly FileListSettingsValidator Validator = new FileListSettingsValidator(); - - public FileListSettings() - { - BaseUrl = "https://filelist.io"; - } + private static readonly FileListSettingsValidator Validator = new (); [FieldDefinition(2, Label = "Username", HelpText = "Site Username", Privacy = PrivacyLevel.UserName)] public string Username { get; set; }