From b002a612ec0a187fbc183299b8201eb173628bf8 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 7 Feb 2021 23:14:14 -0500 Subject: [PATCH] Jackett to Prowlarr C# conversion (Digital Core) --- .../Indexers/Definitions/DigitalCore.cs | 293 ++++++++++++++++++ .../Definitions/Nyaa/NyaaRequestGenerator.cs | 5 - .../Indexers/Definitions/Nyaa/NyaaSettings.cs | 3 - .../Indexers/IndexerCapabilitiesCategories.cs | 97 +++--- 4 files changed, 346 insertions(+), 52 deletions(-) create mode 100644 src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs diff --git a/src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs b/src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs new file mode 100644 index 000000000..f73f77e33 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs @@ -0,0 +1,293 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using FluentValidation; +using Newtonsoft.Json; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.Definitions +{ + public class DigitalCore : HttpIndexerBase + { + public override string Name => "DigitalCore"; + public override DownloadProtocol Protocol => DownloadProtocol.Torrent; + public override IndexerPrivacy Privacy => IndexerPrivacy.Private; + public override IndexerCapabilities Capabilities => SetCapabilities(); + + public DigitalCore(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger) + : base(httpClient, indexerStatusService, configService, logger) + { + } + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new DigitalCoreRequestGenerator() { Settings = Settings, PageSize = PageSize, Capabilities = Capabilities }; + } + + public override IParseIndexerResponse GetParser() + { + return new DigitalCoreParser(Settings, Capabilities.Categories); + } + + private IndexerCapabilities SetCapabilities() + { + var caps = new IndexerCapabilities + { + TvSearchParams = new List + { + TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep + }, + MovieSearchParams = new List + { + MovieSearchParam.Q, MovieSearchParam.ImdbId + }, + MusicSearchParams = new List + { + MusicSearchParam.Q + }, + BookSearchParams = new List + { + BookSearchParam.Q + } + }; + + caps.Categories.AddCategoryMapping(1.ToString(), NewznabStandardCategory.MoviesDVD, "Movies/DVDR"); + caps.Categories.AddCategoryMapping(2.ToString(), NewznabStandardCategory.MoviesSD, "Movies/SD"); + caps.Categories.AddCategoryMapping(3.ToString(), NewznabStandardCategory.MoviesBluRay, "Movies/BluRay"); + caps.Categories.AddCategoryMapping(4.ToString(), NewznabStandardCategory.MoviesUHD, "Movies/4K"); + caps.Categories.AddCategoryMapping(5.ToString(), NewznabStandardCategory.MoviesHD, "Movies/720p"); + caps.Categories.AddCategoryMapping(6.ToString(), NewznabStandardCategory.MoviesHD, "Movies/1080p"); + caps.Categories.AddCategoryMapping(7.ToString(), NewznabStandardCategory.MoviesHD, "Movies/PACKS"); + + caps.Categories.AddCategoryMapping(8.ToString(), NewznabStandardCategory.TVHD, "TV/720p"); + caps.Categories.AddCategoryMapping(9.ToString(), NewznabStandardCategory.TVHD, "TV/1080p"); + caps.Categories.AddCategoryMapping(10.ToString(), NewznabStandardCategory.TVSD, "TV/SD"); + caps.Categories.AddCategoryMapping(11.ToString(), NewznabStandardCategory.TVSD, "TV/DVDR"); + caps.Categories.AddCategoryMapping(12.ToString(), NewznabStandardCategory.TVHD, "TV/PACKS"); + caps.Categories.AddCategoryMapping(13.ToString(), NewznabStandardCategory.TVUHD, "TV/4K"); + caps.Categories.AddCategoryMapping(14.ToString(), NewznabStandardCategory.TVHD, "TV/BluRay"); + + caps.Categories.AddCategoryMapping(17.ToString(), NewznabStandardCategory.Other, "Unknown"); + caps.Categories.AddCategoryMapping(18.ToString(), NewznabStandardCategory.PC0day, "Apps/0day"); + caps.Categories.AddCategoryMapping(20.ToString(), NewznabStandardCategory.PCISO, "Apps/PC"); + caps.Categories.AddCategoryMapping(21.ToString(), NewznabStandardCategory.PCMac, "Apps/Mac"); + caps.Categories.AddCategoryMapping(33.ToString(), NewznabStandardCategory.PC, "Apps/Tutorials"); + + caps.Categories.AddCategoryMapping(22.ToString(), NewznabStandardCategory.AudioMP3, "Music/MP3"); + caps.Categories.AddCategoryMapping(23.ToString(), NewznabStandardCategory.AudioLossless, "Music/FLAC"); + caps.Categories.AddCategoryMapping(24.ToString(), NewznabStandardCategory.Audio, "Music/MTV"); + caps.Categories.AddCategoryMapping(29.ToString(), NewznabStandardCategory.Audio, "Music/PACKS"); + + caps.Categories.AddCategoryMapping(25.ToString(), NewznabStandardCategory.PCGames, "Games/PC"); + caps.Categories.AddCategoryMapping(26.ToString(), NewznabStandardCategory.Console, "Games/NSW"); + caps.Categories.AddCategoryMapping(27.ToString(), NewznabStandardCategory.PCMac, "Games/Mac"); + + caps.Categories.AddCategoryMapping(28.ToString(), NewznabStandardCategory.Books, "Ebooks"); + + caps.Categories.AddCategoryMapping(30.ToString(), NewznabStandardCategory.XXXSD, "XXX/SD"); + caps.Categories.AddCategoryMapping(31.ToString(), NewznabStandardCategory.XXX, "XXX/HD"); + caps.Categories.AddCategoryMapping(32.ToString(), NewznabStandardCategory.XXXUHD, "XXX/4K"); + caps.Categories.AddCategoryMapping(35.ToString(), NewznabStandardCategory.XXXSD, "XXX/Movies/SD"); + caps.Categories.AddCategoryMapping(36.ToString(), NewznabStandardCategory.XXX, "XXX/Movies/HD"); + caps.Categories.AddCategoryMapping(37.ToString(), NewznabStandardCategory.XXXUHD, "XXX/Movies/4K"); + caps.Categories.AddCategoryMapping(34.ToString(), NewznabStandardCategory.XXXImageSet, "XXX/Imagesets"); + + return caps; + } + } + + public class DigitalCoreRequestGenerator : IIndexerRequestGenerator + { + public DigitalCoreSettings Settings { get; set; } + public IndexerCapabilities Capabilities { get; set; } + + public int MaxPages { get; set; } + public int PageSize { get; set; } + + public DigitalCoreRequestGenerator() + { + MaxPages = 30; + PageSize = 100; + } + + private IEnumerable GetPagedRequests(string term, int[] categories) + { + var baseUrl = string.Format("{0}/api/v1/torrents?extendedSearch=false", Settings.BaseUrl.TrimEnd('/')); + + var parameters = string.Empty; + + parameters += "&freeleech=false"; + parameters += "&index=0"; + parameters += "&limit=100"; + parameters += "&order=desc"; + parameters += "&page=search"; + parameters += string.Format("&searchText={0}", term); + + parameters += "&sort=d"; + parameters += "§ion=all"; + parameters += "&stereoscopic=false"; + parameters += "&watchview=false"; + + var searchUrl = baseUrl + parameters; + + foreach (var cat in Capabilities.Categories.MapTorznabCapsToTrackers(categories)) + { + searchUrl += "&categories[]=" + cat; + } + + var request = new IndexerRequest(searchUrl, HttpAccept.Rss); + request.HttpRequest.Cookies.Add("uid", Settings.UId); + request.HttpRequest.Cookies.Add("pass", Settings.Passphrase); + + yield return request; + } + + public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + + pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories)); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria) + { + return new IndexerPageableRequestChain(); + } + + public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria) + { + return new IndexerPageableRequestChain(); + } + + public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria) + { + return new IndexerPageableRequestChain(); + } + + public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + + pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories)); + + return pageableRequests; + } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } + } + + public class DigitalCoreParser : IParseIndexerResponse + { + private readonly DigitalCoreSettings _settings; + private readonly IndexerCapabilitiesCategories _categories; + + public DigitalCoreParser(DigitalCoreSettings settings, IndexerCapabilitiesCategories categories) + { + _settings = settings; + _categories = categories; + } + + public IList ParseResponse(IndexerResponse indexerResponse) + { + var torrentInfos = new List(); + + if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new IndexerException(indexerResponse, + "Unexpected response status {0} code from API request", + indexerResponse.HttpResponse.StatusCode); + } + + try + { + //var json = JArray.Parse(results.Content); + var json = JsonConvert.DeserializeObject(indexerResponse.Content); + + foreach (var row in json ?? Enumerable.Empty()) + { + var release = new TorrentInfo(); + var descriptions = new List(); + var tags = new List(); + + release.Title = row.name; + release.Category = _categories.MapTrackerCatToNewznab(row.category.ToString()); + release.Size = row.size; + release.Seeders = row.seeders; + release.Peers = row.leechers + release.Seeders; + release.PublishDate = DateTime.ParseExact(row.added.ToString() + " +01:00", "yyyy-MM-dd HH:mm:ss zzz", CultureInfo.InvariantCulture); + release.Files = row.numfiles; + release.Grabs = row.times_completed; + + release.Guid = new Uri(_settings.BaseUrl + "torrent/" + row.id.ToString() + "/").ToString(); + release.DownloadUrl = _settings.BaseUrl + "api/v1/torrents/download/" + row.id.ToString(); + + if (row.imdbid2 != null && row.imdbid2.ToString().StartsWith("tt")) + { + if (int.TryParse((string)row.imdbid2, out int imdbNumber)) + { + release.ImdbId = imdbNumber; + } + } + + torrentInfos.Add(release); + } + } + catch (Exception ex) + { + throw new IndexerException(indexerResponse, + "Unable to parse response from DigitalCore: {0}", + ex.Message); + } + + return torrentInfos.ToArray(); + } + + public Action, DateTime?> CookiesUpdater { get; set; } + } + + public class DigitalCoreSettingsValidator : AbstractValidator + { + public DigitalCoreSettingsValidator() + { + RuleFor(c => c.UId).NotEmpty(); + RuleFor(c => c.Passphrase).NotEmpty(); + } + } + + public class DigitalCoreSettings : IIndexerSettings + { + private static readonly DigitalCoreSettingsValidator Validator = new DigitalCoreSettingsValidator(); + + public DigitalCoreSettings() + { + BaseUrl = "https://digitalcore.club/"; + UId = ""; + Passphrase = ""; + } + + public string BaseUrl { get; set; } + + [FieldDefinition(1, Label = "UID", Advanced = true, HelpText = "Uid from login cookie")] + public string UId { get; set; } + + [FieldDefinition(2, Label = "Passphrase", Advanced = true, HelpText = "Pass from login cookie")] + public string Passphrase { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Nyaa/NyaaRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Nyaa/NyaaRequestGenerator.cs index 8e6d9b25a..0d9e03bf5 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Nyaa/NyaaRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Nyaa/NyaaRequestGenerator.cs @@ -34,11 +34,6 @@ namespace NzbDrone.Core.Indexers.Nyaa else { yield return new IndexerRequest(baseUrl, HttpAccept.Rss); - - for (var page = 1; page < maxPages; page++) - { - yield return new IndexerRequest($"{baseUrl}&offset={page + 1}", HttpAccept.Rss); - } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Nyaa/NyaaSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Nyaa/NyaaSettings.cs index d9b811df7..a3d76ac15 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Nyaa/NyaaSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Nyaa/NyaaSettings.cs @@ -1,9 +1,6 @@ -using System.Collections.Generic; using System.Text.RegularExpressions; using FluentValidation; using NzbDrone.Core.Annotations; -using NzbDrone.Core.Languages; -using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Nyaa diff --git a/src/NzbDrone.Core/Indexers/IndexerCapabilitiesCategories.cs b/src/NzbDrone.Core/Indexers/IndexerCapabilitiesCategories.cs index 4bd6a21a3..1c93e23e3 100644 --- a/src/NzbDrone.Core/Indexers/IndexerCapabilitiesCategories.cs +++ b/src/NzbDrone.Core/Indexers/IndexerCapabilitiesCategories.cs @@ -77,26 +77,28 @@ namespace NzbDrone.Core.Indexers AddTorznabCategoryTree(customCat); } - //public List MapTorznabCapsToTrackers(TorznabQuery query, bool mapChildrenCatsToParent = false) - //{ - // var expandedQueryCats = ExpandTorznabQueryCategories(query, mapChildrenCatsToParent); - // var result = _categoryMapping - // .Where(c => expandedQueryCats.Contains(c.NewzNabCategory)) - // .Select(mapping => mapping.TrackerCategory).Distinct().ToList(); - // return result; - //} - public ICollection MapTrackerCatToNewznab(string trackerCategory) + public List MapTorznabCapsToTrackers(int[] queryCategories, bool mapChildrenCatsToParent = false) { - if (string.IsNullOrWhiteSpace(trackerCategory)) + var expandedQueryCats = ExpandTorznabQueryCategories(queryCategories, mapChildrenCatsToParent); + var result = _categoryMapping + .Where(c => expandedQueryCats.Contains(c.NewzNabCategory)) + .Select(mapping => mapping.TrackerCategory).Distinct().ToList(); + return result; + } + + public ICollection MapTrackerCatToNewznab(string input) + { + if (string.IsNullOrWhiteSpace(input)) { - return new List(); + return new List(); } var cats = _categoryMapping .Where(m => !string.IsNullOrWhiteSpace(m.TrackerCategory) && - string.Equals(m.TrackerCategory, trackerCategory, StringComparison.InvariantCultureIgnoreCase)) - .Select(c => c.NewzNabCategory).ToList(); + string.Equals(m.TrackerCategory, input, StringComparison.InvariantCultureIgnoreCase)) + .Select(c => NewznabStandardCategory.AllCats.FirstOrDefault(n => n.Id == c.NewzNabCategory) ?? new IndexerCategory { Id = c.NewzNabCategory }) + .ToList(); return cats; } @@ -134,37 +136,44 @@ namespace NzbDrone.Core.Indexers rhs.GetTorznabCategoryList().Where(x => x.Id < 100000).ToList().ForEach(AddTorznabCategoryTree); } - //public List ExpandTorznabQueryCategories(TorznabQuery query, bool mapChildrenCatsToParent = false) - //{ - // var expandedQueryCats = new List(); - // foreach (var queryCategory in query.Categories) - // { - // expandedQueryCats.Add(queryCategory); - // if (queryCategory >= 100000) - // { - // continue; - // } - - // var parentCat = _torznabCategoryTree.FirstOrDefault(c => c.Id == queryCategory); - // if (parentCat != null) - // { - // // if it's parent cat we add all the children - // expandedQueryCats.AddRange(parentCat.SubCategories.Select(c => c.Id)); - // } - // else if (mapChildrenCatsToParent) - // { - // // if it's child cat and mapChildrenCatsToParent is enabled we add the parent - // var queryCategoryTorznab = new IndexerCategory(queryCategory, ""); - // parentCat = _torznabCategoryTree.FirstOrDefault(c => c.Contains(queryCategoryTorznab)); - // if (parentCat != null) - // { - // expandedQueryCats.Add(parentCat.Id); - // } - // } - // } - - // return expandedQueryCats.Distinct().ToList(); - //} + public List ExpandTorznabQueryCategories(int[] queryCategories, bool mapChildrenCatsToParent = false) + { + var expandedQueryCats = new List(); + + if (queryCategories == null) + { + return expandedQueryCats; + } + + foreach (var queryCategory in queryCategories) + { + expandedQueryCats.Add(queryCategory); + if (queryCategory >= 100000) + { + continue; + } + + var parentCat = _torznabCategoryTree.FirstOrDefault(c => c.Id == queryCategory); + if (parentCat != null) + { + // if it's parent cat we add all the children + expandedQueryCats.AddRange(parentCat.SubCategories.Select(c => c.Id)); + } + else if (mapChildrenCatsToParent) + { + // if it's child cat and mapChildrenCatsToParent is enabled we add the parent + var queryCategoryTorznab = new IndexerCategory(queryCategory, ""); + parentCat = _torznabCategoryTree.FirstOrDefault(c => c.Contains(queryCategoryTorznab)); + if (parentCat != null) + { + expandedQueryCats.Add(parentCat.Id); + } + } + } + + return expandedQueryCats.Distinct().ToList(); + } + private void AddTorznabCategoryTree(IndexerCategory torznabCategory) { // build the category tree