New Indexer: TorrentDay

src/NzbDrone.Common/Extensions/StringExtensions.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.Linq;
+using System.Net;
 using System.Text;
 using System.Text.RegularExpressions;
 
@@ -10,8 +11,6 @@ namespace NzbDrone.Common.Extensions
     public static class StringExtensions
     {
         private static readonly Regex CamelCaseRegex = new Regex("(?<!^)[A-Z]", RegexOptions.Compiled);
-        private static readonly Regex InvalidXmlChars = new Regex(@"(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F\uFEFF\uFFFE\uFFFF]", RegexOptions.Compiled);
-        private static readonly Regex ImdbId = new Regex(@"^(?:tt)?(\d{1,8})$", RegexOptions.Compiled);
 
         public static string NullSafe(this string target)
         {
@@ -174,12 +173,26 @@ namespace NzbDrone.Common.Extensions
             return source.Contains(value, StringComparer.InvariantCultureIgnoreCase);
         }
 
-        public static int CoerceInt(this string str) => int.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
+        public static string UrlEncode(this string searchString, Encoding encoding)
+        {
+            if (string.IsNullOrEmpty(searchString))
+            {
+                return string.Empty;
+            }
 
-        public static string NormalizeSpace(this string s) => s.Trim();
+            var bytes = encoding.GetBytes(searchString);
+            return encoding.GetString(WebUtility.UrlEncodeToBytes(bytes, 0, bytes.Length));
+        }
 
-        public static string NormalizeMultiSpaces(this string s) => new Regex(@"\s+").Replace(NormalizeSpace(s), " ");
+        public static string UrlDecode(this string searchString, Encoding encoding)
+        {
+            if (string.IsNullOrEmpty(searchString))
+            {
+                return string.Empty;
+            }
 
-        public static string NormalizeNumber(this string s) => NormalizeSpace(s).Replace("-", "0").Replace(",", "");
+            var inputBytes = encoding.GetBytes(searchString);
+            return encoding.GetString(WebUtility.UrlDecodeToBytes(inputBytes, 0, inputBytes.Length));
+        }
     }
 }

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs

@@ -10,6 +10,7 @@ using Newtonsoft.Json.Linq;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Parser;
 
 namespace NzbDrone.Core.Indexers.Cardigann
 {
@@ -608,10 +609,10 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         data = data.ToUpper();
                         break;
                     case "urldecode":
-                        data = WebUtilityHelpers.UrlDecode(data, _encoding);
+                        data = data.UrlDecode(_encoding);
                         break;
                     case "urlencode":
-                        data = WebUtilityHelpers.UrlEncode(data, _encoding);
+                        data = data.UrlEncode(_encoding);
                         break;
                     case "timeago":
                     case "reltime":

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs

@@ -8,6 +8,7 @@ using AngleSharp.Html.Parser;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers.Exceptions;
+using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers.Cardigann

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannReleaseInfo.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using NzbDrone.Core.Parser;
 
 namespace NzbDrone.Core.Indexers.Cardigann
 {

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs

@@ -10,6 +10,7 @@ using NLog;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Indexers.Definitions.Cardigann;
 using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser;
 
 namespace NzbDrone.Core.Indexers.Cardigann
 {

src/NzbDrone.Core/Indexers/Definitions/Cardigann/WebUtilityHelpers.cs

@@ -1,30 +0,0 @@
-using System.Net;
-using System.Text;
-
-namespace NzbDrone.Core.Indexers.Cardigann
-{
-    public static class WebUtilityHelpers
-    {
-        public static string UrlEncode(string searchString, Encoding encoding)
-        {
-            if (string.IsNullOrEmpty(searchString))
-            {
-                return string.Empty;
-            }
-
-            var bytes = encoding.GetBytes(searchString);
-            return encoding.GetString(WebUtility.UrlEncodeToBytes(bytes, 0, bytes.Length));
-        }
-
-        public static string UrlDecode(string searchString, Encoding encoding)
-        {
-            if (string.IsNullOrEmpty(searchString))
-            {
-                return string.Empty;
-            }
-
-            var inputBytes = encoding.GetBytes(searchString);
-            return encoding.GetString(WebUtility.UrlDecodeToBytes(inputBytes, 0, inputBytes.Length));
-        }
-    }
-}

src/NzbDrone.Core/Indexers/Definitions/DigitalCore.cs

@@ -226,7 +226,6 @@ namespace NzbDrone.Core.Indexers.Definitions
                     var descriptions = new List<string>();
                     var tags = new List<string>();
 
-
                     release.MinimumRatio = 1.1;
                     release.MinimumSeedTime = 432000; // 120 hours
                     release.Title = row.name;

src/NzbDrone.Core/Indexers/Definitions/TorrentDay.cs

@@ -0,0 +1,295 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using FluentValidation;
+using Newtonsoft.Json;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.Parser.Model;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Definitions
+{
+    public class TorrentDay : HttpIndexerBase<TorrentDaySettings>
+    {
+        public override string Name => "TorrentDay";
+
+        public override string BaseUrl => "https://torrentday.cool/";
+        public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
+        public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
+        public override IndexerCapabilities Capabilities => SetCapabilities();
+
+        public TorrentDay(IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
+            : base(httpClient, indexerStatusService, configService, logger)
+        {
+        }
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new TorrentDayRequestGenerator() { Settings = Settings, Capabilities = Capabilities, BaseUrl = BaseUrl };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new TorrentDayParser(Settings, Capabilities.Categories, BaseUrl);
+        }
+
+        private IndexerCapabilities SetCapabilities()
+        {
+            var caps = new IndexerCapabilities
+            {
+                TvSearchParams = new List<TvSearchParam>
+                       {
+                           TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep, TvSearchParam.ImdbId
+                       },
+                MovieSearchParams = new List<MovieSearchParam>
+                       {
+                           MovieSearchParam.Q, MovieSearchParam.ImdbId
+                       },
+                MusicSearchParams = new List<MusicSearchParam>
+                       {
+                           MusicSearchParam.Q
+                       },
+                BookSearchParams = new List<BookSearchParam>
+                       {
+                           BookSearchParam.Q
+                       }
+            };
+
+            caps.Categories.AddCategoryMapping(29, NewznabStandardCategory.TVAnime, "Anime");
+            caps.Categories.AddCategoryMapping(28, NewznabStandardCategory.PC, "Appz/Packs");
+            caps.Categories.AddCategoryMapping(42, NewznabStandardCategory.AudioAudiobook, "Audio Books");
+            caps.Categories.AddCategoryMapping(20, NewznabStandardCategory.Books, "Books");
+            caps.Categories.AddCategoryMapping(30, NewznabStandardCategory.TVDocumentary, "Documentary");
+            caps.Categories.AddCategoryMapping(47, NewznabStandardCategory.Other, "Fonts");
+            caps.Categories.AddCategoryMapping(43, NewznabStandardCategory.PCMac, "Mac");
+
+            caps.Categories.AddCategoryMapping(96, NewznabStandardCategory.MoviesUHD, "Movie/4K");
+            caps.Categories.AddCategoryMapping(25, NewznabStandardCategory.MoviesSD, "Movies/480p");
+            caps.Categories.AddCategoryMapping(11, NewznabStandardCategory.MoviesBluRay, "Movies/Bluray");
+            caps.Categories.AddCategoryMapping(5, NewznabStandardCategory.MoviesBluRay, "Movies/Bluray-Full");
+            caps.Categories.AddCategoryMapping(3, NewznabStandardCategory.MoviesDVD, "Movies/DVD-R");
+            caps.Categories.AddCategoryMapping(21, NewznabStandardCategory.MoviesSD, "Movies/MP4");
+            caps.Categories.AddCategoryMapping(22, NewznabStandardCategory.MoviesForeign, "Movies/Non-English");
+            caps.Categories.AddCategoryMapping(13, NewznabStandardCategory.Movies, "Movies/Packs");
+            caps.Categories.AddCategoryMapping(44, NewznabStandardCategory.MoviesSD, "Movies/SD/x264");
+            caps.Categories.AddCategoryMapping(48, NewznabStandardCategory.Movies, "Movies/x265");
+            caps.Categories.AddCategoryMapping(1, NewznabStandardCategory.MoviesSD, "Movies/XviD");
+
+            caps.Categories.AddCategoryMapping(17, NewznabStandardCategory.AudioMP3, "Music/Audio");
+            caps.Categories.AddCategoryMapping(23, NewznabStandardCategory.AudioForeign, "Music/Non-English");
+            caps.Categories.AddCategoryMapping(41, NewznabStandardCategory.Audio, "Music/Packs");
+            caps.Categories.AddCategoryMapping(16, NewznabStandardCategory.AudioVideo, "Music/Video");
+            caps.Categories.AddCategoryMapping(27, NewznabStandardCategory.Audio, "Music/Flac");
+
+            caps.Categories.AddCategoryMapping(45, NewznabStandardCategory.AudioOther, "Podcast");
+
+            caps.Categories.AddCategoryMapping(4, NewznabStandardCategory.PCGames, "PC/Games");
+            caps.Categories.AddCategoryMapping(18, NewznabStandardCategory.ConsolePS3, "PS3");
+            caps.Categories.AddCategoryMapping(8, NewznabStandardCategory.ConsolePSP, "PSP");
+            caps.Categories.AddCategoryMapping(10, NewznabStandardCategory.ConsoleWii, "Wii");
+            caps.Categories.AddCategoryMapping(9, NewznabStandardCategory.ConsoleXBox360, "Xbox-360");
+
+            caps.Categories.AddCategoryMapping(24, NewznabStandardCategory.TVSD, "TV/480p");
+            caps.Categories.AddCategoryMapping(32, NewznabStandardCategory.TVHD, "TV/Bluray");
+            caps.Categories.AddCategoryMapping(31, NewznabStandardCategory.TVSD, "TV/DVD-R");
+            caps.Categories.AddCategoryMapping(33, NewznabStandardCategory.TVSD, "TV/DVD-Rip");
+            caps.Categories.AddCategoryMapping(46, NewznabStandardCategory.TVSD, "TV/Mobile");
+            caps.Categories.AddCategoryMapping(14, NewznabStandardCategory.TV, "TV/Packs");
+            caps.Categories.AddCategoryMapping(26, NewznabStandardCategory.TVSD, "TV/SD/x264");
+            caps.Categories.AddCategoryMapping(7, NewznabStandardCategory.TVHD, "TV/x264");
+            caps.Categories.AddCategoryMapping(34, NewznabStandardCategory.TVUHD, "TV/x265");
+            caps.Categories.AddCategoryMapping(2, NewznabStandardCategory.TVSD, "TV/XviD");
+
+            caps.Categories.AddCategoryMapping(6, NewznabStandardCategory.XXX, "XXX/Movies");
+            caps.Categories.AddCategoryMapping(15, NewznabStandardCategory.XXXPack, "XXX/Packs");
+
+            return caps;
+        }
+    }
+
+    public class TorrentDayRequestGenerator : IIndexerRequestGenerator
+    {
+        public TorrentDaySettings Settings { get; set; }
+        public IndexerCapabilities Capabilities { get; set; }
+        public string BaseUrl { get; set; }
+
+        public TorrentDayRequestGenerator()
+        {
+        }
+
+        private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories, string imdbId = null)
+        {
+            var searchUrl = BaseUrl + "t.json";
+
+            var cats = Capabilities.Categories.MapTorznabCapsToTrackers(categories);
+            if (cats.Count == 0)
+            {
+                cats = Capabilities.Categories.GetTrackerCategories();
+            }
+
+            var catStr = string.Join(";", cats);
+            searchUrl = searchUrl + "?" + catStr;
+
+            if (imdbId.IsNotNullOrWhiteSpace())
+            {
+                searchUrl += ";q=" + imdbId;
+            }
+            else
+            {
+                searchUrl += ";q=" + term.UrlEncode(Encoding.UTF8);
+            }
+
+            var request = new IndexerRequest(searchUrl, HttpAccept.Rss);
+
+            foreach (var cookie in CookieUtil.CookieHeaderToDictionary(Settings.Cookie))
+            {
+                request.HttpRequest.Cookies.Add(cookie.Key, cookie.Value);
+            }
+
+            yield return request;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories, searchCriteria.ImdbId));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories, searchCriteria.ImdbId));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SearchTerm), searchCriteria.Categories));
+
+            return pageableRequests;
+        }
+
+        public Func<IDictionary<string, string>> GetCookies { get; set; }
+        public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
+    }
+
+    public class TorrentDayParser : IParseIndexerResponse
+    {
+        private readonly TorrentDaySettings _settings;
+        private readonly IndexerCapabilitiesCategories _categories;
+        private readonly string _baseUrl;
+
+        public TorrentDayParser(TorrentDaySettings settings, IndexerCapabilitiesCategories categories, string baseUrl)
+        {
+            _settings = settings;
+            _categories = categories;
+            _baseUrl = baseUrl;
+        }
+
+        public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
+        {
+            var torrentInfos = new List<TorrentInfo>();
+
+            var rows = JsonConvert.DeserializeObject<dynamic>(indexerResponse.Content);
+
+            foreach (var row in rows)
+            {
+                var title = (string)row.name;
+
+                var torrentId = (long)row.t;
+                var details = new Uri(_baseUrl + "details.php?id=" + torrentId);
+                var seeders = (int)row.seeders;
+                var imdbId = (string)row["imdb-id"];
+                var downloadMultiplier = (double?)row["download-multiplier"] ?? 1;
+                var link = new Uri(_baseUrl + "download.php/" + torrentId + "/" + torrentId + ".torrent");
+                var publishDate = DateTimeUtil.UnixTimestampToDateTime((long)row.ctime).ToLocalTime();
+                var imdb = ParseUtil.GetImdbID(imdbId) ?? 0;
+
+                var release = new TorrentInfo
+                {
+                    Title = title,
+                    Guid = details.AbsoluteUri,
+                    DownloadUrl = link.AbsoluteUri,
+                    PublishDate = publishDate,
+                    Category = _categories.MapTrackerCatToNewznab(row.c.ToString()),
+                    Size = (long)row.size,
+                    Files = (int)row.files,
+                    Grabs = (int)row.completed,
+                    Seeders = seeders,
+                    Peers = seeders + (int)row.leechers,
+                    ImdbId = imdb,
+                    DownloadVolumeFactor = downloadMultiplier,
+                    UploadVolumeFactor = 1,
+                    MinimumRatio = 1,
+                    MinimumSeedTime = 172800 // 48 hours
+                };
+
+                torrentInfos.Add(release);
+            }
+
+            return torrentInfos.ToArray();
+        }
+
+        public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
+    }
+
+    public class TorrentDaySettingsValidator : AbstractValidator<TorrentDaySettings>
+    {
+        public TorrentDaySettingsValidator()
+        {
+            RuleFor(c => c.Cookie).NotEmpty();
+        }
+    }
+
+    public class TorrentDaySettings : IProviderConfig
+    {
+        private static readonly TorrentDaySettingsValidator Validator = new TorrentDaySettingsValidator();
+
+        public TorrentDaySettings()
+        {
+            Cookie = "";
+        }
+
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(1, Label = "Cookie", HelpText = "Site Cookie")]
+        public string Cookie { get; set; }
+
+        public NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/TorrentLeech.cs

@@ -12,6 +12,7 @@ using NzbDrone.Common.Http;
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
@@ -294,7 +295,7 @@ namespace NzbDrone.Core.Indexers.Definitions
 
                 // freeleech #6579 #6624 #7367
                 string dlMultiplier = row.download_multiplier.ToString();
-                var dlVolumeFactor = dlMultiplier.IsNullOrWhiteSpace() ? 1 : dlMultiplier.CoerceInt();
+                var dlVolumeFactor = dlMultiplier.IsNullOrWhiteSpace() ? 1 : ParseUtil.CoerceInt(dlMultiplier);
 
                 var release = new TorrentInfo
                 {
@@ -345,7 +346,7 @@ namespace NzbDrone.Core.Indexers.Definitions
 
         public string BaseUrl { get; set; }
 
-        [FieldDefinition(1, Label = "Username", Advanced = true, HelpText = "Site username")]
+        [FieldDefinition(1, Label = "Username", HelpText = "Site username")]
         public string Username { get; set; }
 
         [FieldDefinition(2, Label = "Password", Type = FieldType.Password, HelpText = "Site password", Privacy = PrivacyLevel.Password)]

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Text;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Extensions;

src/NzbDrone.Core/Parser/DateTimeUtil.cs

@@ -2,7 +2,7 @@ using System;
 using System.Globalization;
 using System.Text.RegularExpressions;
 
-namespace NzbDrone.Core.Indexers.Cardigann
+namespace NzbDrone.Core.Parser
 {
     public static class DateTimeUtil
     {

src/NzbDrone.Core/Parser/ParseUtil.cs

@@ -1,7 +1,7 @@
 using System.Globalization;
 using System.Text.RegularExpressions;
 
-namespace NzbDrone.Core.Indexers.Cardigann
+namespace NzbDrone.Core.Parser
 {
     public static class ParseUtil
     {

src/NzbDrone.Core/Parser/StringUtil.cs

@@ -6,8 +6,9 @@ using System.Security.Cryptography;
 using System.Text;
 using AngleSharp.Dom;
 using AngleSharp.Html;
+using NzbDrone.Common.Extensions;
 
-namespace NzbDrone.Core.Indexers.Cardigann
+namespace NzbDrone.Core.Parser
 {
     public static class StringUtil
     {
@@ -171,7 +172,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                                             Encoding encoding = null,
                                             string separator = "&") =>
             string.Join(separator,
-                        collection.Select(a => $"{a.Key}={WebUtilityHelpers.UrlEncode(a.Value, encoding ?? Encoding.UTF8)}"));
+                        collection.Select(a => $"{a.Key}={a.Value.UrlEncode(encoding ?? Encoding.UTF8)}"));
 
         public static void Add(this ICollection<KeyValuePair<string, string>> collection, string key, string value) => collection.Add(new KeyValuePair<string, string>(key, value));