Fix Rarbg search and parsing

src/NzbDrone.Core/Applications/ApplicationIndexerSyncCommand.cs

@@ -1,8 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using NzbDrone.Core.Messaging.Commands;
 
 namespace NzbDrone.Core.Applications

src/NzbDrone.Core/IndexerSearch/Definitions/BasicSearchCriteria.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {
     public class BasicSearchCriteria : SearchCriteriaBase

src/NzbDrone.Core/IndexerSearch/Definitions/BookSearchCriteria.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {
     public class BookSearchCriteria : SearchCriteriaBase

src/NzbDrone.Core/IndexerSearch/Definitions/MusicSearchCriteria.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {
     public class MusicSearchCriteria : SearchCriteriaBase

src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs

@@ -1,8 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
-using NzbDrone.Common.EnsureThat;
-using NzbDrone.Common.Extensions;
 
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {

src/NzbDrone.Core/IndexerSearch/Definitions/TvSearchCriteria.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {
     public class TvSearchCriteria : SearchCriteriaBase

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs

@@ -6,7 +6,6 @@ using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Disk;
 using NzbDrone.Common.EnvironmentInfo;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Indexers.Cardigann;
 using NzbDrone.Core.Messaging.Commands;

src/NzbDrone.Core/Indexers/CategoryMapping.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.Indexers
 {
     public class CategoryMapping

src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs

@@ -3,13 +3,10 @@ using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Globalization;
 using System.Linq;
-using System.Net;
 using System.Text.RegularExpressions;
 using AngleSharp.Html.Parser;
 using FluentValidation;
-using Newtonsoft.Json;
 using NLog;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.Configuration;

src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNetRequestGenerator.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using System.Linq;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.IndexerSearch.Definitions;
 

src/NzbDrone.Core/Indexers/Definitions/BrokenStones.cs

@@ -1,4 +1,3 @@
-using System.Collections.Generic;
 using NLog;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using System.Linq;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Http;

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs

@@ -1,6 +1,5 @@
 using System.Collections.Generic;
 using FluentValidation;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;

src/NzbDrone.Core/Indexers/Definitions/FileList/FileListSettings.cs

@@ -1,8 +1,5 @@
-using System.Collections.Generic;
 using FluentValidation;
 using NzbDrone.Core.Annotations;
-using NzbDrone.Core.Languages;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 

src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleRequestGenerator.cs

@@ -1,11 +1,7 @@
 using System;
 using System.Collections.Generic;
-using System.Collections.Specialized;
 using NLog;
-using NzbDrone.Common.Cache;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
-using NzbDrone.Common.Serializer;
 using NzbDrone.Core.IndexerSearch.Definitions;
 
 namespace NzbDrone.Core.Indexers.Gazelle

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Xml;
 using System.Xml.Linq;
 using NLog;

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs

@@ -4,7 +4,6 @@ using System.Text.RegularExpressions;
 using FluentValidation;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Annotations;
-using NzbDrone.Core.Languages;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 

src/NzbDrone.Core/Indexers/Definitions/PassThePopcorn/PassThePopcornSettings.cs

@@ -1,8 +1,5 @@
-using System.Collections.Generic;
 using FluentValidation;
 using NzbDrone.Core.Annotations;
-using NzbDrone.Core.Languages;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 

src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs

@@ -21,25 +21,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
         public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
 
-        public override IndexerCapabilities Capabilities => new IndexerCapabilities
+        public override IndexerCapabilities Capabilities => SetCapabilities();
-        {
-            TvSearchParams = new List<TvSearchParam>
-                       {
-                           TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep
-                       },
-            MovieSearchParams = new List<MovieSearchParam>
-                       {
-                           MovieSearchParam.Q, MovieSearchParam.ImdbId
-                       },
-            MusicSearchParams = new List<MusicSearchParam>
-                       {
-                           MusicSearchParam.Q
-                       },
-            BookSearchParams = new List<BookSearchParam>
-                       {
-                           BookSearchParam.Q
-                       }
-        };
 
         public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
 
@@ -56,7 +38,62 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
         public override IParseIndexerResponse GetParser()
         {
-            return new RarbgParser();
+            return new RarbgParser(Capabilities);
+        }
+
+        private IndexerCapabilities SetCapabilities()
+        {
+            var caps = new IndexerCapabilities
+            {
+                TvSearchParams = new List<TvSearchParam>
+                       {
+                           TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep
+                       },
+                MovieSearchParams = new List<MovieSearchParam>
+                       {
+                           MovieSearchParam.Q, MovieSearchParam.ImdbId
+                       },
+                MusicSearchParams = new List<MusicSearchParam>
+                       {
+                           MusicSearchParam.Q
+                       },
+                BookSearchParams = new List<BookSearchParam>
+                       {
+                           BookSearchParam.Q
+                       }
+            };
+
+            caps.Categories.AddCategoryMapping(4, NewznabStandardCategory.XXX, "XXX (18+)");
+            caps.Categories.AddCategoryMapping(14, NewznabStandardCategory.MoviesSD, "Movies/XVID");
+            caps.Categories.AddCategoryMapping(17, NewznabStandardCategory.MoviesSD, "Movies/x264");
+            caps.Categories.AddCategoryMapping(18, NewznabStandardCategory.TVSD, "TV Episodes");
+            caps.Categories.AddCategoryMapping(23, NewznabStandardCategory.AudioMP3, "Music/MP3");
+            caps.Categories.AddCategoryMapping(25, NewznabStandardCategory.AudioLossless, "Music/FLAC");
+            caps.Categories.AddCategoryMapping(27, NewznabStandardCategory.PCGames, "Games/PC ISO");
+            caps.Categories.AddCategoryMapping(28, NewznabStandardCategory.PCGames, "Games/PC RIP");
+            caps.Categories.AddCategoryMapping(32, NewznabStandardCategory.ConsoleXBox360, "Games/XBOX-360");
+            caps.Categories.AddCategoryMapping(33, NewznabStandardCategory.PCISO, "Software/PC ISO");
+            caps.Categories.AddCategoryMapping(35, NewznabStandardCategory.BooksEBook, "e-Books");
+            caps.Categories.AddCategoryMapping(40, NewznabStandardCategory.ConsolePS3, "Games/PS3");
+            caps.Categories.AddCategoryMapping(41, NewznabStandardCategory.TVHD, "TV HD Episodes");
+            caps.Categories.AddCategoryMapping(42, NewznabStandardCategory.MoviesBluRay, "Movies/Full BD");
+            caps.Categories.AddCategoryMapping(44, NewznabStandardCategory.MoviesHD, "Movies/x264/1080");
+            caps.Categories.AddCategoryMapping(45, NewznabStandardCategory.MoviesHD, "Movies/x264/720");
+            caps.Categories.AddCategoryMapping(46, NewznabStandardCategory.MoviesBluRay, "Movies/BD Remux");
+            caps.Categories.AddCategoryMapping(47, NewznabStandardCategory.Movies3D, "Movies/x264/3D");
+            caps.Categories.AddCategoryMapping(48, NewznabStandardCategory.MoviesHD, "Movies/XVID/720");
+            caps.Categories.AddCategoryMapping(49, NewznabStandardCategory.TVUHD, "TV UHD Episodes");
+
+            // torrentapi.org returns "Movies/TV-UHD-episodes" for some reason
+            // possibly because thats what the category is called on the /top100.php page
+            caps.Categories.AddCategoryMapping(49, NewznabStandardCategory.TVUHD, "Movies/TV-UHD-episodes");
+            caps.Categories.AddCategoryMapping(50, NewznabStandardCategory.MoviesUHD, "Movies/x264/4k");
+            caps.Categories.AddCategoryMapping(51, NewznabStandardCategory.MoviesUHD, "Movies/x265/4k");
+            caps.Categories.AddCategoryMapping(52, NewznabStandardCategory.MoviesUHD, "Movs/x265/4k/HDR");
+            caps.Categories.AddCategoryMapping(53, NewznabStandardCategory.ConsolePS4, "Games/PS4");
+            caps.Categories.AddCategoryMapping(54, NewznabStandardCategory.MoviesHD, "Movies/x265/1080");
+
+            return caps;
         }
 
         public override object RequestAction(string action, IDictionary<string, string> query)

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs

@@ -12,6 +12,13 @@ namespace NzbDrone.Core.Indexers.Rarbg
     {
         private static readonly Regex RegexGuid = new Regex(@"^magnet:\?xt=urn:btih:([a-f0-9]+)", RegexOptions.Compiled);
 
+        private readonly IndexerCapabilities _capabilities;
+
+        public RarbgParser(IndexerCapabilities capabilities)
+        {
+            _capabilities = capabilities;
+        }
+
         public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
         {
             var results = new List<ReleaseInfo>();
@@ -51,6 +58,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 var torrentInfo = new TorrentInfo();
 
                 torrentInfo.Guid = GetGuid(torrent);
+                torrentInfo.Category = _capabilities.Categories.MapTrackerCatDescToNewznab(torrent.category);
                 torrentInfo.Title = torrent.title;
                 torrentInfo.Size = torrent.size;
                 torrentInfo.DownloadUrl = torrent.download;

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs

@@ -20,14 +20,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
             _tokenProvider = tokenProvider;
         }
 
-        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        private IEnumerable<IndexerRequest> GetRequest(string term, int[] categories, string imdbId = null, int? tmdbId = null)
-        {
-            var pageableRequests = new IndexerPageableRequestChain();
-            pageableRequests.Add(GetMovieRequest(searchCriteria));
-            return pageableRequests;
-        }
-
-        private IEnumerable<IndexerRequest> GetMovieRequest(MovieSearchCriteria searchCriteria)
         {
             var requestBuilder = new HttpRequestBuilder(BaseUrl)
                 .Resource("/pubapi_v2.php")
@@ -41,17 +34,17 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
             requestBuilder.AddQueryParam("mode", "search");
 
-            if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace())
+            if (imdbId.IsNotNullOrWhiteSpace())
             {
-                requestBuilder.AddQueryParam("search_imdb", searchCriteria.ImdbId);
+                requestBuilder.AddQueryParam("search_imdb", imdbId);
             }
-            else if (searchCriteria.TmdbId > 0)
+            else if (tmdbId.HasValue && tmdbId > 0)
             {
-                requestBuilder.AddQueryParam("search_themoviedb", searchCriteria.TmdbId);
+                requestBuilder.AddQueryParam("search_themoviedb", tmdbId);
             }
-            else if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace())
+            else if (term.IsNotNullOrWhiteSpace())
             {
-                requestBuilder.AddQueryParam("search_string", $"{searchCriteria.SearchTerm}");
+                requestBuilder.AddQueryParam("search_string", $"{term}");
             }
 
             if (!Settings.RankedOnly)
@@ -59,9 +52,12 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 requestBuilder.AddQueryParam("ranked", "0");
             }
 
-            var categoryParam = string.Join(";", searchCriteria.Categories.Distinct());
+            if (categories != null && categories.Length > 0)
+            {
+                var categoryParam = string.Join(";", categories.Distinct());
+                requestBuilder.AddQueryParam("category", categoryParam);
+            }
 
-            requestBuilder.AddQueryParam("category", categoryParam);
             requestBuilder.AddQueryParam("limit", "100");
             requestBuilder.AddQueryParam("token", _tokenProvider.GetToken(Settings, BaseUrl));
             requestBuilder.AddQueryParam("format", "json_extended");
@@ -70,24 +66,39 @@ namespace NzbDrone.Core.Indexers.Rarbg
             yield return new IndexerRequest(requestBuilder.Build());
         }
 
+        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories, searchCriteria.ImdbId, searchCriteria.TmdbId));
+            return pageableRequests;
+        }
+
         public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
         {
-            return new IndexerPageableRequestChain();
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
         {
-            return new IndexerPageableRequestChain();
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories, searchCriteria.ImdbId));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
         {
-            return new IndexerPageableRequestChain();
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
         {
-            return new IndexerPageableRequestChain();
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories));
+            return pageableRequests;
         }
 
         public Func<IDictionary<string, string>> GetCookies { get; set; }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgSettings.cs

@@ -1,8 +1,5 @@
-using System.Collections.Generic;
 using FluentValidation;
 using NzbDrone.Core.Annotations;
-using NzbDrone.Core.Languages;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 

src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotatoRequestGenerator.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.IndexerSearch.Definitions;

src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs

@@ -1,8 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using NzbDrone.Common.Messaging;
 
 namespace NzbDrone.Core.Indexers.Events

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -9,10 +9,8 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Http.CloudFlare;
-using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.IndexerSearch.Definitions;
-using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Extensions;

src/NzbDrone.Core/Indexers/IndexerPageableQueryResult.cs

@@ -1,8 +1,4 @@
-using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers

src/NzbDrone.Core/Indexers/IndexerQueryResult.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Generic;
 using NzbDrone.Core.Parser.Model;
 

src/NzbDrone.Core/Parser/Model/TorrentInfo.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Text;
 
 namespace NzbDrone.Core.Parser.Model