Fix Rarbg search and parsing

src/NzbDrone.Core/Applications/ApplicationIndexerSyncCommand.cs

@@ -1,8 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using NzbDrone.Core.Messaging.Commands;
 
 namespace NzbDrone.Core.Applications

src/NzbDrone.Core/IndexerSearch/Definitions/BasicSearchCriteria.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {
     public class BasicSearchCriteria : SearchCriteriaBase

src/NzbDrone.Core/IndexerSearch/Definitions/BookSearchCriteria.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {
     public class BookSearchCriteria : SearchCriteriaBase

src/NzbDrone.Core/IndexerSearch/Definitions/MusicSearchCriteria.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {
     public class MusicSearchCriteria : SearchCriteriaBase

src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs

@@ -1,8 +1,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text.RegularExpressions;
-using NzbDrone.Common.EnsureThat;
-using NzbDrone.Common.Extensions;
 
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {

src/NzbDrone.Core/IndexerSearch/Definitions/TvSearchCriteria.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.IndexerSearch.Definitions
 {
     public class TvSearchCriteria : SearchCriteriaBase

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs

@@ -6,7 +6,6 @@ using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Disk;
 using NzbDrone.Common.EnvironmentInfo;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Indexers.Cardigann;
 using NzbDrone.Core.Messaging.Commands;

src/NzbDrone.Core/Indexers/CategoryMapping.cs

@@ -1,9 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
 namespace NzbDrone.Core.Indexers
 {
     public class CategoryMapping

src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs

@@ -3,13 +3,10 @@ using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Globalization;
 using System.Linq;
-using System.Net;
 using System.Text.RegularExpressions;
 using AngleSharp.Html.Parser;
 using FluentValidation;
-using Newtonsoft.Json;
 using NLog;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.Configuration;

src/NzbDrone.Core/Indexers/Definitions/BroadcastheNet/BroadcastheNetRequestGenerator.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using System.Linq;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.IndexerSearch.Definitions;
 

src/NzbDrone.Core/Indexers/Definitions/BrokenStones.cs

@@ -1,4 +1,3 @@
-using System.Collections.Generic;
 using NLog;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -1,5 +1,4 @@
 using System.Collections.Generic;
-using System.Linq;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Http;

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs

@@ -1,6 +1,5 @@
 using System.Collections.Generic;
 using FluentValidation;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;

src/NzbDrone.Core/Indexers/Definitions/FileList/FileListSettings.cs

@@ -1,8 +1,5 @@
-using System.Collections.Generic;
 using FluentValidation;
 using NzbDrone.Core.Annotations;
-using NzbDrone.Core.Languages;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 

src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleRequestGenerator.cs

@@ -1,11 +1,7 @@
 using System;
 using System.Collections.Generic;
-using System.Collections.Specialized;
 using NLog;
-using NzbDrone.Common.Cache;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
-using NzbDrone.Common.Serializer;
 using NzbDrone.Core.IndexerSearch.Definitions;
 
 namespace NzbDrone.Core.Indexers.Gazelle

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Xml;
 using System.Xml.Linq;
 using NLog;

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs

@@ -4,7 +4,6 @@ using System.Text.RegularExpressions;
 using FluentValidation;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Annotations;
-using NzbDrone.Core.Languages;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 

src/NzbDrone.Core/Indexers/Definitions/PassThePopcorn/PassThePopcornSettings.cs

@@ -1,8 +1,5 @@
-using System.Collections.Generic;
 using FluentValidation;
 using NzbDrone.Core.Annotations;
-using NzbDrone.Core.Languages;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 

src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs

@@ -21,7 +21,29 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
         public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
 
-        public override IndexerCapabilities Capabilities => new IndexerCapabilities
+        public override IndexerCapabilities Capabilities => SetCapabilities();
+
+        public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
+
+        public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
+            : base(httpClient, indexerStatusService, configService, logger)
+        {
+            _tokenProvider = tokenProvider;
+        }
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new RarbgRequestGenerator(_tokenProvider) { Settings = Settings, BaseUrl = BaseUrl };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new RarbgParser(Capabilities);
+        }
+
+        private IndexerCapabilities SetCapabilities()
+        {
+            var caps = new IndexerCapabilities
             {
                 TvSearchParams = new List<TvSearchParam>
                        {
@@ -41,22 +63,37 @@ namespace NzbDrone.Core.Indexers.Rarbg
                        }
             };
 
-        public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
-
-        public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
-            : base(httpClient, indexerStatusService, configService, logger)
-        {
-            _tokenProvider = tokenProvider;
-        }
-
-        public override IIndexerRequestGenerator GetRequestGenerator()
-        {
-            return new RarbgRequestGenerator(_tokenProvider) { Settings = Settings, BaseUrl = BaseUrl };
-        }
-
-        public override IParseIndexerResponse GetParser()
-        {
-            return new RarbgParser();
+            caps.Categories.AddCategoryMapping(4, NewznabStandardCategory.XXX, "XXX (18+)");
+            caps.Categories.AddCategoryMapping(14, NewznabStandardCategory.MoviesSD, "Movies/XVID");
+            caps.Categories.AddCategoryMapping(17, NewznabStandardCategory.MoviesSD, "Movies/x264");
+            caps.Categories.AddCategoryMapping(18, NewznabStandardCategory.TVSD, "TV Episodes");
+            caps.Categories.AddCategoryMapping(23, NewznabStandardCategory.AudioMP3, "Music/MP3");
+            caps.Categories.AddCategoryMapping(25, NewznabStandardCategory.AudioLossless, "Music/FLAC");
+            caps.Categories.AddCategoryMapping(27, NewznabStandardCategory.PCGames, "Games/PC ISO");
+            caps.Categories.AddCategoryMapping(28, NewznabStandardCategory.PCGames, "Games/PC RIP");
+            caps.Categories.AddCategoryMapping(32, NewznabStandardCategory.ConsoleXBox360, "Games/XBOX-360");
+            caps.Categories.AddCategoryMapping(33, NewznabStandardCategory.PCISO, "Software/PC ISO");
+            caps.Categories.AddCategoryMapping(35, NewznabStandardCategory.BooksEBook, "e-Books");
+            caps.Categories.AddCategoryMapping(40, NewznabStandardCategory.ConsolePS3, "Games/PS3");
+            caps.Categories.AddCategoryMapping(41, NewznabStandardCategory.TVHD, "TV HD Episodes");
+            caps.Categories.AddCategoryMapping(42, NewznabStandardCategory.MoviesBluRay, "Movies/Full BD");
+            caps.Categories.AddCategoryMapping(44, NewznabStandardCategory.MoviesHD, "Movies/x264/1080");
+            caps.Categories.AddCategoryMapping(45, NewznabStandardCategory.MoviesHD, "Movies/x264/720");
+            caps.Categories.AddCategoryMapping(46, NewznabStandardCategory.MoviesBluRay, "Movies/BD Remux");
+            caps.Categories.AddCategoryMapping(47, NewznabStandardCategory.Movies3D, "Movies/x264/3D");
+            caps.Categories.AddCategoryMapping(48, NewznabStandardCategory.MoviesHD, "Movies/XVID/720");
+            caps.Categories.AddCategoryMapping(49, NewznabStandardCategory.TVUHD, "TV UHD Episodes");
+
+            // torrentapi.org returns "Movies/TV-UHD-episodes" for some reason
+            // possibly because thats what the category is called on the /top100.php page
+            caps.Categories.AddCategoryMapping(49, NewznabStandardCategory.TVUHD, "Movies/TV-UHD-episodes");
+            caps.Categories.AddCategoryMapping(50, NewznabStandardCategory.MoviesUHD, "Movies/x264/4k");
+            caps.Categories.AddCategoryMapping(51, NewznabStandardCategory.MoviesUHD, "Movies/x265/4k");
+            caps.Categories.AddCategoryMapping(52, NewznabStandardCategory.MoviesUHD, "Movs/x265/4k/HDR");
+            caps.Categories.AddCategoryMapping(53, NewznabStandardCategory.ConsolePS4, "Games/PS4");
+            caps.Categories.AddCategoryMapping(54, NewznabStandardCategory.MoviesHD, "Movies/x265/1080");
+
+            return caps;
         }
 
         public override object RequestAction(string action, IDictionary<string, string> query)

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs

@@ -12,6 +12,13 @@ namespace NzbDrone.Core.Indexers.Rarbg
     {
         private static readonly Regex RegexGuid = new Regex(@"^magnet:\?xt=urn:btih:([a-f0-9]+)", RegexOptions.Compiled);
 
+        private readonly IndexerCapabilities _capabilities;
+
+        public RarbgParser(IndexerCapabilities capabilities)
+        {
+            _capabilities = capabilities;
+        }
+
         public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
         {
             var results = new List<ReleaseInfo>();
@@ -51,6 +58,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 var torrentInfo = new TorrentInfo();
 
                 torrentInfo.Guid = GetGuid(torrent);
+                torrentInfo.Category = _capabilities.Categories.MapTrackerCatDescToNewznab(torrent.category);
                 torrentInfo.Title = torrent.title;
                 torrentInfo.Size = torrent.size;
                 torrentInfo.DownloadUrl = torrent.download;

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs

@@ -20,14 +20,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
             _tokenProvider = tokenProvider;
         }
 
-        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
-        {
-            var pageableRequests = new IndexerPageableRequestChain();
-            pageableRequests.Add(GetMovieRequest(searchCriteria));
-            return pageableRequests;
-        }
-
-        private IEnumerable<IndexerRequest> GetMovieRequest(MovieSearchCriteria searchCriteria)
+        private IEnumerable<IndexerRequest> GetRequest(string term, int[] categories, string imdbId = null, int? tmdbId = null)
         {
             var requestBuilder = new HttpRequestBuilder(BaseUrl)
                 .Resource("/pubapi_v2.php")
@@ -41,17 +34,17 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
             requestBuilder.AddQueryParam("mode", "search");
 
-            if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace())
+            if (imdbId.IsNotNullOrWhiteSpace())
             {
-                requestBuilder.AddQueryParam("search_imdb", searchCriteria.ImdbId);
+                requestBuilder.AddQueryParam("search_imdb", imdbId);
             }
-            else if (searchCriteria.TmdbId > 0)
+            else if (tmdbId.HasValue && tmdbId > 0)
             {
-                requestBuilder.AddQueryParam("search_themoviedb", searchCriteria.TmdbId);
+                requestBuilder.AddQueryParam("search_themoviedb", tmdbId);
             }
-            else if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace())
+            else if (term.IsNotNullOrWhiteSpace())
             {
-                requestBuilder.AddQueryParam("search_string", $"{searchCriteria.SearchTerm}");
+                requestBuilder.AddQueryParam("search_string", $"{term}");
             }
 
             if (!Settings.RankedOnly)
@@ -59,9 +52,12 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 requestBuilder.AddQueryParam("ranked", "0");
             }
 
-            var categoryParam = string.Join(";", searchCriteria.Categories.Distinct());
-
+            if (categories != null && categories.Length > 0)
+            {
+                var categoryParam = string.Join(";", categories.Distinct());
                 requestBuilder.AddQueryParam("category", categoryParam);
+            }
+
             requestBuilder.AddQueryParam("limit", "100");
             requestBuilder.AddQueryParam("token", _tokenProvider.GetToken(Settings, BaseUrl));
             requestBuilder.AddQueryParam("format", "json_extended");
@@ -70,24 +66,39 @@ namespace NzbDrone.Core.Indexers.Rarbg
             yield return new IndexerRequest(requestBuilder.Build());
         }
 
+        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        {
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories, searchCriteria.ImdbId, searchCriteria.TmdbId));
+            return pageableRequests;
+        }
+
         public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
         {
-            return new IndexerPageableRequestChain();
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
         {
-            return new IndexerPageableRequestChain();
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories, searchCriteria.ImdbId));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
         {
-            return new IndexerPageableRequestChain();
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
         {
-            return new IndexerPageableRequestChain();
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SearchTerm, searchCriteria.Categories));
+            return pageableRequests;
         }
 
         public Func<IDictionary<string, string>> GetCookies { get; set; }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgSettings.cs

@@ -1,8 +1,5 @@
-using System.Collections.Generic;
 using FluentValidation;
 using NzbDrone.Core.Annotations;
-using NzbDrone.Core.Languages;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
 

src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotatoRequestGenerator.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.IndexerSearch.Definitions;

src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs

@@ -1,8 +1,3 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using NzbDrone.Common.Messaging;
 
 namespace NzbDrone.Core.Indexers.Events

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -9,10 +9,8 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Http.CloudFlare;
-using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.IndexerSearch.Definitions;
-using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Text;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Extensions;

src/NzbDrone.Core/Indexers/IndexerPageableQueryResult.cs

@@ -1,8 +1,4 @@
-using System;
 using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers

src/NzbDrone.Core/Indexers/IndexerQueryResult.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Collections.Generic;
 using NzbDrone.Core.Parser.Model;
 

src/NzbDrone.Core/Parser/Model/TorrentInfo.cs

@@ -1,4 +1,3 @@
-using System;
 using System.Text;
 
 namespace NzbDrone.Core.Parser.Model