Cleanup SearchCriteria, Pass Cats from Search to Indexers

pull/6/head
Qstick 4 years ago
parent 529102ebcf
commit 6100adc515

@ -17,8 +17,10 @@ namespace NzbDrone.Core.Test.IndexerSearchTests
[TestCase("Kourtney And Khlo\u00E9 Take The Hamptons", "Kourtney+And+Khloe+Take+The+Hamptons")]
public void should_replace_some_special_characters(string input, string expected)
{
Subject.SceneTitles = new List<string> { input };
Subject.QueryTitles.First().Should().Be(expected);
Subject.SearchTerm = input;
// TODO: Clean Query Terms
//Subject.QueryTitles.First().Should().Be(expected);
}
}
}

@ -25,7 +25,7 @@ namespace NzbDrone.Core.Test.IndexerTests.FileListTests
_movieSearchCriteria = new MovieSearchCriteria
{
SceneTitles = new List<string> { "Star Wars" }
SearchTerm = "Star Wars"
};
}

@ -24,7 +24,7 @@ namespace NzbDrone.Core.Test.IndexerTests.IntegrationTests
_singleSearchCriteria = new MovieSearchCriteria()
{
SceneTitles = new List<string> { "Person of Interest" },
SearchTerm = "Person of Interest",
};
}

@ -24,8 +24,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
Name = "Newznab",
Settings = new NewznabSettings()
{
BaseUrl = "http://indexer.local/",
Categories = new int[] { 1 }
BaseUrl = "http://indexer.local/"
}
};

@ -21,13 +21,12 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
Subject.Settings = new NewznabSettings()
{
BaseUrl = "http://127.0.0.1:1234/",
Categories = new[] { 1, 2 },
ApiKey = "abcd",
};
_movieSearchCriteria = new MovieSearchCriteria
{
SceneTitles = new List<string> { "Star Wars" }
SearchTerm = "Star Wars"
};
_capabilities = new IndexerCapabilities();
@ -52,8 +51,6 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
[Test]
public void should_not_have_duplicate_categories()
{
Subject.Settings.Categories = new[] { 1, 2, 2, 3 };
var results = Subject.GetRecentRequests();
results.GetAllTiers().Should().HaveCount(1);

@ -25,8 +25,7 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests
Name = "Torznab",
Settings = new TorznabSettings()
{
BaseUrl = "http://indexer.local/",
Categories = new int[] { 1 }
BaseUrl = "http://indexer.local/"
}
};

@ -90,7 +90,7 @@ namespace NzbDrone.Core.History
};
history.Data.Add("ElapsedTime", message.Time.ToString());
history.Data.Add("Query", message.Query.SceneTitles.FirstOrDefault() ?? string.Empty);
history.Data.Add("Query", message.Query.SearchTerm ?? string.Empty);
history.Data.Add("Successful", message.Successful.ToString());
history.Data.Add("QueryResults", message.Results.HasValue ? message.Results.ToString() : null);

@ -2,6 +2,8 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
{
public class MovieSearchCriteria : SearchCriteriaBase
{
public string ImdbId { get; set; }
public int TmdbId { get; set; }
public override string ToString()
{
return string.Format("[{0}]", ImdbId);

@ -12,28 +12,9 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
private static readonly Regex NonWord = new Regex(@"[\W]", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex BeginningThe = new Regex(@"^the\s", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public List<string> SceneTitles { get; set; }
public virtual bool InteractiveSearch { get; set; }
public List<int> IndexerIds { get; set; }
public string ImdbId { get; set; }
public int TmdbId { get; set; }
public List<string> QueryTitles => SceneTitles.Select(GetQueryTitle).ToList();
public static string GetQueryTitle(string title)
{
Ensure.That(title, () => title).IsNotNullOrWhiteSpace();
var cleanTitle = BeginningThe.Replace(title, string.Empty);
cleanTitle = cleanTitle.Replace("&", "and");
cleanTitle = SpecialCharacter.Replace(cleanTitle, "");
cleanTitle = NonWord.Replace(cleanTitle, "+");
//remove any repeating +s
cleanTitle = Regex.Replace(cleanTitle, @"\+{2,}", "+");
cleanTitle = cleanTitle.RemoveAccent();
return cleanTitle.Trim('+', ' ');
}
public string SearchTerm { get; set; }
public int[] Categories { get; set; }
}
}

@ -37,31 +37,36 @@ namespace NzbDrone.Core.IndexerSearch
public List<ReleaseInfo> Search(string query, List<int> indexerIds, bool interactiveSearch)
{
var searchSpec = Get<MovieSearchCriteria>(query, indexerIds, interactiveSearch);
var searchSpec = Get<MovieSearchCriteria>(new NewznabRequest { q = query }, indexerIds, interactiveSearch);
return Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec);
}
public NewznabResults Search(NewznabRequest request, List<int> indexerIds, bool interactiveSearch)
{
var searchSpec = Get<MovieSearchCriteria>(request.q, indexerIds, interactiveSearch);
var searchSpec = Get<MovieSearchCriteria>(request, indexerIds, interactiveSearch);
return new NewznabResults { Releases = Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec) };
}
private TSpec Get<TSpec>(string query, List<int> indexerIds, bool interactiveSearch)
private TSpec Get<TSpec>(NewznabRequest query, List<int> indexerIds, bool interactiveSearch)
where TSpec : SearchCriteriaBase, new()
{
var spec = new TSpec()
{
InteractiveSearch = interactiveSearch,
SceneTitles = new List<string>()
InteractiveSearch = interactiveSearch
};
if (query.IsNotNullOrWhiteSpace())
if (query.cat != null)
{
spec.SceneTitles.Add(query);
spec.Categories = query.cat.Split(',').Where(s => !string.IsNullOrWhiteSpace(s)).Select(s => int.Parse(s)).ToArray();
}
else
{
spec.Categories = Array.Empty<int>();
}
spec.SearchTerm = query.q;
spec.IndexerIds = indexerIds;
@ -81,7 +86,7 @@ namespace NzbDrone.Core.IndexerSearch
var reports = new List<ReleaseInfo>();
_logger.ProgressInfo("Searching {0} indexers for {1}", indexers.Count, criteriaBase.QueryTitles.Join(", "));
_logger.ProgressInfo("Searching {0} indexers for {1}", indexers.Count, criteriaBase.SearchTerm);
var taskList = new List<Task>();
var taskFactory = new TaskFactory(TaskCreationOptions.LongRunning, TaskContinuationOptions.None);

@ -82,20 +82,5 @@ namespace NzbDrone.Core.Indexers.Cardigann
return;
}
}
protected static List<int> CategoryIds(List<IndexerCategory> categories)
{
var l = categories.Select(c => c.Id).ToList();
foreach (var category in categories)
{
if (category.SubCategories != null)
{
l.AddRange(CategoryIds(category.SubCategories));
}
}
return l;
}
}
}

@ -9,6 +9,7 @@ using Newtonsoft.Json.Linq;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.IndexerSearch.Definitions;
namespace NzbDrone.Core.Indexers.Cardigann
{
@ -207,6 +208,27 @@ namespace NzbDrone.Core.Indexers.Cardigann
return cats;
}*/
public List<string> MapTorznabCapsToTrackers(int[] searchCategories, bool mapChildrenCatsToParent = false)
{
var queryCats = new List<string>();
foreach (var searchCat in searchCategories)
{
var match = TorznabCatType.AllCats.FirstOrDefault(c => c.Id == searchCat);
if (match != null)
{
queryCats.Add(match.Name);
}
}
var result = _definition.Caps.Categorymappings
.Where(c => queryCats.Contains(c.cat))
.Select(mapping => mapping.id).Distinct().ToList();
return result;
}
protected delegate string TemplateTextModifier(string str);
protected string ApplyGoTemplateText(string template, Dictionary<string, object> variables = null, TemplateTextModifier modifier = null)

@ -28,7 +28,9 @@ namespace NzbDrone.Core.Indexers.Cardigann
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest(null));
var variables = GetBaseTemplateVariables();
pageableRequests.Add(GetRequest(variables));
return pageableRequests;
}
@ -39,23 +41,24 @@ namespace NzbDrone.Core.Indexers.Cardigann
var pageableRequests = new IndexerPageableRequestChain();
foreach (var queryTitle in searchCriteria.QueryTitles)
{
pageableRequests.Add(GetRequest(string.Format("{0}", queryTitle)));
}
var variables = GetBaseTemplateVariables();
variables[".Query.Type"] = "movie";
variables[".Query.Q"] = searchCriteria.SearchTerm;
variables[".Query.Categories"] = searchCriteria.Categories;
variables[".Query.IMDBID"] = searchCriteria.ImdbId;
variables[".Query.IMDBIDShort"] = null;
variables[".Query.TMDBID"] = searchCriteria.TmdbId;
pageableRequests.Add(GetRequest(variables));
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetRequest(string searchCriteria)
private IEnumerable<IndexerRequest> GetRequest(Dictionary<string, object> variables)
{
var search = _definition.Search;
// init template context
var variables = GetBaseTemplateVariables();
variables[".Query.Type"] = null;
variables[".Query.Q"] = searchCriteria;
variables[".Query.Series"] = null;
variables[".Query.Ep"] = null;
variables[".Query.Season"] = null;
@ -64,13 +67,9 @@ namespace NzbDrone.Core.Indexers.Cardigann
variables[".Query.Limit"] = null;
variables[".Query.Offset"] = null;
variables[".Query.Extended"] = null;
variables[".Query.Categories"] = null;
variables[".Query.APIKey"] = null;
variables[".Query.TVDBID"] = null;
variables[".Query.TVRageID"] = null;
variables[".Query.IMDBID"] = null;
variables[".Query.IMDBIDShort"] = null;
variables[".Query.TMDBID"] = null;
variables[".Query.TVMazeID"] = null;
variables[".Query.TraktID"] = null;
variables[".Query.Album"] = null;
@ -81,15 +80,11 @@ namespace NzbDrone.Core.Indexers.Cardigann
variables[".Query.Author"] = null;
variables[".Query.Title"] = null;
/*
var mappedCategories = MapTorznabCapsToTrackers(query);
var mappedCategories = MapTorznabCapsToTrackers((int[])variables[".Query.Categories"]);
if (mappedCategories.Count == 0)
{
mappedCategories = _defaultCategories;
}
*/
var mappedCategories = _defaultCategories;
variables[".Categories"] = mappedCategories;

@ -11,13 +11,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
{
public CardigannSettingsValidator()
{
RuleFor(c => c).Custom((c, context) =>
{
if (c.Categories.Empty())
{
context.AddFailure("'Categories' must be provided");
}
});
}
}
@ -27,7 +20,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
public CardigannSettings()
{
Categories = new[] { 2000, 2010, 2020, 2030, 2035, 2040, 2045, 2050, 2060 };
MultiLanguages = new List<int>();
ExtraFieldData = new Dictionary<string, object>();
}
@ -42,9 +34,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
[FieldDefinition(1000, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
public IEnumerable<int> MultiLanguages { get; set; }
[FieldDefinition(1001, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable all categories", Advanced = true)]
public IEnumerable<int> Categories { get; set; }
// Field 8 is used by TorznabSettings MinimumSeeders
// If you need to add another field here, update TorznabSettings as well and this comment
public virtual NzbDroneValidationResult Validate()

@ -30,11 +30,8 @@ namespace NzbDrone.Core.Indexers.FileList
}
else
{
foreach (var queryTitle in searchCriteria.QueryTitles)
{
var titleYearSearchQuery = string.Format("{0}", queryTitle);
pageableRequests.Add(GetRequest("search-torrents", string.Format("&type=name&query={0}", titleYearSearchQuery.Trim())));
}
var titleYearSearchQuery = string.Format("{0}", searchCriteria.SearchTerm);
pageableRequests.Add(GetRequest("search-torrents", string.Format("&type=name&query={0}", titleYearSearchQuery.Trim())));
}
return pageableRequests;

@ -32,7 +32,7 @@ namespace NzbDrone.Core.Indexers.HDBits
return pageableRequests;
}
private bool TryAddSearchParameters(TorrentQuery query, SearchCriteriaBase searchCriteria)
private bool TryAddSearchParameters(TorrentQuery query, MovieSearchCriteria searchCriteria)
{
if (searchCriteria.ImdbId.IsNullOrWhiteSpace())
{

@ -82,15 +82,10 @@ namespace NzbDrone.Core.Indexers.Newznab
};
}
private NewznabSettings GetSettings(string url, string apiPath = null, int[] categories = null)
private NewznabSettings GetSettings(string url, string apiPath = null)
{
var settings = new NewznabSettings { BaseUrl = url };
if (categories != null)
{
settings.Categories = categories;
}
if (apiPath.IsNotNullOrWhiteSpace())
{
settings.ApiPath = apiPath;
@ -123,17 +118,6 @@ namespace NzbDrone.Core.Indexers.Newznab
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
var notSupported = Settings.Categories.Except(CategoryIds(capabilities.Categories));
if (notSupported.Any())
{
_logger.Warn($"{Definition.Name} does not support the following categories: {string.Join(", ", notSupported)}. You should probably remove them.");
if (notSupported.Count() == Settings.Categories.Count())
{
return new ValidationFailure(string.Empty, $"This indexer does not support any of the selected categories! (You may need to turn on advanced settings to see them)");
}
}
if (capabilities.SearchParams != null && capabilities.SearchParams.Contains(SearchParam.Q))
{
return null;

@ -75,11 +75,11 @@ namespace NzbDrone.Core.Indexers.Newznab
// Some indexers might forget to enable movie search, but normal search still works fine. Thus we force a normal search.
if (capabilities.MovieSearchParams != null)
{
pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "movie", ""));
pageableRequests.Add(GetPagedRequests(MaxPages, new int[] { 2000 }, "movie", ""));
}
else if (capabilities.SearchParams != null)
{
pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories, "search", ""));
pageableRequests.Add(GetPagedRequests(MaxPages, new int[] { 2000 }, "search", ""));
}
return pageableRequests;
@ -89,12 +89,12 @@ namespace NzbDrone.Core.Indexers.Newznab
{
var pageableRequests = new IndexerPageableRequestChain();
AddMovieIdPageableRequests(pageableRequests, MaxPages, Settings.Categories, searchCriteria);
AddMovieIdPageableRequests(pageableRequests, MaxPages, searchCriteria.Categories, searchCriteria);
return pageableRequests;
}
private void AddMovieIdPageableRequests(IndexerPageableRequestChain chain, int maxPages, IEnumerable<int> categories, SearchCriteriaBase searchCriteria)
private void AddMovieIdPageableRequests(IndexerPageableRequestChain chain, int maxPages, IEnumerable<int> categories, MovieSearchCriteria searchCriteria)
{
var includeTmdbSearch = SupportsTmdbSearch && searchCriteria.TmdbId > 0;
var includeImdbSearch = SupportsImdbSearch && searchCriteria.ImdbId.IsNotNullOrWhiteSpace();
@ -136,20 +136,18 @@ namespace NzbDrone.Core.Indexers.Newznab
if (SupportsSearch)
{
chain.AddTier();
foreach (var queryTitle in searchCriteria.QueryTitles)
{
var searchQuery = queryTitle;
if (!Settings.RemoveYear)
{
searchQuery = string.Format("{0}", searchQuery);
}
var searchQuery = searchCriteria.SearchTerm;
chain.Add(GetPagedRequests(MaxPages,
Settings.Categories,
"movie",
string.Format("&q={0}", NewsnabifyTitle(searchQuery))));
if (!Settings.RemoveYear)
{
searchQuery = string.Format("{0}", searchQuery);
}
chain.Add(GetPagedRequests(MaxPages,
categories,
"movie",
string.Format("&q={0}", NewsnabifyTitle(searchQuery))));
}
}

@ -36,14 +36,6 @@ namespace NzbDrone.Core.Indexers.Newznab
public NewznabSettingsValidator()
{
RuleFor(c => c).Custom((c, context) =>
{
if (c.Categories.Empty())
{
context.AddFailure("'Categories' must be provided");
}
});
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.ApiPath).ValidUrlBase("/api");
RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
@ -59,7 +51,6 @@ namespace NzbDrone.Core.Indexers.Newznab
public NewznabSettings()
{
ApiPath = "/api";
Categories = new[] { 2000, 2010, 2020, 2030, 2035, 2040, 2045, 2050, 2060 };
MultiLanguages = new List<int>();
}
@ -75,9 +66,6 @@ namespace NzbDrone.Core.Indexers.Newznab
[FieldDefinition(2, Label = "API Key", Privacy = PrivacyLevel.ApiKey)]
public string ApiKey { get; set; }
[FieldDefinition(3, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable all categories", Advanced = true)]
public IEnumerable<int> Categories { get; set; }
[FieldDefinition(5, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)]
public string AdditionalParameters { get; set; }

@ -60,10 +60,7 @@ namespace NzbDrone.Core.Indexers.Nyaa
{
var pageableRequests = new IndexerPageableRequestChain();
foreach (var queryTitle in searchCriteria.QueryTitles)
{
pageableRequests.Add(GetPagedRequests(MaxPages, PrepareQuery(string.Format("{0}", queryTitle))));
}
pageableRequests.Add(GetPagedRequests(MaxPages, PrepareQuery(string.Format("{0}", searchCriteria.SearchTerm))));
return pageableRequests;
}

@ -35,10 +35,7 @@ namespace NzbDrone.Core.Indexers.PassThePopcorn
}
else
{
foreach (var queryTitle in searchCriteria.QueryTitles)
{
pageableRequests.Add(GetRequest(string.Format("{0}", queryTitle)));
}
pageableRequests.Add(GetRequest(string.Format("{0}", searchCriteria.SearchTerm)));
}
return pageableRequests;

@ -95,9 +95,9 @@ namespace NzbDrone.Core.Indexers.Rarbg
{
requestBuilder.AddQueryParam("search_themoviedb", searchCriteria.TmdbId);
}
else if (searchCriteria.QueryTitles.Count > 0)
else if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace())
{
requestBuilder.AddQueryParam("search_string", $"{searchCriteria.QueryTitles.First()}");
requestBuilder.AddQueryParam("search_string", $"{searchCriteria.SearchTerm}");
}
if (!Settings.RankedOnly)

@ -64,10 +64,10 @@ namespace NzbDrone.Core.Indexers.TorrentPotato
{
requestBuilder.AddQueryParam("imdbid", searchCriteria.ImdbId);
}
else if (searchCriteria.QueryTitles.Count > 0)
else if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace())
{
//TODO: Hack for now
requestBuilder.AddQueryParam("search", $"{searchCriteria.QueryTitles.First()}");
requestBuilder.AddQueryParam("search", $"{searchCriteria.SearchTerm}");
}
yield return new IndexerRequest(requestBuilder.Build());

@ -69,15 +69,10 @@ namespace NzbDrone.Core.Indexers.Torznab
};
}
private TorznabSettings GetSettings(string url, string apiPath = null, int[] categories = null)
private TorznabSettings GetSettings(string url, string apiPath = null)
{
var settings = new TorznabSettings { BaseUrl = url };
if (categories != null)
{
settings.Categories = categories;
}
if (apiPath.IsNotNullOrWhiteSpace())
{
settings.ApiPath = apiPath;
@ -110,17 +105,6 @@ namespace NzbDrone.Core.Indexers.Torznab
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
var notSupported = Settings.Categories.Except(CategoryIds(capabilities.Categories));
if (notSupported.Any())
{
_logger.Warn($"{Definition.Name} does not support the following categories: {string.Join(", ", notSupported)}. You should probably remove them.");
if (notSupported.Count() == Settings.Categories.Count())
{
return new ValidationFailure(string.Empty, $"This indexer does not support any of the selected categories! (You may need to turn on advanced settings to see them)");
}
}
if (capabilities.SearchParams != null && capabilities.SearchParams.Contains(SearchParam.Q))
{
return null;

@ -28,14 +28,6 @@ namespace NzbDrone.Core.Indexers.Torznab
public TorznabSettingsValidator()
{
RuleFor(c => c).Custom((c, context) =>
{
if (c.Categories.Empty())
{
context.AddFailure("'Categories' must be provided");
}
});
RuleFor(c => c.BaseUrl).ValidRootUrl();
RuleFor(c => c.ApiPath).ValidUrlBase("/api");
RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);

Loading…
Cancel
Save