New: (Cardigann) Pagination Support

pull/1592/head
Bogdan 1 year ago
parent c19802c471
commit c5584ac70f

@ -170,8 +170,8 @@ namespace NzbDrone.Core.History
history.Data.Add("Genre", bookSearchCriteria.Genre);
}
history.Data.Add("Limit", message.Query.Limit?.ToString());
history.Data.Add("Offset", message.Query.Offset?.ToString());
history.Data.Add("Limit", message.Query.Limit.ToString());
history.Data.Add("Offset", message.Query.Offset.ToString());
// Clean empty data
history.Data = history.Data.Where(d => d.Value != null).ToDictionary(x => x.Key, x => x.Value);

@ -15,8 +15,8 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
public string SearchTerm { get; set; }
public int[] Categories { get; set; }
public string SearchType { get; set; }
public int? Limit { get; set; }
public int? Offset { get; set; }
public int Limit { get; set; }
public int Offset { get; set; }
public int? MinAge { get; set; }
public int? MaxAge { get; set; }
public long? MinSize { get; set; }
@ -24,7 +24,13 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
public string Source { get; set; }
public string Host { get; set; }
public override string ToString() => $"{SearchQuery}, Offset: {Offset ?? 0}, Limit: {Limit ?? 0}, Categories: [{string.Join(", ", Categories)}]";
protected SearchCriteriaBase()
{
Limit = 100;
Offset = 0;
}
public override string ToString() => $"{SearchQuery}, Offset: {Offset}, Limit: {Limit}, Categories: [{string.Join(", ", Categories)}]";
public virtual string SearchQuery => $"Term: [{SearchTerm}]";

@ -142,8 +142,8 @@ namespace NzbDrone.Core.IndexerSearch
spec.SearchTerm = query.q?.Trim();
spec.SearchType = query.t;
spec.Limit = query.limit;
spec.Offset = query.offset;
spec.Limit = query.limit ?? 100;
spec.Offset = query.offset ?? 0;
spec.MinAge = query.minage;
spec.MaxAge = query.maxage;
spec.MinSize = query.minsize;

@ -28,8 +28,8 @@ namespace NzbDrone.Core.IndexerVersions
{
/* Update Service will fall back if version # does not exist for an indexer per Ta */
private const string DEFINITION_BRANCH = "master";
private const int DEFINITION_VERSION = 9;
private const string DEFINITION_BRANCH = "cardigann-pagination";
private const int DEFINITION_VERSION = 10;
// Used when moving yml to C#
private readonly List<string> _definitionBlocklist = new ()

@ -212,7 +212,7 @@ namespace NzbDrone.Core.Indexers.Definitions
// replace non-word characters with % (wildcard)
var searchString = Regex.Replace(term.Trim(), @"[\W]+", "%");
var page = searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0 ? (int)(searchCriteria.Offset / searchCriteria.Limit) + 1 : 1;
var page = searchCriteria.Limit > 0 && searchCriteria.Offset > 0 ? (int)(searchCriteria.Offset / searchCriteria.Limit) + 1 : 1;
var refererUri = new HttpUri(_settings.BaseUrl)
.CombinePath("/torrents.php")

@ -33,10 +33,10 @@ namespace NzbDrone.Core.Indexers.Definitions.Avistaz
{
{ "in", "1" },
{ "type", categoryMapping.FirstIfSingleOrDefault("0") },
{ "limit", Math.Min(PageSize, searchCriteria.Limit.GetValueOrDefault(PageSize)).ToString() }
{ "limit", Math.Min(PageSize, searchCriteria.Limit).ToString() }
};
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
if (searchCriteria.Limit > 0 && searchCriteria.Offset > 0)
{
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
qc.Add("page", page.ToString());

@ -29,6 +29,7 @@ namespace NzbDrone.Core.Indexers.Definitions
public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
public override bool SupportsRss => false;
public override bool SupportsPagination => true;
public override int PageSize => 100;
public override IndexerCapabilities Capabilities => SetCapabilities();
public BinSearch(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)
@ -86,7 +87,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{ "adv_sort", "date" },
{ "q", term },
{ "m", searchCriteria.Offset.ToString() },
{ "max", searchCriteria.Limit?.ToString() ?? "100" }
{ "max", searchCriteria.Limit.ToString() }
};
var searchUrl = string.Format("{0}/?{1}", Settings.BaseUrl.TrimEnd('/'), qc.GetQueryString());

@ -52,13 +52,13 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet
var searchString = searchCriteria.SearchTerm ?? string.Empty;
var btnResults = searchCriteria.Limit.GetValueOrDefault();
var btnResults = searchCriteria.Limit;
if (btnResults == 0)
{
btnResults = Capabilities.LimitsDefault.GetValueOrDefault(PageSize);
}
var btnOffset = searchCriteria.Offset.GetValueOrDefault(0);
var btnOffset = searchCriteria.Offset;
if (searchCriteria.TvdbId > 0)
{
@ -124,13 +124,13 @@ namespace NzbDrone.Core.Indexers.BroadcastheNet
var searchString = searchCriteria.SearchTerm ?? "";
var btnResults = searchCriteria.Limit.GetValueOrDefault();
var btnResults = searchCriteria.Limit;
if (btnResults == 0)
{
btnResults = Capabilities.LimitsDefault.GetValueOrDefault(PageSize);
}
var btnOffset = searchCriteria.Offset.GetValueOrDefault(0);
var btnOffset = searchCriteria.Offset;
parameters.Search = searchString.Replace(" ", "%");

@ -26,10 +26,6 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
public override string Description => "";
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
// Page size is different per indexer, setting to 1 ensures we don't break out of paging logic
// thinking its a partial page and instead all search_path requests are run for each indexer
public override int PageSize => 1;
public override TimeSpan RateLimit
{
get
@ -45,6 +41,36 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
}
}
public override int PageSize
{
get
{
if (Definition is null)
{
return 0;
}
var definition = _definitionService.GetCachedDefinition(Settings.DefinitionFile);
return definition.Search is { PageSize: > 0 } ? definition.Search.PageSize : 1;
}
}
public override bool SupportsPagination
{
get
{
if (Definition is null)
{
return false;
}
var definition = _definitionService.GetCachedDefinition(Settings.DefinitionFile);
return definition.Search is { PageSize: > 0 };
}
}
public override IIndexerRequestGenerator GetRequestGenerator()
{
var generator = _generatorCache.Get(Settings.DefinitionFile, () =>

@ -74,6 +74,8 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
public class CapabilitiesBlock
{
public int? LimitsMax { get; set; }
public int? LimitsDefault { get; set; }
public Dictionary<string, string> Categories { get; set; }
public List<CategorymappingBlock> Categorymappings { get; set; }
public Dictionary<string, List<string>> Modes { get; set; }
@ -143,6 +145,9 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
public class SearchBlock
{
public int PageSize { get; set; }
public string Pageable { get; set; }
public int FirstPageNumber { get; set; }
public string Path { get; set; }
public List<SearchPathBlock> Paths { get; set; }
public Dictionary<string, List<string>> Headers { get; set; }

@ -407,12 +407,6 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
}
}
/*
if (query.Limit > 0)
{
releases = releases.Take(query.Limit).ToList();
}*/
releases.ForEach(c =>
{
// generate magnet link from info hash (not allowed for private sites)

@ -62,7 +62,16 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
variables[".Query.TraktID"] = searchCriteria.TraktId?.ToString() ?? null;
variables[".Query.DoubanID"] = searchCriteria.DoubanId?.ToString() ?? null;
pageableRequests.Add(GetRequest(variables, searchCriteria));
var request = GetRequest(variables, searchCriteria);
if (bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
{
pageableRequests.AddTier(request);
}
else
{
pageableRequests.Add(request);
}
return pageableRequests;
}
@ -82,7 +91,16 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
variables[".Query.Year"] = searchCriteria.Year?.ToString() ?? null;
variables[".Query.Track"] = searchCriteria.Track;
pageableRequests.Add(GetRequest(variables, searchCriteria));
var request = GetRequest(variables, searchCriteria);
if (bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
{
pageableRequests.AddTier(request);
}
else
{
pageableRequests.Add(request);
}
return pageableRequests;
}
@ -110,7 +128,16 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
variables[".Query.DoubanID"] = searchCriteria.DoubanId?.ToString() ?? null;
variables[".Query.Episode"] = searchCriteria.EpisodeSearchString;
pageableRequests.Add(GetRequest(variables, searchCriteria));
var request = GetRequest(variables, searchCriteria);
if (bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
{
pageableRequests.AddTier(request);
}
else
{
pageableRequests.Add(request);
}
return pageableRequests;
}
@ -129,7 +156,16 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
variables[".Query.Publisher"] = searchCriteria.Publisher;
variables[".Query.Year"] = searchCriteria.Year?.ToString() ?? null;
pageableRequests.Add(GetRequest(variables, searchCriteria));
var request = GetRequest(variables, searchCriteria);
if (bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
{
pageableRequests.AddTier(request);
}
else
{
pageableRequests.Add(request);
}
return pageableRequests;
}
@ -142,7 +178,16 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
var variables = GetQueryVariableDefaults(searchCriteria);
pageableRequests.Add(GetRequest(variables, searchCriteria));
var request = GetRequest(variables, searchCriteria);
if ((_definition?.Search?.Pageable.IsNotNullOrWhiteSpace() ?? true) || bool.TryParse(ApplyGoTemplateText(_definition.Search.Pageable, variables), out _))
{
pageableRequests.AddTier(request);
}
else
{
pageableRequests.Add(request);
}
return pageableRequests;
}
@ -154,8 +199,8 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
variables[".Query.Type"] = searchCriteria.SearchType;
variables[".Query.Q"] = searchCriteria.SearchTerm;
variables[".Query.Categories"] = searchCriteria.Categories;
variables[".Query.Limit"] = searchCriteria.Limit?.ToString() ?? null;
variables[".Query.Offset"] = searchCriteria.Offset?.ToString() ?? null;
variables[".Query.Limit"] = GetQueryLimit(searchCriteria).ToString();
variables[".Query.Offset"] = searchCriteria.Offset.ToString();
variables[".Query.Extended"] = null;
variables[".Query.APIKey"] = null;
variables[".Query.Genre"] = null;
@ -1086,6 +1131,20 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
variables[".Query.Keywords"] = string.Join(" ", keywordTokens);
variables[".Keywords"] = ApplyFilters((string)variables[".Query.Keywords"], search.Keywordsfilters, variables);
if (search.PageSize == 0 && searchCriteria.Offset > 0)
{
// Indexer doesn't support pagination
yield break;
}
var pageSize = Math.Min(GetQueryLimit(searchCriteria), search.PageSize);
if (pageSize > 0)
{
variables[".PageSize"] = pageSize;
variables[".Query.Page"] = (int)(searchCriteria.Offset / pageSize) + search.FirstPageNumber;
}
var searchUrls = new List<string>();
foreach (var searchPath in search.Paths)
@ -1221,5 +1280,10 @@ namespace NzbDrone.Core.Indexers.Definitions.Cardigann
yield return new CardigannRequest(request, variables, searchPath);
}
}
private int GetQueryLimit(SearchCriteriaBase searchCriteria)
{
return Math.Min(Math.Min(searchCriteria.Limit, _definition?.Caps?.LimitsDefault ?? searchCriteria.Limit), _definition?.Caps?.LimitsMax ?? searchCriteria.Limit);
}
}
}

@ -120,12 +120,12 @@ namespace NzbDrone.Core.Indexers.Headphones
baseUrl += "&apikey=" + Settings.ApiKey;
}
if (searchCriteria.Limit.HasValue)
if (searchCriteria.Limit > 0)
{
parameters.Add("limit", searchCriteria.Limit.ToString());
}
if (searchCriteria.Offset.HasValue)
if (searchCriteria.Offset > 0)
{
parameters.Add("offset", searchCriteria.Offset.ToString());
}

@ -205,7 +205,7 @@ namespace NzbDrone.Core.Indexers.Definitions
qc.Add("q", "+(" + term + ")");
}
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
if (searchCriteria.Limit > 0 && searchCriteria.Offset > 0)
{
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
qc.Add("p", page.ToString());

@ -206,7 +206,7 @@ public class LibbleRequestGenerator : IIndexerRequestGenerator
queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1"));
}
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
if (searchCriteria.Limit > 0 && searchCriteria.Offset > 0)
{
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
parameters.Set("page", page.ToString());

@ -236,8 +236,8 @@ namespace NzbDrone.Core.Indexers.Definitions
{ "tor[srchIn][narrator]", "true" },
{ "tor[searchIn]", "torrents" },
{ "tor[sortType]", "default" },
{ "tor[perpage]", searchCriteria.Limit?.ToString() ?? "100" },
{ "tor[startNumber]", searchCriteria.Offset?.ToString() ?? "0" },
{ "tor[perpage]", searchCriteria.Limit.ToString() },
{ "tor[startNumber]", searchCriteria.Offset.ToString() },
{ "thumbnails", "1" }, // gives links for thumbnail sized versions of their posters
{ "description", "1" } // include the description
};

@ -263,12 +263,12 @@ namespace NzbDrone.Core.Indexers.Newznab
searchUrl += "&apikey=" + Settings.ApiKey;
}
if (searchCriteria.Limit.HasValue)
if (searchCriteria.Limit > 0)
{
parameters.Set("limit", searchCriteria.Limit.ToString());
}
if (searchCriteria.Offset.HasValue)
if (searchCriteria.Offset >= 0)
{
parameters.Set("offset", searchCriteria.Offset.ToString());
}

@ -1035,7 +1035,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit, searchCriteria.Offset));
return pageableRequests;
}
@ -1044,7 +1044,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit, searchCriteria.Offset));
return pageableRequests;
}
@ -1053,7 +1053,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}", searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}", searchCriteria.Categories, searchCriteria.Limit, searchCriteria.Offset));
return pageableRequests;
}
@ -1062,7 +1062,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit, searchCriteria.Offset));
return pageableRequests;
}
@ -1071,7 +1071,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit ?? 100, searchCriteria.Offset ?? 0));
pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}", searchCriteria.Categories, searchCriteria.Limit, searchCriteria.Offset));
return pageableRequests;
}

@ -97,7 +97,7 @@ namespace NzbDrone.Core.Indexers.Definitions.PassThePopcorn
queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1"));
}
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
if (searchCriteria.Limit > 0 && searchCriteria.Offset > 0)
{
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
parameters.Set("page", page.ToString());

@ -178,12 +178,12 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var parameters = new NameValueCollection
{
{ "itemsPerPage", Math.Min(_pageSize, searchCriteria.Limit.GetValueOrDefault(_pageSize)).ToString() },
{ "itemsPerPage", Math.Min(_pageSize, searchCriteria.Limit).ToString() },
{ "sort", "torrent.createdAt" },
{ "direction", "desc" }
};
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
if (searchCriteria.Limit > 0 && searchCriteria.Offset > 0)
{
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;
parameters.Set("page", page.ToString());

@ -290,7 +290,7 @@ public class SpeedCDRequestGenerator : IIndexerRequestGenerator
parameters.Add(term.UrlEncode(_encoding));
}
if (searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0)
if (searchCriteria.Limit > 0 && searchCriteria.Offset > 0)
{
var page = (int)(searchCriteria.Offset / searchCriteria.Limit) + 1;

@ -61,7 +61,7 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
if (!SupportsPagination && searchCriteria.Offset > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
@ -90,7 +90,7 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
if (!SupportsPagination && searchCriteria.Offset > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
@ -119,7 +119,7 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
if (!SupportsPagination && searchCriteria.Offset > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
@ -151,7 +151,7 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
if (!SupportsPagination && searchCriteria.Offset > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}
@ -179,7 +179,7 @@ namespace NzbDrone.Core.Indexers
return Task.FromResult(new IndexerPageableQueryResult());
}
if (!SupportsPagination && searchCriteria.Offset is > 0)
if (!SupportsPagination && searchCriteria.Offset > 0)
{
return Task.FromResult(new IndexerPageableQueryResult());
}

@ -123,6 +123,16 @@ namespace NzbDrone.Core.Indexers
definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes);
definition.Capabilities.SupportsRawSearch = defFile.Caps.Allowrawsearch;
MapCardigannCategories(definition, defFile);
if (defFile.Caps?.LimitsDefault is > 0)
{
definition.Capabilities.LimitsDefault = defFile.Caps.LimitsDefault;
}
if (defFile.Caps?.LimitsMax is > 0)
{
definition.Capabilities.LimitsMax = defFile.Caps.LimitsMax;
}
}
private void MapCardigannCategories(IndexerDefinition def, CardigannDefinition defFile)

Loading…
Cancel
Save