open-source-mirrors
/
Readarr
mirror of https://github.com/Readarr/Readarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added searchEngine support in Newznab/Torznab caps

Browse Source 
(cherry picked from commit 1cbcad6960c035603e1dc3cf8ddb0c487a29969d)
pull/2439/head
Taloth Saldono 3 years ago committed by Bogdan
parent d7b1a36a50
commit c1500aa321
11 changed files with 137 additions and 54 deletions
Show Stats Download Patch File Download Diff File

6
src/NzbDrone.Core.Test/IndexerSearchTests/SearchDefinitionFixture.cs
Unescape View File

@ -13,7 +13,7 @@ namespace NzbDrone.Core.Test.IndexerSearchTests
public void should_replace_some_special_characters_author(string author, string expected)
{
Subject.Author = new Author { Name = author };
Subject.AuthorQuery.Should().Be(expected);
Subject.CleanAuthorQuery.Should().Be(expected);
}
[TestCase("…and Justice for All", "and+Justice+for+All")]
@ -26,7 +26,7 @@ namespace NzbDrone.Core.Test.IndexerSearchTests
{
Subject.Author = new Author { Name = "Author" };
Subject.BookTitle = book;
Subject.BookQuery.Should().Be(expected);
Subject.CleanBookQuery.Should().Be(expected);
}
[TestCase("+", "+")]
@ -34,7 +34,7 @@ namespace NzbDrone.Core.Test.IndexerSearchTests
{
Subject.Author = new Author { Name = "Author" };
Subject.BookTitle = book;
Subject.BookQuery.Should().Be(expected);
Subject.CleanBookQuery.Should().Be(expected);
}
}
}

23
src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs
Unescape View File

@ -97,5 +97,28 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
ExceptionVerification.ExpectedErrors(1);
}
[Test]
public void should_use_default_searchengine_if_missing()
{
GivenCapsResponse(_caps);
var caps = Subject.GetCapabilities(_settings);
caps.TextSearchEngine.Should().Be("sphinx");
caps.BookTextSearchEngine.Should().Be("sphinx");
}
[Test]
public void should_use_specified_searchengine()
{
GivenCapsResponse(_caps.Replace("<search ", "<search searchEngine=\"raw\" ")
.Replace("<tv-search ", "<tv-search searchEngine=\"raw2\" "));
var caps = Subject.GetCapabilities(_settings);
caps.TextSearchEngine.Should().Be("raw");
caps.BookTextSearchEngine.Should().Be("raw2");
}
}
}

32
src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs
Unescape View File

@ -2,6 +2,7 @@ using System.Linq;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.Books;
using NzbDrone.Core.Indexers.Newznab;
using NzbDrone.Core.IndexerSearch.Definitions;
using NzbDrone.Core.Test.Framework;
@ -25,7 +26,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
_singleBookSearchCriteria = new BookSearchCriteria
{
Author = new Books.Author { Name = "Alien Ant Farm" },
Author = new Author { Name = "Alien Ant Farm" },
BookTitle = "TruANT"
};
@ -64,38 +65,45 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
}
[Test]
[Ignore("TODO: add raw search support")]
public void should_encode_raw_title()
{
_capabilities.SupportedBookSearchParameters = new[] { "q", "author", "title" };
_capabilities.BookTextSearchEngine = "raw";
// _capabilities.BookTextSearchEngine = "raw";
_singleBookSearchCriteria.BookTitle = "Daisy Jones & The Six";
var bookRawSearchCriteria = new BookSearchCriteria
{
Author = new Author { Name = "Taylor Jenkins Reid" },
BookTitle = "Daisy Jones & The Six"
};
var results = Subject.GetSearchRequests(_singleBookSearchCriteria);
results.Tiers.Should().Be(1);
var results = Subject.GetSearchRequests(bookRawSearchCriteria);
results.Tiers.Should().Be(2);
var pageTier = results.GetTier(0).First().First();
pageTier.Url.Query.Should().Contain("q=Daisy%20Jones%20%26%20The%20Six");
pageTier.Url.Query.Should().Contain("q=Daisy%20Jones%20The%20Six+Taylor%20Jenkins%20Reid");
pageTier.Url.Query.Should().NotContain(" & ");
pageTier.Url.Query.Should().Contain("%26");
pageTier.Url.Query.Should().NotContain("%26");
}
[Test]
public void should_use_clean_title_and_encode()
{
_capabilities.SupportedBookSearchParameters = new[] { "q", "author", "title" };
_capabilities.BookTextSearchEngine = "sphinx";
// _capabilities.BookTextSearchEngine = "sphinx";
_singleBookSearchCriteria.BookTitle = "Daisy Jones & The Six";
var bookRawSearchCriteria = new BookSearchCriteria
{
Author = new Author { Name = "Taylor Jenkins Reid" },
BookTitle = "Daisy Jones & The Six"
};
var results = Subject.GetSearchRequests(_singleBookSearchCriteria);
var results = Subject.GetSearchRequests(bookRawSearchCriteria);
results.Tiers.Should().Be(2);
var pageTier = results.GetTier(0).First().First();
pageTier.Url.Query.Should().Contain("q=Daisy%20Jones%20The%20Six");
pageTier.Url.Query.Should().Contain("q=Daisy%20Jones%20The%20Six+Taylor%20Jenkins%20Reid");
pageTier.Url.Query.Should().NotContain("and");
pageTier.Url.Query.Should().NotContain(" & ");
pageTier.Url.Query.Should().NotContain("%26");

3
src/NzbDrone.Core/IndexerSearch/Definitions/BookSearchCriteria.cs
Unescape View File

@ -9,7 +9,8 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
public string BookIsbn { get; set; }
public string Disambiguation { get; set; }
public string BookQuery => GetQueryTitle(BookTitle.SplitBookTitle(Author.Name).Item1);
public string BookQuery => BookTitle.SplitBookTitle(Author.Name).Item1;
public string CleanBookQuery => GetQueryTitle(BookQuery);
public override string ToString()
{

12
src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs
Unescape View File

@ -8,8 +8,9 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
{
public abstract class SearchCriteriaBase
{
private static readonly Regex NonWord = new Regex(@"[^\w`'’]", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex NonWord = new Regex(@"[^\w']+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex BeginningThe = new Regex(@"^the\s", RegexOptions.IgnoreCase | RegexOptions.Compiled);
private static readonly Regex StandardizeSingleQuotesRegex = new Regex(@"[\u0060\u00B4\u2018\u2019]", RegexOptions.IgnoreCase | RegexOptions.Compiled);
public virtual bool MonitoredBooksOnly { get; set; }
public virtual bool UserInvokedSearch { get; set; }
@ -18,7 +19,8 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
public Author Author { get; set; }
public List<Book> Books { get; set; }
public string AuthorQuery => GetQueryTitle(Author.Name);
public string AuthorQuery => Author?.Name;
public string CleanAuthorQuery => GetQueryTitle(AuthorQuery);
public static string GetQueryTitle(string title)
{
@ -32,12 +34,10 @@ namespace NzbDrone.Core.IndexerSearch.Definitions
}
var cleanTitle = BeginningThe.Replace(title, string.Empty);
cleanTitle = cleanTitle.Replace(" & ", " ");
cleanTitle = cleanTitle.Replace(".", " ");
cleanTitle = StandardizeSingleQuotesRegex.Replace(cleanTitle, "'");
cleanTitle = NonWord.Replace(cleanTitle, "+");
//remove any repeating +s
// remove any repeating +s
cleanTitle = Regex.Replace(cleanTitle, @"\+{2,}", "+");
cleanTitle = cleanTitle.RemoveAccent();
cleanTitle = cleanTitle.Trim('+', ' ');

4
src/NzbDrone.Core/Indexers/FileList/FileListRequestGenerator.cs
Unescape View File

@ -23,7 +23,7 @@ namespace NzbDrone.Core.Indexers.FileList
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest("search-torrents", Settings.Categories, string.Format("&type=name&query={0}+{1}", Uri.EscapeDataString(searchCriteria.AuthorQuery.Trim()), Uri.EscapeDataString(searchCriteria.BookQuery.Trim()))));
pageableRequests.Add(GetRequest("search-torrents", Settings.Categories, string.Format("&type=name&query={0}+{1}", Uri.EscapeDataString(searchCriteria.CleanAuthorQuery.Trim()), Uri.EscapeDataString(searchCriteria.CleanBookQuery.Trim()))));
return pageableRequests;
}
@ -32,7 +32,7 @@ namespace NzbDrone.Core.Indexers.FileList
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest("search-torrents", Settings.Categories, string.Format("&type=name&query={0}", Uri.EscapeDataString(searchCriteria.AuthorQuery.Trim()))));
pageableRequests.Add(GetRequest("search-torrents", Settings.Categories, string.Format("&type=name&query={0}", Uri.EscapeDataString(searchCriteria.CleanAuthorQuery.Trim()))));
return pageableRequests;
}

4
src/NzbDrone.Core/Indexers/Gazelle/GazelleRequestGenerator.cs
Unescape View File

@ -30,14 +30,14 @@ namespace NzbDrone.Core.Indexers.Gazelle
public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest(string.Format("&authorname={0}&groupname={1}", searchCriteria.AuthorQuery, searchCriteria.BookQuery)));
pageableRequests.Add(GetRequest(string.Format("&authorname={0}&groupname={1}", searchCriteria.CleanAuthorQuery, searchCriteria.CleanBookQuery)));
return pageableRequests;
}
public IndexerPageableRequestChain GetSearchRequests(AuthorSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetRequest(string.Format("&authorname={0}", searchCriteria.AuthorQuery)));
pageableRequests.Add(GetRequest(string.Format("&authorname={0}", searchCriteria.CleanAuthorQuery)));
return pageableRequests;
}

9
src/NzbDrone.Core/Indexers/Newznab/Newznab.cs
Unescape View File

@ -115,15 +115,14 @@ namespace NzbDrone.Core.Indexers.Newznab
return null;
}
if (capabilities.SupportedBookSearchParameters != null &&
new[] { "author", "title" }.All(v => capabilities.SupportedBookSearchParameters.Contains(v)))
if (capabilities.SupportedAudioSearchParameters != null &&
new[] { "artist", "album" }.All(v => capabilities.SupportedAudioSearchParameters.Contains(v)))
{
return null;
}
if (capabilities.SupportedTvSearchParameters != null &&
new[] { "q", "tvdbid", "rid" }.Any(v => capabilities.SupportedTvSearchParameters.Contains(v)) &&
new[] { "season", "ep" }.All(v => capabilities.SupportedTvSearchParameters.Contains(v)))
if (capabilities.SupportedBookSearchParameters != null &&
new[] { "author", "title" }.All(v => capabilities.SupportedBookSearchParameters.Contains(v)))
{
return null;
}

10
src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilities.cs
Unescape View File

@ -7,9 +7,12 @@ namespace NzbDrone.Core.Indexers.Newznab
public int DefaultPageSize { get; set; }
public int MaxPageSize { get; set; }
public string[] SupportedSearchParameters { get; set; }
public string[] SupportedTvSearchParameters { get; set; }
public string[] SupportedAudioSearchParameters { get; set; }
public string[] SupportedBookSearchParameters { get; set; }
public bool SupportsAggregateIdSearch { get; set; }
public string TextSearchEngine { get; set; }
public string AudioTextSearchEngine { get; set; }
public string BookTextSearchEngine { get; set; }
public List<NewznabCategory> Categories { get; set; }
public NewznabCapabilities()
@ -17,9 +20,12 @@ namespace NzbDrone.Core.Indexers.Newznab
DefaultPageSize = 100;
MaxPageSize = 100;
SupportedSearchParameters = new[] { "q" };
SupportedTvSearchParameters = new[] { "q", "rid", "season", "ep" }; // This should remain 'rid' for older newznab installs.
SupportedAudioSearchParameters = new[] { "q", "artist", "album" };
SupportedBookSearchParameters = new[] { "q", "author", "title" };
SupportsAggregateIdSearch = false;
TextSearchEngine = "sphinx"; // This should remain 'sphinx' for older newznab installs
AudioTextSearchEngine = "sphinx"; // This should remain 'sphinx' for older newznab installs
BookTextSearchEngine = "sphinx"; // This should remain 'sphinx' for older newznab installs
Categories = new List<NewznabCategory>();
}
}

40
src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs
Unescape View File

@ -118,30 +118,46 @@ namespace NzbDrone.Core.Indexers.Newznab
{
capabilities.SupportedSearchParameters = null;
}
else if (xmlBasicSearch.Attribute("supportedParams") != null)
else
{
capabilities.SupportedSearchParameters = xmlBasicSearch.Attribute("supportedParams").Value.Split(',');
if (xmlBasicSearch.Attribute("supportedParams") != null)
{
capabilities.SupportedSearchParameters = xmlBasicSearch.Attribute("supportedParams").Value.Split(',');
}
capabilities.TextSearchEngine = xmlBasicSearch.Attribute("searchEngine")?.Value ?? capabilities.TextSearchEngine;
}
var xmlTvSearch = xmlSearching.Element("tv-search");
if (xmlTvSearch == null || xmlTvSearch.Attribute("available").Value != "yes")
var xmlAudioSearch = xmlSearching.Element("audio-search");
if (xmlAudioSearch == null || xmlAudioSearch.Attribute("available").Value != "yes")
{
capabilities.SupportedTvSearchParameters = null;
capabilities.SupportedAudioSearchParameters = null;
}
else if (xmlTvSearch.Attribute("supportedParams") != null)
else
{
capabilities.SupportedTvSearchParameters = xmlTvSearch.Attribute("supportedParams").Value.Split(',');
capabilities.SupportsAggregateIdSearch = true;
if (xmlAudioSearch.Attribute("supportedParams") != null)
{
capabilities.SupportedAudioSearchParameters = xmlAudioSearch.Attribute("supportedParams").Value.Split(',');
capabilities.SupportsAggregateIdSearch = true;
}
capabilities.AudioTextSearchEngine = xmlAudioSearch.Attribute("searchEngine")?.Value ?? capabilities.AudioTextSearchEngine;
}
var xmlAudioSearch = xmlSearching.Element("book-search");
if (xmlAudioSearch == null || xmlAudioSearch.Attribute("available").Value != "yes")
var xmlBookSearch = xmlSearching.Element("book-search");
if (xmlBookSearch == null || xmlBookSearch.Attribute("available").Value != "yes")
{
capabilities.SupportedBookSearchParameters = null;
}
else if (xmlAudioSearch.Attribute("supportedParams") != null)
else
{
capabilities.SupportedBookSearchParameters = xmlAudioSearch.Attribute("supportedParams").Value.Split(',');
if (xmlBookSearch.Attribute("supportedParams") != null)
{
capabilities.SupportedBookSearchParameters = xmlBookSearch.Attribute("supportedParams").Value.Split(',');
capabilities.SupportsAggregateIdSearch = true;
}
capabilities.BookTextSearchEngine = xmlBookSearch.Attribute("searchEngine")?.Value ?? capabilities.BookTextSearchEngine;
}
}

48
src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs
Unescape View File

@ -35,6 +35,26 @@ namespace NzbDrone.Core.Indexers.Newznab
protected virtual bool SupportsBookSearch => false;
private string TextSearchEngine
{
get
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
return capabilities.TextSearchEngine;
}
}
private string BookTextSearchEngine
{
get
{
var capabilities = _capabilitiesProvider.GetCapabilities(Settings);
return capabilities.BookTextSearchEngine;
}
}
public virtual IndexerPageableRequestChain GetRecentRequests()
{
var pageableRequests = new IndexerPageableRequestChain();
@ -59,35 +79,41 @@ namespace NzbDrone.Core.Indexers.Newznab
if (SupportsBookSearch)
{
var authorQuery = BookTextSearchEngine == "raw" ? searchCriteria.AuthorQuery : searchCriteria.CleanAuthorQuery;
var bookQuery = BookTextSearchEngine == "raw" ? searchCriteria.BookQuery : searchCriteria.CleanBookQuery;
AddBookPageableRequests(pageableRequests,
searchCriteria,
$"&author={NewsnabifyTitle(searchCriteria.AuthorQuery)}&title={NewsnabifyTitle(searchCriteria.BookQuery)}");
$"&author={NewsnabifyTitle(authorQuery)}&title={NewsnabifyTitle(bookQuery)}");
AddBookPageableRequests(pageableRequests,
searchCriteria,
$"&title={NewsnabifyTitle(searchCriteria.BookQuery)}");
$"&title={NewsnabifyTitle(bookQuery)}");
}
if (SupportsSearch)
{
pageableRequests.AddTier();
var authorQuery = TextSearchEngine == "raw" ? searchCriteria.AuthorQuery : searchCriteria.CleanAuthorQuery;
var bookQuery = TextSearchEngine == "raw" ? searchCriteria.BookQuery : searchCriteria.CleanBookQuery;
pageableRequests.Add(GetPagedRequests(MaxPages,
Settings.Categories,
"search",
$"&q={NewsnabifyTitle(searchCriteria.BookQuery)}+{NewsnabifyTitle(searchCriteria.AuthorQuery)}"));
$"&q={NewsnabifyTitle(bookQuery)}+{NewsnabifyTitle(authorQuery)}"));
pageableRequests.AddTier();
pageableRequests.Add(GetPagedRequests(MaxPages,
Settings.Categories,
"search",
$"&q={NewsnabifyTitle(searchCriteria.AuthorQuery)}+{NewsnabifyTitle(searchCriteria.BookQuery)}"));
pageableRequests.AddTier();
$"&q={NewsnabifyTitle(authorQuery)}+{NewsnabifyTitle(bookQuery)}"));
pageableRequests.Add(GetPagedRequests(MaxPages,
Settings.Categories,
"search",
$"&q={NewsnabifyTitle(searchCriteria.BookQuery)}"));
$"&q={NewsnabifyTitle(bookQuery)}"));
}
return pageableRequests;
@ -99,19 +125,23 @@ namespace NzbDrone.Core.Indexers.Newznab
if (SupportsBookSearch)
{
var authorQuery = BookTextSearchEngine == "raw" ? searchCriteria.AuthorQuery : searchCriteria.CleanAuthorQuery;
AddBookPageableRequests(pageableRequests,
searchCriteria,
$"&author={NewsnabifyTitle(searchCriteria.AuthorQuery)}");
$"&author={NewsnabifyTitle(authorQuery)}");
}
if (SupportsSearch)
{
pageableRequests.AddTier();
var authorQuery = TextSearchEngine == "raw" ? searchCriteria.AuthorQuery : searchCriteria.CleanAuthorQuery;
pageableRequests.Add(GetPagedRequests(MaxPages,
Settings.Categories,
"search",
$"&q={NewsnabifyTitle(searchCriteria.AuthorQuery)}"));
$"&q={NewsnabifyTitle(authorQuery)}"));
}
return pageableRequests;

Write Preview
Loading…
Cancel
Save