open-source-mirrors
/
Prowlarr
mirror of https://github.com/Prowlarr/Prowlarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed: (AnimeTorrents) Add DownloadableOnly/FreeleechOnly settings

Browse Source
pull/1540/head
Bogdan 1 year ago
parent 1a7b6aecf1
commit d4c5e39c9c
2 changed files with 170 additions and 116 deletions
Show Stats Download Patch File Download Diff File

14
src/NzbDrone.Core/Datastore/Migration/030_animetorrents_use_custom_config_contract.cs
Unescape View File

@ -0,0 +1,14 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(030)]
public class animetorrents_use_custom_config_contract : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Update.Table("Indexers").Set(new { ConfigContract = "AnimeTorrentsSettings" }).Where(new { Implementation = "AnimeTorrents" });
}
}
}

272
src/NzbDrone.Core/Indexers/Definitions/AnimeTorrents.cs
Unescape View File

@ -1,14 +1,14 @@
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using AngleSharp.Html.Parser;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Indexers.Settings;
@ -19,15 +19,14 @@ using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers.Definitions
{
public class AnimeTorrents : TorrentIndexerBase<UserPassTorrentBaseSettings>
public class AnimeTorrents : TorrentIndexerBase<AnimeTorrentsSettings>
{
public override string Name => "AnimeTorrents";
public override string[] IndexerUrls => new[] { "https://animetorrents.me/" };
public override string Description => "Definitive source for anime and manga";
private string LoginUrl => Settings.BaseUrl + "login.php";
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override TimeSpan RateLimit => TimeSpan.FromSeconds(4);
public override IndexerCapabilities Capabilities => SetCapabilities();
public AnimeTorrents(IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
@ -37,7 +36,7 @@ namespace NzbDrone.Core.Indexers.Definitions
public override IIndexerRequestGenerator GetRequestGenerator()
{
return new AnimeTorrentsRequestGenerator { Settings = Settings, Capabilities = Capabilities };
return new AnimeTorrentsRequestGenerator(Settings, Capabilities);
}
public override IParseIndexerResponse GetParser()
@ -49,36 +48,37 @@ namespace NzbDrone.Core.Indexers.Definitions
{
UpdateCookies(null, null);
var loginPage = await ExecuteAuth(new HttpRequest(LoginUrl));
var loginUrl = Settings.BaseUrl + "login.php";
var loginPage = await ExecuteAuth(new HttpRequest(loginUrl));
var requestBuilder = new HttpRequestBuilder(LoginUrl)
var requestBuilder = new HttpRequestBuilder(loginUrl)
{
LogResponseContent = true,
AllowAutoRedirect = true,
Method = HttpMethod.Post
AllowAutoRedirect = true
};
var authLoginRequest = requestBuilder
.Post()
.SetCookies(loginPage.GetCookies())
.AddFormParameter("username", Settings.Username)
.AddFormParameter("password", Settings.Password)
.AddFormParameter("form", "login")
.AddFormParameter("rememberme[]", "1")
.SetHeader("Content-Type", "application/x-www-form-urlencoded")
.SetHeader("Referer", loginUrl)
.Build();
var response = await ExecuteAuth(authLoginRequest);
if (response.Content != null && response.Content.Contains("logout.php"))
{
UpdateCookies(response.GetCookies(), DateTime.Now.AddDays(30));
_logger.Debug("AnimeTorrents authentication succeeded");
}
else
if (response.Content == null || !response.Content.Contains("logout.php"))
{
throw new IndexerAuthException("AnimeTorrents authentication failed");
}
UpdateCookies(response.GetCookies(), DateTime.Now.AddDays(30));
_logger.Debug("AnimeTorrents authentication succeeded");
}
protected override bool CheckIfLoginNeeded(HttpResponse httpResponse)
@ -126,49 +126,25 @@ namespace NzbDrone.Core.Indexers.Definitions
public class AnimeTorrentsRequestGenerator : IIndexerRequestGenerator
{
public UserPassTorrentBaseSettings Settings { get; set; }
public IndexerCapabilities Capabilities { get; set; }
private readonly AnimeTorrentsSettings _settings;
private readonly IndexerCapabilities _capabilities;
private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories)
public AnimeTorrentsRequestGenerator(AnimeTorrentsSettings settings, IndexerCapabilities capabilities)
{
var searchString = term;
// replace any space, special char, etc. with % (wildcard)
var replaceRegex = new Regex("[^a-zA-Z0-9]+");
searchString = replaceRegex.Replace(searchString, "%");
var searchUrl = Settings.BaseUrl + "ajax/torrents_data.php";
var searchUrlReferer = Settings.BaseUrl + "torrents.php?cat=0&searchin=filename&search=";
var trackerCats = Capabilities.Categories.MapTorznabCapsToTrackers(categories) ?? new List<string>();
var queryCollection = new NameValueCollection
{
{ "total", "146" }, // Not sure what this is about but its required!
{ "cat", trackerCats.Count == 1 ? trackerCats.First() : "0" },
{ "page", "1" },
{ "searchin", "filename" },
{ "search", searchString }
};
searchUrl += "?" + queryCollection.GetQueryString();
var extraHeaders = new NameValueCollection
{
{ "X-Requested-With", "XMLHttpRequest" },
{ "Referer", searchUrlReferer }
};
var request = new IndexerRequest(searchUrl, null);
request.HttpRequest.Headers.Add(extraHeaders);
yield return request;
_settings = settings;
_capabilities = capabilities;
}
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
{
pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
}
return pageableRequests;
}
@ -177,7 +153,12 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
{
pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
}
return pageableRequests;
}
@ -186,7 +167,12 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
{
pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
}
return pageableRequests;
}
@ -195,7 +181,12 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
{
pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
}
return pageableRequests;
}
@ -204,21 +195,76 @@ namespace NzbDrone.Core.Indexers.Definitions
{
var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories));
var searchTerm = $"{searchCriteria.SanitizedSearchTerm}";
foreach (var category in GetTrackerCategories(searchTerm, searchCriteria))
{
pageableRequests.Add(GetPagedRequests(searchTerm, category, searchCriteria));
}
return pageableRequests;
}
private IEnumerable<IndexerRequest> GetPagedRequests(string term, string category, SearchCriteriaBase searchCriteria)
{
var searchUrl = _settings.BaseUrl + "ajax/torrents_data.php";
// replace non-alphanumeric characters with % (wildcard)
var searchString = Regex.Replace(term.Trim(), "[^a-zA-Z0-9]+", "%");
var page = searchCriteria.Limit is > 0 && searchCriteria.Offset is > 0 ? (int)(searchCriteria.Offset / searchCriteria.Limit) + 1 : 1;
var refererUri = new HttpUri(_settings.BaseUrl)
.CombinePath("/torrents.php")
.AddQueryParam("cat", $"{category}");
if (_settings.DownloadableOnly)
{
refererUri = refererUri.AddQueryParam("dlable", "1");
}
var requestBuilder = new HttpRequestBuilder(searchUrl)
.AddQueryParam("total", "100") // Assuming the total number of pages
.AddQueryParam("cat", $"{category}")
.AddQueryParam("searchin", "filename")
.AddQueryParam("search", searchString)
.AddQueryParam("page", page)
.SetHeader("X-Requested-With", "XMLHttpRequest")
.SetHeader("Referer", refererUri.FullUri)
.Accept(HttpAccept.Html);
if (_settings.DownloadableOnly)
{
requestBuilder.AddQueryParam("dlable", "1");
}
yield return new IndexerRequest(requestBuilder.Build());
}
private IEnumerable<string> GetTrackerCategories(string term, SearchCriteriaBase searchCriteria)
{
var searchTerm = term.Trim();
var categoryMapping = _capabilities.Categories
.MapTorznabCapsToTrackers(searchCriteria.Categories)
.Distinct()
.ToList();
return searchTerm.IsNullOrWhiteSpace() && categoryMapping.Count == 2
? categoryMapping
: new List<string> { categoryMapping.FirstIfSingleOrDefault("0") };
}
public Func<IDictionary<string, string>> GetCookies { get; set; }
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
public class AnimeTorrentsParser : IParseIndexerResponse
{
private readonly UserPassTorrentBaseSettings _settings;
private readonly AnimeTorrentsSettings _settings;
private readonly IndexerCapabilitiesCategories _categories;
public AnimeTorrentsParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories)
public AnimeTorrentsParser(AnimeTorrentsSettings settings, IndexerCapabilitiesCategories categories)
{
_settings = settings;
_categories = categories;
@ -226,94 +272,88 @@ namespace NzbDrone.Core.Indexers.Definitions
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
var torrentInfos = new List<TorrentInfo>();
var releaseInfos = new List<ReleaseInfo>();
var parser = new HtmlParser();
var dom = parser.ParseDocument(indexerResponse.Content);
var rows = dom.QuerySelectorAll("tr");
var rows = dom.QuerySelectorAll("table tr");
foreach (var row in rows.Skip(1))
{
var release = new TorrentInfo();
var qTitleLink = row.QuerySelector("td:nth-of-type(2) a:nth-of-type(1)");
release.Title = qTitleLink.TextContent.Trim();
var downloadVolumeFactor = row.QuerySelector("img[alt=\"Gold Torrent\"]") != null ? 0 : row.QuerySelector("img[alt=\"Silver Torrent\"]") != null ? 0.5 : 1;
// If we search an get no results, we still get a table just with no info.
if (string.IsNullOrWhiteSpace(release.Title))
// skip non-freeleech results when freeleech only is set
if (_settings.FreeleechOnly && downloadVolumeFactor != 0)
{
break;
continue;
}
release.Guid = qTitleLink.GetAttribute("href");
release.InfoUrl = release.Guid;
var dateString = row.QuerySelector("td:nth-of-type(5)").TextContent;
release.PublishDate = DateTime.ParseExact(dateString, "dd MMM yy", CultureInfo.InvariantCulture);
var qTitleLink = row.QuerySelector("td:nth-of-type(2) a:nth-of-type(1)");
var title = qTitleLink?.TextContent.Trim();
// newbie users don't see DL links
var qLink = row.QuerySelector("td:nth-of-type(3) a");
if (qLink != null)
{
release.DownloadUrl = qLink.GetAttribute("href");
}
else
// If we search an get no results, we still get a table just with no info.
if (title.IsNullOrWhiteSpace())
{
// use details link as placeholder
// null causes errors during export to torznab
// skipping the release prevents newbie users from adding the tracker (empty result)
release.DownloadUrl = release.InfoUrl;
break;
}
var sizeStr = row.QuerySelector("td:nth-of-type(6)").TextContent;
release.Size = ParseUtil.GetBytes(sizeStr);
var connections = row.QuerySelector("td:nth-of-type(8)").TextContent.Trim().Split("/".ToCharArray(), StringSplitOptions.RemoveEmptyEntries);
var infoUrl = qTitleLink?.GetAttribute("href");
release.Seeders = ParseUtil.CoerceInt(connections[0].Trim());
release.Peers = ParseUtil.CoerceInt(connections[1].Trim()) + release.Seeders;
release.Grabs = ParseUtil.CoerceInt(connections[2].Trim());
// newbie users don't see DL links
// use details link as placeholder
// skipping the release prevents newbie users from adding the tracker (empty result)
var downloadUrl = row.QuerySelector("td:nth-of-type(3) a")?.GetAttribute("href") ?? infoUrl;
var rCat = row.QuerySelector("td:nth-of-type(1) a").GetAttribute("href");
var rCatIdx = rCat.IndexOf("cat=");
if (rCatIdx > -1)
{
rCat = rCat.Substring(rCatIdx + 4);
}
var connections = row.QuerySelector("td:nth-of-type(8)").TextContent.Trim().Split('/', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
var seeders = ParseUtil.CoerceInt(connections[0]);
release.Categories = _categories.MapTrackerCatToNewznab(rCat);
var categoryLink = row.QuerySelector("td:nth-of-type(1) a")?.GetAttribute("href") ?? string.Empty;
var categoryId = ParseUtil.GetArgumentFromQueryString(categoryLink, "cat");
if (row.QuerySelector("img[alt=\"Gold Torrent\"]") != null)
var release = new TorrentInfo
{
release.DownloadVolumeFactor = 0;
}
else if (row.QuerySelector("img[alt=\"Silver Torrent\"]") != null)
{
release.DownloadVolumeFactor = 0.5;
}
else
{
release.DownloadVolumeFactor = 1;
}
Guid = infoUrl,
InfoUrl = infoUrl,
DownloadUrl = downloadUrl,
Title = title,
Categories = _categories.MapTrackerCatToNewznab(categoryId),
PublishDate = DateTime.ParseExact(row.QuerySelector("td:nth-of-type(5)").TextContent, "dd MMM yy", CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal),
Size = ParseUtil.GetBytes(row.QuerySelector("td:nth-of-type(6)").TextContent.Trim()),
Seeders = seeders,
Peers = ParseUtil.CoerceInt(connections[1]) + seeders,
Grabs = ParseUtil.CoerceInt(connections[2]),
DownloadVolumeFactor = downloadVolumeFactor,
UploadVolumeFactor = 1,
Genres = row.QuerySelectorAll("td:nth-of-type(2) a.tortags").Select(t => t.TextContent.Trim()).ToList()
};
var uLFactorImg = row.QuerySelector("img[alt*=\"x Multiplier Torrent\"]");
if (uLFactorImg != null)
{
release.UploadVolumeFactor = ParseUtil.CoerceDouble(uLFactorImg.GetAttribute("alt").Split('x')[0]);
}
else
{
release.UploadVolumeFactor = 1;
}
qTitleLink.Remove();
//release.Description = row.QuerySelector("td:nth-of-type(2)").TextContent;
torrentInfos.Add(release);
releaseInfos.Add(release);
}
return torrentInfos.ToArray();
return releaseInfos.ToArray();
}
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
}
public class AnimeTorrentsSettings : UserPassTorrentBaseSettings
{
public AnimeTorrentsSettings()
{
FreeleechOnly = false;
DownloadableOnly = false;
}
[FieldDefinition(4, Label = "Freeleech Only", Type = FieldType.Checkbox, HelpText = "Show freeleech torrents only")]
public bool FreeleechOnly { get; set; }
[FieldDefinition(5, Label = "Downloadable Only", Type = FieldType.Checkbox, HelpText = "Search downloadable torrents only (enable this only if your account class is Newbie)")]
public bool DownloadableOnly { get; set; }
}
}

Write Preview
Loading…
Cancel
Save