Nyuels
3 years ago
committed by
Qstick
parent
5e7f4f3fc1
commit
ba1c1baeb5
@ -0,0 +1,317 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Collections.Specialized;
|
||||||
|
using System.Net;
|
||||||
|
using Newtonsoft.Json;
|
||||||
|
using NLog;
|
||||||
|
using NzbDrone.Common.Extensions;
|
||||||
|
using NzbDrone.Common.Http;
|
||||||
|
using NzbDrone.Core.Annotations;
|
||||||
|
using NzbDrone.Core.Configuration;
|
||||||
|
using NzbDrone.Core.Indexers.Exceptions;
|
||||||
|
using NzbDrone.Core.IndexerSearch.Definitions;
|
||||||
|
using NzbDrone.Core.Messaging.Events;
|
||||||
|
using NzbDrone.Core.Parser;
|
||||||
|
using NzbDrone.Core.Parser.Model;
|
||||||
|
using NzbDrone.Core.Validation;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Indexers.Definitions
|
||||||
|
{
|
||||||
|
public class InternetArchive : TorrentIndexerBase<InternetArchiveSettings>
|
||||||
|
{
|
||||||
|
public override string Name => "Internet Archive";
|
||||||
|
|
||||||
|
public override string[] IndexerUrls => new string[] { "https://archive.org/" };
|
||||||
|
|
||||||
|
public override string Description => "Internet Archive is a non-profit library of millions of free books, movies, software, music, websites, and more.";
|
||||||
|
|
||||||
|
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
|
||||||
|
|
||||||
|
public override IndexerPrivacy Privacy => IndexerPrivacy.Public;
|
||||||
|
|
||||||
|
public override IndexerCapabilities Capabilities => SetCapabilities();
|
||||||
|
|
||||||
|
public InternetArchive(IHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
|
||||||
|
: base(httpClient, eventAggregator, indexerStatusService, configService, logger)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public override IParseIndexerResponse GetParser()
|
||||||
|
{
|
||||||
|
return new InternetArchiveParser(Settings, Capabilities.Categories);
|
||||||
|
}
|
||||||
|
|
||||||
|
public override IIndexerRequestGenerator GetRequestGenerator()
|
||||||
|
{
|
||||||
|
return new InternetArchiveRequestGenerator() { Settings = Settings, Capabilities = Capabilities };
|
||||||
|
}
|
||||||
|
|
||||||
|
private IndexerCapabilities SetCapabilities()
|
||||||
|
{
|
||||||
|
var caps = new IndexerCapabilities
|
||||||
|
{
|
||||||
|
TvSearchParams = new List<TvSearchParam>
|
||||||
|
{
|
||||||
|
TvSearchParam.Q
|
||||||
|
},
|
||||||
|
MovieSearchParams = new List<MovieSearchParam>
|
||||||
|
{
|
||||||
|
MovieSearchParam.Q
|
||||||
|
},
|
||||||
|
BookSearchParams = new List<BookSearchParam>
|
||||||
|
{
|
||||||
|
BookSearchParam.Q
|
||||||
|
},
|
||||||
|
MusicSearchParams = new List<MusicSearchParam>
|
||||||
|
{
|
||||||
|
MusicSearchParam.Q
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// c.f. https://archive.org/services/docs/api/metadata-schema/index.html?highlight=mediatype#mediatype
|
||||||
|
caps.Categories.AddCategoryMapping("texts", NewznabStandardCategory.Books);
|
||||||
|
caps.Categories.AddCategoryMapping("etree", NewznabStandardCategory.Audio);
|
||||||
|
caps.Categories.AddCategoryMapping("audio", NewznabStandardCategory.Audio);
|
||||||
|
caps.Categories.AddCategoryMapping("movies", NewznabStandardCategory.Movies);
|
||||||
|
caps.Categories.AddCategoryMapping("software", NewznabStandardCategory.PC);
|
||||||
|
caps.Categories.AddCategoryMapping("image", NewznabStandardCategory.OtherMisc);
|
||||||
|
caps.Categories.AddCategoryMapping("data", NewznabStandardCategory.Other);
|
||||||
|
caps.Categories.AddCategoryMapping("web", NewznabStandardCategory.Other);
|
||||||
|
caps.Categories.AddCategoryMapping("collection", NewznabStandardCategory.Other);
|
||||||
|
caps.Categories.AddCategoryMapping("account", NewznabStandardCategory.Other);
|
||||||
|
|
||||||
|
caps.Categories.AddCategoryMapping("other", NewznabStandardCategory.Other);
|
||||||
|
return caps;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class InternetArchiveRequestGenerator : IIndexerRequestGenerator
|
||||||
|
{
|
||||||
|
public InternetArchiveSettings Settings { get; set; }
|
||||||
|
public IndexerCapabilities Capabilities { get; set; }
|
||||||
|
|
||||||
|
public InternetArchiveRequestGenerator()
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
private IEnumerable<IndexerRequest> GetPagedRequests(string searchTerm, SearchCriteriaBase searchCriteria)
|
||||||
|
{
|
||||||
|
var query = "format:(\"Archive BitTorrent\")";
|
||||||
|
|
||||||
|
if (searchTerm.IsNotNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
if (Settings.TitleOnly)
|
||||||
|
{
|
||||||
|
query = string.Format("title:({0}) AND {1}", searchTerm, query);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
query = string.Format("{0} AND {1}", searchTerm, query);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
var categories = Capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories);
|
||||||
|
if (categories.Count > 0)
|
||||||
|
{
|
||||||
|
query = string.Format("{0} AND mediatype:({1})", query, string.Join(" OR ", categories));
|
||||||
|
}
|
||||||
|
|
||||||
|
string sortBy = (InternetArchiveSort)Settings.SortBy switch
|
||||||
|
{
|
||||||
|
InternetArchiveSort.PublicDate => "publicdate",
|
||||||
|
InternetArchiveSort.Downloads => "downloads",
|
||||||
|
InternetArchiveSort.Size => "item_size",
|
||||||
|
_ => "publicdate",
|
||||||
|
};
|
||||||
|
|
||||||
|
string sortOrder = (InternetArchiveSortOrder)Settings.SortOrder switch
|
||||||
|
{
|
||||||
|
InternetArchiveSortOrder.Descending => "desc",
|
||||||
|
InternetArchiveSortOrder.Ascending => "asc",
|
||||||
|
_ => "desc",
|
||||||
|
};
|
||||||
|
|
||||||
|
var parameters = new NameValueCollection
|
||||||
|
{
|
||||||
|
{ "q", query },
|
||||||
|
{ "fields", "btih,downloads,identifier,item_size,mediatype,publicdate,title" },
|
||||||
|
{ "count", searchCriteria.Limit.GetValueOrDefault(100).ToString() }, // API default is 5000, don't think thats viable at all.
|
||||||
|
{ "sorts", string.Format("{0} {1}", sortBy, sortOrder) }
|
||||||
|
};
|
||||||
|
|
||||||
|
var searchUrl = string.Format("{0}/services/search/v1/scrape?{1}", Settings.BaseUrl.TrimEnd('/'), parameters.GetQueryString());
|
||||||
|
var request = new IndexerRequest(searchUrl, HttpAccept.Json);
|
||||||
|
|
||||||
|
yield return request;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
var pageableRequests = new IndexerPageableRequestChain();
|
||||||
|
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria));
|
||||||
|
return pageableRequests;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
var pageableRequests = new IndexerPageableRequestChain();
|
||||||
|
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria));
|
||||||
|
return pageableRequests;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
var pageableRequests = new IndexerPageableRequestChain();
|
||||||
|
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria));
|
||||||
|
return pageableRequests;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
var pageableRequests = new IndexerPageableRequestChain();
|
||||||
|
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria));
|
||||||
|
return pageableRequests;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
|
||||||
|
{
|
||||||
|
var pageableRequests = new IndexerPageableRequestChain();
|
||||||
|
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria));
|
||||||
|
return pageableRequests;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Func<IDictionary<string, string>> GetCookies { get; set; }
|
||||||
|
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public class InternetArchiveParser : IParseIndexerResponse
|
||||||
|
{
|
||||||
|
private readonly InternetArchiveSettings _settings;
|
||||||
|
private readonly IndexerCapabilitiesCategories _categories;
|
||||||
|
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
|
||||||
|
|
||||||
|
public InternetArchiveParser(InternetArchiveSettings settings, IndexerCapabilitiesCategories categories)
|
||||||
|
{
|
||||||
|
_settings = settings;
|
||||||
|
_categories = categories;
|
||||||
|
}
|
||||||
|
|
||||||
|
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
|
||||||
|
{
|
||||||
|
var torrentInfos = new List<TorrentInfo>();
|
||||||
|
|
||||||
|
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
|
||||||
|
{
|
||||||
|
throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!indexerResponse.HttpResponse.Headers.ContentType.Contains(HttpAccept.Json.Value))
|
||||||
|
{
|
||||||
|
throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}");
|
||||||
|
}
|
||||||
|
|
||||||
|
var jsonResponse = new HttpResponse<InternetArchiveResponse>(indexerResponse.HttpResponse);
|
||||||
|
|
||||||
|
foreach (var searchResult in jsonResponse.Resource.SearchResults)
|
||||||
|
{
|
||||||
|
var title = searchResult.Title ?? searchResult.Identifier;
|
||||||
|
|
||||||
|
var downloadUrl = string.Format("{0}/download/{1}/{1}_archive.torrent", _settings.BaseUrl.TrimEnd('/'), searchResult.Identifier);
|
||||||
|
var detailsUrl = string.Format("{0}/details/{1}", _settings.BaseUrl.TrimEnd('/'), searchResult.Identifier);
|
||||||
|
|
||||||
|
var category = _categories.MapTrackerCatToNewznab(searchResult.MediaType);
|
||||||
|
|
||||||
|
var release = new TorrentInfo
|
||||||
|
{
|
||||||
|
Categories = category,
|
||||||
|
CommentUrl = detailsUrl,
|
||||||
|
DownloadUrl = downloadUrl,
|
||||||
|
DownloadVolumeFactor = 0,
|
||||||
|
Guid = detailsUrl,
|
||||||
|
Grabs = searchResult.Downloads,
|
||||||
|
InfoHash = searchResult.InfoHash,
|
||||||
|
InfoUrl = detailsUrl,
|
||||||
|
MagnetUrl = MagnetLinkBuilder.BuildPublicMagnetLink(searchResult.InfoHash, title),
|
||||||
|
Peers = 2,
|
||||||
|
PublishDate = searchResult.PublicDate,
|
||||||
|
Seeders = 1,
|
||||||
|
Size = searchResult.Size,
|
||||||
|
Title = title,
|
||||||
|
UploadVolumeFactor = 1
|
||||||
|
};
|
||||||
|
|
||||||
|
torrentInfos.Add(release);
|
||||||
|
}
|
||||||
|
|
||||||
|
return torrentInfos.ToArray();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public class InternetArchiveResponse
|
||||||
|
{
|
||||||
|
[JsonProperty(PropertyName = "items")]
|
||||||
|
public List<InternetArchiveTorrent> SearchResults { get; set; }
|
||||||
|
|
||||||
|
public string Cursor { get; set; }
|
||||||
|
public int Count { get; set; }
|
||||||
|
public int Total { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public class InternetArchiveTorrent
|
||||||
|
{
|
||||||
|
public int Downloads { get; set; }
|
||||||
|
public string Identifier { get; set; }
|
||||||
|
|
||||||
|
[JsonProperty(PropertyName = "btih")]
|
||||||
|
public string InfoHash { get; set; }
|
||||||
|
public string MediaType { get; set; }
|
||||||
|
public DateTime PublicDate { get; set; }
|
||||||
|
|
||||||
|
[JsonProperty(PropertyName = "item_size")]
|
||||||
|
public long Size { get; set; }
|
||||||
|
public string Title { get; set; }
|
||||||
|
}
|
||||||
|
|
||||||
|
public class InternetArchiveSettings : IIndexerSettings
|
||||||
|
{
|
||||||
|
[FieldDefinition(1, Label = "Base Url", Type = FieldType.Select, SelectOptionsProviderAction = "getUrls", HelpText = "Select which baseurl Prowlarr will use for requests to the site")]
|
||||||
|
public string BaseUrl { get; set; }
|
||||||
|
|
||||||
|
[FieldDefinition(2, Label = "Sort By", Type = FieldType.Select, Advanced = true, SelectOptions = typeof(InternetArchiveSort), HelpText = "Field used to sort the search results.")]
|
||||||
|
public int SortBy { get; set; }
|
||||||
|
|
||||||
|
[FieldDefinition(3, Label = "Sort Order", Type = FieldType.Select, Advanced = true, SelectOptions = typeof(InternetArchiveSortOrder), HelpText = "Order to use when sorting results.")]
|
||||||
|
public int SortOrder { get; set; }
|
||||||
|
|
||||||
|
[FieldDefinition(4, Label = "Title Only", Type = FieldType.Checkbox, Advanced = true, HelpText = "Whether to search in title only.")]
|
||||||
|
public bool TitleOnly { get; set; }
|
||||||
|
|
||||||
|
[FieldDefinition(5)]
|
||||||
|
public IndexerBaseSettings BaseSettings { get; set; } = new IndexerBaseSettings();
|
||||||
|
|
||||||
|
public InternetArchiveSettings()
|
||||||
|
{
|
||||||
|
SortBy = (int)InternetArchiveSort.PublicDate;
|
||||||
|
SortOrder = (int)InternetArchiveSortOrder.Descending;
|
||||||
|
TitleOnly = false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public NzbDroneValidationResult Validate()
|
||||||
|
{
|
||||||
|
return new NzbDroneValidationResult();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum InternetArchiveSort
|
||||||
|
{
|
||||||
|
PublicDate,
|
||||||
|
Downloads,
|
||||||
|
Size
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum InternetArchiveSortOrder
|
||||||
|
{
|
||||||
|
Descending,
|
||||||
|
Ascending
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in new issue