New: Sync Indexers with Mylar3

pull/380/head
Qstick 3 years ago
parent 09d839ffb1
commit 77892a3885

@ -7,5 +7,6 @@ namespace NzbDrone.Core.Applications
public int IndexerId { get; set; }
public int AppId { get; set; }
public int RemoteIndexerId { get; set; }
public string RemoteIndexerName { get; set; }
}
}

@ -8,6 +8,7 @@ namespace NzbDrone.Core.Applications
{
List<AppIndexerMap> GetMappingsForApp(int appId);
AppIndexerMap Insert(AppIndexerMap appIndexerMap);
AppIndexerMap Update(AppIndexerMap appIndexerMap);
void Delete(int mappingId);
void DeleteAllForApp(int appId);
}
@ -41,6 +42,11 @@ namespace NzbDrone.Core.Applications
return _appIndexerMapRepository.Insert(appIndexerMap);
}
public AppIndexerMap Update(AppIndexerMap appIndexerMap)
{
return _appIndexerMapRepository.Update(appIndexerMap);
}
public void Handle(ProviderDeletedEvent<IApplication> message)
{
_appIndexerMapRepository.DeleteAllForApp(message.ProviderId);

@ -14,7 +14,7 @@ namespace NzbDrone.Core.Applications
protected readonly IAppIndexerMapService _appIndexerMapService;
protected readonly Logger _logger;
protected static readonly Regex AppIndexerRegex = new Regex(@"\/(?<indexer>\d.)\/",
protected static readonly Regex AppIndexerRegex = new Regex(@"\/(?<indexer>\d.)\/?$",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
public abstract string Name { get; }
@ -58,7 +58,7 @@ namespace NzbDrone.Core.Applications
public abstract void AddIndexer(IndexerDefinition indexer);
public abstract void UpdateIndexer(IndexerDefinition indexer);
public abstract void RemoveIndexer(int indexerId);
public abstract Dictionary<int, int> GetIndexerMappings();
public abstract List<AppIndexerMap> GetIndexerMappings();
public virtual object RequestAction(string action, IDictionary<string, string> query)
{

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Net;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration.Events;
using NzbDrone.Core.Indexers;
@ -110,9 +111,6 @@ namespace NzbDrone.Core.Applications
{
var indexerMappings = _appIndexerMapService.GetMappingsForApp(app.Definition.Id);
//Remote-Local mappings currently stored by Prowlarr
var prowlarrMappings = indexerMappings.ToDictionary(i => i.RemoteIndexerId, i => i.IndexerId);
//Get Dictionary of Remote Indexers point to Prowlarr and what they are mapped to
var remoteMappings = ExecuteAction(a => a.GetIndexerMappings(), app);
@ -124,9 +122,16 @@ namespace NzbDrone.Core.Applications
//Add mappings if not already in db, these were setup manually in the app or orphaned by a table wipe
foreach (var mapping in remoteMappings)
{
if (!prowlarrMappings.ContainsKey(mapping.Key))
if (!indexerMappings.Any(m => (m.RemoteIndexerId > 0 && m.RemoteIndexerId == mapping.RemoteIndexerId) || (m.RemoteIndexerName.IsNotNullOrWhiteSpace() && m.RemoteIndexerName == mapping.RemoteIndexerName)))
{
var addMapping = new AppIndexerMap { AppId = app.Definition.Id, RemoteIndexerId = mapping.Key, IndexerId = mapping.Value };
var addMapping = new AppIndexerMap
{
AppId = app.Definition.Id,
RemoteIndexerId = mapping.RemoteIndexerId,
RemoteIndexerName = mapping.RemoteIndexerName,
IndexerId = mapping.IndexerId
};
_appIndexerMapService.Insert(addMapping);
indexerMappings.Add(addMapping);
}

@ -9,6 +9,6 @@ namespace NzbDrone.Core.Applications
void AddIndexer(IndexerDefinition indexer);
void UpdateIndexer(IndexerDefinition indexer);
void RemoveIndexer(int indexerId);
Dictionary<int, int> GetIndexerMappings();
List<AppIndexerMap> GetIndexerMappings();
}
}

@ -55,12 +55,12 @@ namespace NzbDrone.Core.Applications.Lidarr
return new ValidationResult(failures);
}
public override Dictionary<int, int> GetIndexerMappings()
public override List<AppIndexerMap> GetIndexerMappings()
{
var indexers = _lidarrV1Proxy.GetIndexers(Settings)
.Where(i => i.Implementation == "Newznab" || i.Implementation == "Torznab");
var mappings = new Dictionary<int, int>();
var mappings = new List<AppIndexerMap>();
foreach (var indexer in indexers)
{
@ -71,7 +71,7 @@ namespace NzbDrone.Core.Applications.Lidarr
if (match.Groups["indexer"].Success && int.TryParse(match.Groups["indexer"].Value, out var indexerId))
{
//Add parsed mapping if it's mapped to a Indexer in this Prowlarr instance
mappings.Add(indexer.Id, indexerId);
mappings.Add(new AppIndexerMap { RemoteIndexerId = indexer.Id, IndexerId = indexerId });
}
}
}

@ -0,0 +1,154 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using FluentValidation.Results;
using Newtonsoft.Json.Linq;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Indexers;
namespace NzbDrone.Core.Applications.Mylar
{
public class Mylar : ApplicationBase<MylarSettings>
{
public override string Name => "Mylar";
private readonly IMylarV3Proxy _mylarV3Proxy;
private readonly IConfigFileProvider _configFileProvider;
public Mylar(IMylarV3Proxy lidarrV1Proxy, IConfigFileProvider configFileProvider, IAppIndexerMapService appIndexerMapService, Logger logger)
: base(appIndexerMapService, logger)
{
_mylarV3Proxy = lidarrV1Proxy;
_configFileProvider = configFileProvider;
}
public override ValidationResult Test()
{
var failures = new List<ValidationFailure>();
try
{
failures.AddIfNotNull(_mylarV3Proxy.TestConnection(Settings));
}
catch (WebException ex)
{
_logger.Error(ex, "Unable to send test message");
failures.AddIfNotNull(new ValidationFailure("BaseUrl", "Unable to complete application test, cannot connect to Mylar"));
}
return new ValidationResult(failures);
}
public override List<AppIndexerMap> GetIndexerMappings()
{
var indexers = _mylarV3Proxy.GetIndexers(Settings);
var mappings = new List<AppIndexerMap>();
foreach (var indexer in indexers)
{
if (indexer.Apikey == _configFileProvider.ApiKey)
{
var match = AppIndexerRegex.Match(indexer.Host);
if (match.Groups["indexer"].Success && int.TryParse(match.Groups["indexer"].Value, out var indexerId))
{
//Add parsed mapping if it's mapped to a Indexer in this Prowlarr instance
mappings.Add(new AppIndexerMap { RemoteIndexerName = $"{indexer.Type},{indexer.Name}", IndexerId = indexerId });
}
}
}
return mappings;
}
public override void AddIndexer(IndexerDefinition indexer)
{
if (indexer.Capabilities.Categories.SupportedCategories(Settings.SyncCategories.ToArray()).Any())
{
var lidarrIndexer = BuildMylarIndexer(indexer, indexer.Protocol);
var remoteIndexer = _mylarV3Proxy.AddIndexer(lidarrIndexer, Settings);
_appIndexerMapService.Insert(new AppIndexerMap { AppId = Definition.Id, IndexerId = indexer.Id, RemoteIndexerName = $"{remoteIndexer.Type},{remoteIndexer.Name}" });
}
}
public override void RemoveIndexer(int indexerId)
{
var appMappings = _appIndexerMapService.GetMappingsForApp(Definition.Id);
var indexerMapping = appMappings.FirstOrDefault(m => m.IndexerId == indexerId);
if (indexerMapping != null)
{
//Remove Indexer remotely and then remove the mapping
var indexerProps = indexerMapping.RemoteIndexerName.Split(",");
_mylarV3Proxy.RemoveIndexer(indexerProps[1], (MylarProviderType)Enum.Parse(typeof(MylarProviderType), indexerProps[0]), Settings);
_appIndexerMapService.Delete(indexerMapping.Id);
}
}
public override void UpdateIndexer(IndexerDefinition indexer)
{
_logger.Debug("Updating indexer {0} [{1}]", indexer.Name, indexer.Id);
var appMappings = _appIndexerMapService.GetMappingsForApp(Definition.Id);
var indexerMapping = appMappings.FirstOrDefault(m => m.IndexerId == indexer.Id);
var indexerProps = indexerMapping.RemoteIndexerName.Split(",");
var mylarIndexer = BuildMylarIndexer(indexer, indexer.Protocol, indexerProps[1]);
//Use the old remote id to find the indexer on Mylar incase the update was from a name change in Prowlarr
var remoteIndexer = _mylarV3Proxy.GetIndexer(indexerProps[1], mylarIndexer.Type, Settings);
if (remoteIndexer != null)
{
_logger.Debug("Remote indexer found, syncing with current settings");
if (!mylarIndexer.Equals(remoteIndexer))
{
_mylarV3Proxy.UpdateIndexer(mylarIndexer, Settings);
indexerMapping.RemoteIndexerName = $"{mylarIndexer.Type},{mylarIndexer.Altername}";
_appIndexerMapService.Update(indexerMapping);
}
}
else
{
_appIndexerMapService.Delete(indexerMapping.Id);
if (indexer.Capabilities.Categories.SupportedCategories(Settings.SyncCategories.ToArray()).Any())
{
_logger.Debug("Remote indexer not found, re-adding {0} to Mylar", indexer.Name);
var newRemoteIndexer = _mylarV3Proxy.AddIndexer(mylarIndexer, Settings);
_appIndexerMapService.Insert(new AppIndexerMap { AppId = Definition.Id, IndexerId = indexer.Id, RemoteIndexerName = $"{newRemoteIndexer.Type},{newRemoteIndexer.Name}" });
}
else
{
_logger.Debug("Remote indexer not found for {0}, skipping re-add to Mylar due to indexer capabilities", indexer.Name);
}
}
}
private MylarIndexer BuildMylarIndexer(IndexerDefinition indexer, DownloadProtocol protocol, string originalName = null)
{
var schema = protocol == DownloadProtocol.Usenet ? MylarProviderType.Newznab : MylarProviderType.Torznab;
var lidarrIndexer = new MylarIndexer
{
Name = originalName ?? $"{indexer.Name} (Prowlarr)",
Altername = $"{indexer.Name} (Prowlarr)",
Host = $"{Settings.ProwlarrUrl.TrimEnd('/')}/{indexer.Id}",
Apikey = _configFileProvider.ApiKey,
Categories = string.Join(",", indexer.Capabilities.Categories.SupportedCategories(Settings.SyncCategories.ToArray())),
Enabled = indexer.Enable,
Type = schema,
};
return lidarrIndexer;
}
}
}

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace NzbDrone.Core.Applications.Mylar
{
public class MylarError
{
public int Code { get; set; }
public string Message { get; set; }
}
}

@ -0,0 +1,23 @@
using System;
using NzbDrone.Common.Exceptions;
namespace NzbDrone.Core.Applications.Mylar
{
public class MylarException : NzbDroneException
{
public MylarException(string message)
: base(message)
{
}
public MylarException(string message, params object[] args)
: base(message, args)
{
}
public MylarException(string message, Exception innerException)
: base(message, innerException)
{
}
}
}

@ -0,0 +1,22 @@
namespace NzbDrone.Core.Applications.Mylar
{
public class MylarField
{
public int Order { get; set; }
public string Name { get; set; }
public string Label { get; set; }
public string Unit { get; set; }
public string HelpText { get; set; }
public string HelpLink { get; set; }
public object Value { get; set; }
public string Type { get; set; }
public bool Advanced { get; set; }
public string Section { get; set; }
public string Hidden { get; set; }
public MylarField Clone()
{
return (MylarField)MemberwiseClone();
}
}
}

@ -0,0 +1,49 @@
using System.Collections.Generic;
namespace NzbDrone.Core.Applications.Mylar
{
public class MylarIndexerResponse
{
public bool Success { get; set; }
public MylarIndexerData Data { get; set; }
public MylarError Error { get; set; }
}
public class MylarIndexerData
{
public List<MylarIndexer> Torznabs { get; set; }
public List<MylarIndexer> Newznabs { get; set; }
}
public enum MylarProviderType
{
Newznab,
Torznab
}
public class MylarIndexer
{
public string Name { get; set; }
public string Host { get; set; }
public string Apikey { get; set; }
public string Categories { get; set; }
public bool Enabled { get; set; }
public string Altername { get; set; }
public MylarProviderType Type { get; set; }
public bool Equals(MylarIndexer other)
{
if (ReferenceEquals(null, other))
{
return false;
}
return other.Host == Host &&
other.Apikey == Apikey &&
other.Name == Name &&
other.Categories == Categories &&
other.Enabled == Enabled &&
other.Altername == Altername;
}
}
}

@ -0,0 +1,46 @@
using System.Collections.Generic;
using FluentValidation;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Applications.Mylar
{
public class MylarSettingsValidator : AbstractValidator<MylarSettings>
{
public MylarSettingsValidator()
{
RuleFor(c => c.BaseUrl).IsValidUrl();
RuleFor(c => c.ProwlarrUrl).IsValidUrl();
RuleFor(c => c.ApiKey).NotEmpty();
}
}
public class MylarSettings : IApplicationSettings
{
private static readonly MylarSettingsValidator Validator = new MylarSettingsValidator();
public MylarSettings()
{
ProwlarrUrl = "http://localhost:9696";
BaseUrl = "http://localhost:8090";
SyncCategories = new[] { NewznabStandardCategory.BooksComics.Id };
}
public IEnumerable<int> SyncCategories { get; set; }
[FieldDefinition(0, Label = "Prowlarr Server", HelpText = "Prowlarr server URL as Mylar sees it, including http(s)://, port, and urlbase if needed")]
public string ProwlarrUrl { get; set; }
[FieldDefinition(1, Label = "Mylar Server", HelpText = "Mylar server URL, including http(s):// and port if needed")]
public string BaseUrl { get; set; }
[FieldDefinition(2, Label = "ApiKey", Privacy = PrivacyLevel.ApiKey, HelpText = "The ApiKey generated by Mylar in Settings/Web Interface")]
public string ApiKey { get; set; }
public NzbDroneValidationResult Validate()
{
return new NzbDroneValidationResult(Validator.Validate(this));
}
}
}

@ -0,0 +1,8 @@
namespace NzbDrone.Core.Applications.Mylar
{
public class MylarStatus
{
public bool Success { get; set; }
public MylarError Error { get; set; }
}
}

@ -0,0 +1,190 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using FluentValidation.Results;
using Newtonsoft.Json;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Indexers;
namespace NzbDrone.Core.Applications.Mylar
{
public interface IMylarV3Proxy
{
MylarIndexer AddIndexer(MylarIndexer indexer, MylarSettings settings);
List<MylarIndexer> GetIndexers(MylarSettings settings);
MylarIndexer GetIndexer(string indexerName, MylarProviderType indexerType, MylarSettings settings);
void RemoveIndexer(string indexerName, MylarProviderType indexerType, MylarSettings settings);
MylarIndexer UpdateIndexer(MylarIndexer indexer, MylarSettings settings);
ValidationFailure TestConnection(MylarSettings settings);
}
public class MylarV1Proxy : IMylarV3Proxy
{
private readonly IHttpClient _httpClient;
private readonly Logger _logger;
public MylarV1Proxy(IHttpClient httpClient, Logger logger)
{
_httpClient = httpClient;
_logger = logger;
}
public MylarStatus GetStatus(MylarSettings settings)
{
var request = BuildRequest(settings, "/api", "getVersion", HttpMethod.GET);
return Execute<MylarStatus>(request);
}
public List<MylarIndexer> GetIndexers(MylarSettings settings)
{
var request = BuildRequest(settings, "/api", "listProviders", HttpMethod.GET);
var response = Execute<MylarIndexerResponse>(request);
if (!response.Success)
{
throw new MylarException(string.Format("Mylar Error - Code {0}: {1}", response.Error.Code, response.Error.Message));
}
var indexers = new List<MylarIndexer>();
var torIndexers = response.Data.Torznabs;
torIndexers.ForEach(i => i.Type = MylarProviderType.Torznab);
var nzbIndexers = response.Data.Newznabs;
nzbIndexers.ForEach(i => i.Type = MylarProviderType.Newznab);
indexers.AddRange(torIndexers);
indexers.AddRange(nzbIndexers);
indexers.ForEach(i => i.Altername = i.Name);
return indexers;
}
public MylarIndexer GetIndexer(string indexerName, MylarProviderType indexerType, MylarSettings settings)
{
var indexers = GetIndexers(settings);
return indexers.SingleOrDefault(i => i.Name == indexerName && i.Type == indexerType);
}
public void RemoveIndexer(string indexerName, MylarProviderType indexerType, MylarSettings settings)
{
var parameters = new Dictionary<string, string>
{
{ "name", indexerName },
{ "providertype", indexerType.ToString().ToLower() }
};
var request = BuildRequest(settings, "/api", "delProvider", HttpMethod.GET, parameters);
CheckForError(Execute<MylarStatus>(request));
}
public MylarIndexer AddIndexer(MylarIndexer indexer, MylarSettings settings)
{
var parameters = new Dictionary<string, string>
{
{ "name", indexer.Name },
{ "providertype", indexer.Type.ToString().ToLower() },
{ "host", indexer.Host },
{ "prov_apikey", indexer.Apikey },
{ "enabled", indexer.Enabled.ToString().ToLower() },
{ "categories", indexer.Categories }
};
var request = BuildRequest(settings, "/api", "addProvider", HttpMethod.GET, parameters);
CheckForError(Execute<MylarStatus>(request));
return indexer;
}
public MylarIndexer UpdateIndexer(MylarIndexer indexer, MylarSettings settings)
{
var parameters = new Dictionary<string, string>
{
{ "name", indexer.Name },
{ "providertype", indexer.Type.ToString().ToLower() },
{ "host", indexer.Host },
{ "prov_apikey", indexer.Apikey },
{ "enabled", indexer.Enabled.ToString().ToLower() },
{ "categories", indexer.Categories },
{ "altername", indexer.Altername }
};
var request = BuildRequest(settings, "/api", "changeProvider", HttpMethod.GET, parameters);
CheckForError(Execute<MylarStatus>(request));
return indexer;
}
private void CheckForError(MylarStatus response)
{
if (!response.Success)
{
throw new MylarException(string.Format("Mylar Error - Code {0}: {1}", response.Error.Code, response.Error.Message));
}
}
public ValidationFailure TestConnection(MylarSettings settings)
{
try
{
var status = GetStatus(settings);
if (!status.Success)
{
return new ValidationFailure("ApiKey", status.Error.Message);
}
}
catch (HttpException ex)
{
_logger.Error(ex, "Unable to send test message");
return new ValidationFailure("BaseUrl", "Unable to complete application test");
}
catch (Exception ex)
{
_logger.Error(ex, "Unable to send test message");
return new ValidationFailure("", "Unable to send test message");
}
return null;
}
private HttpRequest BuildRequest(MylarSettings settings, string resource, string command, HttpMethod method, Dictionary<string, string> parameters = null)
{
var baseUrl = settings.BaseUrl.TrimEnd('/');
var requestBuilder = new HttpRequestBuilder(baseUrl).Resource(resource)
.AddQueryParam("cmd", command)
.AddQueryParam("apikey", settings.ApiKey);
if (parameters != null)
{
foreach (var param in parameters)
{
requestBuilder.AddQueryParam(param.Key, param.Value);
}
}
var request = requestBuilder.Build();
request.Headers.ContentType = "application/json";
request.Method = method;
request.AllowAutoRedirect = true;
return request;
}
private TResource Execute<TResource>(HttpRequest request)
where TResource : new()
{
var response = _httpClient.Execute(request);
var results = JsonConvert.DeserializeObject<TResource>(response.Content);
return results;
}
}
}

@ -55,12 +55,12 @@ namespace NzbDrone.Core.Applications.Radarr
return new ValidationResult(failures);
}
public override Dictionary<int, int> GetIndexerMappings()
public override List<AppIndexerMap> GetIndexerMappings()
{
var indexers = _radarrV3Proxy.GetIndexers(Settings)
.Where(i => i.Implementation == "Newznab" || i.Implementation == "Torznab");
var mappings = new Dictionary<int, int>();
var mappings = new List<AppIndexerMap>();
foreach (var indexer in indexers)
{
@ -71,7 +71,7 @@ namespace NzbDrone.Core.Applications.Radarr
if (match.Groups["indexer"].Success && int.TryParse(match.Groups["indexer"].Value, out var indexerId))
{
//Add parsed mapping if it's mapped to a Indexer in this Prowlarr instance
mappings.Add(indexer.Id, indexerId);
mappings.Add(new AppIndexerMap { RemoteIndexerId = indexer.Id, IndexerId = indexerId });
}
}
}

@ -55,12 +55,12 @@ namespace NzbDrone.Core.Applications.Readarr
return new ValidationResult(failures);
}
public override Dictionary<int, int> GetIndexerMappings()
public override List<AppIndexerMap> GetIndexerMappings()
{
var indexers = _readarrV1Proxy.GetIndexers(Settings)
.Where(i => i.Implementation == "Newznab" || i.Implementation == "Torznab");
.Where(i => i.Implementation == "Newznab" || i.Implementation == "Torznab");
var mappings = new Dictionary<int, int>();
var mappings = new List<AppIndexerMap>();
foreach (var indexer in indexers)
{
@ -71,7 +71,7 @@ namespace NzbDrone.Core.Applications.Readarr
if (match.Groups["indexer"].Success && int.TryParse(match.Groups["indexer"].Value, out var indexerId))
{
//Add parsed mapping if it's mapped to a Indexer in this Prowlarr instance
mappings.Add(indexer.Id, indexerId);
mappings.Add(new AppIndexerMap { RemoteIndexerId = indexer.Id, IndexerId = indexerId });
}
}
}

@ -55,12 +55,12 @@ namespace NzbDrone.Core.Applications.Sonarr
return new ValidationResult(failures);
}
public override Dictionary<int, int> GetIndexerMappings()
public override List<AppIndexerMap> GetIndexerMappings()
{
var indexers = _sonarrV3Proxy.GetIndexers(Settings)
.Where(i => i.Implementation == "Newznab" || i.Implementation == "Torznab");
var mappings = new Dictionary<int, int>();
var mappings = new List<AppIndexerMap>();
foreach (var indexer in indexers)
{
@ -71,7 +71,7 @@ namespace NzbDrone.Core.Applications.Sonarr
if (match.Groups["indexer"].Success && int.TryParse(match.Groups["indexer"].Value, out var indexerId))
{
//Add parsed mapping if it's mapped to a Indexer in this Prowlarr instance
mappings.Add(indexer.Id, indexerId);
mappings.Add(new AppIndexerMap { RemoteIndexerId = indexer.Id, IndexerId = indexerId });
}
}
}

@ -0,0 +1,14 @@
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(11)]
public class app_indexer_remote_name : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Alter.Table("ApplicationIndexerMapping").AddColumn("RemoteIndexerName").AsString().Nullable();
}
}
}
Loading…
Cancel
Save