From abcf4f7d8ffed5d884406f5981ddb36c78abec47 Mon Sep 17 00:00:00 2001 From: Robert Dailey Date: Sun, 2 Jul 2023 17:25:11 -0500 Subject: [PATCH] feat: api_key and base_url are now optional These can be implicitly set via secrets that follow a naming convention. --- CHANGELOG.md | 5 + schemas/config-schema.json | 2 - .../Config/ConfigAutofacModule.cs | 4 + .../ConfigYamlDataObjectsValidation.cs | 22 +- .../Config/Parsing/ConfigurationLoader.cs | 19 +- .../PostProcessing/IConfigPostProcessor.cs | 6 + .../ImplicitUrlAndKeyPostProcessor.cs | 57 +++++ .../Config/Secrets/ISecretsProvider.cs | 4 +- .../Config/Secrets/SecretsProvider.cs | 26 +- .../ImplicitUrlAndKeyPostProcessorTest.cs | 242 ++++++++++++++++++ .../Config/YamlConfigValidatorTest.cs | 39 ++- 11 files changed, 387 insertions(+), 39 deletions(-) create mode 100644 src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/IConfigPostProcessor.cs create mode 100644 src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessor.cs create mode 100644 src/tests/Recyclarr.TrashLib.Tests/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessorTest.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 181ad4ce..a173ffce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `base_url` and `api_key` are now optional. These can be implicitly set via secrets that follow a + naming convention. See the Secrets reference page on the wiki for details. + ## [5.1.1] - 2023-06-29 ### Fixed diff --git a/schemas/config-schema.json b/schemas/config-schema.json index fd145c32..b38e9871 100644 --- a/schemas/config-schema.json +++ b/schemas/config-schema.json @@ -120,7 +120,6 @@ "radarr_instance": { "type": "object", "additionalProperties": false, - "required": ["base_url", "api_key"], "properties": { "base_url": { "$ref": "#/$defs/base_url", @@ -155,7 +154,6 @@ "sonarr_instance": { "type": "object", "additionalProperties": false, - "required": ["base_url", "api_key"], "properties": { "base_url": { "$ref": "#/$defs/base_url", diff --git a/src/Recyclarr.TrashLib/Config/ConfigAutofacModule.cs b/src/Recyclarr.TrashLib/Config/ConfigAutofacModule.cs index 9319c492..0bcd4948 100644 --- a/src/Recyclarr.TrashLib/Config/ConfigAutofacModule.cs +++ b/src/Recyclarr.TrashLib/Config/ConfigAutofacModule.cs @@ -2,6 +2,7 @@ using Autofac; using FluentValidation; using Recyclarr.TrashLib.Config.Listers; using Recyclarr.TrashLib.Config.Parsing; +using Recyclarr.TrashLib.Config.Parsing.PostProcessing; using Recyclarr.TrashLib.Config.Secrets; using Recyclarr.TrashLib.Config.Services; using Recyclarr.TrashLib.Config.Yaml; @@ -38,5 +39,8 @@ public class ConfigAutofacModule : Module // Config Listers builder.RegisterType().Keyed(ConfigCategory.Templates); builder.RegisterType().Keyed(ConfigCategory.Local); + + // Config Post Processors + builder.RegisterType().As(); } } diff --git a/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs b/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs index 24222f3f..3e52b8b7 100644 --- a/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs +++ b/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs @@ -9,19 +9,21 @@ public class ServiceConfigYamlValidator : AbstractValidator { public ServiceConfigYamlValidator() { - RuleFor(x => x.BaseUrl).NotEmpty().NotNull() - .WithMessage("'base_url' is required and must not be empty"); + RuleFor(x => x.BaseUrl).Cascade(CascadeMode.Stop) + .NotEmpty().Must(x => x!.StartsWith("http")) + .WithMessage("{PropertyName} must start with 'http' or 'https'") + .WithName("base_url"); - RuleFor(x => x.BaseUrl).NotEmpty().Must(x => x is not null && x.StartsWith("http")) - .WithMessage("'base_url' must start with 'http' or 'https'"); + // RuleFor(x => x.BaseUrl) + // .When(x => x.BaseUrl is {Length: > 0}, ApplyConditionTo.CurrentValidator) + // .WithMessage("{PropertyName} must start with 'http' or 'https'"); - RuleFor(x => x.ApiKey).NotEmpty() - .WithMessage("'api_key' is required"); + RuleFor(x => x.ApiKey).NotEmpty().WithName("api_key"); - RuleFor(x => x.CustomFormats).NotEmpty() - .When(x => x.CustomFormats is not null) - .WithName("custom_formats") - .ForEach(x => x.SetValidator(new CustomFormatConfigYamlValidator())); + RuleFor(x => x.CustomFormats) + .NotEmpty().When(x => x.CustomFormats is not null) + .ForEach(x => x.SetValidator(new CustomFormatConfigYamlValidator())) + .WithName("custom_formats"); RuleFor(x => x.QualityDefinition) .SetNonNullableValidator(new QualitySizeConfigYamlValidator()); diff --git a/src/Recyclarr.TrashLib/Config/Parsing/ConfigurationLoader.cs b/src/Recyclarr.TrashLib/Config/Parsing/ConfigurationLoader.cs index 8648c0b2..3b4238f7 100644 --- a/src/Recyclarr.TrashLib/Config/Parsing/ConfigurationLoader.cs +++ b/src/Recyclarr.TrashLib/Config/Parsing/ConfigurationLoader.cs @@ -1,5 +1,6 @@ using System.IO.Abstractions; using AutoMapper; +using Recyclarr.TrashLib.Config.Parsing.PostProcessing; using Recyclarr.TrashLib.Config.Services; namespace Recyclarr.TrashLib.Config.Parsing; @@ -9,12 +10,18 @@ public class ConfigurationLoader : IConfigurationLoader private readonly ConfigParser _parser; private readonly IMapper _mapper; private readonly ConfigValidationExecutor _validator; + private readonly IEnumerable _postProcessors; - public ConfigurationLoader(ConfigParser parser, IMapper mapper, ConfigValidationExecutor validator) + public ConfigurationLoader( + ConfigParser parser, + IMapper mapper, + ConfigValidationExecutor validator, + IEnumerable postProcessors) { _parser = parser; _mapper = mapper; _validator = validator; + _postProcessors = postProcessors; } public ICollection LoadMany( @@ -52,6 +59,8 @@ public class ConfigurationLoader : IConfigurationLoader return Array.Empty(); } + config = _postProcessors.Aggregate(config, (current, processor) => processor.Process(current)); + if (!_validator.Validate(config)) { return Array.Empty(); @@ -61,20 +70,18 @@ public class ConfigurationLoader : IConfigurationLoader if (desiredServiceType is null or SupportedServices.Radarr) { - convertedConfigs.AddRange( - ValidateAndMap(config.Radarr)); + convertedConfigs.AddRange(MapConfigs(config.Radarr)); } if (desiredServiceType is null or SupportedServices.Sonarr) { - convertedConfigs.AddRange( - ValidateAndMap(config.Sonarr)); + convertedConfigs.AddRange(MapConfigs(config.Sonarr)); } return convertedConfigs; } - private IEnumerable ValidateAndMap( + private IEnumerable MapConfigs( IReadOnlyDictionary? configs) where TServiceConfig : ServiceConfiguration where TConfigYaml : ServiceConfigYaml diff --git a/src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/IConfigPostProcessor.cs b/src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/IConfigPostProcessor.cs new file mode 100644 index 00000000..00b13720 --- /dev/null +++ b/src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/IConfigPostProcessor.cs @@ -0,0 +1,6 @@ +namespace Recyclarr.TrashLib.Config.Parsing.PostProcessing; + +public interface IConfigPostProcessor +{ + RootConfigYaml Process(RootConfigYaml config); +} diff --git a/src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessor.cs b/src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessor.cs new file mode 100644 index 00000000..752e70de --- /dev/null +++ b/src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessor.cs @@ -0,0 +1,57 @@ +using Recyclarr.TrashLib.Config.Secrets; + +namespace Recyclarr.TrashLib.Config.Parsing.PostProcessing; + +public class ImplicitUrlAndKeyPostProcessor : IConfigPostProcessor +{ + private readonly ILogger _log; + private readonly ISecretsProvider _secrets; + + public ImplicitUrlAndKeyPostProcessor(ILogger log, ISecretsProvider secrets) + { + _log = log; + _secrets = secrets; + } + + public RootConfigYaml Process(RootConfigYaml config) + { + return new RootConfigYaml + { + Radarr = ProcessService(config.Radarr), + Sonarr = ProcessService(config.Sonarr) + }; + } + + private IReadOnlyDictionary? ProcessService(IReadOnlyDictionary? services) + where T : ServiceConfigYaml + { + if (services is null) + { + return null; + } + + var updatedServices = new Dictionary(); + foreach (var (name, service) in services) + { + updatedServices.Add(name, FillUrlAndKey(name, service)); + } + + return updatedServices; + } + + private T FillUrlAndKey(string instanceName, T config) + where T : ServiceConfigYaml + { + return config with + { + ApiKey = config.ApiKey ?? GetSecret(instanceName, "api_key"), + BaseUrl = config.BaseUrl ?? GetSecret(instanceName, "base_url") + }; + } + + private string? GetSecret(string instanceName, string property) + { + _log.Debug("Obtain {Property} implicitly for instance {InstanceName}", property, instanceName); + return _secrets.Secrets.GetValueOrDefault($"{instanceName}_{property}"); + } +} diff --git a/src/Recyclarr.TrashLib/Config/Secrets/ISecretsProvider.cs b/src/Recyclarr.TrashLib/Config/Secrets/ISecretsProvider.cs index 528f9ef0..74231142 100644 --- a/src/Recyclarr.TrashLib/Config/Secrets/ISecretsProvider.cs +++ b/src/Recyclarr.TrashLib/Config/Secrets/ISecretsProvider.cs @@ -1,8 +1,6 @@ -using System.Collections.Immutable; - namespace Recyclarr.TrashLib.Config.Secrets; public interface ISecretsProvider { - IImmutableDictionary Secrets { get; } + IReadOnlyDictionary Secrets { get; } } diff --git a/src/Recyclarr.TrashLib/Config/Secrets/SecretsProvider.cs b/src/Recyclarr.TrashLib/Config/Secrets/SecretsProvider.cs index 2c4324d1..446e8c9d 100644 --- a/src/Recyclarr.TrashLib/Config/Secrets/SecretsProvider.cs +++ b/src/Recyclarr.TrashLib/Config/Secrets/SecretsProvider.cs @@ -1,4 +1,3 @@ -using System.Collections.Immutable; using Recyclarr.Common.Extensions; using Recyclarr.TrashLib.Startup; using YamlDotNet.Serialization; @@ -7,33 +6,28 @@ namespace Recyclarr.TrashLib.Config.Secrets; public class SecretsProvider : ISecretsProvider { - public IImmutableDictionary Secrets => _secrets.Value; + public IReadOnlyDictionary Secrets => _secrets.Value; private readonly IAppPaths _paths; - private readonly Lazy> _secrets; + private readonly Lazy> _secrets; public SecretsProvider(IAppPaths paths) { _paths = paths; - _secrets = new Lazy>(LoadSecretsFile); + _secrets = new Lazy>(LoadSecretsFile); } - private IImmutableDictionary LoadSecretsFile() + private Dictionary LoadSecretsFile() { - var result = new Dictionary(); - var yamlPath = _paths.AppDataDirectory.YamlFile("secrets"); - if (yamlPath is not null) + if (yamlPath is null) { - using var stream = yamlPath.OpenText(); - var deserializer = new DeserializerBuilder().Build(); - var dict = deserializer.Deserialize?>(stream); - if (dict is not null) - { - result = dict; - } + return new Dictionary(); } - return result.ToImmutableDictionary(); + using var stream = yamlPath.OpenText(); + var deserializer = new DeserializerBuilder().Build(); + var result = deserializer.Deserialize?>(stream); + return result ?? new Dictionary(); } } diff --git a/src/tests/Recyclarr.TrashLib.Tests/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessorTest.cs b/src/tests/Recyclarr.TrashLib.Tests/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessorTest.cs new file mode 100644 index 00000000..9e1148ab --- /dev/null +++ b/src/tests/Recyclarr.TrashLib.Tests/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessorTest.cs @@ -0,0 +1,242 @@ +using Recyclarr.TrashLib.Config.Parsing; +using Recyclarr.TrashLib.Config.Parsing.PostProcessing; +using Recyclarr.TrashLib.Config.Secrets; + +namespace Recyclarr.TrashLib.Tests.Config.Parsing.PostProcessing; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ImplicitUrlAndKeyPostProcessorTest +{ + [Test, AutoMockData] + public void Update_only_base_url_when_absent( + [Frozen] ISecretsProvider secrets, + ImplicitUrlAndKeyPostProcessor sut) + { + secrets.Secrets.Returns(new Dictionary + { + {"instance1_base_url", "secret_base_url_1"}, + {"instance1_api_key", "secret_api_key_1"}, + {"instance2_base_url", "secret_base_url_2"}, + {"instance2_api_key", "secret_api_key_2"} + }); + + var result = sut.Process(new RootConfigYaml + { + Radarr = new Dictionary + { + { + "instance1", new RadarrConfigYaml + { + ApiKey = "explicit_base_url" + } + } + }, + Sonarr = new Dictionary + { + { + "instance2", new SonarrConfigYaml + { + ApiKey = "explicit_base_url" + } + } + } + }); + + result.Should().BeEquivalentTo(new RootConfigYaml + { + Radarr = new Dictionary + { + { + "instance1", new RadarrConfigYaml + { + ApiKey = "explicit_base_url", + BaseUrl = "secret_base_url_1" + } + } + }, + Sonarr = new Dictionary + { + { + "instance2", new SonarrConfigYaml + { + ApiKey = "explicit_base_url", + BaseUrl = "secret_base_url_2" + } + } + } + }); + } + + [Test, AutoMockData] + public void Update_only_api_key_when_absent( + [Frozen] ISecretsProvider secrets, + ImplicitUrlAndKeyPostProcessor sut) + { + secrets.Secrets.Returns(new Dictionary + { + {"instance1_base_url", "secret_base_url_1"}, + {"instance1_api_key", "secret_api_key_1"}, + {"instance2_base_url", "secret_base_url_2"}, + {"instance2_api_key", "secret_api_key_2"} + }); + + var result = sut.Process(new RootConfigYaml + { + Radarr = new Dictionary + { + { + "instance1", new RadarrConfigYaml + { + BaseUrl = "explicit_base_url" + } + } + }, + Sonarr = new Dictionary + { + { + "instance2", new SonarrConfigYaml + { + BaseUrl = "explicit_base_url" + } + } + } + }); + + result.Should().BeEquivalentTo(new RootConfigYaml + { + Radarr = new Dictionary + { + { + "instance1", new RadarrConfigYaml + { + ApiKey = "secret_api_key_1", + BaseUrl = "explicit_base_url" + } + } + }, + Sonarr = new Dictionary + { + { + "instance2", new SonarrConfigYaml + { + ApiKey = "secret_api_key_2", + BaseUrl = "explicit_base_url" + } + } + } + }); + } + + [Test, AutoMockData] + public void Update_base_url_and_api_key_when_absent( + [Frozen] ISecretsProvider secrets, + ImplicitUrlAndKeyPostProcessor sut) + { + secrets.Secrets.Returns(new Dictionary + { + {"instance1_base_url", "secret_base_url_1"}, + {"instance1_api_key", "secret_api_key_1"}, + {"instance2_base_url", "secret_base_url_2"}, + {"instance2_api_key", "secret_api_key_2"} + }); + + var result = sut.Process(new RootConfigYaml + { + Radarr = new Dictionary + { + {"instance1", new RadarrConfigYaml()} + }, + Sonarr = new Dictionary + { + {"instance2", new SonarrConfigYaml()} + } + }); + + result.Should().BeEquivalentTo(new RootConfigYaml + { + Radarr = new Dictionary + { + { + "instance1", new RadarrConfigYaml + { + ApiKey = "secret_api_key_1", + BaseUrl = "secret_base_url_1" + } + } + }, + Sonarr = new Dictionary + { + { + "instance2", new SonarrConfigYaml + { + ApiKey = "secret_api_key_2", + BaseUrl = "secret_base_url_2" + } + } + } + }); + } + + [Test, AutoMockData] + public void Change_nothing_when_all_explicitly_provided( + [Frozen] ISecretsProvider secrets, + ImplicitUrlAndKeyPostProcessor sut) + { + secrets.Secrets.Returns(new Dictionary + { + {"instance1_base_url", "secret_base_url_1"}, + {"instance1_api_key", "secret_api_key_1"}, + {"instance2_base_url", "secret_base_url_2"}, + {"instance2_api_key", "secret_api_key_2"} + }); + + var result = sut.Process(new RootConfigYaml + { + Radarr = new Dictionary + { + { + "instance1", new RadarrConfigYaml + { + BaseUrl = "explicit_base_url", + ApiKey = "explicit_api_key" + } + } + }, + Sonarr = new Dictionary + { + { + "instance2", new SonarrConfigYaml + { + BaseUrl = "explicit_base_url", + ApiKey = "explicit_api_key" + } + } + } + }); + + result.Should().BeEquivalentTo(new RootConfigYaml + { + Radarr = new Dictionary + { + { + "instance1", new RadarrConfigYaml + { + ApiKey = "explicit_api_key", + BaseUrl = "explicit_base_url" + } + } + }, + Sonarr = new Dictionary + { + { + "instance2", new SonarrConfigYaml + { + ApiKey = "explicit_api_key", + BaseUrl = "explicit_base_url" + } + } + } + }); + } +} diff --git a/src/tests/Recyclarr.TrashLib.Tests/Config/YamlConfigValidatorTest.cs b/src/tests/Recyclarr.TrashLib.Tests/Config/YamlConfigValidatorTest.cs index ec1a46b9..b1222671 100644 --- a/src/tests/Recyclarr.TrashLib.Tests/Config/YamlConfigValidatorTest.cs +++ b/src/tests/Recyclarr.TrashLib.Tests/Config/YamlConfigValidatorTest.cs @@ -81,7 +81,7 @@ public class YamlConfigValidatorTest : TrashLibIntegrationFixture var config = new ServiceConfigYaml { ApiKey = "valid", - BaseUrl = "about:empty", + BaseUrl = "", CustomFormats = new List { new() @@ -105,7 +105,42 @@ public class YamlConfigValidatorTest : TrashLibIntegrationFixture var validator = Resolve(); var result = validator.TestValidate(config); - result.ShouldHaveValidationErrorFor(x => x.BaseUrl); + result.ShouldHaveValidationErrorFor(x => x.BaseUrl) + .WithErrorMessage("'base_url' must not be empty."); + } + + [Test] + public void Validation_failure_when_base_url_not_start_with_http() + { + var config = new ServiceConfigYaml + { + ApiKey = "valid", + BaseUrl = "ftp://foo.com", + CustomFormats = new List + { + new() + { + TrashIds = new[] {"valid"}, + QualityProfiles = new List + { + new() + { + Name = "valid" + } + } + } + }, + QualityDefinition = new QualitySizeConfigYaml + { + Type = "valid" + } + }; + + var validator = Resolve(); + var result = validator.TestValidate(config); + + result.ShouldHaveValidationErrorFor(x => x.BaseUrl) + .WithErrorMessage("base_url must start with 'http' or 'https'"); } public static string FirstCf { get; } = $"{nameof(ServiceConfigYaml.CustomFormats)}[0].";