feat: api_key and base_url are now optional

These can be implicitly set via secrets that follow a naming convention.

CHANGELOG.md

@@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- `base_url` and `api_key` are now optional. These can be implicitly set via secrets that follow a
+  naming convention. See the Secrets reference page on the wiki for details.
+
 ## [5.1.1] - 2023-06-29
 
 ### Fixed

schemas/config-schema.json

@@ -120,7 +120,6 @@
     "radarr_instance": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["base_url", "api_key"],
       "properties": {
         "base_url": {
           "$ref": "#/$defs/base_url",
@@ -155,7 +154,6 @@
     "sonarr_instance": {
       "type": "object",
       "additionalProperties": false,
-      "required": ["base_url", "api_key"],
       "properties": {
         "base_url": {
           "$ref": "#/$defs/base_url",

src/Recyclarr.TrashLib/Config/ConfigAutofacModule.cs

@@ -2,6 +2,7 @@ using Autofac;
 using FluentValidation;
 using Recyclarr.TrashLib.Config.Listers;
 using Recyclarr.TrashLib.Config.Parsing;
+using Recyclarr.TrashLib.Config.Parsing.PostProcessing;
 using Recyclarr.TrashLib.Config.Secrets;
 using Recyclarr.TrashLib.Config.Services;
 using Recyclarr.TrashLib.Config.Yaml;
@@ -38,5 +39,8 @@ public class ConfigAutofacModule : Module
         // Config Listers
         builder.RegisterType<ConfigTemplateLister>().Keyed<IConfigLister>(ConfigCategory.Templates);
         builder.RegisterType<ConfigLocalLister>().Keyed<IConfigLister>(ConfigCategory.Local);
+
+        // Config Post Processors
+        builder.RegisterType<ImplicitUrlAndKeyPostProcessor>().As<IConfigPostProcessor>();
     }
 }

src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs

@@ -9,19 +9,21 @@ public class ServiceConfigYamlValidator : AbstractValidator<ServiceConfigYaml>
 {
     public ServiceConfigYamlValidator()
     {
-        RuleFor(x => x.BaseUrl).NotEmpty().NotNull()
-            .WithMessage("'base_url' is required and must not be empty");
+        RuleFor(x => x.BaseUrl).Cascade(CascadeMode.Stop)
+            .NotEmpty().Must(x => x!.StartsWith("http"))
+            .WithMessage("{PropertyName} must start with 'http' or 'https'")
+            .WithName("base_url");
 
-        RuleFor(x => x.BaseUrl).NotEmpty().Must(x => x is not null && x.StartsWith("http"))
-            .WithMessage("'base_url' must start with 'http' or 'https'");
+        // RuleFor(x => x.BaseUrl)
+        //     .When(x => x.BaseUrl is {Length: > 0}, ApplyConditionTo.CurrentValidator)
+        //     .WithMessage("{PropertyName} must start with 'http' or 'https'");
 
-        RuleFor(x => x.ApiKey).NotEmpty()
-            .WithMessage("'api_key' is required");
+        RuleFor(x => x.ApiKey).NotEmpty().WithName("api_key");
 
-        RuleFor(x => x.CustomFormats).NotEmpty()
-            .When(x => x.CustomFormats is not null)
-            .WithName("custom_formats")
-            .ForEach(x => x.SetValidator(new CustomFormatConfigYamlValidator()));
+        RuleFor(x => x.CustomFormats)
+            .NotEmpty().When(x => x.CustomFormats is not null)
+            .ForEach(x => x.SetValidator(new CustomFormatConfigYamlValidator()))
+            .WithName("custom_formats");
 
         RuleFor(x => x.QualityDefinition)
             .SetNonNullableValidator(new QualitySizeConfigYamlValidator());

src/Recyclarr.TrashLib/Config/Parsing/ConfigurationLoader.cs

@@ -1,5 +1,6 @@
 using System.IO.Abstractions;
 using AutoMapper;
+using Recyclarr.TrashLib.Config.Parsing.PostProcessing;
 using Recyclarr.TrashLib.Config.Services;
 
 namespace Recyclarr.TrashLib.Config.Parsing;
@@ -9,12 +10,18 @@ public class ConfigurationLoader : IConfigurationLoader
     private readonly ConfigParser _parser;
     private readonly IMapper _mapper;
     private readonly ConfigValidationExecutor _validator;
+    private readonly IEnumerable<IConfigPostProcessor> _postProcessors;
 
-    public ConfigurationLoader(ConfigParser parser, IMapper mapper, ConfigValidationExecutor validator)
+    public ConfigurationLoader(
+        ConfigParser parser,
+        IMapper mapper,
+        ConfigValidationExecutor validator,
+        IEnumerable<IConfigPostProcessor> postProcessors)
     {
         _parser = parser;
         _mapper = mapper;
         _validator = validator;
+        _postProcessors = postProcessors;
     }
 
     public ICollection<IServiceConfiguration> LoadMany(
@@ -52,6 +59,8 @@ public class ConfigurationLoader : IConfigurationLoader
             return Array.Empty<IServiceConfiguration>();
         }
 
+        config = _postProcessors.Aggregate(config, (current, processor) => processor.Process(current));
+
         if (!_validator.Validate(config))
         {
             return Array.Empty<IServiceConfiguration>();
@@ -61,20 +70,18 @@ public class ConfigurationLoader : IConfigurationLoader
 
         if (desiredServiceType is null or SupportedServices.Radarr)
         {
-            convertedConfigs.AddRange(
-                ValidateAndMap<RadarrConfigYaml, RadarrConfiguration>(config.Radarr));
+            convertedConfigs.AddRange(MapConfigs<RadarrConfigYaml, RadarrConfiguration>(config.Radarr));
         }
 
         if (desiredServiceType is null or SupportedServices.Sonarr)
         {
-            convertedConfigs.AddRange(
-                ValidateAndMap<SonarrConfigYaml, SonarrConfiguration>(config.Sonarr));
+            convertedConfigs.AddRange(MapConfigs<SonarrConfigYaml, SonarrConfiguration>(config.Sonarr));
         }
 
         return convertedConfigs;
     }
 
-    private IEnumerable<IServiceConfiguration> ValidateAndMap<TConfigYaml, TServiceConfig>(
+    private IEnumerable<IServiceConfiguration> MapConfigs<TConfigYaml, TServiceConfig>(
         IReadOnlyDictionary<string, TConfigYaml>? configs)
         where TServiceConfig : ServiceConfiguration
         where TConfigYaml : ServiceConfigYaml

src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/IConfigPostProcessor.cs

@@ -0,0 +1,6 @@
+namespace Recyclarr.TrashLib.Config.Parsing.PostProcessing;
+
+public interface IConfigPostProcessor
+{
+    RootConfigYaml Process(RootConfigYaml config);
+}

src/Recyclarr.TrashLib/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessor.cs

@@ -0,0 +1,57 @@
+using Recyclarr.TrashLib.Config.Secrets;
+
+namespace Recyclarr.TrashLib.Config.Parsing.PostProcessing;
+
+public class ImplicitUrlAndKeyPostProcessor : IConfigPostProcessor
+{
+    private readonly ILogger _log;
+    private readonly ISecretsProvider _secrets;
+
+    public ImplicitUrlAndKeyPostProcessor(ILogger log, ISecretsProvider secrets)
+    {
+        _log = log;
+        _secrets = secrets;
+    }
+
+    public RootConfigYaml Process(RootConfigYaml config)
+    {
+        return new RootConfigYaml
+        {
+            Radarr = ProcessService(config.Radarr),
+            Sonarr = ProcessService(config.Sonarr)
+        };
+    }
+
+    private IReadOnlyDictionary<string, T>? ProcessService<T>(IReadOnlyDictionary<string, T>? services)
+        where T : ServiceConfigYaml
+    {
+        if (services is null)
+        {
+            return null;
+        }
+
+        var updatedServices = new Dictionary<string, T>();
+        foreach (var (name, service) in services)
+        {
+            updatedServices.Add(name, FillUrlAndKey(name, service));
+        }
+
+        return updatedServices;
+    }
+
+    private T FillUrlAndKey<T>(string instanceName, T config)
+        where T : ServiceConfigYaml
+    {
+        return config with
+        {
+            ApiKey = config.ApiKey ?? GetSecret(instanceName, "api_key"),
+            BaseUrl = config.BaseUrl ?? GetSecret(instanceName, "base_url")
+        };
+    }
+
+    private string? GetSecret(string instanceName, string property)
+    {
+        _log.Debug("Obtain {Property} implicitly for instance {InstanceName}", property, instanceName);
+        return _secrets.Secrets.GetValueOrDefault($"{instanceName}_{property}");
+    }
+}

src/Recyclarr.TrashLib/Config/Secrets/ISecretsProvider.cs

@@ -1,8 +1,6 @@
-using System.Collections.Immutable;
-
 namespace Recyclarr.TrashLib.Config.Secrets;
 
 public interface ISecretsProvider
 {
-    IImmutableDictionary<string, string> Secrets { get; }
+    IReadOnlyDictionary<string, string> Secrets { get; }
 }

src/Recyclarr.TrashLib/Config/Secrets/SecretsProvider.cs

@@ -1,4 +1,3 @@
-using System.Collections.Immutable;
 using Recyclarr.Common.Extensions;
 using Recyclarr.TrashLib.Startup;
 using YamlDotNet.Serialization;
@@ -7,33 +6,28 @@ namespace Recyclarr.TrashLib.Config.Secrets;
 
 public class SecretsProvider : ISecretsProvider
 {
-    public IImmutableDictionary<string, string> Secrets => _secrets.Value;
+    public IReadOnlyDictionary<string, string> Secrets => _secrets.Value;
 
     private readonly IAppPaths _paths;
-    private readonly Lazy<IImmutableDictionary<string, string>> _secrets;
+    private readonly Lazy<Dictionary<string, string>> _secrets;
 
     public SecretsProvider(IAppPaths paths)
     {
         _paths = paths;
-        _secrets = new Lazy<IImmutableDictionary<string, string>>(LoadSecretsFile);
+        _secrets = new Lazy<Dictionary<string, string>>(LoadSecretsFile);
     }
 
-    private IImmutableDictionary<string, string> LoadSecretsFile()
+    private Dictionary<string, string> LoadSecretsFile()
     {
-        var result = new Dictionary<string, string>();
-
         var yamlPath = _paths.AppDataDirectory.YamlFile("secrets");
-        if (yamlPath is not null)
+        if (yamlPath is null)
         {
-            using var stream = yamlPath.OpenText();
-            var deserializer = new DeserializerBuilder().Build();
-            var dict = deserializer.Deserialize<Dictionary<string, string>?>(stream);
-            if (dict is not null)
-            {
-                result = dict;
-            }
+            return new Dictionary<string, string>();
         }
 
-        return result.ToImmutableDictionary();
+        using var stream = yamlPath.OpenText();
+        var deserializer = new DeserializerBuilder().Build();
+        var result = deserializer.Deserialize<Dictionary<string, string>?>(stream);
+        return result ?? new Dictionary<string, string>();
     }
 }

src/tests/Recyclarr.TrashLib.Tests/Config/Parsing/PostProcessing/ImplicitUrlAndKeyPostProcessorTest.cs

@@ -0,0 +1,242 @@
+using Recyclarr.TrashLib.Config.Parsing;
+using Recyclarr.TrashLib.Config.Parsing.PostProcessing;
+using Recyclarr.TrashLib.Config.Secrets;
+
+namespace Recyclarr.TrashLib.Tests.Config.Parsing.PostProcessing;
+
+[TestFixture]
+[Parallelizable(ParallelScope.All)]
+public class ImplicitUrlAndKeyPostProcessorTest
+{
+    [Test, AutoMockData]
+    public void Update_only_base_url_when_absent(
+        [Frozen] ISecretsProvider secrets,
+        ImplicitUrlAndKeyPostProcessor sut)
+    {
+        secrets.Secrets.Returns(new Dictionary<string, string>
+        {
+            {"instance1_base_url", "secret_base_url_1"},
+            {"instance1_api_key", "secret_api_key_1"},
+            {"instance2_base_url", "secret_base_url_2"},
+            {"instance2_api_key", "secret_api_key_2"}
+        });
+
+        var result = sut.Process(new RootConfigYaml
+        {
+            Radarr = new Dictionary<string, RadarrConfigYaml>
+            {
+                {
+                    "instance1", new RadarrConfigYaml
+                    {
+                        ApiKey = "explicit_base_url"
+                    }
+                }
+            },
+            Sonarr = new Dictionary<string, SonarrConfigYaml>
+            {
+                {
+                    "instance2", new SonarrConfigYaml
+                    {
+                        ApiKey = "explicit_base_url"
+                    }
+                }
+            }
+        });
+
+        result.Should().BeEquivalentTo(new RootConfigYaml
+        {
+            Radarr = new Dictionary<string, RadarrConfigYaml>
+            {
+                {
+                    "instance1", new RadarrConfigYaml
+                    {
+                        ApiKey = "explicit_base_url",
+                        BaseUrl = "secret_base_url_1"
+                    }
+                }
+            },
+            Sonarr = new Dictionary<string, SonarrConfigYaml>
+            {
+                {
+                    "instance2", new SonarrConfigYaml
+                    {
+                        ApiKey = "explicit_base_url",
+                        BaseUrl = "secret_base_url_2"
+                    }
+                }
+            }
+        });
+    }
+
+    [Test, AutoMockData]
+    public void Update_only_api_key_when_absent(
+        [Frozen] ISecretsProvider secrets,
+        ImplicitUrlAndKeyPostProcessor sut)
+    {
+        secrets.Secrets.Returns(new Dictionary<string, string>
+        {
+            {"instance1_base_url", "secret_base_url_1"},
+            {"instance1_api_key", "secret_api_key_1"},
+            {"instance2_base_url", "secret_base_url_2"},
+            {"instance2_api_key", "secret_api_key_2"}
+        });
+
+        var result = sut.Process(new RootConfigYaml
+        {
+            Radarr = new Dictionary<string, RadarrConfigYaml>
+            {
+                {
+                    "instance1", new RadarrConfigYaml
+                    {
+                        BaseUrl = "explicit_base_url"
+                    }
+                }
+            },
+            Sonarr = new Dictionary<string, SonarrConfigYaml>
+            {
+                {
+                    "instance2", new SonarrConfigYaml
+                    {
+                        BaseUrl = "explicit_base_url"
+                    }
+                }
+            }
+        });
+
+        result.Should().BeEquivalentTo(new RootConfigYaml
+        {
+            Radarr = new Dictionary<string, RadarrConfigYaml>
+            {
+                {
+                    "instance1", new RadarrConfigYaml
+                    {
+                        ApiKey = "secret_api_key_1",
+                        BaseUrl = "explicit_base_url"
+                    }
+                }
+            },
+            Sonarr = new Dictionary<string, SonarrConfigYaml>
+            {
+                {
+                    "instance2", new SonarrConfigYaml
+                    {
+                        ApiKey = "secret_api_key_2",
+                        BaseUrl = "explicit_base_url"
+                    }
+                }
+            }
+        });
+    }
+
+    [Test, AutoMockData]
+    public void Update_base_url_and_api_key_when_absent(
+        [Frozen] ISecretsProvider secrets,
+        ImplicitUrlAndKeyPostProcessor sut)
+    {
+        secrets.Secrets.Returns(new Dictionary<string, string>
+        {
+            {"instance1_base_url", "secret_base_url_1"},
+            {"instance1_api_key", "secret_api_key_1"},
+            {"instance2_base_url", "secret_base_url_2"},
+            {"instance2_api_key", "secret_api_key_2"}
+        });
+
+        var result = sut.Process(new RootConfigYaml
+        {
+            Radarr = new Dictionary<string, RadarrConfigYaml>
+            {
+                {"instance1", new RadarrConfigYaml()}
+            },
+            Sonarr = new Dictionary<string, SonarrConfigYaml>
+            {
+                {"instance2", new SonarrConfigYaml()}
+            }
+        });
+
+        result.Should().BeEquivalentTo(new RootConfigYaml
+        {
+            Radarr = new Dictionary<string, RadarrConfigYaml>
+            {
+                {
+                    "instance1", new RadarrConfigYaml
+                    {
+                        ApiKey = "secret_api_key_1",
+                        BaseUrl = "secret_base_url_1"
+                    }
+                }
+            },
+            Sonarr = new Dictionary<string, SonarrConfigYaml>
+            {
+                {
+                    "instance2", new SonarrConfigYaml
+                    {
+                        ApiKey = "secret_api_key_2",
+                        BaseUrl = "secret_base_url_2"
+                    }
+                }
+            }
+        });
+    }
+
+    [Test, AutoMockData]
+    public void Change_nothing_when_all_explicitly_provided(
+        [Frozen] ISecretsProvider secrets,
+        ImplicitUrlAndKeyPostProcessor sut)
+    {
+        secrets.Secrets.Returns(new Dictionary<string, string>
+        {
+            {"instance1_base_url", "secret_base_url_1"},
+            {"instance1_api_key", "secret_api_key_1"},
+            {"instance2_base_url", "secret_base_url_2"},
+            {"instance2_api_key", "secret_api_key_2"}
+        });
+
+        var result = sut.Process(new RootConfigYaml
+        {
+            Radarr = new Dictionary<string, RadarrConfigYaml>
+            {
+                {
+                    "instance1", new RadarrConfigYaml
+                    {
+                        BaseUrl = "explicit_base_url",
+                        ApiKey = "explicit_api_key"
+                    }
+                }
+            },
+            Sonarr = new Dictionary<string, SonarrConfigYaml>
+            {
+                {
+                    "instance2", new SonarrConfigYaml
+                    {
+                        BaseUrl = "explicit_base_url",
+                        ApiKey = "explicit_api_key"
+                    }
+                }
+            }
+        });
+
+        result.Should().BeEquivalentTo(new RootConfigYaml
+        {
+            Radarr = new Dictionary<string, RadarrConfigYaml>
+            {
+                {
+                    "instance1", new RadarrConfigYaml
+                    {
+                        ApiKey = "explicit_api_key",
+                        BaseUrl = "explicit_base_url"
+                    }
+                }
+            },
+            Sonarr = new Dictionary<string, SonarrConfigYaml>
+            {
+                {
+                    "instance2", new SonarrConfigYaml
+                    {
+                        ApiKey = "explicit_api_key",
+                        BaseUrl = "explicit_base_url"
+                    }
+                }
+            }
+        });
+    }
+}

src/tests/Recyclarr.TrashLib.Tests/Config/YamlConfigValidatorTest.cs

@@ -81,7 +81,7 @@ public class YamlConfigValidatorTest : TrashLibIntegrationFixture
         var config = new ServiceConfigYaml
         {
             ApiKey = "valid",
-            BaseUrl = "about:empty",
+            BaseUrl = "",
             CustomFormats = new List<CustomFormatConfigYaml>
             {
                 new()
@@ -105,7 +105,42 @@ public class YamlConfigValidatorTest : TrashLibIntegrationFixture
         var validator = Resolve<ServiceConfigYamlValidator>();
         var result = validator.TestValidate(config);
 
-        result.ShouldHaveValidationErrorFor(x => x.BaseUrl);
+        result.ShouldHaveValidationErrorFor(x => x.BaseUrl)
+            .WithErrorMessage("'base_url' must not be empty.");
+    }
+
+    [Test]
+    public void Validation_failure_when_base_url_not_start_with_http()
+    {
+        var config = new ServiceConfigYaml
+        {
+            ApiKey = "valid",
+            BaseUrl = "ftp://foo.com",
+            CustomFormats = new List<CustomFormatConfigYaml>
+            {
+                new()
+                {
+                    TrashIds = new[] {"valid"},
+                    QualityProfiles = new List<QualityScoreConfigYaml>
+                    {
+                        new()
+                        {
+                            Name = "valid"
+                        }
+                    }
+                }
+            },
+            QualityDefinition = new QualitySizeConfigYaml
+            {
+                Type = "valid"
+            }
+        };
+
+        var validator = Resolve<ServiceConfigYamlValidator>();
+        var result = validator.TestValidate(config);
+
+        result.ShouldHaveValidationErrorFor(x => x.BaseUrl)
+            .WithErrorMessage("base_url must start with 'http' or 'https'");
     }
 
     public static string FirstCf { get; } = $"{nameof(ServiceConfigYaml.CustomFormats)}[0].";