From aa523a0e147e8d23252707b8a47fcbef6e7fc7a7 Mon Sep 17 00:00:00 2001 From: voltron4lyfe Date: Tue, 13 Dec 2022 22:40:58 -0800 Subject: [PATCH] feat: Enable referencing environment variables in config Closes #145 Closes #154 --- CHANGELOG.md | 4 + .../IntegrationFixture.cs | 2 + .../Parsing/ConfigurationLoaderEnvVarTest.cs | 171 ++++++++++++++++++ .../Parsing/ConfigurationLoaderSecretsTest.cs | 139 ++++++++++++++ .../Config/Parsing/ConfigurationLoaderTest.cs | 125 ------------- .../EnvironmentVariableNotDefinedException.cs | 15 ++ .../EnvironmentVariablesDeserializer.cs | 41 +++++ .../EnvironmentVariablesYamlBehavior.cs | 24 +++ 8 files changed, 396 insertions(+), 125 deletions(-) create mode 100644 src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderEnvVarTest.cs create mode 100644 src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderSecretsTest.cs create mode 100644 src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariableNotDefinedException.cs create mode 100644 src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariablesDeserializer.cs create mode 100644 src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariablesYamlBehavior.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 61d0cf55..332acdb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Environment variables may now be used in YAML configuration (#145). + ## [4.2.0] - 2023-01-13 ### Added diff --git a/src/Recyclarr.Cli.TestLibrary/IntegrationFixture.cs b/src/Recyclarr.Cli.TestLibrary/IntegrationFixture.cs index c87dd5f4..7068836b 100644 --- a/src/Recyclarr.Cli.TestLibrary/IntegrationFixture.cs +++ b/src/Recyclarr.Cli.TestLibrary/IntegrationFixture.cs @@ -6,6 +6,7 @@ using Autofac; using Autofac.Features.ResolveAnything; using NSubstitute; using NUnit.Framework; +using Recyclarr.Common; using Recyclarr.Common.TestLibrary; using Recyclarr.TestLibrary; using Recyclarr.TrashLib; @@ -43,6 +44,7 @@ public abstract class IntegrationFixture : IDisposable builder.RegisterMockFor(); builder.RegisterMockFor(); builder.RegisterMockFor(); + builder.RegisterMockFor(); builder.RegisterMockFor(m => { // By default, choose some extremely high number so that all the newest features are enabled. diff --git a/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderEnvVarTest.cs b/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderEnvVarTest.cs new file mode 100644 index 00000000..8c3a8858 --- /dev/null +++ b/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderEnvVarTest.cs @@ -0,0 +1,171 @@ +using FluentAssertions; +using NSubstitute; +using NUnit.Framework; +using Recyclarr.Cli.TestLibrary; +using Recyclarr.Common; +using Recyclarr.TrashLib.Config; +using Recyclarr.TrashLib.Config.Parsing; +using Recyclarr.TrashLib.Services.Sonarr.Config; + +namespace Recyclarr.TrashLib.Tests.Config.Parsing; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ConfigurationLoaderEnvVarTest : IntegrationFixture +{ + [Test] + public void Test_successful_environment_variable_loading() + { + var env = Resolve(); + env.GetEnvironmentVariable("SONARR_API_KEY").Returns("the_api_key"); + env.GetEnvironmentVariable("SONARR_URL").Returns("the_url"); + + var sut = Resolve(); + + const string testYml = @" +sonarr: + instance: + api_key: !env_var SONARR_API_KEY + base_url: !env_var SONARR_URL http://sonarr:1233 +"; + + var configCollection = sut.LoadFromStream(new StringReader(testYml)); + + var config = configCollection.Get(SupportedServices.Sonarr); + config.Should().BeEquivalentTo(new[] + { + new + { + BaseUrl = "the_url", + ApiKey = "the_api_key" + } + }); + } + + [Test] + public void Use_default_value_if_env_var_not_defined() + { + var sut = Resolve(); + + const string testYml = @" +sonarr: + instance: + base_url: !env_var SONARR_URL http://sonarr:1233 + api_key: value +"; + + var configCollection = sut.LoadFromStream(new StringReader(testYml)); + + var config = configCollection.Get(SupportedServices.Sonarr); + config.Should().BeEquivalentTo(new[] + { + new + { + BaseUrl = "http://sonarr:1233" + } + }); + } + + [Test] + public void Default_value_with_spaces_is_allowed() + { + var env = Resolve(); + env.GetEnvironmentVariable("SONARR_URL").Returns(""); + + var sut = Resolve(); + + const string testYml = @" +sonarr: + instance: + base_url: !env_var SONARR_URL some value + api_key: value +"; + + var configCollection = sut.LoadFromStream(new StringReader(testYml)); + + var config = configCollection.Get(SupportedServices.Sonarr); + config.Should().BeEquivalentTo(new[] + { + new + { + BaseUrl = "some value" + } + }); + } + + [Test] + public void Quotation_characters_are_stripped_from_default_value() + { + var env = Resolve(); + env.GetEnvironmentVariable("SONARR_URL").Returns(""); + + var sut = Resolve(); + + const string testYml = @" +sonarr: + instance: + base_url: !env_var SONARR_URL ""the url"" + api_key: !env_var SONARR_API 'the key' +"; + + var configCollection = sut.LoadFromStream(new StringReader(testYml)); + + var config = configCollection.Get(SupportedServices.Sonarr); + config.Should().BeEquivalentTo(new[] + { + new + { + BaseUrl = "the url", + ApiKey = "the key" + } + }); + } + + [Test] + public void Multiple_spaces_between_default_and_env_var_work() + { + var sut = Resolve(); + + const string testYml = @" +sonarr: + instance: + base_url: !env_var SONARR_URL some value + api_key: value +"; + + var configCollection = sut.LoadFromStream(new StringReader(testYml)); + + var config = configCollection.Get(SupportedServices.Sonarr); + config.Should().BeEquivalentTo(new[] + { + new + { + BaseUrl = "some value" + } + }); + } + + [Test] + public void Tab_characters_are_stripped() + { + var sut = Resolve(); + + const string testYml = $@" +sonarr: + instance: + base_url: !env_var SONARR_URL {"\t"}some value + api_key: value +"; + + var configCollection = sut.LoadFromStream(new StringReader(testYml)); + + var config = configCollection.Get(SupportedServices.Sonarr); + config.Should().BeEquivalentTo(new[] + { + new + { + BaseUrl = "some value" + } + }); + } +} diff --git a/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderSecretsTest.cs b/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderSecretsTest.cs new file mode 100644 index 00000000..c5ed5324 --- /dev/null +++ b/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderSecretsTest.cs @@ -0,0 +1,139 @@ +using System.IO.Abstractions.TestingHelpers; +using FluentAssertions; +using NUnit.Framework; +using Recyclarr.Cli.TestLibrary; +using Recyclarr.TrashLib.Config; +using Recyclarr.TrashLib.Config.Parsing; +using Recyclarr.TrashLib.Config.Secrets; +using Recyclarr.TrashLib.Services.Sonarr.Config; +using Serilog.Sinks.TestCorrelator; +using YamlDotNet.Core; + +namespace Recyclarr.TrashLib.Tests.Config.Parsing; + +[TestFixture] +[Parallelizable(ParallelScope.All)] +public class ConfigurationLoaderSecretsTest : IntegrationFixture +{ + [Test] + public void Test_secret_loading() + { + var configLoader = Resolve(); + + const string testYml = @" +sonarr: + instance1: + api_key: !secret api_key + base_url: !secret 123GARBAGE_ + release_profiles: + - trash_ids: + - !secret secret_rp +"; + + const string secretsYml = @" +api_key: 95283e6b156c42f3af8a9b16173f876b +123GARBAGE_: 'https://radarr:7878' +secret_rp: 1234567 +"; + + Fs.AddFile(Paths.SecretsPath.FullName, new MockFileData(secretsYml)); + var expected = new List + { + new() + { + InstanceName = "instance1", + ApiKey = "95283e6b156c42f3af8a9b16173f876b", + BaseUrl = "https://radarr:7878", + ReleaseProfiles = new List + { + new() + { + TrashIds = new[] {"1234567"} + } + } + } + }; + + var parsedSecret = configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); + parsedSecret.Get(SupportedServices.Sonarr) + .Should().BeEquivalentTo(expected, o => o.Excluding(x => x.LineNumber)); + } + + [Test] + public void Throw_when_referencing_invalid_secret() + { + using var logContext = TestCorrelator.CreateContext(); + var configLoader = Resolve(); + + const string testYml = @" +sonarr: + instance2: + api_key: !secret api_key + base_url: fake_url +"; + + const string secretsYml = "no_api_key: 95283e6b156c42f3af8a9b16173f876b"; + + Fs.AddFile(Paths.SecretsPath.FullName, new MockFileData(secretsYml)); + + var act = () => configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); + + act.Should().Throw() + .WithInnerException() + .WithMessage("*api_key is not defined in secrets.yml"); + } + + [Test] + public void Throw_when_referencing_secret_without_secrets_file() + { + var configLoader = Resolve(); + + const string testYml = @" +sonarr: + instance3: + api_key: !secret api_key + base_url: fake_url +"; + + Action act = () => configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); + act.Should().Throw() + .WithInnerException() + .WithMessage("*api_key is not defined in secrets.yml"); + } + + [Test] + public void Throw_when_secret_value_is_not_scalar() + { + var configLoader = Resolve(); + + const string testYml = @" +sonarr: + instance4: + api_key: !secret { property: value } + base_url: fake_url +"; + + Action act = () => configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); + act.Should().Throw().WithMessage("Expected 'Scalar'*"); + } + + [Test] + public void Throw_when_expected_value_is_not_scalar() + { + var configLoader = Resolve(); + + const string testYml = @" +sonarr: + instance5: + api_key: fake_key + base_url: fake_url + release_profiles: !secret bogus_profile +"; + + const string secretsYml = @"bogus_profile: 95283e6b156c42f3af8a9b16173f876b"; + + Fs.AddFile(Paths.SecretsPath.FullName, new MockFileData(secretsYml)); + Action act = () => configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); + act.Should().Throw().WithMessage("Exception during deserialization"); + } +} diff --git a/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderTest.cs b/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderTest.cs index 4c9fc325..df1ccb50 100644 --- a/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderTest.cs +++ b/src/Recyclarr.TrashLib.Tests/Config/Parsing/ConfigurationLoaderTest.cs @@ -13,13 +13,10 @@ using Recyclarr.TestLibrary; using Recyclarr.TestLibrary.AutoFixture; using Recyclarr.TrashLib.Config; using Recyclarr.TrashLib.Config.Parsing; -using Recyclarr.TrashLib.Config.Secrets; using Recyclarr.TrashLib.Config.Yaml; using Recyclarr.TrashLib.Services.Radarr.Config; using Recyclarr.TrashLib.Services.Sonarr.Config; using Recyclarr.TrashLib.TestLibrary; -using Serilog.Sinks.TestCorrelator; -using YamlDotNet.Core; namespace Recyclarr.TrashLib.Tests.Config.Parsing; @@ -128,128 +125,6 @@ public class ConfigurationLoaderTest : IntegrationFixture }, o => o.Excluding(x => x.LineNumber)); } - [Test] - public void Test_secret_loading() - { - var configLoader = Resolve(); - - const string testYml = @" -sonarr: - instance1: - api_key: !secret api_key - base_url: !secret 123GARBAGE_ - release_profiles: - - trash_ids: - - !secret secret_rp -"; - - const string secretsYml = @" -api_key: 95283e6b156c42f3af8a9b16173f876b -123GARBAGE_: 'https://radarr:7878' -secret_rp: 1234567 -"; - - Fs.AddFile(Paths.SecretsPath.FullName, new MockFileData(secretsYml)); - var expected = new List - { - new() - { - InstanceName = "instance1", - ApiKey = "95283e6b156c42f3af8a9b16173f876b", - BaseUrl = "https://radarr:7878", - ReleaseProfiles = new List - { - new() - { - TrashIds = new[] {"1234567"} - } - } - } - }; - - var parsedSecret = configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); - parsedSecret.Get(SupportedServices.Sonarr) - .Should().BeEquivalentTo(expected, o => o.Excluding(x => x.LineNumber)); - } - - [Test] - public void Throw_when_referencing_invalid_secret() - { - using var logContext = TestCorrelator.CreateContext(); - var configLoader = Resolve(); - - const string testYml = @" -sonarr: - instance2: - api_key: !secret api_key - base_url: fake_url -"; - - const string secretsYml = "no_api_key: 95283e6b156c42f3af8a9b16173f876b"; - - Fs.AddFile(Paths.SecretsPath.FullName, new MockFileData(secretsYml)); - - var act = () => configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); - - act.Should().Throw() - .WithInnerException() - .WithMessage("*api_key is not defined in secrets.yml"); - } - - [Test] - public void Throw_when_referencing_secret_without_secrets_file() - { - var configLoader = Resolve(); - - const string testYml = @" -sonarr: - instance3: - api_key: !secret api_key - base_url: fake_url -"; - - Action act = () => configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); - act.Should().Throw() - .WithInnerException() - .WithMessage("*api_key is not defined in secrets.yml"); - } - - [Test] - public void Throw_when_secret_value_is_not_scalar() - { - var configLoader = Resolve(); - - const string testYml = @" -sonarr: - instance4: - api_key: !secret { property: value } - base_url: fake_url -"; - - Action act = () => configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); - act.Should().Throw().WithMessage("Expected 'Scalar'*"); - } - - [Test] - public void Throw_when_expected_value_is_not_scalar() - { - var configLoader = Resolve(); - - const string testYml = @" -sonarr: - instance5: - api_key: fake_key - base_url: fake_url - release_profiles: !secret bogus_profile -"; - - const string secretsYml = @"bogus_profile: 95283e6b156c42f3af8a9b16173f876b"; - - Fs.AddFile(Paths.SecretsPath.FullName, new MockFileData(secretsYml)); - Action act = () => configLoader.LoadFromStream(new StringReader(testYml), "sonarr"); - act.Should().Throw().WithMessage("Exception during deserialization"); - } - [Test, AutoMockData] public void Throw_when_yaml_file_only_has_comment(ConfigurationLoader sut) { diff --git a/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariableNotDefinedException.cs b/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariableNotDefinedException.cs new file mode 100644 index 00000000..08014dc5 --- /dev/null +++ b/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariableNotDefinedException.cs @@ -0,0 +1,15 @@ +namespace Recyclarr.TrashLib.Config.EnvironmentVariables; + +public class EnvironmentVariableNotDefinedException : Exception +{ + public int Line { get; } + public string EnvironmentVariableName { get; } + + public EnvironmentVariableNotDefinedException(int line, string environmentVariableName) + : base( + $"Line {line} refers to undefined environment variable {environmentVariableName} and no default is specified.") + { + Line = line; + EnvironmentVariableName = environmentVariableName; + } +} diff --git a/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariablesDeserializer.cs b/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariablesDeserializer.cs new file mode 100644 index 00000000..9ca0c73b --- /dev/null +++ b/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariablesDeserializer.cs @@ -0,0 +1,41 @@ +using Recyclarr.Common; +using YamlDotNet.Core; +using YamlDotNet.Core.Events; +using YamlDotNet.Serialization; + +namespace Recyclarr.TrashLib.Config.EnvironmentVariables; + +public record EnvironmentVariableTag; + +public class EnvironmentVariablesDeserializer : INodeDeserializer +{ + private readonly IEnvironment _environment; + + public EnvironmentVariablesDeserializer(IEnvironment environment) + { + _environment = environment; + } + + public bool Deserialize(IParser reader, Type expectedType, Func nestedObjectDeserializer, + out object? value) + { + // Only process items flagged as Environment Variables + if (expectedType != typeof(EnvironmentVariableTag)) + { + value = null; + return false; + } + + var scalar = reader.Consume(); + var split = scalar.Value.Trim().Split(' ', 2); + var envVarValue = _environment.GetEnvironmentVariable(split[0]); + if (string.IsNullOrWhiteSpace(envVarValue)) + { + // Trim whitespace + quotation characters + envVarValue = split.ElementAtOrDefault(1)?.Trim().Trim('\'', '"'); + } + + value = envVarValue ?? throw new EnvironmentVariableNotDefinedException(scalar.Start.Line, scalar.Value); + return true; + } +} diff --git a/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariablesYamlBehavior.cs b/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariablesYamlBehavior.cs new file mode 100644 index 00000000..43bf3cde --- /dev/null +++ b/src/Recyclarr.TrashLib/Config/EnvironmentVariables/EnvironmentVariablesYamlBehavior.cs @@ -0,0 +1,24 @@ +using JetBrains.Annotations; +using Recyclarr.Common; +using Recyclarr.TrashLib.Config.Yaml; +using YamlDotNet.Serialization; + +namespace Recyclarr.TrashLib.Config.EnvironmentVariables; + +[UsedImplicitly] +public class EnvironmentVariablesYamlBehavior : IYamlBehavior +{ + private readonly IEnvironment _environment; + + public EnvironmentVariablesYamlBehavior(IEnvironment environment) + { + _environment = environment; + } + + public void Setup(DeserializerBuilder builder) + { + builder + .WithNodeDeserializer(new EnvironmentVariablesDeserializer(_environment)) + .WithTagMapping("!env_var", typeof(EnvironmentVariableTag)); + } +}