From 8b350b5bce48648a6cff05b31834fdfa3af44ab8 Mon Sep 17 00:00:00 2001 From: Robert Dailey Date: Sun, 20 Aug 2023 09:43:17 -0500 Subject: [PATCH] feat: Add exclusions support to 'reset_unmatched_scores' A new `except` property is provided under `reset_unmatched_scores` to list one or more custom format names to exclude when resetting scores. --- CHANGELOG.md | 8 ++ schemas/config-schema.json | 23 +++- .../QualityProfileNoticePhase.cs | 11 ++ .../QualityProfileTransactionPhase.cs | 22 +++ .../QualityProfile/UpdatedFormatScore.cs | 9 +- .../QualityProfile/UpdatedQualityProfile.cs | 1 + .../ResetUnmatchedScoresYamlTypeConverter.cs | 22 +++ .../Config/Parsing/ConfigYamlDataObjects.cs | 17 ++- .../ConfigYamlDataObjectsValidation.cs | 20 +++ .../Config/Parsing/ConfigYamlMapperProfile.cs | 2 + .../Config/Services/ServiceConfiguration.cs | 8 +- src/tests/Recyclarr.Cli.TestLibrary/NewQp.cs | 5 +- .../QualityProfileTransactionPhaseTest.cs | 128 +++++++++++++++++- .../UpdatedQualityProfileTest.cs | 3 +- 14 files changed, 261 insertions(+), 18 deletions(-) create mode 100644 src/Recyclarr.TrashLib/Config/Parsing/BackwardCompatibility/ResetUnmatchedScoresYamlTypeConverter.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index e29ccc49..ca4016bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,11 +11,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - New `delete` command added for deleting one, many, or all custom formats from Radarr or Sonarr. +- Exclusions are now supported under `reset_unmatched_scores`. This is used to prevent score resets + to specific custom formats. See [the docs][except] for more info. ### Changed - Program now exits when invalid instances are specified. +### Deprecated + +- `reset_unmatched_scores` has a new syntax. The old syntax [has been deprecated][resetdeprecate]. + ### Fixed - If multiple configuration files refer to the same `base_url` (i.e. the same instance), this is now @@ -23,6 +29,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Radarr or Sonarr, you need to manually merge those config files. See [this page][configmerge]. [configmerge]: https://recyclarr.dev/wiki/yaml/config-examples/#merge-single-instance +[except]: https://recyclarr.dev/wiki/yaml/config-reference/#qp-reset-unmatched-scores +[resetdeprecate]: https://recyclarr.dev/wiki/upgrade-guide/v6.0/#breaking-changes ## [5.2.1] - 2023-08-07 diff --git a/schemas/config-schema.json b/schemas/config-schema.json index df8aae67..60f722f0 100644 --- a/schemas/config-schema.json +++ b/schemas/config-schema.json @@ -109,12 +109,27 @@ "properties": { "name": { "type": "string", - "description": "The name of the quality profile to which settings should apply" + "description": "The name of the quality profile to which settings should apply." }, "reset_unmatched_scores": { - "type": "boolean", - "description": "If set to true, enables setting scores to 0 in quality profiles where either a CF was not mentioned in the trash_ids array or it was in that list but did not get a score (e.g. no score in guide).", - "default": false + "type": "object", + "description": "Settings related to resetting unmatched CF scores.", + "additionalProperties": false, + "required": ["enabled"], + "properties": { + "enabled": { + "type": "boolean", + "description": "If set to true, enables setting scores to 0 in quality profiles where either a CF was not mentioned in the trash_ids array or it was in that list but did not get a score (e.g. no score in guide)." + }, + "except": { + "type": "array", + "description": "An array of strings that match custom formats to exclude when resetting scores. Matching is case-insensitive.", + "minItems": 1, + "items": { + "type": "string" + } + } + } }, "upgrade": { "type": "object", diff --git a/src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileNoticePhase.cs b/src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileNoticePhase.cs index e2af0920..4f3852fe 100644 --- a/src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileNoticePhase.cs +++ b/src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileNoticePhase.cs @@ -53,5 +53,16 @@ public class QualityProfileNoticePhase _log.Warning("Quality profile '{ProfileName}' references invalid quality names: {InvalidNames}", profileName, invalidNames); } + + var invalidCfExceptNames = transactions.UpdatedProfiles + .Where(x => x.InvalidExceptCfNames.Any()) + .Select(x => (x.ProfileName, x.InvalidExceptCfNames)); + + foreach (var (profileName, invalidNames) in invalidCfExceptNames) + { + _log.Warning( + "`except` under `reset_unmatched_scores` in quality profile '{ProfileName}' has invalid " + + "CF names: {CfNames}", profileName, invalidNames); + } } } diff --git a/src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhase.cs b/src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhase.cs index 0bbd7584..063beae3 100644 --- a/src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhase.cs +++ b/src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhase.cs @@ -1,7 +1,9 @@ using System.Diagnostics.CodeAnalysis; using FluentValidation.Results; using Recyclarr.Cli.Pipelines.QualityProfile.Api; +using Recyclarr.Common.Extensions; using Recyclarr.Common.FluentValidation; +using Recyclarr.TrashLib.Config.Services; namespace Recyclarr.Cli.Pipelines.QualityProfile.PipelinePhases; @@ -91,10 +93,30 @@ public class QualityProfileTransactionPhase { foreach (var profile in updatedProfiles) { + profile.InvalidExceptCfNames = GetInvalidExceptCfNames( + profile.ProfileConfig.Profile.ResetUnmatchedScores, profile.ProfileDto); + profile.UpdatedScores = ProcessScoreUpdates(profile.ProfileConfig, profile.ProfileDto); } } + private static IReadOnlyCollection GetInvalidExceptCfNames( + ResetUnmatchedScoresConfig resetConfig, + QualityProfileDto profileDto) + { + var except = resetConfig.Except; + if (!except.Any()) + { + return Array.Empty(); + } + + var serviceCfNames = profileDto.FormatItems.Select(x => x.Name).ToList(); + return except + .Distinct(StringComparer.InvariantCultureIgnoreCase) + .Where(x => serviceCfNames.TrueForAll(y => !y.EqualsIgnoreCase(x))) + .ToList(); + } + private static List ProcessScoreUpdates( ProcessedQualityProfileData profileData, QualityProfileDto profileDto) diff --git a/src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedFormatScore.cs b/src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedFormatScore.cs index 3e466156..a709652f 100644 --- a/src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedFormatScore.cs +++ b/src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedFormatScore.cs @@ -1,5 +1,6 @@ using Recyclarr.Cli.Pipelines.QualityProfile.Api; using Recyclarr.Cli.Pipelines.QualityProfile.PipelinePhases; +using Recyclarr.Common.Extensions; namespace Recyclarr.Cli.Pipelines.QualityProfile; @@ -38,8 +39,12 @@ public record UpdatedFormatScore(ProfileFormatItemDto Dto, int NewScore, FormatS public static UpdatedFormatScore Reset(ProfileFormatItemDto dto, ProcessedQualityProfileData profileData) { - var score = profileData.Profile.ResetUnmatchedScores ? 0 : dto.Score; - return new UpdatedFormatScore(dto, score, FormatScoreUpdateReason.Reset); + var reset = profileData.Profile.ResetUnmatchedScores; + var shouldReset = reset.Enabled && reset.Except.All(x => !dto.Name.EqualsIgnoreCase(x)); + + var score = shouldReset ? 0 : dto.Score; + var reason = shouldReset ? FormatScoreUpdateReason.Reset : FormatScoreUpdateReason.NoChange; + return new UpdatedFormatScore(dto, score, reason); } public static UpdatedFormatScore Updated(ProfileFormatItemDto dto, ProcessedQualityProfileScore score) diff --git a/src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedQualityProfile.cs b/src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedQualityProfile.cs index c931f36d..8c67d935 100644 --- a/src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedQualityProfile.cs +++ b/src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedQualityProfile.cs @@ -17,6 +17,7 @@ public record UpdatedQualityProfile public required QualityProfileUpdateReason UpdateReason { get; set; } public IReadOnlyCollection UpdatedScores { get; set; } = Array.Empty(); public UpdatedQualities UpdatedQualities { get; init; } = new(); + public IReadOnlyCollection InvalidExceptCfNames { get; set; } = Array.Empty(); public string ProfileName { diff --git a/src/Recyclarr.TrashLib/Config/Parsing/BackwardCompatibility/ResetUnmatchedScoresYamlTypeConverter.cs b/src/Recyclarr.TrashLib/Config/Parsing/BackwardCompatibility/ResetUnmatchedScoresYamlTypeConverter.cs new file mode 100644 index 00000000..a6842768 --- /dev/null +++ b/src/Recyclarr.TrashLib/Config/Parsing/BackwardCompatibility/ResetUnmatchedScoresYamlTypeConverter.cs @@ -0,0 +1,22 @@ +using System.ComponentModel; +using System.Globalization; + +namespace Recyclarr.TrashLib.Config.Parsing.BackwardCompatibility; + +public class ResetUnmatchedScoresYamlTypeConverter : TypeConverter +{ + public override bool CanConvertFrom(ITypeDescriptorContext? context, Type sourceType) + { + return sourceType == typeof(bool) || sourceType == typeof(string); + } + + public override object ConvertFrom(ITypeDescriptorContext? context, CultureInfo? culture, object value) + { + var enabledFlag = Convert.ToBoolean(value); + return new ResetUnmatchedScoresConfigYaml + { + FromBool = true, + Enabled = enabledFlag + }; + } +} diff --git a/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjects.cs b/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjects.cs index 33dca1b9..d3495be2 100644 --- a/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjects.cs +++ b/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjects.cs @@ -1,5 +1,7 @@ +using System.ComponentModel; using System.Diagnostics.CodeAnalysis; using JetBrains.Annotations; +using Recyclarr.TrashLib.Config.Parsing.BackwardCompatibility; using Recyclarr.TrashLib.Config.Services; using YamlDotNet.Serialization; @@ -42,15 +44,28 @@ public record QualityProfileQualityConfigYaml public IReadOnlyCollection? Qualities { get; init; } } +[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] +[TypeConverter(typeof(ResetUnmatchedScoresYamlTypeConverter))] +public record ResetUnmatchedScoresConfigYaml +{ + // This exists so that we know if this value came from the legacy 'true|false' value. + [YamlIgnore] + public bool FromBool { get; set; } + + public bool? Enabled { get; init; } + public IReadOnlyCollection? Except { get; init; } +} + [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] public record QualityProfileConfigYaml { public string? Name { get; init; } + public ResetUnmatchedScoresConfigYaml? ResetUnmatchedScores { get; init; } public QualityProfileFormatUpgradeYaml? Upgrade { get; init; } public int? MinFormatScore { get; init; } - public bool ResetUnmatchedScores { get; init; } public QualitySortAlgorithm? QualitySort { get; init; } public IReadOnlyCollection? Qualities { get; init; } + } [UsedImplicitly(ImplicitUseTargetFlags.WithMembers)] diff --git a/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs b/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs index 7fe92f83..26384fc7 100644 --- a/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs +++ b/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlDataObjectsValidation.cs @@ -87,6 +87,23 @@ public class QualityProfileFormatUpgradeYamlValidator : AbstractValidator +{ + public ResetUnmatchedScoresConfigYamlValidator() + { + RuleFor(x => x.Enabled) + .NotNull() + .WithMessage("Under `reset_unmatched_scores`, the `enabled` property is required."); + + RuleFor(x => x.FromBool) + .Must(x => !x) // must be false + .WithMessage( + "Using true/false with `reset_unmatched_scores` is deprecated. " + + "See: https://recyclarr.dev/wiki/upgrade-guide/v6.0/#reset-scores") + .WithSeverity(Severity.Warning); + } +} + public class QualityProfileConfigYamlValidator : AbstractValidator { public QualityProfileConfigYamlValidator() @@ -102,6 +119,9 @@ public class QualityProfileConfigYamlValidator : AbstractValidator x.Upgrade) .SetNonNullableValidator(x => new QualityProfileFormatUpgradeYamlValidator(x)); + RuleFor(x => x.ResetUnmatchedScores) + .SetNonNullableValidator(new ResetUnmatchedScoresConfigYamlValidator()); + RuleFor(x => x.Qualities) .Custom(ValidateHaveNoDuplicates!) .Must(x => x!.Any(y => y.Enabled is true or null)) diff --git a/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlMapperProfile.cs b/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlMapperProfile.cs index 99add1fb..a13bab1b 100644 --- a/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlMapperProfile.cs +++ b/src/Recyclarr.TrashLib/Config/Parsing/ConfigYamlMapperProfile.cs @@ -17,6 +17,8 @@ public class ConfigYamlMapperProfile : Profile CreateMap() .ForMember(x => x.Enabled, o => o.NullSubstitute(true)); + CreateMap(); + CreateMap() .ForMember(x => x.UpgradeAllowed, o => o.MapFrom(x => x.Upgrade!.Allowed)) .ForMember(x => x.UpgradeUntilQuality, o => o.MapFrom(x => x.Upgrade!.UntilQuality)) diff --git a/src/Recyclarr.TrashLib/Config/Services/ServiceConfiguration.cs b/src/Recyclarr.TrashLib/Config/Services/ServiceConfiguration.cs index 12f41126..0f20d2ce 100644 --- a/src/Recyclarr.TrashLib/Config/Services/ServiceConfiguration.cs +++ b/src/Recyclarr.TrashLib/Config/Services/ServiceConfiguration.cs @@ -59,6 +59,12 @@ public enum QualitySortAlgorithm Bottom } +public record ResetUnmatchedScoresConfig +{ + public bool Enabled { get; init; } + public IReadOnlyCollection Except { get; init; } = Array.Empty(); +} + public record QualityProfileConfig { public string Name { get; init; } = ""; @@ -66,7 +72,7 @@ public record QualityProfileConfig public string? UpgradeUntilQuality { get; init; } public int? UpgradeUntilScore { get; init; } public int? MinFormatScore { get; init; } - public bool ResetUnmatchedScores { get; init; } + public ResetUnmatchedScoresConfig ResetUnmatchedScores { get; init; } = new(); public QualitySortAlgorithm QualitySort { get; init; } public IReadOnlyCollection Qualities { get; init; } = Array.Empty(); diff --git a/src/tests/Recyclarr.Cli.TestLibrary/NewQp.cs b/src/tests/Recyclarr.Cli.TestLibrary/NewQp.cs index 4f4b530a..aa04e70e 100644 --- a/src/tests/Recyclarr.Cli.TestLibrary/NewQp.cs +++ b/src/tests/Recyclarr.Cli.TestLibrary/NewQp.cs @@ -32,7 +32,10 @@ public static class NewQp var profileConfig = new QualityProfileConfig { Name = profileName, - ResetUnmatchedScores = resetUnmatchedScores + ResetUnmatchedScores = new ResetUnmatchedScoresConfig + { + Enabled = resetUnmatchedScores + } }; return Processed(profileConfig, scores); diff --git a/src/tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhaseTest.cs b/src/tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhaseTest.cs index 2c29f207..b619158f 100644 --- a/src/tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhaseTest.cs +++ b/src/tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhaseTest.cs @@ -293,7 +293,18 @@ public class QualityProfileTransactionPhaseTest { var guideData = new[] { - NewQp.Processed("profile1", false, ("quality3", "id3", 3, 100), ("quality4", "id4", 4, 500)) + NewQp.Processed(new QualityProfileConfig + { + Name = "profile1", + ResetUnmatchedScores = new ResetUnmatchedScoresConfig + { + Enabled = false, + // Throw in some exceptions here, just to test whether or not these somehow affect the result + // despite Enable being set to false. + Except = new[] {"cf1"} + } + }, + ("cf3", "id3", 3, 100), ("cf4", "id4", 4, 500)) }; var dtos = new[] @@ -305,13 +316,13 @@ public class QualityProfileTransactionPhaseTest { new ProfileFormatItemDto { - Name = "quality1", + Name = "cf1", Format = 1, Score = 200 }, new ProfileFormatItemDto { - Name = "quality2", + Name = "cf2", Format = 2, Score = 300 } @@ -327,10 +338,113 @@ public class QualityProfileTransactionPhaseTest .ContainSingle().Which.UpdatedScores.Should() .BeEquivalentTo(new[] { - NewQp.UpdatedScore("quality1", 200, 200, FormatScoreUpdateReason.Reset), - NewQp.UpdatedScore("quality2", 300, 300, FormatScoreUpdateReason.Reset), - NewQp.UpdatedScore("quality3", 0, 100, FormatScoreUpdateReason.New), - NewQp.UpdatedScore("quality4", 0, 500, FormatScoreUpdateReason.New) + NewQp.UpdatedScore("cf1", 200, 200, FormatScoreUpdateReason.NoChange), + NewQp.UpdatedScore("cf2", 300, 300, FormatScoreUpdateReason.NoChange), + NewQp.UpdatedScore("cf3", 0, 100, FormatScoreUpdateReason.New), + NewQp.UpdatedScore("cf4", 0, 500, FormatScoreUpdateReason.New) + }, o => o.Excluding(x => x.Dto.Format)); + } + + [Test, AutoMockData] + public void Reset_scores_with_reset_exceptions(QualityProfileTransactionPhase sut) + { + var guideData = new[] + { + NewQp.Processed(new QualityProfileConfig + { + Name = "profile1", + ResetUnmatchedScores = new ResetUnmatchedScoresConfig + { + Enabled = true, + Except = new[] {"cf1"} + } + }, + ("cf3", "id3", 3, 100), ("cf4", "id4", 4, 500)) + }; + + var dtos = new[] + { + new QualityProfileDto + { + Name = "profile1", + FormatItems = new[] + { + new ProfileFormatItemDto + { + Name = "cf1", + Format = 1, + Score = 200 + }, + new ProfileFormatItemDto + { + Name = "cf2", + Format = 2, + Score = 300 + } + } + } + }; + + var serviceData = new QualityProfileServiceData(dtos, new QualityProfileDto()); + + var result = sut.Execute(guideData, serviceData); + + result.UpdatedProfiles.Should() + .ContainSingle().Which.UpdatedScores.Should() + .BeEquivalentTo(new[] + { + NewQp.UpdatedScore("cf1", 200, 200, FormatScoreUpdateReason.NoChange), + NewQp.UpdatedScore("cf2", 300, 0, FormatScoreUpdateReason.Reset), + NewQp.UpdatedScore("cf3", 0, 100, FormatScoreUpdateReason.New), + NewQp.UpdatedScore("cf4", 0, 500, FormatScoreUpdateReason.New) }, o => o.Excluding(x => x.Dto.Format)); } + + [Test, AutoMockData] + public void Reset_scores_with_invalid_except_list_items(QualityProfileTransactionPhase sut) + { + var guideData = new[] + { + NewQp.Processed(new QualityProfileConfig + { + Name = "profile1", + ResetUnmatchedScores = new ResetUnmatchedScoresConfig + { + Enabled = true, + Except = new[] {"cf50"} + } + }) + }; + + var dtos = new[] + { + new QualityProfileDto + { + Name = "profile1", + FormatItems = new[] + { + new ProfileFormatItemDto + { + Name = "cf1", + Format = 1, + Score = 200 + }, + new ProfileFormatItemDto + { + Name = "cf2", + Format = 2, + Score = 300 + } + } + } + }; + + var serviceData = new QualityProfileServiceData(dtos, new QualityProfileDto()); + + var result = sut.Execute(guideData, serviceData); + + result.UpdatedProfiles.Should() + .ContainSingle().Which.InvalidExceptCfNames.Should() + .BeEquivalentTo("cf50"); + } } diff --git a/src/tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/UpdatedQualityProfileTest.cs b/src/tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/UpdatedQualityProfileTest.cs index 07e7b1f1..fe5a8bf7 100644 --- a/src/tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/UpdatedQualityProfileTest.cs +++ b/src/tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/UpdatedQualityProfileTest.cs @@ -229,8 +229,7 @@ public class UpdatedQualityProfileTest }, ProfileConfig = NewQp.Processed(new QualityProfileConfig { - // todo: Why is this commented out? - // UpgradeUntilQuality = "Quality Item 9" + // Do not specify an `UpgradeUntilQuality` here to simulate fallback }), UpdatedQualities = new UpdatedQualities {