feat: Repair quality profiles missing required qualities

In rare circumstances outside of Recyclarr, quality profiles become
invalid due to missing required qualities. When this happens, users are
not even able to save the profile using the Sonarr or Radarr UI.
Recyclarr now detects this situation and automatically repairs the
quality profile by re-adding these missing qualities for users.

See: https://github.com/Radarr/Radarr/issues/9738

CHANGELOG.md

@@ -8,6 +8,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- In rare circumstances outside of Recyclarr, quality profiles become invalid due to missing
+  required qualities. When this happens, users are not even able to save the profile using the
+  Sonarr or Radarr UI. Recyclarr now detects this situation and automatically repairs the quality
+  profile by re-adding these missing qualities for users. See [this issue][9738].
+
+[9738]: https://github.com/Radarr/Radarr/issues/9738
+
 ## [7.0.0] - 2024-06-27
 
 This release contains **BREAKING CHANGES**. See the [v7.0 Upgrade Guide][breaking7] for required

src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileLogPhase.cs

@@ -49,26 +49,30 @@ public class QualityProfileLogPhase(ILogger log) : ILogPipelinePhase<QualityProf
             }
         }
 
-        var invalidQualityNames = transactions.ChangedProfiles
+        foreach (var profile in transactions.ChangedProfiles.Select(x => x.Profile))
-            .Select(x => (x.Profile.ProfileName, x.Profile.UpdatedQualities.InvalidQualityNames))
+        {
-            .Where(x => x.InvalidQualityNames.Count != 0)
+            var invalidQualityNames = profile.UpdatedQualities.InvalidQualityNames;
-            .ToList();
+            if (invalidQualityNames.Count != 0)
-
-        foreach (var (profileName, invalidNames) in invalidQualityNames)
             {
                 log.Warning("Quality profile '{ProfileName}' references invalid quality names: {InvalidNames}",
-                profileName, invalidNames);
+                    profile.ProfileName, invalidQualityNames);
             }
 
-        var invalidCfExceptNames = transactions.ChangedProfiles
+            var invalidCfExceptNames = profile.InvalidExceptCfNames;
-            .Where(x => x.Profile.InvalidExceptCfNames.Count != 0)
+            if (invalidCfExceptNames.Count != 0)
-            .Select(x => (x.Profile.ProfileName, x.Profile.InvalidExceptCfNames));
-
-        foreach (var (profileName, invalidNames) in invalidCfExceptNames)
             {
                 log.Warning(
                     "`except` under `reset_unmatched_scores` in quality profile '{ProfileName}' has invalid " +
-                "CF names: {CfNames}", profileName, invalidNames);
+                    "CF names: {CfNames}", profile.ProfileName, invalidCfExceptNames);
+            }
+
+            var missingQualities = profile.MissingQualities;
+            if (missingQualities.Count != 0)
+            {
+                log.Information(
+                    "Recyclarr detected that the following required qualities are missing from profile " +
+                    "'{ProfileName}' and will re-add them: {QualityNames}", profile.ProfileName, missingQualities);
+            }
         }
     }
 

src/Recyclarr.Cli/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhase.cs

@@ -49,14 +49,14 @@ public class QualityProfileTransactionPhase(QualityProfileStatCalculator statCal
 
     private static List<UpdatedQualityProfile> BuildUpdatedProfiles(
         QualityProfileTransactionData transactions,
-        IEnumerable<ProcessedQualityProfileData> guideData,
+        IEnumerable<ProcessedQualityProfileData> processedConfig,
         QualityProfileServiceData serviceData)
     {
         // Match quality profiles in the user's config to profiles in the service.
         // For each match, we return a tuple including the list of custom format scores ("formatItems").
         // Using GroupJoin() because we want a LEFT OUTER JOIN so we can list which quality profiles in config
         // do not match profiles in Radarr.
-        var matchedProfiles = guideData
+        var matchedProfiles = processedConfig
             .GroupJoin(serviceData.Profiles,
                 x => x.Profile.Name,
                 x => x.Name,
@@ -74,20 +74,58 @@ public class QualityProfileTransactionPhase(QualityProfileStatCalculator statCal
             }
 
             var organizer = new QualityItemOrganizer();
-            var newDto = dto ?? serviceData.Schema;
 
+            if (dto is null)
+            {
+                AddDto(serviceData.Schema, QualityProfileUpdateReason.New);
+            }
+            else
+            {
+                var missingQualities = FixupMissingQualities(dto, serviceData.Schema);
+                AddDto(dto, QualityProfileUpdateReason.Changed);
+                updatedProfiles[^1].MissingQualities = missingQualities;
+            }
+
+            continue;
+
+            void AddDto(QualityProfileDto newDto, QualityProfileUpdateReason reason)
+            {
                 updatedProfiles.Add(new UpdatedQualityProfile
                 {
                     ProfileConfig = config,
                     ProfileDto = newDto,
-                UpdateReason = dto is null ? QualityProfileUpdateReason.New : QualityProfileUpdateReason.Changed,
+                    UpdateReason = reason,
                     UpdatedQualities = organizer.OrganizeItems(newDto, config.Profile)
                 });
             }
+        }
 
         return updatedProfiles;
     }
 
+    private static List<string> FixupMissingQualities(
+        QualityProfileDto dto,
+        QualityProfileDto schema)
+    {
+        // There's a very rare bug in Sonarr & Radarr that results in core qualities being lost in an existing profile.
+        // It's unclear how this happens; but what ends up happening is that you get an error "Must contain all
+        // qualities" in the Sonarr frontend when you open a QP and simply click save. In Recyclarr, you also see this
+        // error when attempting to sync changes to that profile. While this bug is not caused by recyclarr, we do not
+        // want this to prevent users from having to sync. The workaround to fix this (linked below) is very cumbersome,
+        // so there's value in having Recyclarr transparently fix this for users.
+        //
+        // See: https://github.com/Radarr/Radarr/issues/9738
+        var missingQualities = schema.Items.FlattenQualities().LeftOuterHashJoin(dto.Items.FlattenQualities(),
+                l => l.Quality!.Id,
+                r => r.Quality!.Id)
+            .Where(x => x.Right is null)
+            .Select(x => x.Left)
+            .ToList();
+
+        dto.Items = dto.Items.Concat(missingQualities).ToList();
+        return missingQualities.Select(x => x.Quality!.Name ?? $"(id: {x.Quality.Id})").ToList();
+    }
+
     private static void UpdateProfileScores(IEnumerable<UpdatedQualityProfile> updatedProfiles)
     {
         foreach (var profile in updatedProfiles)

src/Recyclarr.Cli/Pipelines/QualityProfile/QualityProfileExtensions.cs

@@ -10,6 +10,12 @@ public static class QualityProfileExtensions
         return items.Flatten(x => x.Items);
     }
 
+    public static IEnumerable<ProfileItemDto> FlattenQualities(this IEnumerable<ProfileItemDto> items)
+    {
+        return FlattenItems(items)
+            .Where(x => x.Quality is not null);
+    }
+
     public static ProfileItemDto? FindGroupById(this IEnumerable<ProfileItemDto> items, int? id)
     {
         if (id is null)

src/Recyclarr.Cli/Pipelines/QualityProfile/QualityProfileStatCalculator.cs

@@ -51,7 +51,7 @@ public class QualityProfileStatCalculator(ILogger log)
     {
         using var oldJson = JsonSerializer.SerializeToDocument(oldDto.Items);
         using var newJson = JsonSerializer.SerializeToDocument(newDto.Items);
-        stats.QualitiesChanged = !oldJson.DeepEquals(newJson);
+        stats.QualitiesChanged = stats.Profile.MissingQualities.Count > 0 || !oldJson.DeepEquals(newJson);
     }
 
     private void ScoreUpdates(

src/Recyclarr.Cli/Pipelines/QualityProfile/UpdatedQualityProfile.cs

@@ -18,6 +18,7 @@ public record UpdatedQualityProfile
     public IReadOnlyCollection<UpdatedFormatScore> UpdatedScores { get; set; } = Array.Empty<UpdatedFormatScore>();
     public UpdatedQualities UpdatedQualities { get; init; } = new();
     public IReadOnlyCollection<string> InvalidExceptCfNames { get; set; } = Array.Empty<string>();
+    public IReadOnlyCollection<string> MissingQualities { get; set; } = Array.Empty<string>();
 
     public string ProfileName
     {

tests/Recyclarr.Cli.Tests/Pipelines/QualityProfile/PipelinePhases/QualityProfileTransactionPhaseTest.cs

@@ -471,4 +471,65 @@ public class QualityProfileTransactionPhaseTest
             .ContainSingle().Which.Profile.InvalidExceptCfNames.Should()
             .BeEquivalentTo("cf50");
     }
+
+    [Test, AutoMockData]
+    public void Missing_required_qualities_are_readded(QualityProfileTransactionPhase sut)
+    {
+        var dtos = new[]
+        {
+            new QualityProfileDto
+            {
+                Name = "profile1",
+                Items = new[]
+                {
+                    new ProfileItemDto
+                    {
+                        Quality = new ProfileItemQualityDto {Id = 1, Name = "One"}
+                    },
+                    new ProfileItemDto
+                    {
+                        Quality = new ProfileItemQualityDto {Id = 2, Name = "Two"}
+                    }
+                }
+            }
+        };
+
+        var context = new QualityProfilePipelineContext
+        {
+            ConfigOutput = new[]
+            {
+                new ProcessedQualityProfileData
+                {
+                    Profile = new QualityProfileConfig
+                    {
+                        Name = "profile1"
+                    }
+                }
+            },
+            ApiFetchOutput = new QualityProfileServiceData(dtos, new QualityProfileDto())
+            {
+                Schema = new QualityProfileDto
+                {
+                    Items = new[]
+                    {
+                        new ProfileItemDto {Quality = new ProfileItemQualityDto {Id = 1, Name = "One"}},
+                        new ProfileItemDto {Quality = new ProfileItemQualityDto {Id = 2, Name = "Two"}},
+                        new ProfileItemDto {Quality = new ProfileItemQualityDto {Id = 3, Name = "Three"}}
+                    }
+                }
+            }
+        };
+
+        sut.Execute(context);
+
+        var profiles = context.TransactionOutput.ChangedProfiles;
+        profiles.Should().ContainSingle();
+        profiles.First().Profile.MissingQualities.Should().BeEquivalentTo("Three");
+        profiles.First().Profile.ProfileDto.Items.Should().BeEquivalentTo(
+        [
+            new ProfileItemDto {Quality = new ProfileItemQualityDto {Id = 1, Name = "One"}},
+            new ProfileItemDto {Quality = new ProfileItemQualityDto {Id = 2, Name = "Two"}},
+            new ProfileItemDto {Quality = new ProfileItemQualityDto {Id = 3, Name = "Three"}}
+        ]);
+    }
 }