Fixed: Unable to load UI if Quality Profiles contain removed Custom Format items

Manually cherry picked from Sonarr commits:

- 2c004e1f9665763111fcd964b81338bdbe735865
- 4b4301a076488c595969921697d7002ca427c955

src/NzbDrone.Core.Test/DecisionEngineTests/UpgradeAllowedSpecificationFixture.cs

@@ -40,13 +40,11 @@ namespace NzbDrone.Core.Test.DecisionEngineTests
                 {
                     new ProfileFormatItem
                     {
-                        Id = 1,
                         Format = _customFormatOne,
                         Score = 50
                     },
                     new ProfileFormatItem
                     {
-                        Id = 1,
                         Format = _customFormatTwo,
                         Score = 100
                     }

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupQualityProfileFormatItemsFixture.cs

@@ -0,0 +1,133 @@
+using System.Collections.Generic;
+using System.Linq;
+using FizzWare.NBuilder;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.CustomFormats;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Housekeeping.Housekeepers;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Profiles;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
+{
+    [TestFixture]
+    public class CleanupQualityProfileFormatItemsFixture : DbTest<CleanupQualityProfileFormatItems, Profile>
+    {
+        [SetUp]
+        public void Setup()
+        {
+            Mocker.SetConstant<IQualityProfileFormatItemsCleanupRepository>(
+                new QualityProfileFormatItemsCleanupRepository(Mocker.Resolve<IMainDatabase>(), Mocker.Resolve<IEventAggregator>()));
+
+            Mocker.SetConstant<ICustomFormatRepository>(
+                new CustomFormatRepository(Mocker.Resolve<IMainDatabase>(), Mocker.Resolve<IEventAggregator>()));
+        }
+
+        [Test]
+        public void should_remove_orphaned_custom_formats()
+        {
+            var qualityProfile = Builder<Profile>.CreateNew()
+                .With(h => h.Items = Qualities.QualityFixture.GetDefaultQualities())
+                .With(h => h.MinFormatScore = 50)
+                .With(h => h.CutoffFormatScore = 100)
+                .With(h => h.FormatItems = new List<ProfileFormatItem>
+                {
+                    Builder<ProfileFormatItem>.CreateNew()
+                        .With(c => c.Format = new CustomFormat("My Custom Format") { Id = 0 })
+                        .Build()
+                })
+                .BuildNew();
+
+            Db.Insert(qualityProfile);
+            Subject.Clean();
+
+            var result = AllStoredModels;
+
+            result.Should().HaveCount(1);
+            result.First().FormatItems.Should().BeEmpty();
+            result.First().MinFormatScore.Should().Be(0);
+            result.First().CutoffFormatScore.Should().Be(0);
+        }
+
+        [Test]
+        public void should_not_remove_unorphaned_custom_formats()
+        {
+            var minFormatScore = 50;
+            var cutoffFormatScore = 100;
+
+            var customFormat = Builder<CustomFormat>.CreateNew()
+                .With(h => h.Specifications = new List<ICustomFormatSpecification>())
+                .BuildNew();
+
+            Db.Insert(customFormat);
+
+            var qualityProfile = Builder<Profile>.CreateNew()
+                .With(h => h.Items = Qualities.QualityFixture.GetDefaultQualities())
+                .With(h => h.MinFormatScore = minFormatScore)
+                .With(h => h.CutoffFormatScore = cutoffFormatScore)
+                .With(h => h.FormatItems = new List<ProfileFormatItem>
+                {
+                    Builder<ProfileFormatItem>.CreateNew()
+                        .With(c => c.Format = customFormat)
+                        .Build()
+                })
+                .BuildNew();
+
+            Db.Insert(qualityProfile);
+
+            Subject.Clean();
+            var result = AllStoredModels;
+
+            result.Should().HaveCount(1);
+            result.First().FormatItems.Should().HaveCount(1);
+            result.First().MinFormatScore.Should().Be(minFormatScore);
+            result.First().CutoffFormatScore.Should().Be(cutoffFormatScore);
+        }
+
+        [Test]
+        public void should_add_missing_custom_formats()
+        {
+            var minFormatScore = 50;
+            var cutoffFormatScore = 100;
+
+            var customFormat1 = Builder<CustomFormat>.CreateNew()
+                .With(h => h.Id = 1)
+                .With(h => h.Name = "Custom Format 1")
+                .With(h => h.Specifications = new List<ICustomFormatSpecification>())
+                .BuildNew();
+
+            var customFormat2 = Builder<CustomFormat>.CreateNew()
+                .With(h => h.Id = 2)
+                .With(h => h.Name = "Custom Format 2")
+                .With(h => h.Specifications = new List<ICustomFormatSpecification>())
+                .BuildNew();
+
+            Db.Insert(customFormat1);
+            Db.Insert(customFormat2);
+
+            var qualityProfile = Builder<Profile>.CreateNew()
+                .With(h => h.Items = Qualities.QualityFixture.GetDefaultQualities())
+                .With(h => h.MinFormatScore = minFormatScore)
+                .With(h => h.CutoffFormatScore = cutoffFormatScore)
+                .With(h => h.FormatItems = new List<ProfileFormatItem>
+                {
+                    Builder<ProfileFormatItem>.CreateNew()
+                        .With(c => c.Format = customFormat1)
+                        .Build()
+                })
+                .BuildNew();
+
+            Db.Insert(qualityProfile);
+
+            Subject.Clean();
+            var result = AllStoredModels;
+
+            result.Should().HaveCount(1);
+            result.First().FormatItems.Should().HaveCount(2);
+            result.First().MinFormatScore.Should().Be(minFormatScore);
+            result.First().CutoffFormatScore.Should().Be(cutoffFormatScore);
+        }
+    }
+}

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupQualityProfileFormatItems.cs

@@ -0,0 +1,91 @@
+using System.Collections.Generic;
+using System.Linq;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.CustomFormats;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Profiles;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class CleanupQualityProfileFormatItems : IHousekeepingTask
+    {
+        private readonly IQualityProfileFormatItemsCleanupRepository _repository;
+        private readonly ICustomFormatRepository _customFormatRepository;
+
+        public CleanupQualityProfileFormatItems(IQualityProfileFormatItemsCleanupRepository repository,
+                                                ICustomFormatRepository customFormatRepository)
+        {
+            _repository = repository;
+            _customFormatRepository = customFormatRepository;
+        }
+
+        public void Clean()
+        {
+            var customFormats = _customFormatRepository.All().ToDictionary(c => c.Id);
+            var profiles = _repository.All();
+            var updatedProfiles = new List<Profile>();
+
+            foreach (var profile in profiles)
+            {
+                var formatItems = new List<ProfileFormatItem>();
+
+                // Make sure the profile doesn't include formats that have been removed
+                profile.FormatItems.ForEach(p =>
+                {
+                    if (p.Format != null && customFormats.ContainsKey(p.Format.Id))
+                    {
+                        formatItems.Add(p);
+                    }
+                });
+
+                // Make sure the profile includes all available formats
+                foreach (var customFormat in customFormats)
+                {
+                    if (formatItems.None(f => f.Format.Id == customFormat.Key))
+                    {
+                        formatItems.Insert(0, new ProfileFormatItem
+                        {
+                            Format = customFormat.Value,
+                            Score = 0
+                        });
+                    }
+                }
+
+                var previousIds = profile.FormatItems.Select(i => i.Format.Id).ToList();
+                var ids = formatItems.Select(i => i.Format.Id).ToList();
+
+                // Update the profile if any formats were added or removed
+                if (ids.Except(previousIds).Any() || previousIds.Except(ids).Any())
+                {
+                    profile.FormatItems = formatItems;
+
+                    if (profile.FormatItems.Empty())
+                    {
+                        profile.MinFormatScore = 0;
+                        profile.CutoffFormatScore = 0;
+                    }
+
+                    updatedProfiles.Add(profile);
+                }
+            }
+
+            if (updatedProfiles.Any())
+            {
+                _repository.SetFields(updatedProfiles, p => p.FormatItems, p => p.MinFormatScore, p => p.CutoffFormatScore);
+            }
+        }
+    }
+
+    public interface IQualityProfileFormatItemsCleanupRepository : IBasicRepository<Profile>
+    {
+    }
+
+    public class QualityProfileFormatItemsCleanupRepository : BasicRepository<Profile>, IQualityProfileFormatItemsCleanupRepository
+    {
+        public QualityProfileFormatItemsCleanupRepository(IMainDatabase database, IEventAggregator eventAggregator)
+            : base(database, eventAggregator)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Profiles/ProfileFormatItem.cs

@@ -1,4 +1,3 @@
-using System.Text.Json.Serialization;
 using NzbDrone.Core.CustomFormats;
 using NzbDrone.Core.Datastore;
 
@@ -6,8 +5,6 @@ namespace NzbDrone.Core.Profiles
 {
     public class ProfileFormatItem : IEmbeddedDocument
     {
-        [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingDefault)]
-        public int Id { get; set; }
         public CustomFormat Format { get; set; }
         public int Score { get; set; }
     }

src/NzbDrone.Core/Profiles/ProfileRepository.cs

@@ -34,10 +34,20 @@ namespace NzbDrone.Core.Profiles
             // all the custom formats
             foreach (var profile in profiles)
             {
+                var formatItems = new List<ProfileFormatItem>();
+
                 foreach (var formatItem in profile.FormatItems)
                 {
-                    formatItem.Format = cfs[formatItem.Format.Id];
+                    // Skip any format that has been removed, but the profile wasn't updated properly
+                    if (cfs.ContainsKey(formatItem.Format.Id))
+                    {
+                        formatItem.Format = cfs[formatItem.Format.Id];
+
+                        formatItems.Add(formatItem);
+                    }
                 }
+
+                profile.FormatItems = formatItems;
             }
 
             return profiles;

src/NzbDrone.Core/Profiles/ProfileService.cs

@@ -250,7 +250,6 @@ namespace NzbDrone.Core.Profiles
 
             var formatItems = _formatService.All().Select(format => new ProfileFormatItem
             {
-                Id = format.Id,
                 Score = 0,
                 Format = format
             }).ToList();