From 4d193b2279b63fa71ba36e0ac8c2f23f9d70168b Mon Sep 17 00:00:00 2001 From: Qstick Date: Thu, 30 Jul 2020 22:46:18 -0400 Subject: [PATCH] Fixed: Fix Movies, Lists with Invalid Profile on Migration --- ...0_fix_invalid_profile_referencesFixture.cs | 221 +++++++++++++ .../180_fix_invalid_profile_references.cs | 308 ++++++++++++++++++ 2 files changed, 529 insertions(+) create mode 100644 src/NzbDrone.Core.Test/Datastore/Migration/180_fix_invalid_profile_referencesFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/180_fix_invalid_profile_references.cs diff --git a/src/NzbDrone.Core.Test/Datastore/Migration/180_fix_invalid_profile_referencesFixture.cs b/src/NzbDrone.Core.Test/Datastore/Migration/180_fix_invalid_profile_referencesFixture.cs new file mode 100644 index 000000000..ee6218366 --- /dev/null +++ b/src/NzbDrone.Core.Test/Datastore/Migration/180_fix_invalid_profile_referencesFixture.cs @@ -0,0 +1,221 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Datastore.Migration; +using NzbDrone.Core.Languages; +using NzbDrone.Core.Movies; +using NzbDrone.Core.Profiles; +using NzbDrone.Core.Qualities; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Datastore.Migration +{ + [TestFixture] + public class fix_invalid_profile_referencesFixture : MigrationTest + { + private void AddDefaultProfile(fix_invalid_profile_references m, string name, int profileId) + { + var allowed = new Quality[] { Quality.WEBDL720p }; + + var items = Quality.DefaultQualityDefinitions + .OrderBy(v => v.Weight) + .Select(v => new { Quality = (int)v.Quality, Allowed = allowed.Contains(v.Quality) }) + .ToList(); + + var profile = new + { + Id = profileId, + Name = name, + FormatItems = new List().ToJson(), + Cutoff = (int)Quality.WEBDL720p, + Items = items.ToJson(), + Language = (int)Language.English, + MinFormatScore = 0, + CutOffFormatScore = 0 + }; + + m.Insert.IntoTable("Profiles").Row(profile); + } + + private void AddMovie(fix_invalid_profile_references m, string movieTitle, int tmdbId, int profileId) + { + var movie = new + { + Id = tmdbId, + Monitored = true, + Title = movieTitle, + CleanTitle = movieTitle, + Status = MovieStatusType.Announced, + MinimumAvailability = MovieStatusType.Announced, + Images = new[] { new { CoverType = "Poster" } }.ToJson(), + Recommendations = new[] { 1 }.ToJson(), + HasPreDBEntry = false, + Runtime = 90, + OriginalLanguage = 1, + ProfileId = profileId, + MovieFileId = 1, + Path = string.Format("/Movies/{0}", movieTitle), + TitleSlug = movieTitle, + TmdbId = tmdbId + }; + + m.Insert.IntoTable("Movies").Row(movie); + } + + private void AddCustomFormat(fix_invalid_profile_references c, int id, string name) + { + var customFormat = new + { + Id = id, + Name = name, + Specifications = "[]" + }; + + c.Insert.IntoTable("CustomFormats").Row(customFormat); + } + + [Test] + public void should_add_default_profiles_if_none_exist_but_movies_exist() + { + var profileId = 18; + + var db = WithMigrationTestDb(c => + { + AddMovie(c, "movie", 123456, profileId); + }); + + var items = db.Query("SELECT Id, ProfileId FROM Movies"); + var profiles = db.Query("SELECT Id FROM Profiles"); + + items.Should().HaveCount(1); + profiles.Should().HaveCount(6); + items.First().ProfileId.Should().BeOneOf(profiles.Select(p => p.Id)); + } + + [Test] + public void should_not_add_default_profiles_if_one_exist() + { + var profileId = 18; + + var db = WithMigrationTestDb(c => + { + AddDefaultProfile(c, "My Custom Profile", profileId); + AddMovie(c, "movie", 123456, 17); + }); + + var items = db.Query("SELECT Id, ProfileId FROM Movies"); + var profiles = db.Query("SELECT Id FROM Profiles"); + + items.Should().HaveCount(1); + profiles.Should().HaveCount(1); + items.First().ProfileId.Should().BeOneOf(profiles.Select(p => p.Id)); + } + + [Test] + public void should_add_custom_formats_to_default_profiles_if_some_exist() + { + var profileId = 18; + var formatId = 3; + + var db = WithMigrationTestDb(c => + { + AddCustomFormat(c, formatId, "SomeFormat"); + AddMovie(c, "movie", 123456, profileId); + }); + + var items = db.Query("SELECT Id, ProfileId FROM Movies"); + var profiles = db.Query("SELECT Id, FormatItems FROM Profiles"); + + items.Should().HaveCount(1); + profiles.Should().HaveCount(6); + profiles.First().FormatItems.Should().HaveCount(1); + profiles.First().FormatItems.First().Format.Should().Be(formatId); + items.First().ProfileId.Should().BeOneOf(profiles.Select(p => p.Id)); + } + + [Test] + public void should_not_change_movies_with_valid_profile() + { + var profileId = 2; + + var db = WithMigrationTestDb(c => + { + AddDefaultProfile(c, "My Custom Profile", profileId); + AddMovie(c, "movie", 123456, profileId); + }); + + var items = db.Query("SELECT Id, ProfileId FROM Movies"); + + items.Should().HaveCount(1); + items.First().ProfileId.Should().Be(profileId); + } + + [Test] + public void should_change_movies_with_bad_profile_id() + { + var profileId = 2; + + var db = WithMigrationTestDb(c => + { + AddDefaultProfile(c, "My Custom Profile", profileId); + AddMovie(c, "movie", 123456, 1); + }); + + var items = db.Query("SELECT Id, ProfileId FROM Movies"); + + items.Should().HaveCount(1); + items.First().ProfileId.Should().Be(profileId); + } + + [Test] + public void should_change_to_most_common_valid_profile_in_library() + { + var commonProfileId = 2; + var otherProfileId = 3; + + var db = WithMigrationTestDb(c => + { + AddDefaultProfile(c, "My Custom Profile", commonProfileId); + AddDefaultProfile(c, "My Custom Profile 2", otherProfileId); + AddMovie(c, "movie1", 123451, 1); + AddMovie(c, "movie2", 123452, 1); + AddMovie(c, "movie3", 123453, 1); + AddMovie(c, "movie4", 123454, 1); + AddMovie(c, "movie5", 123455, commonProfileId); + AddMovie(c, "movie6", 123456, commonProfileId); + AddMovie(c, "movie7", 123457, commonProfileId); + AddMovie(c, "movie8", 123458, otherProfileId); + AddMovie(c, "movie9", 123459, otherProfileId); + }); + + var items = db.Query("SELECT Id, ProfileId FROM Movies"); + + items.Should().HaveCount(9); + items.Where(x => x.ProfileId == commonProfileId).Should().HaveCount(7); + } + } + + public class Movie179 + { + public int Id { get; set; } + public int ProfileId { get; set; } + } + + public class Profile179 + { + public int Id { get; set; } + public List FormatItems { get; set; } + } + + public class ProfileFormatItem179 + { + public int Id { get; set; } + public int Format { get; set; } + public int Score { get; set; } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/180_fix_invalid_profile_references.cs b/src/NzbDrone.Core/Datastore/Migration/180_fix_invalid_profile_references.cs new file mode 100644 index 000000000..3826d79ed --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/180_fix_invalid_profile_references.cs @@ -0,0 +1,308 @@ +using System.Collections.Generic; +using System.Data; +using System.Linq; +using Dapper; +using FluentMigrator; +using NzbDrone.Core.Datastore.Converters; +using NzbDrone.Core.Datastore.Migration.Framework; +using NzbDrone.Core.Languages; +using NzbDrone.Core.Qualities; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(180)] + public class fix_invalid_profile_references : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Execute.WithConnection(FixMovies); + } + + private void FixMovies(IDbConnection conn, IDbTransaction tran) + { + var profiles = GetProfileIds(conn); + var movieRows = conn.Query($"SELECT Id, ProfileId FROM Movies"); + var listRows = conn.Query($"SELECT Id, ProfileId FROM NetImport"); + + // Only process if there are lists or movies existing in the DB + if (movieRows.Any() || listRows.Any()) + { + //If there are no Profiles lets add the defaults + if (!profiles.Any()) + { + InsertDefaultQualityProfiles(conn, tran); + profiles = GetProfileIds(conn); + } + + var mostCommonProfileId = 0; + + //If we have some movies, lets determine the most common profile used and use it for the bad entries + if (movieRows.Any()) + { + mostCommonProfileId = movieRows.Select(x => x.ProfileId) + .Where(x => profiles.Contains(x)) + .GroupBy(p => p) + .OrderByDescending(g => g.Count()) + .Select(g => g.Key) + .FirstOrDefault(); + } + + // If all the movie profiles are bad or there are no movies, just use the first profile for bad movies and lsits + if (mostCommonProfileId == 0) + { + mostCommonProfileId = profiles.First(); + } + + //Correct any Movies that reference profiles that are null + var sql = $"UPDATE Movies SET ProfileId = {mostCommonProfileId} WHERE Id IN(SELECT Movies.Id FROM Movies LEFT OUTER JOIN Profiles ON Movies.ProfileId = Profiles.Id WHERE Profiles.Id IS NULL)"; + conn.Execute(sql, transaction: tran); + + //Correct any Lists that reference profiles that are null + sql = $"UPDATE NetImport SET ProfileId = {mostCommonProfileId} WHERE Id IN(SELECT NetImport.Id FROM NetImport LEFT OUTER JOIN Profiles ON NetImport.ProfileId = Profiles.Id WHERE Profiles.Id IS NULL)"; + conn.Execute(sql, transaction: tran); + } + } + + private List GetProfileIds(IDbConnection conn) + { + return conn.Query("SELECT Id From Profiles").Select(p => p.Id).ToList(); + } + + private void InsertDefaultQualityProfiles(IDbConnection conn, IDbTransaction tran) + { + var profiles = GetDefaultQualityProfiles(conn); + var formatItemConverter = new EmbeddedDocumentConverter>(new CustomFormatIntConverter()); + var profileItemConverter = new EmbeddedDocumentConverter>(new QualityIntConverter()); + var profileId = 1; + + foreach (var profile in profiles.OrderBy(p => p.Id)) + { + using (IDbCommand insertNewLanguageProfileCmd = conn.CreateCommand()) + { + insertNewLanguageProfileCmd.Transaction = tran; + insertNewLanguageProfileCmd.CommandText = "INSERT INTO Profiles (Id, Name, Cutoff, Items, Language, FormatItems, MinFormatScore, CutoffFormatScore, UpgradeAllowed) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"; + insertNewLanguageProfileCmd.AddParameter(profileId); + insertNewLanguageProfileCmd.AddParameter(profile.Name); + insertNewLanguageProfileCmd.AddParameter(profile.Cutoff); + + var paramItems = insertNewLanguageProfileCmd.CreateParameter(); + profileItemConverter.SetValue(paramItems, profile.Items); + + insertNewLanguageProfileCmd.Parameters.Add(paramItems); + insertNewLanguageProfileCmd.AddParameter(profile.Language.Id); + + var paramFormats = insertNewLanguageProfileCmd.CreateParameter(); + formatItemConverter.SetValue(paramFormats, profile.FormatItems); + + insertNewLanguageProfileCmd.Parameters.Add(paramFormats); + insertNewLanguageProfileCmd.AddParameter(profile.MinFormatScore); + insertNewLanguageProfileCmd.AddParameter(profile.CutoffFormatScore); + insertNewLanguageProfileCmd.AddParameter(profile.UpgradeAllowed); + + insertNewLanguageProfileCmd.ExecuteNonQuery(); + } + + profileId += 1; + } + } + + private List GetDefaultQualityProfiles(IDbConnection conn) + { + var profiles = new List(); + + //Grab custom formats if any exist and add them to the new profiles + var formats = conn.Query($"SELECT Id FROM CustomFormats").ToList(); + + profiles.Add(GetDefaultProfile("Any", + formats, + Quality.Bluray480p, + Quality.WORKPRINT, + Quality.CAM, + Quality.TELESYNC, + Quality.TELECINE, + Quality.DVDSCR, + Quality.REGIONAL, + Quality.SDTV, + Quality.DVD, + Quality.DVDR, + Quality.HDTV720p, + Quality.HDTV1080p, + Quality.HDTV2160p, + Quality.WEBDL480p, + Quality.WEBRip480p, + Quality.WEBDL720p, + Quality.WEBRip720p, + Quality.WEBDL1080p, + Quality.WEBRip1080p, + Quality.WEBDL2160p, + Quality.WEBRip2160p, + Quality.Bluray480p, + Quality.Bluray576p, + Quality.Bluray720p, + Quality.Bluray1080p, + Quality.Bluray2160p, + Quality.Remux1080p, + Quality.Remux2160p, + Quality.BRDISK)); + + profiles.Add(GetDefaultProfile("SD", + formats, + Quality.Bluray480p, + Quality.WORKPRINT, + Quality.CAM, + Quality.TELESYNC, + Quality.TELECINE, + Quality.DVDSCR, + Quality.REGIONAL, + Quality.SDTV, + Quality.DVD, + Quality.WEBDL480p, + Quality.WEBRip480p, + Quality.Bluray480p, + Quality.Bluray576p)); + + profiles.Add(GetDefaultProfile("HD-720p", + formats, + Quality.Bluray720p, + Quality.HDTV720p, + Quality.WEBDL720p, + Quality.WEBRip720p, + Quality.Bluray720p)); + + profiles.Add(GetDefaultProfile("HD-1080p", + formats, + Quality.Bluray1080p, + Quality.HDTV1080p, + Quality.WEBDL1080p, + Quality.WEBRip1080p, + Quality.Bluray1080p, + Quality.Remux1080p)); + + profiles.Add(GetDefaultProfile("Ultra-HD", + formats, + Quality.Remux2160p, + Quality.HDTV2160p, + Quality.WEBDL2160p, + Quality.WEBRip2160p, + Quality.Bluray2160p, + Quality.Remux2160p)); + + profiles.Add(GetDefaultProfile("HD - 720p/1080p", + formats, + Quality.Bluray720p, + Quality.HDTV720p, + Quality.HDTV1080p, + Quality.WEBDL720p, + Quality.WEBRip720p, + Quality.WEBDL1080p, + Quality.WEBRip1080p, + Quality.Bluray720p, + Quality.Bluray1080p, + Quality.Remux1080p)); + + return profiles; + } + + private QualityProfile180 GetDefaultProfile(string name, List formats, Quality cutoff = null, params Quality[] allowed) + { + var groupedQualites = Quality.DefaultQualityDefinitions.GroupBy(q => q.Weight); + var items = new List(); + var groupId = 1000; + var profileCutoff = cutoff == null ? Quality.Unknown.Id : cutoff.Id; + + foreach (var group in groupedQualites) + { + if (group.Count() == 1) + { + var quality = group.First().Quality; + + items.Add(new QualityProfileItem111 { Quality = group.First().Quality, Allowed = allowed.Contains(quality), Items = new List() }); + continue; + } + + var groupAllowed = group.Any(g => allowed.Contains(g.Quality)); + + items.Add(new QualityProfileItem111 + { + Id = groupId, + Name = group.First().GroupName, + Items = group.Select(g => new QualityProfileItem111 + { + Quality = g.Quality, + Allowed = groupAllowed, + Items = new List() + }).ToList(), + Allowed = groupAllowed + }); + + if (group.Any(g => g.Quality.Id == profileCutoff)) + { + profileCutoff = groupId; + } + + groupId++; + } + + var formatItems = formats.Select(format => new ProfileFormatItem180 + { + Id = format.Id, + Score = 0, + Format = format.Id + }).ToList(); + + var qualityProfile = new QualityProfile180 + { + Name = name, + Cutoff = profileCutoff, + Items = items, + Language = Language.English, + MinFormatScore = 0, + CutoffFormatScore = 0, + UpgradeAllowed = 0, + FormatItems = formatItems + }; + + return qualityProfile; + } + + private class ProfileEntity179 + { + public int Id { get; set; } + public int ProfileId { get; set; } + } + + private class QualityProfile180 + { + public int Id { get; set; } + public string Name { get; set; } + public int Cutoff { get; set; } + public int MinFormatScore { get; set; } + public int CutoffFormatScore { get; set; } + public int UpgradeAllowed { get; set; } + public Language Language { get; set; } + public List FormatItems { get; set; } + public List Items { get; set; } + } + + private class QualityProfileItem111 + { + public int Id { get; set; } + public string Name { get; set; } + public Quality Quality { get; set; } + public List Items { get; set; } + public bool Allowed { get; set; } + } + + private class ProfileFormatItem180 + { + public int Id { get; set; } + public int Format { get; set; } + public int Score { get; set; } + } + + private class CustomFormat180 + { + public int Id { get; set; } + } + } +}