open-source-mirrors
/
Radarr
mirror of https://github.com/Radarr/Radarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed: Clean DB dupes and add unique constrain

Browse Source
pull/5048/head
Qstick 4 years ago
parent 45d329423c
commit d835358f75
2 changed files with 252 additions and 0 deletions
Show Stats Download Patch File Download Diff File

170
src/NzbDrone.Core.Test/Datastore/Migration/186_fix_tmdb_duplicatesFixture.cs
Unescape View File

@ -0,0 +1,170 @@
using System;
using System.Linq;
using FluentAssertions;
using NUnit.Framework;
using NzbDrone.Common.Serializer;
using NzbDrone.Core.Datastore.Migration;
using NzbDrone.Core.Movies;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.Datastore.Migration
{
[TestFixture]
public class fix_tmdb_duplicatesFixture : MigrationTest<fix_tmdb_duplicates>
{
private void AddMovie(fix_tmdb_duplicates m, int id, string movieTitle, string titleSlug, int tmdbId, int movieFileId, DateTime? lastInfo, DateTime added)
{
var movie = new
{
Id = id,
Monitored = true,
Title = movieTitle,
CleanTitle = movieTitle,
Status = MovieStatusType.Announced,
MinimumAvailability = MovieStatusType.Announced,
Images = new[] { new { CoverType = "Poster" } }.ToJson(),
Recommendations = new[] { 1 }.ToJson(),
HasPreDBEntry = false,
Runtime = 90,
OriginalLanguage = 1,
ProfileId = 1,
MovieFileId = movieFileId,
Path = string.Format("/Movies/{0}", movieTitle),
TitleSlug = titleSlug,
TmdbId = tmdbId,
Added = added,
LastInfoSync = lastInfo,
};
m.Insert.IntoTable("Movies").Row(movie);
}
[Test]
public void should_clean_duplicate_movies()
{
var tmdbId = 123465;
var dateAdded = DateTime.UtcNow;
var db = WithMigrationTestDb(c =>
{
AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 2, "movie", "slug1", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded);
});
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
items.Should().HaveCount(1);
}
[Test]
public void should_not_clean_non_duplicate_movies()
{
var tmdbId = 123465;
var dateAdded = DateTime.UtcNow;
var db = WithMigrationTestDb(c =>
{
AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 2, "movie", "slug1", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 5, "movie2", "slug4", 123457, 0, dateAdded, dateAdded);
});
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
items.Should().HaveCount(2);
items.Where(i => i.TmdbId == tmdbId).Should().HaveCount(1);
}
[Test]
public void should_not_clean_any_if_no_duplicate_movies()
{
var dateAdded = DateTime.UtcNow;
var db = WithMigrationTestDb(c =>
{
AddMovie(c, 1, "movie1", "slug", 1, 0, dateAdded, dateAdded);
AddMovie(c, 2, "movie2", "slug1", 2, 0, dateAdded, dateAdded);
AddMovie(c, 3, "movie3", "slug2", 3, 0, dateAdded, dateAdded);
AddMovie(c, 4, "movie4", "slug3", 4, 0, dateAdded, dateAdded);
AddMovie(c, 5, "movie5", "slug4", 123457, 0, dateAdded, dateAdded);
});
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
items.Should().HaveCount(5);
}
[Test]
public void should_keep_movie_with_file_when_duplicates()
{
var tmdbId = 123465;
var dateAdded = DateTime.UtcNow;
var db = WithMigrationTestDb(c =>
{
AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 2, "movie", "slug1", tmdbId, 1, dateAdded, dateAdded);
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 4, "movie", "slug3", tmdbId, 0, dateAdded, dateAdded);
});
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
items.Should().HaveCount(1);
items.First().Id.Should().Be(2);
}
[Test]
public void should_keep_earliest_added_a_movie_with_file_when_duplicates_and_multiple_have_file()
{
var tmdbId = 123465;
var dateAdded = DateTime.UtcNow;
var db = WithMigrationTestDb(c =>
{
AddMovie(c, 1, "movie", "slug", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 2, "movie", "slug1", tmdbId, 1, dateAdded, dateAdded.AddSeconds(200));
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 4, "movie", "slug3", tmdbId, 2, dateAdded, dateAdded);
});
var items = db.Query<Movie185>("SELECT Id, TmdbId, MovieFileId FROM Movies");
items.Should().HaveCount(1);
items.First().MovieFileId.Should().BeGreaterThan(0);
items.First().Id.Should().Be(4);
}
[Test]
public void should_keep_a_movie_with_info_when_duplicates_and_no_file()
{
var tmdbId = 123465;
var dateAdded = DateTime.UtcNow;
var db = WithMigrationTestDb(c =>
{
AddMovie(c, 1, "movie", "slug", tmdbId, 0, null, dateAdded);
AddMovie(c, 2, "movie", "slug1", tmdbId, 0, null, dateAdded);
AddMovie(c, 3, "movie", "slug2", tmdbId, 0, dateAdded, dateAdded);
AddMovie(c, 4, "movie", "slug3", tmdbId, 0, null, dateAdded);
});
var items = db.Query<Movie185>("SELECT Id, LastInfoSync, TmdbId, MovieFileId FROM Movies");
items.Should().HaveCount(1);
items.First().LastInfoSync.Should().NotBeNull();
}
public class Movie185
{
public int Id { get; set; }
public int TmdbId { get; set; }
public int MovieFileId { get; set; }
public DateTime? LastInfoSync { get; set; }
}
}
}

82
src/NzbDrone.Core/Datastore/Migration/186_fix_tmdb_duplicates.cs
Unescape View File

@ -0,0 +1,82 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Linq;
using Dapper;
using FluentMigrator;
using NzbDrone.Core.Datastore.Migration.Framework;
namespace NzbDrone.Core.Datastore.Migration
{
[Migration(186)]
public class fix_tmdb_duplicates : NzbDroneMigrationBase
{
protected override void MainDbUpgrade()
{
Execute.WithConnection(FixMovies);
Delete.Index("IX_Movies_TmdbId").OnTable("Movies");
Alter.Table("Movies").AlterColumn("TmdbId").AsInt32().Unique();
}
private void FixMovies(IDbConnection conn, IDbTransaction tran)
{
var movieRows = conn.Query<MovieEntity185>($"SELECT Id, TmdbId, Added, LastInfoSync, MovieFileId FROM Movies");
// Only process if there are movies existing in the DB
if (movieRows.Any())
{
var movieGroups = movieRows.GroupBy(m => m.TmdbId);
var problemMovies = movieGroups.Where(g => g.Count() > 1);
var purgeMovies = new List<MovieEntity185>();
// Don't do anything if there are no duplicate movies
if (!problemMovies.Any())
{
return;
}
//Process duplicates to pick which to purge
foreach (var problemGroup in problemMovies)
{
var moviesWithFiles = problemGroup.Where(m => m.MovieFileId > 0);
var moviesWithInfo = problemGroup.Where(m => m.LastInfoSync != null);
// If we only have one with file keep it
if (moviesWithFiles.Count() == 1)
{
purgeMovies.AddRange(problemGroup.Where(m => m.MovieFileId == 0).Select(m => m));
continue;
}
// If we only have one with info keep it
if (moviesWithInfo.Count() == 1)
{
purgeMovies.AddRange(problemGroup.Where(m => m.LastInfoSync == null).Select(m => m));
continue;
}
// Else Prioritize by having file then Added
purgeMovies.AddRange(problemGroup.OrderByDescending(m => m.MovieFileId > 0 ? 1 : 0).ThenBy(m => m.Added).Skip(1).Select(m => m));
}
if (purgeMovies.Count > 0)
{
var deleteSql = "DELETE FROM Movies WHERE Id = @Id";
conn.Execute(deleteSql, purgeMovies, transaction: tran);
}
// Delete duplicates, files, metadata, history, etc...
// (Or just the movie and let housekeeper take the rest)
}
}
private class MovieEntity185
{
public int Id { get; set; }
public int TmdbId { get; set; }
public DateTime Added { get; set; }
public DateTime? LastInfoSync { get; set; }
public int MovieFileId { get; set; }
}
}
}
Write Preview
Loading…
Cancel
Save