diff --git a/src/NzbDrone.Core.Test/Extras/ExtraServiceFixture.cs b/src/NzbDrone.Core.Test/Extras/ExtraServiceFixture.cs new file mode 100644 index 000000000..b56a43fff --- /dev/null +++ b/src/NzbDrone.Core.Test/Extras/ExtraServiceFixture.cs @@ -0,0 +1,614 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Disk; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Extras; +using NzbDrone.Core.Extras.Others; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.MediaFiles.Events; +using NzbDrone.Core.MediaFiles.TrackImport; +using NzbDrone.Core.Music; +using NzbDrone.Core.Organizer; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Profiles.Qualities; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Test.Common; + +namespace NzbDrone.Core.Test.Extras +{ + public class ExtraServiceFixture : CoreTest + { + private string _albumDir; + private Artist _artist; + private Album _album; + + [SetUp] + public void CommonSetup() + { + var artistDir = @"C:\Test\Music\Foo Fooers".AsOsAgnostic(); + _artist = new Artist() + { + QualityProfile = new QualityProfile { Items = Qualities.QualityFixture.GetDefaultQualities() }, + Path = artistDir, + }; + _album = new Album() + { + Id = 15, + Artist = _artist, + Title = "Twenty Thirties", + }; + var release = new AlbumRelease() + { + AlbumId = _album.Id, + Monitored = true, + }; + _album.AlbumReleases = new List { release }; + _albumDir = Path.Combine(_artist.Path, $"{_album.Title} (1995) [FLAC]"); + + Mocker.GetMock() + .Setup(x => x.GetParentFolder(It.IsAny())) + .Returns(arg => Path.GetDirectoryName(arg.AsOsAgnostic())); + + Mocker.GetMock() + .Setup(x => x.ImportExtraFiles).Returns(true); + Mocker.GetMock() + .Setup(x => x.ExtraFileExtensions).Returns(".cue,.nfo,.log,.jpg"); + + // Rename on by default + var cfg = NamingConfig.Default; + cfg.RenameTracks = true; + Mocker.GetMock().Setup(x => x.GetConfig()).Returns(cfg); + } + + public class AlbumImportTests : ExtraServiceFixture + { + private List> _importDecisions; + private List _importDirExtraFiles; + + [SetUp] + public void Setup() + { + var track = NewTrack(_album, _albumDir, "01 - hello world.flac"); + _importDecisions = new () + { + new ImportDecision(track) + }; + _importDirExtraFiles = new List + { + Path.Combine(_albumDir, "album.cue"), + Path.Combine(_albumDir, "albumfoo_barz.jpg"), + Path.Combine(_albumDir, "release.nfo"), + Path.Combine(_albumDir, "eac.log"), + }; + + Mocker.GetMock().Setup(x => x.GetFilesByArtist(_album.ArtistId)) + .Returns(track.Tracks.Select(t => t.TrackFile.Value).ToList()); + Mocker.GetMock().Setup(x => x.GetTracksByArtist(_album.ArtistId)) + .Returns(new List { track.Tracks.Single() }); + } + + [Test] + public void should_import_extras_during_manual_import_with_naming_config_having_rename_on() + { + SetupFilesUnderCommonDir(_albumDir, _importDecisions.Select(d => d.Item.Path).Concat(_importDirExtraFiles)); + + // act + Subject.ImportAlbumExtras(_importDecisions); + + // assert + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == _importDirExtraFiles.Count))); + } + + [TestCase(false)] + [TestCase(true)] + public void should_not_import_extras_when_no_separate_album_dir_set(bool testStandardTrackFormat) + { + SetupFilesUnderCommonDir(_albumDir, _importDecisions.Select(d => d.Item.Path).Concat(_importDirExtraFiles)); + + var cfg = NamingConfig.Default; + cfg.RenameTracks = true; + + // modify either standard or multidisc format to test both branches: + if (testStandardTrackFormat) + { + cfg.StandardTrackFormat = "{Artist Name} - {Album Title} - {track:00} - {Track Title}"; + } + else + { + cfg.MultiDiscTrackFormat = "{Medium Format} {medium:00}/{Artist Name} - {Album Title} - {track:00} - {Track Title}"; + } + + SetupNamingConfig(cfg); + + Subject.ImportAlbumExtras(_importDecisions); + + Mocker.GetMock().VerifyNoOtherCalls(); + } + + [Test] + public void should_import_extra_from_multi_cd_root_dir() + { + var cd1Subdir = Path.Combine(_albumDir, "CD1"); + var cd2Subdir = Path.Combine(_albumDir, "CD2"); + + var cd1Track = NewTrack(_album, cd1Subdir, "101 - Foo Track.flac"); + var cd2Track = NewTrack(_album, cd2Subdir, "201 - bonustrackbar.flac"); + + var extraFileInAlbumRoot = Path.Combine(_albumDir, "album.cue"); + + SetupFilesUnderCommonDir(_albumDir, cd1Track.Path, cd2Track.Path, extraFileInAlbumRoot); + + // act + var decisions = new List> + { + new ImportDecision(cd1Track), + new ImportDecision(cd2Track), + }; + + Subject.ImportAlbumExtras(decisions); + + // assert + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == 1))); + Mocker.GetMock() + .Verify(x => x.Upsert( + It.Is>( + arg => arg.Single().Extension == ".cue" + && arg.Single().RelativePath.AsOsAgnostic() == _artist.Path.GetRelativePath(extraFileInAlbumRoot).AsOsAgnostic()))); + } + + [TestCase("")] + [TestCase("extras_subdir")] + public void should_move_album_extra_to_correct_subdir_on_artist_renamed_event(string extraFilesDir) + { + var newDir = $"{_albumDir} [Release FOO]".AsOsAgnostic(); + var renamed = new List(); + foreach (var import in _importDecisions) + { + renamed.Add(new RenamedTrackFile() + { + PreviousPath = import.Item.Path, + TrackFile = new TrackFile() + { + Id = 11, + Album = _album, + AlbumId = _album.Id, + Path = import.Item.Path.Replace(_albumDir, newDir), + Tracks = new List() + { + new Track() { Album = _album, Artist = _artist, TrackFileId = 11 }, + } + }, + }); + } + + var relativePathBeforeMove = Path.Combine(new DirectoryInfo(_albumDir).Name, extraFilesDir, "album.cue"); + var albumExtra = new OtherExtraFile + { + Id = 251, + AlbumId = _album.Id, + ArtistId = _album.ArtistId, + RelativePath = relativePathBeforeMove, + Extension = ".cue", + Added = DateTime.UtcNow, + TrackFileId = null, + }; + + Mocker.GetMock().Setup(x => x.GetFilesByArtist(_album.ArtistId)) + .Returns(new List() { albumExtra }); + + // act + Subject.Handle(new ArtistRenamedEvent(_artist, renamed)); + + var expectedExtraDir = Path.Combine(newDir, extraFilesDir); + + // assert + Mocker.GetMock() + .Verify(x => x.MoveFile( + It.Is(arg => arg.Contains(relativePathBeforeMove)), + It.Is(arg => arg.Contains(expectedExtraDir)), + It.IsAny()), Times.Once); + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == 1))); + } + + [Test] + public void should_move_album_extras_for_multicd_release_on_artist_renamed_event() + { + var newAlbumDir = $"{_albumDir} 2CDs".AsOsAgnostic(); + + var oldCd1Subdir = Path.Combine(_albumDir, "Disk 1"); + var oldCd2Subdir = Path.Combine(_albumDir, "Disk 2"); + var cd1Subdir = Path.Combine(newAlbumDir, "CD1"); + var cd2Subdir = Path.Combine(newAlbumDir, "CD2"); + var cd1Track = NewTrack(_album, cd1Subdir, "101 - Foo Track.flac"); + var cd2Track = NewTrack(_album, cd2Subdir, "201 - bonustrackbar.flac"); + + var renamed = new List() + { + new RenamedTrackFile + { + PreviousPath = Path.Combine(oldCd1Subdir, "101 - Foo Track.flac"), + TrackFile = cd1Track.Tracks.Single().TrackFile.Value, + }, + new RenamedTrackFile + { + PreviousPath = Path.Combine(oldCd2Subdir, "201 - bonustrackbar.flac"), + TrackFile = cd2Track.Tracks.Single().TrackFile.Value, + }, + }; + + var albumDirExtraOldRelativePath = Path.Combine(new DirectoryInfo(_albumDir).Name, "album.cue"); + var albumExtraAtRoot = new OtherExtraFile + { + Id = 251, + AlbumId = _album.Id, + ArtistId = _album.ArtistId, + RelativePath = albumDirExtraOldRelativePath, + Extension = ".cue", + Added = DateTime.UtcNow, + TrackFileId = null, + }; + + var cd1ExtraOldRelativePath = Path.Combine(_artist.Path.GetRelativePath(oldCd1Subdir), "cd1.log"); + var cd1ExtraFile = new OtherExtraFile() + { + Id = 252, + AlbumId = _album.Id, + ArtistId = _album.ArtistId, + RelativePath = cd1ExtraOldRelativePath, + Extension = ".log", + Added = DateTime.UtcNow, + TrackFileId = null, + }; + + Mocker.GetMock().Setup(x => x.GetFilesByArtist(_album.ArtistId)) + .Returns(new List() { albumExtraAtRoot, cd1ExtraFile }); + + // act + Subject.Handle(new ArtistRenamedEvent(_artist, renamed)); + + // verify + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == 2))); + + // assert + Mocker.GetMock() + .Verify(x => x.MoveFile( + It.Is(arg => arg.EndsWithIgnoreCase(albumDirExtraOldRelativePath)), + It.Is(arg => arg.StartsWith(newAlbumDir)), + It.IsAny()), Times.Once); + + Mocker.GetMock() + .Verify(x => x.MoveFile( + It.Is(arg => arg.EndsWithIgnoreCase(cd1ExtraOldRelativePath)), + It.Is(arg => arg.StartsWith(cd1Subdir)), + It.IsAny()), Times.Once); + } + } + + public class AlbumDownloadTests : ExtraServiceFixture + { + private string _downloadDir; + private List> _approvedDownloadDecisions; + private List _downloadDirExtraFiles; + + [SetUp] + public void Setup() + { + _downloadDir = @"C:\temp\downloads\Artist - TT (1995) FLAC".AsOsAgnostic(); + var downloadedTrack = NewTrack(_album, _albumDir, "01 - First seconds.flac", _downloadDir); + _approvedDownloadDecisions = new List>() + { + new ImportDecision(downloadedTrack), + }; + _downloadDirExtraFiles = new List + { + Path.Combine(_downloadDir, "album.cue"), + Path.Combine(_downloadDir, "cover.nfo"), + Path.Combine(_downloadDir, "eac.log"), + }; + } + + [Test] + public void should_import_extras_from_download_location() + { + SetupFilesUnderCommonDir(_downloadDir, _approvedDownloadDecisions.Select(d => d.Item.Path).Concat(_downloadDirExtraFiles)); + + Subject.ImportAlbumExtras(_approvedDownloadDecisions); + + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == _downloadDirExtraFiles.Count))); + foreach (var sourcePath in _downloadDirExtraFiles) + { + Mocker.GetMock() + .Verify(x => x.TransferFile( + It.Is(arg => arg.AsOsAgnostic() == sourcePath.AsOsAgnostic()), + It.Is(arg => arg.AsOsAgnostic().StartsWith(_albumDir.AsOsAgnostic())), + It.IsAny(), + It.IsAny())); + } + } + + [Test] + public void should_not_import_track_specific_extras() + { + var trackName = Path.GetFileNameWithoutExtension(_approvedDownloadDecisions.First().Item.Path); + var trackExtra = Path.Combine(_downloadDir, $"{trackName}.cue"); + + SetupFilesUnderCommonDir(_downloadDir, + _approvedDownloadDecisions.Select(d => d.Item.Path).Concat(_downloadDirExtraFiles) + .Append(trackExtra)); + + Subject.ImportAlbumExtras(_approvedDownloadDecisions); + + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == _downloadDirExtraFiles.Count))); + + Mocker.GetMock() + .Verify(x => x.TransferFile( + It.Is(arg => arg.AsOsAgnostic() == trackExtra.AsOsAgnostic()), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + + [Test] + public void should_import_with_extensions_from_settings() + { + SetupFilesUnderCommonDir(_downloadDir, _downloadDirExtraFiles); + + Mocker.GetMock() + .Setup(x => x.ExtraFileExtensions) + .Returns(".cue, .txt"); + + Subject.ImportAlbumExtras(_approvedDownloadDecisions); + + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>( + arg => arg.Count == 1 + && arg.Single().Extension == ".cue"))); + } + + [Test] + public void should_not_import_extras_with_naming_cfg_having_rename_off() + { + SetupFilesUnderCommonDir(_downloadDir, + _approvedDownloadDecisions.Select(d => d.Item.Path) + .Concat(_downloadDirExtraFiles)); + + var cfg = NamingConfig.Default; + cfg.RenameTracks = false; // explicitly set for readability + SetupNamingConfig(cfg); + + Subject.ImportAlbumExtras(_approvedDownloadDecisions); + + Mocker.GetMock().VerifyNoOtherCalls(); + } + + [TestCase("{Album Title} ({Release Year})")] + [TestCase("{ALBUM TITLE} ({Release Year})")] + [TestCase("{Album Title}")] + [TestCase("{Album.Title}")] + [TestCase("{Album_Title}")] + public void should_import_extras_rename_pattern_contains_album_title(string albumDirPattern) + { + SetupFilesUnderCommonDir(_downloadDir, + _approvedDownloadDecisions.Select(d => d.Item.Path) + .Concat(_downloadDirExtraFiles)); + + var cfg = NamingConfig.Default; + cfg.RenameTracks = true; + + cfg.StandardTrackFormat = cfg.StandardTrackFormat + .Replace("{Album Title} ({Release Year})", albumDirPattern); + cfg.MultiDiscTrackFormat = cfg.MultiDiscTrackFormat + .Replace("{Album Title} ({Release Year})", albumDirPattern); + + SetupNamingConfig(cfg); + + // act + Subject.ImportAlbumExtras(_approvedDownloadDecisions); + + // assert + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == _downloadDirExtraFiles.Count))); + } + + [Test] + public void should_import_extra_from_multi_cd_subdirs() + { + var cd1Source = Path.Combine(_downloadDir, "CD1"); + var cd2Source = Path.Combine(_downloadDir, "CD2"); + var cd1Destination = Path.Combine(_albumDir, "Disk 1"); + var cd2Destination = Path.Combine(_albumDir, "Disk 2"); + + var cd1Track = NewTrack(_album, cd1Destination, "101 - Foo Track.flac", cd1Source); + var cd2Track = NewTrack(_album, cd2Destination, "201 - bonustrackbar.flac", cd2Source); + var decisions = new List> + { + new ImportDecision(cd1Track), + new ImportDecision(cd2Track), + }; + var cd1Extra = Path.Combine(cd1Source, "cd1_foo.cue"); + var cd2Extra = Path.Combine(cd2Source, "cd2_bar.cue"); + + SetupFilesUnderCommonDir(_downloadDir, cd1Track.Path, cd1Extra, cd2Track.Path, cd2Extra); + + Subject.ImportAlbumExtras(decisions); + + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == 2))); + } + + [Test] + public void should_import_from_separate_extras_dir_having_no_tracks() + { + var cd1Track = NewTrack(_album, _albumDir, "101 - Foo Track.flac", _downloadDir); + var cd2Track = NewTrack(_album, _albumDir, "201 - Bonustrackbar.flac", _downloadDir); + var extraFileInRoot = Path.Combine(_downloadDir, "cuesheet.cue"); + var extraFileInSubdir = Path.Combine(_downloadDir, "artwork", "cover.jpg"); + + SetupFilesUnderCommonDir(_downloadDir, cd1Track.Path, cd2Track.Path, extraFileInRoot, extraFileInSubdir); + var decisions = new List> + { + new ImportDecision(cd1Track), + new ImportDecision(cd2Track), + }; + Subject.ImportAlbumExtras(decisions); + + // assert + Mocker.GetMock() + .Verify(x => x.Upsert(It.Is>(arg => arg.Count == 2))); + } + + [TestCase(new string[] { "" }, null)] + [TestCase(new string[] { "files" }, null)] + [TestCase(new string[] { "first", "second_dir" }, null)] + [TestCase(new string[] { "Disk 1" }, new string[] { "CD1" })] + [TestCase(new string[] { "Disk 2", "cd2_extras" }, new string[] { "CD2", "cd2_extras" })] + public void should_copy_multicd_extra_file_to_correct_subdirectory(string[] sourcePathDirs, string[] destinationPathDirs = null) + { + var relativeSourcePath = Path.Combine(sourcePathDirs); + var relativeDestinationPath = destinationPathDirs != null ? Path.Combine(destinationPathDirs) : relativeSourcePath; + + var cd1Source = Path.Combine(_downloadDir, "Disk 1"); + var cd2Source = Path.Combine(_downloadDir, "Disk 2"); + var cd1Destination = Path.Combine(_albumDir, "CD1"); + var cd2Destination = Path.Combine(_albumDir, "CD2"); + + var cd1Track = NewTrack(_album, cd1Destination, "101 - Foo Track.flac", cd1Source); + var cd2Track = NewTrack(_album, cd2Destination, "201 - bonustrackbar.flac", cd2Source); + var extraFileName = "foobarextra.nfo"; + var extraFilePath = Path.Combine(_downloadDir, relativeSourcePath, extraFileName); + + SetupFilesUnderCommonDir(_downloadDir, cd1Track.Path, cd2Track.Path, extraFilePath); + + var decisions = new List> + { + new ImportDecision(cd1Track), + new ImportDecision(cd2Track), + }; + + Subject.ImportAlbumExtras(decisions); + + var expectedExtraPath = Path.Combine(_albumDir, relativeDestinationPath, extraFileName); + + Mocker.GetMock() + .Verify(x => x.TransferFile( + It.Is(arg => arg.AsOsAgnostic() == extraFilePath.AsOsAgnostic()), + It.Is(arg => arg.AsOsAgnostic() == expectedExtraPath.AsOsAgnostic()), + It.IsAny(), + It.IsAny()), + Times.Once); + } + + [Test] + public void should_copy_multicd_nosubdir_extras_at_destination_root() + { + var cd1Destination = Path.Combine(_albumDir, "CD1"); + var cd2Destination = Path.Combine(_albumDir, "CD2"); + var cd1Track = NewTrack(_album, cd1Destination, "101 - Foo Track.flac", _downloadDir); + var cd2Track = NewTrack(_album, cd2Destination, "201 - bonustrackbar.flac", _downloadDir); + var extraFile = Path.Combine(_downloadDir, "album.jpg"); + + SetupFilesUnderCommonDir(_downloadDir, cd1Track.Path, cd2Track.Path, extraFile); + + var decisions = new List> + { + new ImportDecision(cd1Track), + new ImportDecision(cd2Track), + }; + Subject.ImportAlbumExtras(decisions); + + // assert + var expectedExtraDestination = Path.Combine(_albumDir, "album.jpg"); + Mocker.GetMock() + .Verify(x => x.TransferFile( + It.Is(arg => arg == extraFile), + It.Is(arg => arg == expectedExtraDestination), + It.IsAny(), + It.IsAny())); + } + } + + /// + /// Set as the current naming configuration for the current test. + /// + /// The naming config to return from . + private void SetupNamingConfig(NamingConfig cfg) + { + Mocker.GetMock().Setup(x => x.GetConfig()).Returns(cfg); + } + + /// + /// Create a new track record with a given path and optional source dir for the download. + /// + /// Track album + /// The directory of the track file in the Lidarr library dir. + /// File name. + /// The source dir when the import is from a download. Pass null for track import. + private LocalTrack NewTrack(Album album, string trackDir, string trackFileName, string downloadSourceDir = null) + { + var sourcePath = Path.Combine(downloadSourceDir ?? trackDir, trackFileName); + var destinationPath = Path.Combine(trackDir, trackFileName); + return new LocalTrack + { + Artist = album.Artist, + Album = album, + Release = album.AlbumReleases.Value.First(), + Tracks = new List + { + new Track() + { + Album = album, + TrackFile = new LazyLoaded( + new TrackFile { Album = _album, AlbumId = _album.Id, Path = destinationPath }) + }, + }, + Path = sourcePath, + }; + } + + private void SetupFilesUnderCommonDir(string rootDir, IEnumerable filePath) + { + SetupFilesUnderCommonDir(rootDir, filePath.ToArray()); + } + + private void SetupFilesUnderCommonDir(string rootDir, params string[] filePaths) + { + Mocker.GetMock() + .Setup(x => x.GetFiles(It.Is(arg => arg.AsOsAgnostic() == rootDir.AsOsAgnostic()), true)) + .Returns(filePaths); + + var fileGroups = filePaths.GroupBy(x => Path.GetDirectoryName(x)) + .OrderBy(p => p.Key.Length).ToArray(); + + for (var i = 0; i < fileGroups.Length; i++) + { + var currentDir = fileGroups[i].Key; + + // current dir + Mocker.GetMock() + .Setup(x => x.GetFiles(It.Is(arg => arg.AsOsAgnostic() == currentDir.AsOsAgnostic()), false)) + .Returns(fileGroups[i]); + + // recursive search + var subdirs = fileGroups[i..fileGroups.Length] + .Where(grp => grp.Key.StartsWith(currentDir)); + + Mocker.GetMock() + .Setup(x => x.GetFiles(It.Is(arg => arg.AsOsAgnostic() == currentDir.AsOsAgnostic()), true)) + .Returns(subdirs.SelectMany(f => f)); + } + } + } +} diff --git a/src/NzbDrone.Core/Configuration/ConfigService.cs b/src/NzbDrone.Core/Configuration/ConfigService.cs index 6aa7c5448..b95837719 100644 --- a/src/NzbDrone.Core/Configuration/ConfigService.cs +++ b/src/NzbDrone.Core/Configuration/ConfigService.cs @@ -208,14 +208,14 @@ namespace NzbDrone.Core.Configuration public bool ImportExtraFiles { - get { return GetValueBoolean("ImportExtraFiles", false); } + get { return GetValueBoolean("ImportExtraFiles", true); } set { SetValue("ImportExtraFiles", value); } } public string ExtraFileExtensions { - get { return GetValue("ExtraFileExtensions", "srt"); } + get { return GetValue("ExtraFileExtensions", "log, cue, nfo, jpg, jpeg, png"); } set { SetValue("ExtraFileExtensions", value); } } diff --git a/src/NzbDrone.Core/Datastore/Migration/078_relax_not_null_constraints_extra_files.cs b/src/NzbDrone.Core/Datastore/Migration/078_relax_not_null_constraints_extra_files.cs new file mode 100644 index 000000000..582707a6a --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/078_relax_not_null_constraints_extra_files.cs @@ -0,0 +1,14 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(078)] + public class relax_not_null_constraints_extra_files : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("ExtraFiles").AlterColumn("TrackFileId").AsInt32().Nullable(); + } + } +} diff --git a/src/NzbDrone.Core/Extras/ExtraService.cs b/src/NzbDrone.Core/Extras/ExtraService.cs index c86acc38a..cff7db6f1 100644 --- a/src/NzbDrone.Core/Extras/ExtraService.cs +++ b/src/NzbDrone.Core/Extras/ExtraService.cs @@ -4,11 +4,14 @@ using System.IO; using System.Linq; using NLog; using NzbDrone.Common.Disk; +using NzbDrone.Common.Extensions; using NzbDrone.Core.Configuration; using NzbDrone.Core.Extras.Files; +using NzbDrone.Core.Extras.Others; using NzbDrone.Core.MediaCover; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.MediaFiles.Events; +using NzbDrone.Core.MediaFiles.TrackImport; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; @@ -18,6 +21,7 @@ namespace NzbDrone.Core.Extras public interface IExtraService { void ImportTrack(LocalTrack localTrack, TrackFile trackFile, bool isReadOnly); + void ImportAlbumExtras(List> importedTracks); } public class ExtraService : IExtraService, @@ -32,6 +36,7 @@ namespace NzbDrone.Core.Extras private readonly IDiskProvider _diskProvider; private readonly IConfigService _configService; private readonly List _extraFileManagers; + private readonly AlbumExtraFileManager _albumExtraManager; private readonly Logger _logger; public ExtraService(IMediaFileService mediaFileService, @@ -40,6 +45,7 @@ namespace NzbDrone.Core.Extras IDiskProvider diskProvider, IConfigService configService, IEnumerable extraFileManagers, + AlbumExtraFileManager albumExtraManager, Logger logger) { _mediaFileService = mediaFileService; @@ -48,9 +54,104 @@ namespace NzbDrone.Core.Extras _diskProvider = diskProvider; _configService = configService; _extraFileManagers = extraFileManagers.OrderBy(e => e.Order).ToList(); + _albumExtraManager = albumExtraManager; _logger = logger; } + public void ImportAlbumExtras(List> importedTracks) + { + if (!_configService.ImportExtraFiles) + { + return; + } + + var trackDestinationDirs = importedTracks.SelectMany(x => x.Item.Tracks.Select(t => t.TrackFile.Value.Path)) + .GroupBy(f => _diskProvider.GetParentFolder(f)); + + var sourceDirs = importedTracks.GroupBy(x => _diskProvider.GetParentFolder(x.Item.Path)); + if (!sourceDirs.Any()) + { + return; + } + + string sourceRoot = null; + string destinationRoot = null; + + try + { + sourceRoot = GetCommonParent(sourceDirs.Select(x => x.Key)); + destinationRoot = GetCommonParent(trackDestinationDirs.Select(x => x.Key)); + } + catch (ArgumentException ex) + { + throw new InvalidOperationException("Common parent dir could not be found, extra files will not be imported", ex); + } + + var extraFileImports = new Dictionary(); + var trackNames = importedTracks.Select(f => Path.GetFileNameWithoutExtension(f.Item.Path)); + var wantedExtensions = ExtraFileExtensionsList(); + + // extra files in track dirs for multi-CD releases + foreach (var sourceDirImports in sourceDirs) + { + var trackFilePath = sourceDirImports.First() + .Item?.Tracks?.FirstOrDefault()?.TrackFile?.Value?.Path; + if (trackFilePath == null) + { + continue; + } + + var targetDir = sourceDirs.Count() == 1 + ? destinationRoot + : _diskProvider.GetParentFolder(trackFilePath); + + var trackDirFiles = _diskProvider.GetFiles(sourceDirImports.Key, false); + var trackDirExtraFiles = FilterAlbumExtraFiles(trackDirFiles, trackNames, wantedExtensions); + foreach (var trackDirExtra in trackDirExtraFiles) + { + var import = AlbumExtraFileImport.AtDestinationDir(trackDirExtra, targetDir); + extraFileImports.Add(trackDirExtra, import); + } + + // nested files under track dirs: + var subdirFiles = _diskProvider.GetFiles(sourceDirImports.Key, true); + subdirFiles = FilterAlbumExtraFiles(subdirFiles, trackNames, wantedExtensions); + + foreach (var subdirExtra in subdirFiles.Where(x => !extraFileImports.ContainsKey(x))) + { + var extraFileDirectory = _diskProvider.GetParentFolder(subdirExtra); + var relative = sourceDirImports.Key.GetRelativePath(extraFileDirectory); + var dest = Path.Combine(targetDir, relative); + var import = AlbumExtraFileImport.AtDestinationDir(subdirExtra, dest); + extraFileImports.Add(subdirExtra, import); + } + } + + if (sourceDirs.Count() > 1) + { + // look for common parent dir + var parentDirs = sourceDirs.GroupBy(x => _diskProvider.GetParentFolder(x.Key)); + + if (parentDirs.Count() == 1) + { + var albumDirFiles = _diskProvider.GetFiles(parentDirs.Single().Key, true); + var albumExtras = FilterAlbumExtraFiles(albumDirFiles, trackNames, wantedExtensions); + + foreach (var albumExtraFile in albumExtras.Where(x => !extraFileImports.ContainsKey(x))) + { + var newImport = AlbumExtraFileImport.AtRelativePathFromSource(albumExtraFile, sourceRoot, destinationRoot); + extraFileImports.Add(albumExtraFile, newImport); + } + } + } + + var firstTrack = importedTracks.First(); + var artist = firstTrack.Item.Artist; + var albumId = firstTrack.Item.Album.Id; + + _albumExtraManager.ImportAlbumExtras(artist, albumId, extraFileImports.Values); + } + public void ImportTrack(LocalTrack localTrack, TrackFile trackFile, bool isReadOnly) { ImportExtraFiles(localTrack, trackFile, isReadOnly); @@ -69,10 +170,7 @@ namespace NzbDrone.Core.Extras var sourceFolder = _diskProvider.GetParentFolder(sourcePath); var sourceFileName = Path.GetFileNameWithoutExtension(sourcePath); var files = _diskProvider.GetFiles(sourceFolder, false); - - var wantedExtensions = _configService.ExtraFileExtensions.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) - .Select(e => e.Trim(' ', '.')) - .ToList(); + var wantedExtensions = ExtraFileExtensionsList(); var matchingFilenames = files.Where(f => Path.GetFileNameWithoutExtension(f).StartsWith(sourceFileName, StringComparison.InvariantCultureIgnoreCase)).ToList(); var filteredFilenames = new List(); @@ -176,6 +274,18 @@ namespace NzbDrone.Core.Extras { extraFileManager.MoveFilesAfterRename(artist, trackFiles); } + + _ = _albumExtraManager.MoveFilesAfterRename(artist, message.RenamedFiles); + } + + private static IEnumerable FilterAlbumExtraFiles(IEnumerable files, + IEnumerable trackFileNames, + IEnumerable wantedExtensions) + { + return files + .Where(x => + wantedExtensions.Any(ext => x.EndsWith(ext, StringComparison.InvariantCultureIgnoreCase)) + && !trackFileNames.Any(t => t.Equals(Path.GetFileNameWithoutExtension(x), StringComparison.OrdinalIgnoreCase))); } private List GetTrackFiles(int artistId) @@ -191,5 +301,53 @@ namespace NzbDrone.Core.Extras return trackFiles; } + + private List ExtraFileExtensionsList() + { + return _configService.ExtraFileExtensions + .Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries) + .Select(e => e.Trim(' ', '.')) + .ToList(); + } + + private string GetCommonParent(IEnumerable paths) + { + if (paths.Count() == 1) + { + return paths.Single(); + } + + var parentDirs = paths.GroupBy(p => _diskProvider.GetParentFolder(p)); + if (parentDirs.Count() == 1) + { + return parentDirs.Single().Key; + } + + // search depth limited to 1+1, parent of parent: + var parentOfParent = parentDirs.Select(d => _diskProvider.GetParentFolder(d.Key)).GroupBy(i => i); + if (parentOfParent.Count() == 1) + { + return parentOfParent.Single().Key; + } + + // Look for shortest path and check if this is the parent dir: + var ordered = parentDirs.OrderBy(x => x.Key.Length); + + var commonParent = ordered.First().Key; + foreach (var childDir in ordered.Skip(1)) + { + try + { + _ = commonParent.GetRelativePath(childDir.Key); + } + catch (NotParentException ex) + { + throw new ArgumentException( + $"Unable to find common parent: child path not under parent candidate '{commonParent}'", nameof(paths), ex); + } + } + + return commonParent; + } } } diff --git a/src/NzbDrone.Core/Extras/Files/AlbumExtraFileManager.cs b/src/NzbDrone.Core/Extras/Files/AlbumExtraFileManager.cs new file mode 100644 index 000000000..2ed23cba4 --- /dev/null +++ b/src/NzbDrone.Core/Extras/Files/AlbumExtraFileManager.cs @@ -0,0 +1,211 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text.RegularExpressions; +using NLog; +using NzbDrone.Common.Disk; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Extras.Others; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Music; +using NzbDrone.Core.Organizer; + +namespace NzbDrone.Core.Extras.Files +{ + public class AlbumExtraFileManager + { + private readonly IConfigService _configService; + private readonly INamingConfigService _namingConfigService; + private readonly IDiskTransferService _diskTransferService; + private readonly IDiskProvider _diskProvider; + private readonly IOtherExtraFileService _otherExtraFileService; + private readonly Logger _logger; + + private static readonly Regex _albumDirRegex = new Regex(@"{Album.+?Title}.*?\/.*?track", RegexOptions.IgnoreCase); + + public AlbumExtraFileManager( + IConfigService configService, + INamingConfigService namingConfigService, + IDiskTransferService diskTransferService, + IDiskProvider diskProvider, + IOtherExtraFileService otherExtraFileService, + Logger logger) + { + _configService = configService; + _namingConfigService = namingConfigService; + _diskTransferService = diskTransferService; + _diskProvider = diskProvider; + _otherExtraFileService = otherExtraFileService; + _logger = logger; + } + + public IEnumerable ImportAlbumExtras(Artist artist, int albumId, IEnumerable extraFileImports) + { + var namingConfig = _namingConfigService.GetConfig(); + if (!namingConfig.RenameTracks) + { + _logger.Debug($"File renaming is deactivated, skipping {extraFileImports.Count()} album extras"); + return new List(); + } + + var albumDirInStandardFormat = _albumDirRegex.IsMatch(namingConfig.StandardTrackFormat); + if (!albumDirInStandardFormat) + { + _logger.Debug($"Track template does not include an album dir, skipping {extraFileImports.Count()} album extras"); + return new List(); + } + + var albumDirInMultiDiscFormat = _albumDirRegex.IsMatch(namingConfig.MultiDiscTrackFormat); + if (!albumDirInMultiDiscFormat) + { + _logger.Debug($"Multi-disc template does not include an album dir, skipping {extraFileImports.Count()} album extras"); + return new List(); + } + + try + { + var result = new List(extraFileImports.Count()); + foreach (var extraFileImport in extraFileImports) + { + var file = ImportSingleFile(artist, albumId, extraFileImport.SourcePath, extraFileImport.DestinationPath); + result.Add(file); + } + + _otherExtraFileService.Upsert(result.ToList()); + + return result; + } + catch (Exception ex) + { + _logger.Error(ex, $"Failed to import {extraFileImports.Count()} album extra files for artist '{artist.CleanName}'"); + return new List(); + } + } + + public IEnumerable MoveFilesAfterRename(Artist artist, List trackFiles) + { + var extraFiles = _otherExtraFileService.GetFilesByArtist(artist.Id); + if (!extraFiles.Any()) + { + return new List(); + } + + _logger.Debug($"Found {extraFiles.Count} extra files for artist '{artist.Name}'"); + + var movedFiles = new List(); + + try + { + foreach (var albumTracks in trackFiles.GroupBy(x => x.TrackFile.AlbumId)) + { + var albumFiles = MoveAlbumExtraFiles(artist, extraFiles.Where(x => x.AlbumId == albumTracks.Key), albumTracks); + _otherExtraFileService.Upsert(albumFiles); + + movedFiles.AddRange(albumFiles); + } + } + catch (Exception ex) + { + _logger.Error(ex, $"Moving album extras for artist '{artist.Name}' failed"); + return new List(); + } + + _logger.Info($"Moved {movedFiles.Count} extra files on rename for '{artist.Name}'"); + + return movedFiles; + } + + private OtherExtraFile ImportSingleFile(Artist artist, int albumId, string sourcePath, string destinationPath) + { + var transferMode = _configService.CopyUsingHardlinks ? TransferMode.HardLinkOrCopy : TransferMode.Copy; + + if (!sourcePath.PathEquals(destinationPath)) + { + _diskProvider.CreateFolder(_diskProvider.GetParentFolder(destinationPath)); + _diskTransferService.TransferFile(sourcePath, destinationPath, transferMode, true); + } + + var extension = Path.GetExtension(destinationPath); + + return new OtherExtraFile + { + ArtistId = artist.Id, + AlbumId = albumId, + TrackFileId = null, + RelativePath = artist.Path.GetRelativePath(destinationPath), + Extension = extension, + }; + } + + private List MoveAlbumExtraFiles(Artist artist, IEnumerable extraFiles, IGrouping albumTracks) + { + var movedFiles = new List(); + var previousTrackDirs = albumTracks.GroupBy(x => _diskProvider.GetParentFolder(x.PreviousPath)); + + // extra files in track directories should stay together with the tracks: + foreach (var dir in previousTrackDirs) + { + var relativeTrackDir = artist.Path.GetRelativePath(dir.Key); + var extrasUnderTrackDir = extraFiles.Where( + x => x.RelativePath.StartsWithIgnoreCase(relativeTrackDir)); + + var oldRelative = artist.Path.GetRelativePath(_diskProvider.GetParentFolder(dir.First().PreviousPath)); + var newRelative = artist.Path.GetRelativePath(_diskProvider.GetParentFolder(dir.First().TrackFile.Path)); + foreach (var extraFile in extrasUnderTrackDir) + { + var oldFilePath = Path.Combine(artist.Path, extraFile.RelativePath); + + var updatedRelativePath = extraFile.RelativePath.Replace(oldRelative, newRelative); + extraFile.RelativePath = updatedRelativePath; + + var newFilePath = Path.Combine(artist.Path, updatedRelativePath); + MoveToNewDir(oldFilePath, newFilePath); + + movedFiles.Add(extraFile); + } + } + + // move remaining files to new album dir: + var remainingExtraFiles = extraFiles.Where(x => !movedFiles.Any(f => f.Id == x.Id)); + var newTrackDirs = albumTracks.GroupBy(x => _diskProvider.GetParentFolder(x.TrackFile.Path)); + + if (remainingExtraFiles.Any() + && previousTrackDirs.Count() > 1 + && newTrackDirs.Count() > 1) + { + var oldParentDir = previousTrackDirs.First().Key.GetParentPath(); + var newParentDir = newTrackDirs.First().Key.GetParentPath(); + + if (previousTrackDirs.All(d => d.Key.GetParentPath() == oldParentDir) + && newTrackDirs.All(d => d.Key.GetParentPath() == newParentDir)) + { + var oldRelative = artist.Path.GetRelativePath(oldParentDir); + var newRelative = artist.Path.GetRelativePath(newParentDir); + + foreach (var extraFile in remainingExtraFiles) + { + var oldPath = Path.Combine(artist.Path, extraFile.RelativePath); + + var newExtraRelativePath = extraFile.RelativePath.Replace(oldRelative, newRelative); + var newFilePath = Path.Combine(artist.Path, newExtraRelativePath); + + MoveToNewDir(oldPath, newFilePath); + + extraFile.RelativePath = newExtraRelativePath; + movedFiles.Add(extraFile); + } + } + } + + return movedFiles; + } + + private void MoveToNewDir(string oldFilePath, string newFilePath) + { + _diskProvider.CreateFolder(_diskProvider.GetParentFolder(newFilePath)); + _diskProvider.MoveFile(oldFilePath, newFilePath); + } + } +} diff --git a/src/NzbDrone.Core/Extras/Others/AlbumExtraFileImport.cs b/src/NzbDrone.Core/Extras/Others/AlbumExtraFileImport.cs new file mode 100644 index 000000000..0697cbce9 --- /dev/null +++ b/src/NzbDrone.Core/Extras/Others/AlbumExtraFileImport.cs @@ -0,0 +1,34 @@ +using System.IO; +using NzbDrone.Common.Extensions; + +namespace NzbDrone.Core.Extras.Others +{ + public class AlbumExtraFileImport + { + public AlbumExtraFileImport(string sourceFilePath, string destinationFilePath) + { + SourcePath = sourceFilePath; + DestinationPath = destinationFilePath; + } + + public string SourcePath { get; } + + public string DestinationPath { get; } + + public static AlbumExtraFileImport AtDestinationDir(string sourceFilePath, string destinationDir) + { + var fileName = Path.GetFileName(sourceFilePath); + var destinationPath = Path.Join(destinationDir, fileName); + + return new AlbumExtraFileImport(sourceFilePath, destinationPath); + } + + public static AlbumExtraFileImport AtRelativePathFromSource(string sourceFilePath, string sourceRootDir, string destinationRootDir) + { + var relative = sourceRootDir.GetRelativePath(sourceFilePath); + var destinationPath = Path.Join(destinationRootDir, relative); + + return new AlbumExtraFileImport(sourceFilePath, destinationPath); + } + } +} diff --git a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportApprovedTracks.cs b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportApprovedTracks.cs index 6c15d475e..e7439af8b 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackImport/ImportApprovedTracks.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackImport/ImportApprovedTracks.cs @@ -324,6 +324,12 @@ namespace NzbDrone.Core.MediaFiles.TrackImport var album = _albumService.GetAlbum(albumImport.First().ImportDecision.Item.Album.Id); var artist = albumImport.First().ImportDecision.Item.Artist; + if (album != null) + { + var albumTrackFiles = filesToAdd.Where(x => x.AlbumId == album.Id); + _extraService.ImportAlbumExtras(albumImport.Select(x => x.ImportDecision).ToList()); + } + if (albumImport.Where(e => e.Errors.Count == 0).ToList().Count > 0 && artist != null && album != null) { _eventAggregator.PublishEvent(new AlbumImportedEvent( diff --git a/src/NzbDrone.Test.Common/TestBase.cs b/src/NzbDrone.Test.Common/TestBase.cs index 6d2f2a908..a70a9c72b 100644 --- a/src/NzbDrone.Test.Common/TestBase.cs +++ b/src/NzbDrone.Test.Common/TestBase.cs @@ -103,7 +103,7 @@ namespace NzbDrone.Test.Common [SetUp] public void TestBaseSetup() { - GetType().IsPublic.Should().BeTrue("All Test fixtures should be public to work in mono."); + GetType().Should().Match(t => t.IsPublic || t.IsNestedPublic, "All Test fixtures should be public to work in mono."); LogManager.ReconfigExistingLoggers();