using System; using System.Collections.Generic; using System.IO; using System.IO.Abstractions; using System.Linq; using NLog; using NzbDrone.Common; using NzbDrone.Common.Crypto; using NzbDrone.Common.Disk; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Core.CustomFormats; using NzbDrone.Core.Datastore; using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download; using NzbDrone.Core.Download.TrackedDownloads; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Qualities; using NzbDrone.Core.RootFolders; namespace NzbDrone.Core.MediaFiles.TrackImport.Manual { public interface IManualImportService { List GetMediaFiles(string path, string downloadId, Artist artist, FilterFilesType filter, bool replaceExistingFiles); List UpdateItems(List item); } public class ManualImportService : IExecute, IManualImportService { private readonly IDiskProvider _diskProvider; private readonly IParsingService _parsingService; private readonly IRootFolderService _rootFolderService; private readonly IDiskScanService _diskScanService; private readonly IMakeImportDecision _importDecisionMaker; private readonly ICueSheetService _cueSheetService; private readonly ICustomFormatCalculationService _formatCalculator; private readonly IArtistService _artistService; private readonly IAlbumService _albumService; private readonly IReleaseService _releaseService; private readonly ITrackService _trackService; private readonly IAudioTagService _audioTagService; private readonly IImportApprovedTracks _importApprovedTracks; private readonly ITrackedDownloadService _trackedDownloadService; private readonly IDownloadedTracksImportService _downloadedTracksImportService; private readonly IProvideImportItemService _provideImportItemService; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; public ManualImportService(IDiskProvider diskProvider, IParsingService parsingService, IRootFolderService rootFolderService, IDiskScanService diskScanService, IMakeImportDecision importDecisionMaker, ICueSheetService cueSheetService, ICustomFormatCalculationService formatCalculator, IArtistService artistService, IAlbumService albumService, IReleaseService releaseService, ITrackService trackService, IAudioTagService audioTagService, IImportApprovedTracks importApprovedTracks, ITrackedDownloadService trackedDownloadService, IDownloadedTracksImportService downloadedTracksImportService, IProvideImportItemService provideImportItemService, IEventAggregator eventAggregator, Logger logger) { _diskProvider = diskProvider; _parsingService = parsingService; _rootFolderService = rootFolderService; _diskScanService = diskScanService; _importDecisionMaker = importDecisionMaker; _cueSheetService = cueSheetService; _formatCalculator = formatCalculator; _artistService = artistService; _albumService = albumService; _releaseService = releaseService; _trackService = trackService; _audioTagService = audioTagService; _importApprovedTracks = importApprovedTracks; _trackedDownloadService = trackedDownloadService; _downloadedTracksImportService = downloadedTracksImportService; _provideImportItemService = provideImportItemService; _eventAggregator = eventAggregator; _logger = logger; } public List GetMediaFiles(string path, string downloadId, Artist artist, FilterFilesType filter, bool replaceExistingFiles) { if (downloadId.IsNotNullOrWhiteSpace()) { var trackedDownload = _trackedDownloadService.Find(downloadId); if (trackedDownload == null) { return new List(); } if (trackedDownload.ImportItem == null) { trackedDownload.ImportItem = _provideImportItemService.ProvideImportItem(trackedDownload.DownloadItem, trackedDownload.ImportItem); } path = trackedDownload.ImportItem.OutputPath.FullPath; } if (!_diskProvider.FolderExists(path)) { if (!_diskProvider.FileExists(path)) { return new List(); } var files = new List { _diskProvider.GetFileInfo(path) }; var config = new ImportDecisionMakerConfig { Filter = FilterFilesType.None, NewDownload = true, SingleRelease = false, IncludeExisting = !replaceExistingFiles, AddNewArtists = false }; var decision = _importDecisionMaker.GetImportDecisions(files, null, null, config); var result = MapItem(decision.First(), downloadId, replaceExistingFiles, false); return new List { result }; } return ProcessFolder(path, downloadId, artist, filter, replaceExistingFiles); } private List ProcessFolder(string folder, string downloadId, Artist artist, FilterFilesType filter, bool replaceExistingFiles) { DownloadClientItem downloadClientItem = null; var directoryInfo = new DirectoryInfo(folder); artist = artist ?? _parsingService.GetArtist(directoryInfo.Name); if (downloadId.IsNotNullOrWhiteSpace()) { var trackedDownload = _trackedDownloadService.Find(downloadId); downloadClientItem = trackedDownload.DownloadItem; if (artist == null) { artist = trackedDownload.RemoteAlbum?.Artist; } } var audioFiles = _diskScanService.GetAudioFiles(folder).ToList(); var results = new List(); var idOverrides = new IdentificationOverrides { Artist = artist, Album = null }; results.AddRange(ProcessFolder(downloadId, idOverrides, filter, replaceExistingFiles, downloadClientItem, directoryInfo.Name, audioFiles)); return results; } private List ProcessFolder(string downloadId, IdentificationOverrides idOverrides, FilterFilesType filter, bool replaceExistingFiles, DownloadClientItem downloadClientItem, string albumTitle, List audioFiles, List cueSheetInfos = null) { idOverrides ??= new IdentificationOverrides(); var itemInfo = new ImportDecisionMakerInfo { DownloadClientItem = downloadClientItem, ParsedAlbumInfo = Parser.Parser.ParseAlbumTitle(albumTitle), DetectCueFileEncoding = true, }; var config = new ImportDecisionMakerConfig { Filter = filter, NewDownload = true, SingleRelease = false, IncludeExisting = !replaceExistingFiles, AddNewArtists = false }; var decisions = _cueSheetService.GetImportDecisions(ref audioFiles, null, itemInfo, config); if (!audioFiles.Empty()) { decisions.AddRange(_importDecisionMaker.GetImportDecisions(audioFiles, idOverrides, itemInfo, config)); } // paths will be different for new and old files which is why we need to map separately var newFiles = audioFiles.Join(decisions, f => f.FullName, d => d.Item.Path, (f, d) => new { File = f, Decision = d }, PathEqualityComparer.Instance); var newItemsList = newFiles.Select(x => MapItem(x.Decision, downloadId, replaceExistingFiles, false)).ToList(); var existingDecisions = decisions.Except(newFiles.Select(x => x.Decision)); var existingItems = existingDecisions.Select(x => MapItem(x, null, replaceExistingFiles, false)); var itemsList = newItemsList.Concat(existingItems.ToList()).ToList(); return itemsList; } public List UpdateItems(List items) { var replaceExistingFiles = items.All(x => x.ReplaceExistingFiles); var groupedItems = items.Where(x => !x.AdditionalFile).GroupBy(x => x.Album?.Id); _logger.Debug($"UpdateItems, {groupedItems.Count()} groups, replaceExisting {replaceExistingFiles}"); var result = new List(); foreach (var group in groupedItems) { _logger.Debug("UpdateItems, group key: {0}", group.Key); var disableReleaseSwitching = group.First().DisableReleaseSwitching; var config = new ImportDecisionMakerConfig { Filter = FilterFilesType.None, NewDownload = true, SingleRelease = true, IncludeExisting = !replaceExistingFiles, AddNewArtists = false }; var audioFiles = new List(); foreach (var item in group) { var file = _diskProvider.GetFileInfo(item.Path); audioFiles.Add(file); if (item.CueSheetPath != null) { var cueFile = _diskProvider.GetFileInfo(item.CueSheetPath); audioFiles.Add(cueFile); } } var itemInfo = new ImportDecisionMakerInfo(); var idOverride = new IdentificationOverrides { Artist = group.First().Artist, Album = group.First().Album, AlbumRelease = group.First().Release }; itemInfo.DetectCueFileEncoding = true; var decisions = _cueSheetService.GetImportDecisions(ref audioFiles, idOverride, itemInfo, config); if (audioFiles.Count > 0) { decisions.AddRange(_importDecisionMaker.GetImportDecisions(audioFiles, idOverride, itemInfo, config)); } if (decisions.Count > 0) { result.AddRange(UpdateItems(group, decisions, replaceExistingFiles, disableReleaseSwitching)); } } return result; } private List UpdateItems(IGrouping group, List> decisions, bool replaceExistingFiles, bool disableReleaseSwitching) { var result = new List(); var existingItems = group.Join(decisions, i => i.Path, d => d.Item.Path, (i, d) => new { Item = i, Decision = d }, PathEqualityComparer.Instance); foreach (var pair in existingItems) { var item = pair.Item; var decision = pair.Decision; if (decision.Item.Artist != null) { item.Artist = decision.Item.Artist; } if (decision.Item.Album != null) { item.Album = decision.Item.Album; item.Release = decision.Item.Release; } if (decision.Item.Tracks.Any()) { item.Tracks = decision.Item.Tracks; } if (item.Quality?.Quality == Quality.Unknown) { item.Quality = decision.Item.Quality; } if (item.ReleaseGroup.IsNullOrWhiteSpace()) { item.ReleaseGroup = decision.Item.ReleaseGroup; } item.Rejections = decision.Rejections; item.Size = decision.Item.Size; result.Add(item); } var newDecisions = decisions.Except(existingItems.Select(x => x.Decision)); result.AddRange(newDecisions.Select(x => MapItem(x, null, replaceExistingFiles, disableReleaseSwitching))); return result; } private ManualImportItem MapItem(ImportDecision decision, string downloadId, bool replaceExistingFiles, bool disableReleaseSwitching) { var item = new ManualImportItem(); item.Id = HashConverter.GetHashInt31(decision.Item.Path); item.Path = decision.Item.Path; item.Name = Path.GetFileNameWithoutExtension(decision.Item.Path); item.DownloadId = downloadId; if (decision.Item.Artist != null) { item.Artist = decision.Item.Artist; item.CustomFormats = _formatCalculator.ParseCustomFormat(decision.Item); } if (decision.Item.Album != null) { item.Album = decision.Item.Album; item.Release = decision.Item.Release; } if (decision.Item.Tracks.Any()) { item.Tracks = decision.Item.Tracks; } item.Quality = decision.Item.Quality; item.Size = _diskProvider.GetFileSize(decision.Item.Path); item.Rejections = decision.Rejections; item.Tags = decision.Item.FileTrackInfo; item.AdditionalFile = decision.Item.AdditionalFile; item.ReplaceExistingFiles = replaceExistingFiles; item.DisableReleaseSwitching = disableReleaseSwitching; item.IsSingleFileRelease = decision.Item.IsSingleFileRelease; item.CueSheetPath = decision.Item.CueSheetPath; return item; } public void Execute(ManualImportCommand message) { _logger.ProgressTrace("Manually importing {0} files using mode {1}", message.Files.Count, message.ImportMode); var imported = new List(); var importedTrackedDownload = new List(); var albumIds = message.Files.GroupBy(e => e.AlbumId).ToList(); var fileCount = 0; foreach (var importAlbumId in albumIds) { var albumImportDecisions = new List>(); // turn off anyReleaseOk if specified if (importAlbumId.First().DisableReleaseSwitching) { var album = _albumService.GetAlbum(importAlbumId.First().AlbumId); album.AnyReleaseOk = false; _albumService.UpdateAlbum(album); } foreach (var file in importAlbumId) { _logger.ProgressTrace("Processing file {0} of {1}", fileCount + 1, message.Files.Count); var artist = _artistService.GetArtist(file.ArtistId); var album = _albumService.GetAlbum(file.AlbumId); var release = _releaseService.GetRelease(file.AlbumReleaseId); var tracks = _trackService.GetTracks(file.TrackIds); var fileTrackInfo = _audioTagService.ReadTags(file.Path) ?? new ParsedTrackInfo(); var fileInfo = _diskProvider.GetFileInfo(file.Path); var localTrack = new LocalTrack { ExistingFile = artist.Path.IsParentPath(file.Path), Tracks = tracks, FileTrackInfo = fileTrackInfo, Path = file.Path, Size = fileInfo.Length, Modified = fileInfo.LastWriteTimeUtc, Quality = file.Quality, Artist = artist, Album = album, Release = release, IsSingleFileRelease = file.IsSingleFileRelease, CueSheetPath = file.CueSheetPath, }; if (file.IsSingleFileRelease) { localTrack.Tracks.ForEach(x => x.IsSingleFileRelease = true); } var importDecision = new ImportDecision(localTrack); if (_rootFolderService.GetBestRootFolder(artist.Path) == null) { _logger.Warn($"Destination artist folder {artist.Path} not in a Root Folder, skipping import"); importDecision.Reject(new Rejection($"Destination artist folder {artist.Path} is not in a Root Folder")); } albumImportDecisions.Add(importDecision); fileCount += 1; } var downloadId = importAlbumId.Select(x => x.DownloadId).FirstOrDefault(x => x.IsNotNullOrWhiteSpace()); if (downloadId.IsNullOrWhiteSpace()) { imported.AddRange(_importApprovedTracks.Import(albumImportDecisions, message.ReplaceExistingFiles, null, message.ImportMode)); } else { var trackedDownload = _trackedDownloadService.Find(downloadId); var importResults = _importApprovedTracks.Import(albumImportDecisions, message.ReplaceExistingFiles, trackedDownload.DownloadItem, message.ImportMode); imported.AddRange(importResults); foreach (var importResult in importResults) { importedTrackedDownload.Add(new ManuallyImportedFile { TrackedDownload = trackedDownload, ImportResult = importResult }); } } } _logger.ProgressTrace("Manually imported {0} files", imported.Count); foreach (var groupedTrackedDownload in importedTrackedDownload.GroupBy(i => i.TrackedDownload.DownloadItem.DownloadId).ToList()) { var trackedDownload = groupedTrackedDownload.First().TrackedDownload; var importArtist = groupedTrackedDownload.First().ImportResult.ImportDecision.Item.Artist; var outputPath = trackedDownload.ImportItem.OutputPath.FullPath; if (_diskProvider.FolderExists(outputPath)) { if (_downloadedTracksImportService.ShouldDeleteFolder( _diskProvider.GetDirectoryInfo(outputPath), importArtist) && trackedDownload.DownloadItem.CanMoveFiles) { _diskProvider.DeleteFolder(outputPath, true); } } var remoteTrackCount = Math.Max(1, trackedDownload.RemoteAlbum?.Albums.Sum(x => x.AlbumReleases.Value.Where(y => y.Monitored).Sum(z => z.TrackCount)) ?? 1); var importResults = groupedTrackedDownload.Select(x => x.ImportResult).ToList(); var importedTrackCount = importResults.Where(c => c.Result == ImportResultType.Imported) .SelectMany(c => c.ImportDecision.Item.Tracks) .Count(); var allTracksImported = importResults.All(c => c.Result == ImportResultType.Imported) || importedTrackCount >= remoteTrackCount; if (allTracksImported) { trackedDownload.State = TrackedDownloadState.Imported; _eventAggregator.PublishEvent(new DownloadCompletedEvent(trackedDownload, importArtist.Id)); } } } } }