Lidarr/src/NzbDrone.Core/MediaFiles/TrackImport/Manual/ManualImportService.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
using NLog;
using NzbDrone.Common;
using NzbDrone.Common.Crypto;
using NzbDrone.Common.Disk;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Instrumentation.Extensions;
using NzbDrone.Core.CustomFormats;
using NzbDrone.Core.Datastore;
using NzbDrone.Core.DecisionEngine;
using NzbDrone.Core.Download;
using NzbDrone.Core.Download.TrackedDownloads;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Music;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Qualities;
using NzbDrone.Core.RootFolders;

namespace NzbDrone.Core.MediaFiles.TrackImport.Manual
{
    public interface IManualImportService
    {
        List<ManualImportItem> GetMediaFiles(string path, string downloadId, Artist artist, FilterFilesType filter, bool replaceExistingFiles);
        List<ManualImportItem> UpdateItems(List<ManualImportItem> item);
    }

    public class ManualImportService : IExecute<ManualImportCommand>, IManualImportService
    {
        private readonly IDiskProvider _diskProvider;
        private readonly IParsingService _parsingService;
        private readonly IRootFolderService _rootFolderService;
        private readonly IDiskScanService _diskScanService;
        private readonly IMakeImportDecision _importDecisionMaker;
        private readonly ICueSheetService _cueSheetService;
        private readonly ICustomFormatCalculationService _formatCalculator;
        private readonly IArtistService _artistService;
        private readonly IAlbumService _albumService;
        private readonly IReleaseService _releaseService;
        private readonly ITrackService _trackService;
        private readonly IAudioTagService _audioTagService;
        private readonly IImportApprovedTracks _importApprovedTracks;
        private readonly ITrackedDownloadService _trackedDownloadService;
        private readonly IDownloadedTracksImportService _downloadedTracksImportService;
        private readonly IProvideImportItemService _provideImportItemService;
        private readonly IEventAggregator _eventAggregator;
        private readonly Logger _logger;

        public ManualImportService(IDiskProvider diskProvider,
                                   IParsingService parsingService,
                                   IRootFolderService rootFolderService,
                                   IDiskScanService diskScanService,
                                   IMakeImportDecision importDecisionMaker,
                                   ICueSheetService cueSheetService,
                                   ICustomFormatCalculationService formatCalculator,
                                   IArtistService artistService,
                                   IAlbumService albumService,
                                   IReleaseService releaseService,
                                   ITrackService trackService,
                                   IAudioTagService audioTagService,
                                   IImportApprovedTracks importApprovedTracks,
                                   ITrackedDownloadService trackedDownloadService,
                                   IDownloadedTracksImportService downloadedTracksImportService,
                                   IProvideImportItemService provideImportItemService,
                                   IEventAggregator eventAggregator,
                                   Logger logger)
        {
            _diskProvider = diskProvider;
            _parsingService = parsingService;
            _rootFolderService = rootFolderService;
            _diskScanService = diskScanService;
            _importDecisionMaker = importDecisionMaker;
            _cueSheetService = cueSheetService;
            _formatCalculator = formatCalculator;
            _artistService = artistService;
            _albumService = albumService;
            _releaseService = releaseService;
            _trackService = trackService;
            _audioTagService = audioTagService;
            _importApprovedTracks = importApprovedTracks;
            _trackedDownloadService = trackedDownloadService;
            _downloadedTracksImportService = downloadedTracksImportService;
            _provideImportItemService = provideImportItemService;
            _eventAggregator = eventAggregator;
            _logger = logger;
        }

        public List<ManualImportItem> GetMediaFiles(string path, string downloadId, Artist artist, FilterFilesType filter, bool replaceExistingFiles)
        {
            if (downloadId.IsNotNullOrWhiteSpace())
            {
                var trackedDownload = _trackedDownloadService.Find(downloadId);

                if (trackedDownload == null)
                {
                    return new List<ManualImportItem>();
                }

                if (trackedDownload.ImportItem == null)
                {
                    trackedDownload.ImportItem = _provideImportItemService.ProvideImportItem(trackedDownload.DownloadItem, trackedDownload.ImportItem);
                }

                path = trackedDownload.ImportItem.OutputPath.FullPath;
            }

            if (!_diskProvider.FolderExists(path))
            {
                if (!_diskProvider.FileExists(path))
                {
                    return new List<ManualImportItem>();
                }

                var files = new List<IFileInfo> { _diskProvider.GetFileInfo(path) };

                var config = new ImportDecisionMakerConfig
                {
                    Filter = FilterFilesType.None,
                    NewDownload = true,
                    SingleRelease = false,
                    IncludeExisting = !replaceExistingFiles,
                    AddNewArtists = false
                };

                var decision = _importDecisionMaker.GetImportDecisions(files, null, null, config);
                var result = MapItem(decision.First(), downloadId, replaceExistingFiles, false);

                return new List<ManualImportItem> { result };
            }

            return ProcessFolder(path, downloadId, artist, filter, replaceExistingFiles);
        }

        private List<ManualImportItem> ProcessFolder(string folder, string downloadId, Artist artist, FilterFilesType filter, bool replaceExistingFiles)
        {
            DownloadClientItem downloadClientItem = null;
            var directoryInfo = new DirectoryInfo(folder);
            artist = artist ?? _parsingService.GetArtist(directoryInfo.Name);

            if (downloadId.IsNotNullOrWhiteSpace())
            {
                var trackedDownload = _trackedDownloadService.Find(downloadId);
                downloadClientItem = trackedDownload.DownloadItem;

                if (artist == null)
                {
                    artist = trackedDownload.RemoteAlbum?.Artist;
                }
            }

            var audioFiles = _diskScanService.GetAudioFiles(folder).ToList();
            var results = new List<ManualImportItem>();

            var idOverrides = new IdentificationOverrides
            {
                Artist = artist,
                Album = null
            };

            results.AddRange(ProcessFolder(downloadId, idOverrides, filter, replaceExistingFiles, downloadClientItem, directoryInfo.Name, audioFiles));

            return results;
        }

        private List<ManualImportItem> ProcessFolder(string downloadId, IdentificationOverrides idOverrides, FilterFilesType filter, bool replaceExistingFiles, DownloadClientItem downloadClientItem, string albumTitle, List<IFileInfo> audioFiles, List<CueSheetInfo> cueSheetInfos = null)
        {
            idOverrides ??= new IdentificationOverrides();
            var itemInfo = new ImportDecisionMakerInfo
            {
                DownloadClientItem = downloadClientItem,
                ParsedAlbumInfo = Parser.Parser.ParseAlbumTitle(albumTitle),
                DetectCueFileEncoding = true,
            };

            var config = new ImportDecisionMakerConfig
            {
                Filter = filter,
                NewDownload = true,
                SingleRelease = false,
                IncludeExisting = !replaceExistingFiles,
                AddNewArtists = false
            };

            var decisions = _cueSheetService.GetImportDecisions(ref audioFiles, null, itemInfo, config);
            if (!audioFiles.Empty())
            {
                decisions.AddRange(_importDecisionMaker.GetImportDecisions(audioFiles, idOverrides, itemInfo, config));
            }

            // paths will be different for new and old files which is why we need to map separately
            var newFiles = audioFiles.Join(decisions,
                                            f => f.FullName,
                                            d => d.Item.Path,
                                            (f, d) => new { File = f, Decision = d },
                                            PathEqualityComparer.Instance);

            var newItemsList = newFiles.Select(x => MapItem(x.Decision, downloadId, replaceExistingFiles, false)).ToList();

            var existingDecisions = decisions.Except(newFiles.Select(x => x.Decision));
            var existingItems = existingDecisions.Select(x => MapItem(x, null, replaceExistingFiles, false));

            var itemsList = newItemsList.Concat(existingItems.ToList()).ToList();
            return itemsList;
        }

        public List<ManualImportItem> UpdateItems(List<ManualImportItem> items)
        {
            var replaceExistingFiles = items.All(x => x.ReplaceExistingFiles);
            var groupedItems = items.Where(x => !x.AdditionalFile).GroupBy(x => x.Album?.Id);
            _logger.Debug($"UpdateItems, {groupedItems.Count()} groups, replaceExisting {replaceExistingFiles}");

            var result = new List<ManualImportItem>();

            foreach (var group in groupedItems)
            {
                _logger.Debug("UpdateItems, group key: {0}", group.Key);

                var disableReleaseSwitching = group.First().DisableReleaseSwitching;

                var config = new ImportDecisionMakerConfig
                {
                    Filter = FilterFilesType.None,
                    NewDownload = true,
                    SingleRelease = true,
                    IncludeExisting = !replaceExistingFiles,
                    AddNewArtists = false
                };

                var audioFiles = new List<IFileInfo>();
                foreach (var item in group)
                {
                    var file = _diskProvider.GetFileInfo(item.Path);
                    audioFiles.Add(file);

                    if (item.CueSheetPath != null)
                    {
                        var cueFile = _diskProvider.GetFileInfo(item.CueSheetPath);
                        audioFiles.Add(cueFile);
                    }
                }

                var itemInfo = new ImportDecisionMakerInfo();
                var idOverride = new IdentificationOverrides
                {
                    Artist = group.First().Artist,
                    Album = group.First().Album,
                    AlbumRelease = group.First().Release
                };

                itemInfo.DetectCueFileEncoding = true;
                var decisions = _cueSheetService.GetImportDecisions(ref audioFiles, idOverride, itemInfo, config);
                if (audioFiles.Count > 0)
                {
                    decisions.AddRange(_importDecisionMaker.GetImportDecisions(audioFiles, idOverride, itemInfo, config));
                }

                if (decisions.Count > 0)
                {
                    result.AddRange(UpdateItems(group, decisions, replaceExistingFiles, disableReleaseSwitching));
                }
            }

            return result;
        }

        private List<ManualImportItem> UpdateItems(IGrouping<int?, ManualImportItem> group, List<ImportDecision<LocalTrack>> decisions, bool replaceExistingFiles, bool disableReleaseSwitching)
        {
            var result = new List<ManualImportItem>();

            var existingItems = group.Join(decisions,
                                           i => i.Path,
                                           d => d.Item.Path,
                                           (i, d) => new { Item = i, Decision = d },
                                           PathEqualityComparer.Instance);

            foreach (var pair in existingItems)
            {
                var item = pair.Item;
                var decision = pair.Decision;

                if (decision.Item.Artist != null)
                {
                    item.Artist = decision.Item.Artist;
                }

                if (decision.Item.Album != null)
                {
                    item.Album = decision.Item.Album;
                    item.Release = decision.Item.Release;
                }

                if (decision.Item.Tracks.Any())
                {
                    item.Tracks = decision.Item.Tracks;
                }

                if (item.Quality?.Quality == Quality.Unknown)
                {
                    item.Quality = decision.Item.Quality;
                }

                if (item.ReleaseGroup.IsNullOrWhiteSpace())
                {
                    item.ReleaseGroup = decision.Item.ReleaseGroup;
                }

                item.Rejections = decision.Rejections;
                item.Size = decision.Item.Size;

                result.Add(item);
            }

            var newDecisions = decisions.Except(existingItems.Select(x => x.Decision));
            result.AddRange(newDecisions.Select(x => MapItem(x, null, replaceExistingFiles, disableReleaseSwitching)));

            return result;
        }

        private ManualImportItem MapItem(ImportDecision<LocalTrack> decision, string downloadId, bool replaceExistingFiles, bool disableReleaseSwitching)
        {
            var item = new ManualImportItem();

            item.Id = HashConverter.GetHashInt31(decision.Item.Path);
            item.Path = decision.Item.Path;
            item.Name = Path.GetFileNameWithoutExtension(decision.Item.Path);
            item.DownloadId = downloadId;

            if (decision.Item.Artist != null)
            {
                item.Artist = decision.Item.Artist;

                item.CustomFormats = _formatCalculator.ParseCustomFormat(decision.Item);
            }

            if (decision.Item.Album != null)
            {
                item.Album = decision.Item.Album;
                item.Release = decision.Item.Release;
            }

            if (decision.Item.Tracks.Any())
            {
                item.Tracks = decision.Item.Tracks;
            }

            item.Quality = decision.Item.Quality;
            item.Size = _diskProvider.GetFileSize(decision.Item.Path);
            item.Rejections = decision.Rejections;
            item.Tags = decision.Item.FileTrackInfo;
            item.AdditionalFile = decision.Item.AdditionalFile;
            item.ReplaceExistingFiles = replaceExistingFiles;
            item.DisableReleaseSwitching = disableReleaseSwitching;
            item.IsSingleFileRelease = decision.Item.IsSingleFileRelease;
            item.CueSheetPath = decision.Item.CueSheetPath;

            return item;
        }

        public void Execute(ManualImportCommand message)
        {
            _logger.ProgressTrace("Manually importing {0} files using mode {1}", message.Files.Count, message.ImportMode);

            var imported = new List<ImportResult>();
            var importedTrackedDownload = new List<ManuallyImportedFile>();
            var albumIds = message.Files.GroupBy(e => e.AlbumId).ToList();
            var fileCount = 0;

            foreach (var importAlbumId in albumIds)
            {
                var albumImportDecisions = new List<ImportDecision<LocalTrack>>();

                // turn off anyReleaseOk if specified
                if (importAlbumId.First().DisableReleaseSwitching)
                {
                    var album = _albumService.GetAlbum(importAlbumId.First().AlbumId);
                    album.AnyReleaseOk = false;
                    _albumService.UpdateAlbum(album);
                }

                foreach (var file in importAlbumId)
                {
                    _logger.ProgressTrace("Processing file {0} of {1}", fileCount + 1, message.Files.Count);

                    var artist = _artistService.GetArtist(file.ArtistId);
                    var album = _albumService.GetAlbum(file.AlbumId);
                    var release = _releaseService.GetRelease(file.AlbumReleaseId);
                    var tracks = _trackService.GetTracks(file.TrackIds);
                    var fileTrackInfo = _audioTagService.ReadTags(file.Path) ?? new ParsedTrackInfo();
                    var fileInfo = _diskProvider.GetFileInfo(file.Path);

                    var localTrack = new LocalTrack
                    {
                        ExistingFile = artist.Path.IsParentPath(file.Path),
                        Tracks = tracks,
                        FileTrackInfo = fileTrackInfo,
                        Path = file.Path,
                        Size = fileInfo.Length,
                        Modified = fileInfo.LastWriteTimeUtc,
                        Quality = file.Quality,
                        Artist = artist,
                        Album = album,
                        Release = release,
                        IsSingleFileRelease = file.IsSingleFileRelease,
                        CueSheetPath = file.CueSheetPath,
                    };

                    if (file.IsSingleFileRelease)
                    {
                        localTrack.Tracks.ForEach(x => x.IsSingleFileRelease = true);
                    }

                    var importDecision = new ImportDecision<LocalTrack>(localTrack);
                    if (_rootFolderService.GetBestRootFolder(artist.Path) == null)
                    {
                        _logger.Warn($"Destination artist folder {artist.Path} not in a Root Folder, skipping import");
                        importDecision.Reject(new Rejection($"Destination artist folder {artist.Path} is not in a Root Folder"));
                    }

                    albumImportDecisions.Add(importDecision);
                    fileCount += 1;
                }

                var downloadId = importAlbumId.Select(x => x.DownloadId).FirstOrDefault(x => x.IsNotNullOrWhiteSpace());
                if (downloadId.IsNullOrWhiteSpace())
                {
                    imported.AddRange(_importApprovedTracks.Import(albumImportDecisions, message.ReplaceExistingFiles, null, message.ImportMode));
                }
                else
                {
                    var trackedDownload = _trackedDownloadService.Find(downloadId);
                    var importResults = _importApprovedTracks.Import(albumImportDecisions, message.ReplaceExistingFiles, trackedDownload.DownloadItem, message.ImportMode);

                    imported.AddRange(importResults);

                    foreach (var importResult in importResults)
                    {
                        importedTrackedDownload.Add(new ManuallyImportedFile
                        {
                            TrackedDownload = trackedDownload,
                            ImportResult = importResult
                        });
                    }
                }
            }

            _logger.ProgressTrace("Manually imported {0} files", imported.Count);

            foreach (var groupedTrackedDownload in importedTrackedDownload.GroupBy(i => i.TrackedDownload.DownloadItem.DownloadId).ToList())
            {
                var trackedDownload = groupedTrackedDownload.First().TrackedDownload;
                var importArtist = groupedTrackedDownload.First().ImportResult.ImportDecision.Item.Artist;

                var outputPath = trackedDownload.ImportItem.OutputPath.FullPath;

                if (_diskProvider.FolderExists(outputPath))
                {
                    if (_downloadedTracksImportService.ShouldDeleteFolder(
                            _diskProvider.GetDirectoryInfo(outputPath),
                            importArtist) && trackedDownload.DownloadItem.CanMoveFiles)
                    {
                        _diskProvider.DeleteFolder(outputPath, true);
                    }
                }

                var remoteTrackCount = Math.Max(1,
                    trackedDownload.RemoteAlbum?.Albums.Sum(x =>
                        x.AlbumReleases.Value.Where(y => y.Monitored).Sum(z => z.TrackCount)) ?? 1);

                var importResults = groupedTrackedDownload.Select(x => x.ImportResult).ToList();
                var importedTrackCount = importResults.Where(c => c.Result == ImportResultType.Imported)
                    .SelectMany(c => c.ImportDecision.Item.Tracks)
                    .Count();
                var allTracksImported = importResults.All(c => c.Result == ImportResultType.Imported) || importedTrackCount >= remoteTrackCount;

                if (allTracksImported)
                {
                    trackedDownload.State = TrackedDownloadState.Imported;
                    _eventAggregator.PublishEvent(new DownloadCompletedEvent(trackedDownload, importArtist.Id));
                }
            }
        }
    }
}