From a2ba8e76bb54f166a84329549fe559b8bc690d80 Mon Sep 17 00:00:00 2001 From: ta264 Date: Thu, 27 Feb 2020 21:27:03 +0000 Subject: [PATCH] New: Watch filesystem for changes to library --- .../MediaManagement/MediaManagement.js | 16 + .../Config/MediaManagementConfigResource.cs | 2 + .../TPLTests/DebouncerFixture.cs | 22 ++ .../Extensions/PathExtensions.cs | 12 +- src/NzbDrone.Common/TPL/Debouncer.cs | 9 +- .../Configuration/ConfigService.cs | 7 + .../Configuration/IConfigService.cs | 1 + .../MediaFiles/AudioTagService.cs | 7 + .../MediaFiles/DiskScanService.cs | 6 +- .../MediaFiles/RootFolderWatchingService.cs | 311 ++++++++++++++++++ .../MediaFiles/TrackFileMovingService.cs | 24 +- .../Music/Services/MoveArtistService.cs | 6 + 12 files changed, 406 insertions(+), 17 deletions(-) create mode 100644 src/NzbDrone.Core/MediaFiles/RootFolderWatchingService.cs diff --git a/frontend/src/Settings/MediaManagement/MediaManagement.js b/frontend/src/Settings/MediaManagement/MediaManagement.js index a6fe21fcf..f3dcfedf5 100644 --- a/frontend/src/Settings/MediaManagement/MediaManagement.js +++ b/frontend/src/Settings/MediaManagement/MediaManagement.js @@ -256,6 +256,22 @@ class MediaManagement extends Component { /> + + Watch Root Folders for file changes + + + + path).IsValidPath(); var info = new FileInfo(path.Trim()); + return info.FullName.CleanFilePathBasic(); + } + public static string CleanFilePathBasic(this string path) + { //UNC - if (OsInfo.IsWindows && info.FullName.StartsWith(@"\\")) + if (OsInfo.IsWindows && path.StartsWith(@"\\")) { - return info.FullName.TrimEnd('/', '\\', ' '); + return path.TrimEnd('/', '\\', ' '); } - if (OsInfo.IsNotWindows && info.FullName.TrimEnd('/').Length == 0) + if (OsInfo.IsNotWindows && path.TrimEnd('/').Length == 0) { return "/"; } - return info.FullName.TrimEnd('/').Trim('\\', ' '); + return path.TrimEnd('/').Trim('\\', ' '); } public static bool PathNotEquals(this string firstPath, string secondPath, StringComparison? comparison = null) diff --git a/src/NzbDrone.Common/TPL/Debouncer.cs b/src/NzbDrone.Common/TPL/Debouncer.cs index 0fa101525..f10d4bc4d 100644 --- a/src/NzbDrone.Common/TPL/Debouncer.cs +++ b/src/NzbDrone.Common/TPL/Debouncer.cs @@ -6,15 +6,17 @@ namespace NzbDrone.Common.TPL { private readonly Action _action; private readonly System.Timers.Timer _timer; + private readonly bool _executeRestartsTimer; private volatile int _paused; private volatile bool _triggered; - public Debouncer(Action action, TimeSpan debounceDuration) + public Debouncer(Action action, TimeSpan debounceDuration, bool executeRestartsTimer = false) { _action = action; _timer = new System.Timers.Timer(debounceDuration.TotalMilliseconds); _timer.Elapsed += timer_Elapsed; + _executeRestartsTimer = executeRestartsTimer; } private void timer_Elapsed(object sender, System.Timers.ElapsedEventArgs e) @@ -32,6 +34,11 @@ namespace NzbDrone.Common.TPL lock (_timer) { _triggered = true; + if (_executeRestartsTimer) + { + _timer.Stop(); + } + if (_paused == 0) { _timer.Start(); diff --git a/src/NzbDrone.Core/Configuration/ConfigService.cs b/src/NzbDrone.Core/Configuration/ConfigService.cs index 8f0a5fa26..7e79f1ec9 100644 --- a/src/NzbDrone.Core/Configuration/ConfigService.cs +++ b/src/NzbDrone.Core/Configuration/ConfigService.cs @@ -227,6 +227,13 @@ namespace NzbDrone.Core.Configuration set { SetValue("ExtraFileExtensions", value); } } + public bool WatchLibraryForChanges + { + get { return GetValueBoolean("WatchLibraryForChanges", true); } + + set { SetValue("WatchLibraryForChanges", value); } + } + public RescanAfterRefreshType RescanAfterRefresh { get { return GetValueEnum("RescanAfterRefresh", RescanAfterRefreshType.Always); } diff --git a/src/NzbDrone.Core/Configuration/IConfigService.cs b/src/NzbDrone.Core/Configuration/IConfigService.cs index 7db24c4b4..3ec13925b 100644 --- a/src/NzbDrone.Core/Configuration/IConfigService.cs +++ b/src/NzbDrone.Core/Configuration/IConfigService.cs @@ -36,6 +36,7 @@ namespace NzbDrone.Core.Configuration bool CopyUsingHardlinks { get; set; } bool ImportExtraFiles { get; set; } string ExtraFileExtensions { get; set; } + bool WatchLibraryForChanges { get; set; } RescanAfterRefreshType RescanAfterRefresh { get; set; } AllowFingerprinting AllowFingerprinting { get; set; } diff --git a/src/NzbDrone.Core/MediaFiles/AudioTagService.cs b/src/NzbDrone.Core/MediaFiles/AudioTagService.cs index 8f0dc7d83..67d517251 100644 --- a/src/NzbDrone.Core/MediaFiles/AudioTagService.cs +++ b/src/NzbDrone.Core/MediaFiles/AudioTagService.cs @@ -39,6 +39,7 @@ namespace NzbDrone.Core.MediaFiles private readonly IConfigService _configService; private readonly IMediaFileService _mediaFileService; private readonly IDiskProvider _diskProvider; + private readonly IRootFolderWatchingService _rootFolderWatchingService; private readonly IArtistService _artistService; private readonly IMapCoversToLocal _mediaCoverService; private readonly IEventAggregator _eventAggregator; @@ -47,6 +48,7 @@ namespace NzbDrone.Core.MediaFiles public AudioTagService(IConfigService configService, IMediaFileService mediaFileService, IDiskProvider diskProvider, + IRootFolderWatchingService rootFolderWatchingService, IArtistService artistService, IMapCoversToLocal mediaCoverService, IEventAggregator eventAggregator, @@ -55,6 +57,7 @@ namespace NzbDrone.Core.MediaFiles _configService = configService; _mediaFileService = mediaFileService; _diskProvider = diskProvider; + _rootFolderWatchingService = rootFolderWatchingService; _artistService = artistService; _mediaCoverService = mediaCoverService; _eventAggregator = eventAggregator; @@ -186,6 +189,7 @@ namespace NzbDrone.Core.MediaFiles tags.MusicBrainzAlbumComment = null; tags.MusicBrainzReleaseTrackId = null; + _rootFolderWatchingService.ReportFileSystemChangeBeginning(path); tags.Write(path); } @@ -211,6 +215,8 @@ namespace NzbDrone.Core.MediaFiles var diff = ReadAudioTag(path).Diff(newTags); + _rootFolderWatchingService.ReportFileSystemChangeBeginning(path); + if (_configService.ScrubAudioTags) { _logger.Debug($"Scrubbing tags for {trackfile}"); @@ -218,6 +224,7 @@ namespace NzbDrone.Core.MediaFiles } _logger.Debug($"Writing tags for {trackfile}"); + newTags.Write(path); UpdateTrackfileSizeAndModified(trackfile, path); diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index d7fb36730..1fd5477ff 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -34,6 +34,9 @@ namespace NzbDrone.Core.MediaFiles IDiskScanService, IExecute { + public static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(?:extras|@eadir|extrafanart|plex versions|\.[^\\/]+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + public static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|^Thumbs\.db$|^\.DS_store$|\.partial~$", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private readonly IDiskProvider _diskProvider; private readonly IMediaFileService _mediaFileService; private readonly IMakeImportDecision _importDecisionMaker; @@ -65,9 +68,6 @@ namespace NzbDrone.Core.MediaFiles _logger = logger; } - private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(?:extras|@eadir|extrafanart|plex versions|\.[^\\/]+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|^Thumbs\.db$|^\.DS_store$|\.partial~$", RegexOptions.Compiled | RegexOptions.IgnoreCase); - public void Scan(List folders = null, FilterFilesType filter = FilterFilesType.Known, List artistIds = null) { if (folders == null) diff --git a/src/NzbDrone.Core/MediaFiles/RootFolderWatchingService.cs b/src/NzbDrone.Core/MediaFiles/RootFolderWatchingService.cs new file mode 100644 index 000000000..9ce640b82 --- /dev/null +++ b/src/NzbDrone.Core/MediaFiles/RootFolderWatchingService.cs @@ -0,0 +1,311 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using NLog; +using NzbDrone.Common.Disk; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.TPL; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Configuration.Events; +using NzbDrone.Core.Datastore.Events; +using NzbDrone.Core.Lifecycle; +using NzbDrone.Core.MediaFiles.Commands; +using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.RootFolders; + +namespace NzbDrone.Core.MediaFiles +{ + public interface IRootFolderWatchingService + { + void ReportFileSystemChangeBeginning(params string[] paths); + } + + public sealed class RootFolderWatchingService : IRootFolderWatchingService, + IDisposable, + IHandle>, + IHandle, + IHandle + { + private const int DEBOUNCE_TIMEOUT_SECONDS = 30; + + private readonly ConcurrentDictionary _fileSystemWatchers = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _tempIgnoredPaths = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _changedPaths = new ConcurrentDictionary(); + + private readonly IRootFolderService _rootFolderService; + private readonly IManageCommandQueue _commandQueueManager; + private readonly IConfigService _configService; + private readonly Logger _logger; + + private readonly Debouncer _scanDebouncer; + private bool _watchForChanges; + + public RootFolderWatchingService(IRootFolderService rootFolderService, + IManageCommandQueue commandQueueManager, + IConfigService configService, + Logger logger) + { + _rootFolderService = rootFolderService; + _commandQueueManager = commandQueueManager; + _configService = configService; + _logger = logger; + + _scanDebouncer = new Debouncer(ScanPending, TimeSpan.FromSeconds(DEBOUNCE_TIMEOUT_SECONDS), true); + } + + public void Dispose() + { + foreach (var watcher in _fileSystemWatchers.Values) + { + DisposeWatcher(watcher, false); + } + } + + public void ReportFileSystemChangeBeginning(params string[] paths) + { + foreach (var path in paths.Where(x => x.IsNotNullOrWhiteSpace())) + { + _logger.Trace($"reporting start of change to {path}"); + _tempIgnoredPaths.AddOrUpdate(path.CleanFilePathBasic(), 1, (key, value) => value + 1); + } + } + + public void Handle(ApplicationStartedEvent message) + { + _watchForChanges = _configService.WatchLibraryForChanges; + + if (_watchForChanges) + { + _rootFolderService.All().ForEach(x => StartWatchingPath(x.Path)); + } + } + + public void Handle(ConfigSavedEvent message) + { + var oldWatch = _watchForChanges; + _watchForChanges = _configService.WatchLibraryForChanges; + + if (_watchForChanges != oldWatch) + { + if (_watchForChanges) + { + _rootFolderService.All().ForEach(x => StartWatchingPath(x.Path)); + } + else + { + _rootFolderService.All().ForEach(x => StopWatchingPath(x.Path)); + } + } + } + + public void Handle(ModelEvent message) + { + if (message.Action == ModelAction.Created && _watchForChanges) + { + StartWatchingPath(message.Model.Path); + } + else if (message.Action == ModelAction.Deleted) + { + StopWatchingPath(message.Model.Path); + } + } + + private void StartWatchingPath(string path) + { + // Already being watched + if (_fileSystemWatchers.ContainsKey(path)) + { + return; + } + + // Creating a FileSystemWatcher over the LAN can take hundreds of milliseconds, so wrap it in a Task to do them all in parallel + Task.Run(() => + { + try + { + var newWatcher = new FileSystemWatcher(path, "*") + { + IncludeSubdirectories = true, + InternalBufferSize = 65536, + NotifyFilter = NotifyFilters.DirectoryName | NotifyFilters.FileName | NotifyFilters.LastWrite + }; + + newWatcher.Created += Watcher_Changed; + newWatcher.Deleted += Watcher_Changed; + newWatcher.Renamed += Watcher_Changed; + newWatcher.Changed += Watcher_Changed; + newWatcher.Error += Watcher_Error; + + if (_fileSystemWatchers.TryAdd(path, newWatcher)) + { + newWatcher.EnableRaisingEvents = true; + _logger.Info("Watching directory {0}", path); + } + else + { + DisposeWatcher(newWatcher, false); + } + } + catch (Exception ex) + { + _logger.Error(ex, "Error watching path: {0}", path); + } + }); + } + + private void StopWatchingPath(string path) + { + if (_fileSystemWatchers.TryGetValue(path, out var watcher)) + { + DisposeWatcher(watcher, true); + } + } + + private void Watcher_Error(object sender, ErrorEventArgs e) + { + var ex = e.GetException(); + var dw = (FileSystemWatcher)sender; + + if (ex.GetType() == typeof(InternalBufferOverflowException)) + { + _logger.Warn(ex, "The file system watcher experienced an internal buffer overflow for: {0}", dw.Path); + + _changedPaths.TryAdd(dw.Path, dw.Path); + _scanDebouncer.Execute(); + } + else + { + _logger.Error(ex, "Error in Directory watcher for: {0}" + dw.Path); + + DisposeWatcher(dw, true); + } + } + + private void Watcher_Changed(object sender, FileSystemEventArgs e) + { + try + { + var rootFolder = ((FileSystemWatcher)sender).Path; + var path = e.FullPath; + + if (path.IsNullOrWhiteSpace()) + { + throw new ArgumentNullException("path"); + } + + _changedPaths.TryAdd(path, rootFolder); + + _scanDebouncer.Execute(); + } + catch (Exception ex) + { + _logger.Error(ex, "Exception in ReportFileSystemChanged. Path: {0}", e.FullPath); + } + } + + private void ScanPending() + { + var pairs = _changedPaths.ToArray(); + _changedPaths.Clear(); + + var ignored = _tempIgnoredPaths.Keys.ToArray(); + _tempIgnoredPaths.Clear(); + + var toScan = new HashSet(); + + foreach (var item in pairs) + { + var path = item.Key.CleanFilePathBasic(); + var rootFolder = item.Value; + + if (!ShouldIgnoreChange(path, ignored)) + { + _logger.Trace("Actioning change to {0}", path); + toScan.Add(rootFolder); + } + else + { + _logger.Trace("Ignoring change to {0}", path); + } + } + + if (toScan.Any()) + { + _commandQueueManager.Push(new RescanFoldersCommand(toScan.ToList(), FilterFilesType.Known, true, null)); + } + } + + private bool ShouldIgnoreChange(string cleanPath, string[] ignoredPaths) + { + var cleaned = cleanPath.CleanFilePathBasic(); + + // Skip partial/backup + if (cleanPath.EndsWith(".partial~") || + cleanPath.EndsWith(".backup~")) + { + return true; + } + + // only proceed for directories and files with music extensions + var extension = Path.GetExtension(cleaned); + if (extension.IsNullOrWhiteSpace() && !Directory.Exists(cleaned)) + { + return true; + } + + if (extension.IsNotNullOrWhiteSpace() && !MediaFileExtensions.Extensions.Contains(extension)) + { + return true; + } + + // If the parent of an ignored path has a change event, ignore that too + // Note that we can't afford to use the PathEquals or IsParentPath functions because + // these rely on disk access which is too slow when trying to handle many update events + return ignoredPaths.Any(i => i.Equals(cleaned, DiskProviderBase.PathStringComparison) || + i.StartsWith(cleaned + Path.DirectorySeparatorChar, DiskProviderBase.PathStringComparison) || + Path.GetDirectoryName(i).Equals(cleaned, DiskProviderBase.PathStringComparison)); + } + + private void DisposeWatcher(FileSystemWatcher watcher, bool removeFromList) + { + try + { + using (watcher) + { + _logger.Info("Stopping directory watching for path {0}", watcher.Path); + + watcher.Created -= Watcher_Changed; + watcher.Deleted -= Watcher_Changed; + watcher.Renamed -= Watcher_Changed; + watcher.Changed -= Watcher_Changed; + watcher.Error -= Watcher_Error; + + try + { + watcher.EnableRaisingEvents = false; + } + catch (InvalidOperationException) + { + // Seeing this under mono on linux sometimes + // Collection was modified; enumeration operation may not execute. + } + } + } + catch + { + // we don't care about exceptions disposing + } + finally + { + if (removeFromList) + { + _fileSystemWatchers.TryRemove(watcher.Path, out _); + } + } + } + } +} diff --git a/src/NzbDrone.Core/MediaFiles/TrackFileMovingService.cs b/src/NzbDrone.Core/MediaFiles/TrackFileMovingService.cs index b53d09e03..12947516f 100644 --- a/src/NzbDrone.Core/MediaFiles/TrackFileMovingService.cs +++ b/src/NzbDrone.Core/MediaFiles/TrackFileMovingService.cs @@ -30,21 +30,23 @@ namespace NzbDrone.Core.MediaFiles private readonly IBuildFileNames _buildFileNames; private readonly IDiskTransferService _diskTransferService; private readonly IDiskProvider _diskProvider; + private readonly IRootFolderWatchingService _rootFolderWatchingService; private readonly IMediaFileAttributeService _mediaFileAttributeService; private readonly IEventAggregator _eventAggregator; private readonly IConfigService _configService; private readonly Logger _logger; public TrackFileMovingService(ITrackService trackService, - IAlbumService albumService, - IUpdateTrackFileService updateTrackFileService, - IBuildFileNames buildFileNames, - IDiskTransferService diskTransferService, - IDiskProvider diskProvider, - IMediaFileAttributeService mediaFileAttributeService, - IEventAggregator eventAggregator, - IConfigService configService, - Logger logger) + IAlbumService albumService, + IUpdateTrackFileService updateTrackFileService, + IBuildFileNames buildFileNames, + IDiskTransferService diskTransferService, + IDiskProvider diskProvider, + IRootFolderWatchingService rootFolderWatchingService, + IMediaFileAttributeService mediaFileAttributeService, + IEventAggregator eventAggregator, + IConfigService configService, + Logger logger) { _trackService = trackService; _albumService = albumService; @@ -52,6 +54,7 @@ namespace NzbDrone.Core.MediaFiles _buildFileNames = buildFileNames; _diskTransferService = diskTransferService; _diskProvider = diskProvider; + _rootFolderWatchingService = rootFolderWatchingService; _mediaFileAttributeService = mediaFileAttributeService; _eventAggregator = eventAggregator; _configService = configService; @@ -119,6 +122,7 @@ namespace NzbDrone.Core.MediaFiles throw new SameFilenameException("File not moved, source and destination are the same", trackFilePath); } + _rootFolderWatchingService.ReportFileSystemChangeBeginning(trackFilePath, destinationFilePath); _diskTransferService.TransferFile(trackFilePath, destinationFilePath, mode); trackFile.Path = destinationFilePath; @@ -166,6 +170,8 @@ namespace NzbDrone.Core.MediaFiles var changed = false; var newEvent = new TrackFolderCreatedEvent(artist, trackFile); + _rootFolderWatchingService.ReportFileSystemChangeBeginning(artistFolder, albumFolder, trackFolder); + if (!_diskProvider.FolderExists(artistFolder)) { CreateFolder(artistFolder); diff --git a/src/NzbDrone.Core/Music/Services/MoveArtistService.cs b/src/NzbDrone.Core/Music/Services/MoveArtistService.cs index bb8f9643d..cf3ba7ec8 100644 --- a/src/NzbDrone.Core/Music/Services/MoveArtistService.cs +++ b/src/NzbDrone.Core/Music/Services/MoveArtistService.cs @@ -2,6 +2,7 @@ using System.IO; using NLog; using NzbDrone.Common.Disk; using NzbDrone.Common.Instrumentation.Extensions; +using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Music.Commands; @@ -15,6 +16,7 @@ namespace NzbDrone.Core.Music private readonly IArtistService _artistService; private readonly IBuildFileNames _filenameBuilder; private readonly IDiskProvider _diskProvider; + private readonly IRootFolderWatchingService _rootFolderWatchingService; private readonly IDiskTransferService _diskTransferService; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; @@ -22,6 +24,7 @@ namespace NzbDrone.Core.Music public MoveArtistService(IArtistService artistService, IBuildFileNames filenameBuilder, IDiskProvider diskProvider, + IRootFolderWatchingService rootFolderWatchingService, IDiskTransferService diskTransferService, IEventAggregator eventAggregator, Logger logger) @@ -29,6 +32,7 @@ namespace NzbDrone.Core.Music _artistService = artistService; _filenameBuilder = filenameBuilder; _diskProvider = diskProvider; + _rootFolderWatchingService = rootFolderWatchingService; _diskTransferService = diskTransferService; _eventAggregator = eventAggregator; _logger = logger; @@ -55,6 +59,8 @@ namespace NzbDrone.Core.Music { if (moveFiles) { + _rootFolderWatchingService.ReportFileSystemChangeBeginning(sourcePath, destinationPath); + _diskTransferService.TransferFolder(sourcePath, destinationPath, TransferMode.Move); _logger.ProgressInfo("{0} moved successfully to {1}", artist.Name, artist.Path);