From fe10df693551ded214adc49cbbe4d5ad00de66fb Mon Sep 17 00:00:00 2001
From: Luke Pulverenti <luke.pulverenti@gmail.com>
Date: Sat, 9 Sep 2017 23:18:23 -0400
Subject: [PATCH] improve queries from scheduled tasks

---
 .../Data/SqliteItemRepository.cs              | 48 +++++++++++++++++
 .../Library/LibraryManager.cs                 | 25 ---------
 .../LiveTv/RefreshChannelsScheduledTask.cs    |  2 +-
 .../ScheduledTasks/ChapterImagesTask.cs       | 10 +++-
 .../Services/SwaggerService.cs                |  6 +--
 .../Entities/InternalItemsQuery.cs            |  5 ++
 .../Library/ILibraryManager.cs                |  2 -
 .../MediaInfo/SubtitleDownloader.cs           | 53 ++++++++++---------
 .../MediaInfo/SubtitleScheduledTask.cs        | 47 ++++++++++++----
 9 files changed, 131 insertions(+), 67 deletions(-)

diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs
index b5bc7692ec..ea3845f0c9 100644
--- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs
+++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs
@@ -4254,6 +4254,54 @@ namespace Emby.Server.Implementations.Data
                 }
             }
 
+            if (!string.IsNullOrWhiteSpace(query.HasNoAudioTrackWithLanguage))
+            {
+                whereClauses.Add("((select language from MediaStreams where MediaStreams.ItemId=A.Guid and MediaStreams.StreamType='Audio' and MediaStreams.Language=@HasNoAudioTrackWithLanguage limit 1) is null)");
+                if (statement != null)
+                {
+                    statement.TryBind("@HasNoAudioTrackWithLanguage", query.HasNoAudioTrackWithLanguage);
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(query.HasNoInternalSubtitleTrackWithLanguage))
+            {
+                whereClauses.Add("((select language from MediaStreams where MediaStreams.ItemId=A.Guid and MediaStreams.StreamType='Subtitle' and MediaStreams.IsExternal=0 and MediaStreams.Language=@HasNoInternalSubtitleTrackWithLanguage limit 1) is null)");
+                if (statement != null)
+                {
+                    statement.TryBind("@HasNoInternalSubtitleTrackWithLanguage", query.HasNoInternalSubtitleTrackWithLanguage);
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(query.HasNoExternalSubtitleTrackWithLanguage))
+            {
+                whereClauses.Add("((select language from MediaStreams where MediaStreams.ItemId=A.Guid and MediaStreams.StreamType='Subtitle' and MediaStreams.IsExternal=1 and MediaStreams.Language=@HasNoExternalSubtitleTrackWithLanguage limit 1) is null)");
+                if (statement != null)
+                {
+                    statement.TryBind("@HasNoExternalSubtitleTrackWithLanguage", query.HasNoExternalSubtitleTrackWithLanguage);
+                }
+            }
+
+            if (!string.IsNullOrWhiteSpace(query.HasNoSubtitleTrackWithLanguage))
+            {
+                whereClauses.Add("((select language from MediaStreams where MediaStreams.ItemId=A.Guid and MediaStreams.StreamType='Subtitle' and MediaStreams.Language=@HasNoSubtitleTrackWithLanguage limit 1) is null)");
+                if (statement != null)
+                {
+                    statement.TryBind("@HasNoSubtitleTrackWithLanguage", query.HasNoSubtitleTrackWithLanguage);
+                }
+            }
+
+            if (query.HasChapterImages.HasValue)
+            {
+                if (query.HasChapterImages.Value)
+                {
+                    whereClauses.Add("((select imagepath from Chapters2 where Chapters2.ItemId=A.Guid and imagepath not null limit 1) not null)");
+                }
+                else
+                {
+                    whereClauses.Add("((select imagepath from Chapters2 where Chapters2.ItemId=A.Guid and imagepath not null limit 1) is null)");
+                }
+            }
+
             if (query.HasDeadParentId.HasValue && query.HasDeadParentId.Value)
             {
                 whereClauses.Add("ParentId NOT NULL AND ParentId NOT IN (select guid from TypedBaseItems)");
diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs
index 8c5ecf782e..510ecee76b 100644
--- a/Emby.Server.Implementations/Library/LibraryManager.cs
+++ b/Emby.Server.Implementations/Library/LibraryManager.cs
@@ -620,37 +620,12 @@ namespace Emby.Server.Implementations.Library
             return ResolveItem(args, resolvers);
         }
 
-        private readonly List<string> _ignoredPaths = new List<string>();
-
-        public void RegisterIgnoredPath(string path)
-        {
-            lock (_ignoredPaths)
-            {
-                _ignoredPaths.Add(path);
-            }
-        }
-        public void UnRegisterIgnoredPath(string path)
-        {
-            lock (_ignoredPaths)
-            {
-                _ignoredPaths.Remove(path);
-            }
-        }
-
         public bool IgnoreFile(FileSystemMetadata file, BaseItem parent)
         {
             if (EntityResolutionIgnoreRules.Any(r => r.ShouldIgnore(file, parent)))
             {
                 return true;
             }
-
-            //lock (_ignoredPaths)
-            {
-                if (_ignoredPaths.Contains(file.FullName, StringComparer.OrdinalIgnoreCase))
-                {
-                    return true;
-                }
-            }
             return false;
         }
 
diff --git a/Emby.Server.Implementations/LiveTv/RefreshChannelsScheduledTask.cs b/Emby.Server.Implementations/LiveTv/RefreshChannelsScheduledTask.cs
index cad28c809c..225360159b 100644
--- a/Emby.Server.Implementations/LiveTv/RefreshChannelsScheduledTask.cs
+++ b/Emby.Server.Implementations/LiveTv/RefreshChannelsScheduledTask.cs
@@ -51,7 +51,7 @@ namespace Emby.Server.Implementations.LiveTv
             return new[] { 
             
                 // Every so often
-                new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerInterval, IntervalTicks = TimeSpan.FromHours(12).Ticks}
+                new TaskTriggerInfo { Type = TaskTriggerInfo.TriggerInterval, IntervalTicks = TimeSpan.FromHours(24).Ticks}
             };
         }
 
diff --git a/Emby.Server.Implementations/ScheduledTasks/ChapterImagesTask.cs b/Emby.Server.Implementations/ScheduledTasks/ChapterImagesTask.cs
index ec371c741f..bf7bf9ff8b 100644
--- a/Emby.Server.Implementations/ScheduledTasks/ChapterImagesTask.cs
+++ b/Emby.Server.Implementations/ScheduledTasks/ChapterImagesTask.cs
@@ -58,8 +58,8 @@ namespace Emby.Server.Implementations.ScheduledTasks
         /// </summary>
         public IEnumerable<TaskTriggerInfo> GetDefaultTriggers()
         {
-            return new[] { 
-            
+            return new[] {
+
                 new TaskTriggerInfo
                 {
                     Type = TaskTriggerInfo.TriggerDaily,
@@ -88,6 +88,12 @@ namespace Emby.Server.Implementations.ScheduledTasks
                 IsFolder = false,
                 Recursive = true,
                 DtoOptions = new DtoOptions(false)
+                {
+                    EnableImages = false
+                },
+                SourceTypes = new SourceType[] { SourceType.Library },
+                HasChapterImages = false,
+                IsVirtualItem = false
 
             })
                 .OfType<Video>()
diff --git a/Emby.Server.Implementations/Services/SwaggerService.cs b/Emby.Server.Implementations/Services/SwaggerService.cs
index d684dce5d4..63cbb78dd5 100644
--- a/Emby.Server.Implementations/Services/SwaggerService.cs
+++ b/Emby.Server.Implementations/Services/SwaggerService.cs
@@ -193,13 +193,11 @@ namespace Emby.Server.Implementations.Services
                     summary = info.Summary,
                     produces = new[]
                     {
-                        "application/json",
-                        "application/xml"
+                        "application/json"
                     },
                     consumes = new[]
                     {
-                        "application/json",
-                        "application/xml"
+                        "application/json"
                     },
                     operationId = info.RequestType.Name,
                     tags = new string[] { },
diff --git a/MediaBrowser.Controller/Entities/InternalItemsQuery.cs b/MediaBrowser.Controller/Entities/InternalItemsQuery.cs
index 03dc18b7a7..a7f6c60148 100644
--- a/MediaBrowser.Controller/Entities/InternalItemsQuery.cs
+++ b/MediaBrowser.Controller/Entities/InternalItemsQuery.cs
@@ -160,6 +160,7 @@ namespace MediaBrowser.Controller.Entities
         public bool ForceDirect { get; set; }
         public Dictionary<string, string> ExcludeProviderIds { get; set; }
         public bool EnableGroupByMetadataKey { get; set; }
+        public bool? HasChapterImages { get; set; }
 
         public Tuple<string, SortOrder>[] OrderBy { get; set; }
 
@@ -169,6 +170,10 @@ namespace MediaBrowser.Controller.Entities
 
         public DtoOptions DtoOptions { get; set; }
         public int MinSimilarityScore { get; set; }
+        public string HasNoAudioTrackWithLanguage { get; set; }
+        public string HasNoInternalSubtitleTrackWithLanguage { get; set; }
+        public string HasNoExternalSubtitleTrackWithLanguage { get; set; }
+        public string HasNoSubtitleTrackWithLanguage { get; set; }
 
         public InternalItemsQuery()
         {
diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs
index 05845102b0..3001e33661 100644
--- a/MediaBrowser.Controller/Library/ILibraryManager.cs
+++ b/MediaBrowser.Controller/Library/ILibraryManager.cs
@@ -563,8 +563,6 @@ namespace MediaBrowser.Controller.Library
         QueryResult<Tuple<BaseItem, ItemCounts>> GetAlbumArtists(InternalItemsQuery query);
         QueryResult<Tuple<BaseItem, ItemCounts>> GetAllArtists(InternalItemsQuery query);
 
-        void RegisterIgnoredPath(string path);
-        void UnRegisterIgnoredPath(string path);
         int GetCount(InternalItemsQuery query);
     }
 }
\ No newline at end of file
diff --git a/MediaBrowser.Providers/MediaInfo/SubtitleDownloader.cs b/MediaBrowser.Providers/MediaInfo/SubtitleDownloader.cs
index 79e89a1104..7c6b55d3ac 100644
--- a/MediaBrowser.Providers/MediaInfo/SubtitleDownloader.cs
+++ b/MediaBrowser.Providers/MediaInfo/SubtitleDownloader.cs
@@ -32,16 +32,40 @@ namespace MediaBrowser.Providers.MediaInfo
             bool requirePerfectMatch,
             IEnumerable<string> languages,
             CancellationToken cancellationToken)
+        {
+            var downloadedLanguages = new List<string>();
+
+            foreach (var lang in languages)
+            {
+                var downloaded = await DownloadSubtitles(video, mediaStreams, skipIfEmbeddedSubtitlesPresent,
+                    skipIfAudioTrackMatches, requirePerfectMatch, lang, cancellationToken).ConfigureAwait(false);
+
+                if (downloaded)
+                {
+                    downloadedLanguages.Add(lang);
+                }
+            }
+
+            return downloadedLanguages;
+        }
+
+        public Task<bool> DownloadSubtitles(Video video,
+            List<MediaStream> mediaStreams,
+            bool skipIfEmbeddedSubtitlesPresent,
+            bool skipIfAudioTrackMatches,
+            bool requirePerfectMatch,
+            string lang,
+            CancellationToken cancellationToken)
         {
             if (video.LocationType != LocationType.FileSystem ||
                 video.VideoType != VideoType.VideoFile)
             {
-                return new List<string>();
+                return Task.FromResult(false);
             }
 
             if (!video.IsCompleteMedia)
             {
-                return new List<string>();
+                return Task.FromResult(false);
             }
 
             VideoContentType mediaType;
@@ -57,30 +81,11 @@ namespace MediaBrowser.Providers.MediaInfo
             else
             {
                 // These are the only supported types
-                return new List<string>();
+                return Task.FromResult(false);
             }
 
-            var downloadedLanguages = new List<string>();
-
-            foreach (var lang in languages)
-            {
-                try
-                {
-                    var downloaded = await DownloadSubtitles(video, mediaStreams, skipIfEmbeddedSubtitlesPresent, skipIfAudioTrackMatches, requirePerfectMatch, lang, mediaType, cancellationToken)
-                        .ConfigureAwait(false);
-
-                    if (downloaded)
-                    {
-                        downloadedLanguages.Add(lang);
-                    }
-                }
-                catch (Exception ex)
-                {
-                    _logger.ErrorException("Error downloading subtitles", ex);
-                }
-            }
-
-            return downloadedLanguages;
+            return DownloadSubtitles(video, mediaStreams, skipIfEmbeddedSubtitlesPresent, skipIfAudioTrackMatches,
+                requirePerfectMatch, lang, mediaType, cancellationToken);
         }
 
         private async Task<bool> DownloadSubtitles(Video video,
diff --git a/MediaBrowser.Providers/MediaInfo/SubtitleScheduledTask.cs b/MediaBrowser.Providers/MediaInfo/SubtitleScheduledTask.cs
index 61cf1cfc3c..4d68735b94 100644
--- a/MediaBrowser.Providers/MediaInfo/SubtitleScheduledTask.cs
+++ b/MediaBrowser.Providers/MediaInfo/SubtitleScheduledTask.cs
@@ -80,17 +80,44 @@ namespace MediaBrowser.Providers.MediaInfo
                 return;
             }
 
-            var videos = _libraryManager.GetItemList(new InternalItemsQuery
+            var dict = new Dictionary<Guid, BaseItem>();
+
+            foreach (var lang in options.DownloadLanguages)
             {
-                MediaTypes = new string[] { MediaType.Video },
-                IsVirtualItem = false,
-                IncludeItemTypes = types.ToArray(types.Count),
-                DtoOptions = new DtoOptions(true)
+                var query = new InternalItemsQuery
+                {
+                    MediaTypes = new string[] {MediaType.Video},
+                    IsVirtualItem = false,
+                    IncludeItemTypes = types.ToArray(types.Count),
+                    DtoOptions = new DtoOptions(true),
+                    SourceTypes = new[] {SourceType.Library}
+                };
+
+                if (options.SkipIfAudioTrackMatches)
+                {
+                    query.HasNoAudioTrackWithLanguage = lang;
+                }
+
+                if (options.SkipIfEmbeddedSubtitlesPresent)
+                {
+                    // Exclude if it already has any subtitles of the same language
+                    query.HasNoSubtitleTrackWithLanguage = lang;
+                }
+                else
+                {
+                    // Exclude if it already has external subtitles of the same language
+                    query.HasNoExternalSubtitleTrackWithLanguage = lang;
+                }
 
-            }).OfType<Video>()
-                .Where(i => i.LocationType != LocationType.Remote)
-                .ToList();
+                var videosByLanguage = _libraryManager.GetItemList(query);
 
+                foreach (var video in videosByLanguage)
+                {
+                    dict[video.Id] = video;
+                }
+            }
+
+            var videos = dict.Values.ToList();
             if (videos.Count == 0)
             {
                 return;
@@ -100,9 +127,11 @@ namespace MediaBrowser.Providers.MediaInfo
 
             foreach (var video in videos)
             {
+                cancellationToken.ThrowIfCancellationRequested();
+
                 try
                 {
-                    await DownloadSubtitles(video, options, cancellationToken).ConfigureAwait(false);
+                    await DownloadSubtitles(video as Video, options, cancellationToken).ConfigureAwait(false);
                 }
                 catch (Exception ex)
                 {