Updated migration to attempt associate old grabbed & imported events and associate drone factory imports during CompletedDownloadHandling.

src/NzbDrone.Core.Test/Download/CompletedDownloadServiceFixture.cs

@@ -13,6 +13,8 @@ using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Test.Framework;
 using NzbDrone.Core.MediaFiles;
 using NzbDrone.Test.Common;
+using NzbDrone.Core.Tv;
+using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Test.Download
 {
@@ -28,6 +30,10 @@ namespace NzbDrone.Core.Test.Download
                                              .All()
                                              .With(h => h.Status = DownloadItemStatus.Completed)
                                              .With(h => h.OutputPath = @"C:\DropFolder\MyDownload".AsOsAgnostic())
+                                             .With(h => h.RemoteEpisode = new RemoteEpisode
+                                                {
+                                                    Episodes = new List<Episode> { new Episode { Id = 1 } }
+                                                })
                                              .Build()
                                              .ToList();
             
@@ -256,6 +262,8 @@ namespace NzbDrone.Core.Test.Download
             history.First().Data.Add("downloadClient", "SabnzbdClient");
             history.First().Data.Add("downloadClientId", _completed.First().DownloadClientId);
 
+            Subject.Execute(new CheckForFinishedDownloadCommand());
+
             VerifyNoImports();
         }
 
@@ -278,7 +286,58 @@ namespace NzbDrone.Core.Test.Download
             history.First().Data.Add("downloadClient", "SabnzbdClient");
             history.First().Data.Add("downloadClientId", _completed.First().DownloadClientId);
 
+            Subject.Execute(new CheckForFinishedDownloadCommand());
+
+            VerifyNoImports();
+        }
+
+        [Test]
+        public void should_process_as_already_imported_if_drone_factory_import_history_exists()
+        {
+            GivenCompletedDownloadClientHistory(false);
+
+            _completed.Clear();
+            _completed.AddRange(Builder<DownloadClientItem>.CreateListOfSize(2)
+                                             .All()
+                                             .With(h => h.Status = DownloadItemStatus.Completed)
+                                             .With(h => h.OutputPath = @"C:\DropFolder\MyDownload".AsOsAgnostic())
+                                             .With(h => h.RemoteEpisode = new RemoteEpisode
+                                             {
+                                                 Episodes = new List<Episode> { new Episode { Id = 1 } }
+                                             })
+                                             .Build());
+
+            var grabbedHistory = Builder<History.History>.CreateListOfSize(2)
+                                                  .All()
+                                                  .With(d => d.Data["downloadClient"] = "SabnzbdClient")
+                                                  .TheFirst(1)
+                                                  .With(d => d.Data["downloadClientId"] = _completed.First().DownloadClientId)
+                                                  .With(d => d.SourceTitle = "Droned.S01E01.720p-LAZY")
+                                                  .TheLast(1)
+                                                  .With(d => d.Data["downloadClientId"] = _completed.Last().DownloadClientId)
+                                                  .With(d => d.SourceTitle = "Droned.S01E01.Proper.720p-LAZY")
+                                                  .Build()
+                                                  .ToList();
+
+            var importedHistory = Builder<History.History>.CreateListOfSize(2)
+                                                  .All()
+                                                  .With(d => d.EpisodeId = 1)
+                                                  .TheFirst(1)
+                                                  .With(d => d.Data["droppedPath"] = @"C:\mydownload\Droned.S01E01.720p-LAZY\lzy-dr101.mkv".AsOsAgnostic())
+                                                  .TheLast(1)
+                                                  .With(d => d.Data["droppedPath"] = @"C:\mydownload\Droned.S01E01.Proper.720p-LAZY\lzy-dr101.mkv".AsOsAgnostic())
+                                                  .Build()
+                                                  .ToList();
+
+            GivenGrabbedHistory(grabbedHistory);
+            GivenImportedHistory(importedHistory);
+
+            Subject.Execute(new CheckForFinishedDownloadCommand());
+
             VerifyNoImports();
+
+            Mocker.GetMock<IHistoryService>()
+                .Verify(v => v.UpdateHistoryData(It.IsAny<int>(), It.IsAny<Dictionary<String, String>>()), Times.Exactly(2));
         }
 
         [Test]

src/NzbDrone.Core/Datastore/Migration/051_download_client_import.cs

@@ -0,0 +1,224 @@
+using System;
+using System.Data;
+using System.Linq;
+using System.Collections.Generic;
+using FluentMigrator;
+using Newtonsoft.Json;
+using NzbDrone.Common;
+using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Datastore.Migration.Framework;
+using System.IO;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(51)]
+    public class download_client_import : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Execute.WithConnection(ConvertFolderSettings);
+
+            Execute.WithConnection(AssociateImportedHistoryItems);
+        }
+
+        private void ConvertFolderSettings(IDbConnection conn, IDbTransaction tran)
+        {
+            using (IDbCommand downloadClientsCmd = conn.CreateCommand())
+            {
+                downloadClientsCmd.Transaction = tran;
+                downloadClientsCmd.CommandText = @"SELECT Value FROM Config WHERE Key = 'downloadedepisodesfolder'";
+                var downloadedEpisodesFolder = downloadClientsCmd.ExecuteScalar() as String;
+
+                downloadClientsCmd.Transaction = tran;
+                downloadClientsCmd.CommandText = @"SELECT Id, Implementation, Settings, ConfigContract FROM DownloadClients WHERE ConfigContract = 'FolderSettings'";
+                using (IDataReader downloadClientReader = downloadClientsCmd.ExecuteReader())
+                {
+                    while (downloadClientReader.Read())
+                    {
+                        var id = downloadClientReader.GetInt32(0);
+                        var implementation = downloadClientReader.GetString(1);
+                        var settings = downloadClientReader.GetString(2);
+                        var configContract = downloadClientReader.GetString(3);
+
+                        var settingsJson = JsonConvert.DeserializeObject(settings) as Newtonsoft.Json.Linq.JObject;
+
+                        if (implementation == "Blackhole")
+                        {
+                            var newSettings = new
+                            {
+                                NzbFolder = settingsJson.Value<String>("folder"),
+                                WatchFolder = downloadedEpisodesFolder
+                            }.ToJson();
+
+                            using (IDbCommand updateCmd = conn.CreateCommand())
+                            {
+                                updateCmd.Transaction = tran;
+                                updateCmd.CommandText = "UPDATE DownloadClients SET Implementation = ?, Settings = ?, ConfigContract = ? WHERE Id = ?";
+                                updateCmd.AddParameter("UsenetBlackhole");
+                                updateCmd.AddParameter(newSettings);
+                                updateCmd.AddParameter("UsenetBlackholeSettings");
+                                updateCmd.AddParameter(id);
+
+                                updateCmd.ExecuteNonQuery();
+                            }
+                        }
+                        else if (implementation == "Pneumatic")
+                        {
+                            var newSettings = new
+                            {
+                                NzbFolder = settingsJson.Value<String>("folder")
+                            }.ToJson();
+
+                            using (IDbCommand updateCmd = conn.CreateCommand())
+                            {
+                                updateCmd.Transaction = tran;
+                                updateCmd.CommandText = "UPDATE DownloadClients SET Settings = ?, ConfigContract = ? WHERE Id = ?";
+                                updateCmd.AddParameter(newSettings);
+                                updateCmd.AddParameter("PneumaticSettings");
+                                updateCmd.AddParameter(id);
+
+                                updateCmd.ExecuteNonQuery();
+                            }
+                        }
+                        else
+                        {
+                            using (IDbCommand updateCmd = conn.CreateCommand())
+                            {
+                                updateCmd.Transaction = tran;
+                                updateCmd.CommandText = "DELETE FROM DownloadClients WHERE Id = ?";
+                                updateCmd.AddParameter(id);
+
+                                updateCmd.ExecuteNonQuery();
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        private sealed class MigrationHistoryItem
+        {
+            public Int32 Id { get; set; }
+            public Int32 EpisodeId { get; set; }
+            public Int32 SeriesId { get; set; }
+            public String SourceTitle { get; set; }
+            public DateTime Date { get; set; }
+            public Dictionary<String, String> Data { get; set; }
+            public MigrationHistoryEventType EventType { get; set; }
+        }
+
+        private enum MigrationHistoryEventType
+        {
+            Unknown = 0,
+            Grabbed = 1,
+            SeriesFolderImported = 2,
+            DownloadFolderImported = 3,
+            DownloadFailed = 4
+        }
+
+        private void AssociateImportedHistoryItems(IDbConnection conn, IDbTransaction tran)
+        {
+            var historyItems = new List<MigrationHistoryItem>();
+
+            using (IDbCommand historyCmd = conn.CreateCommand())
+            {
+                historyCmd.Transaction = tran;
+                historyCmd.CommandText = @"SELECT Id, EpisodeId, SeriesId, SourceTitle, Date, Data, EventType FROM History WHERE EventType NOT NULL";
+                using (IDataReader historyRead = historyCmd.ExecuteReader())
+                {
+                    while (historyRead.Read())
+                    {
+                        historyItems.Add(new MigrationHistoryItem
+                            {
+                                Id = historyRead.GetInt32(0),
+                                EpisodeId = historyRead.GetInt32(1),
+                                SeriesId = historyRead.GetInt32(2),
+                                SourceTitle = historyRead.GetString(3),
+                                Date = historyRead.GetDateTime(4),
+                                Data = Json.Deserialize<Dictionary<String, String>>(historyRead.GetString(5)),
+                                EventType = (MigrationHistoryEventType)historyRead.GetInt32(6)
+                            });
+                    }
+                }
+            }
+
+            var numHistoryItemsNotAssociated = historyItems.Count(v => v.EventType == MigrationHistoryEventType.DownloadFolderImported &&
+                                                                       v.Data.GetValueOrDefault("downloadClientId") == null);
+
+            if (numHistoryItemsNotAssociated == 0)
+            {
+                return;
+            }
+
+            var historyItemsToAssociate = new Dictionary<MigrationHistoryItem, MigrationHistoryItem>();
+
+            var historyItemsLookup = historyItems.ToLookup(v => v.EpisodeId);
+
+            foreach (var historyItemGroup in historyItemsLookup)
+            {
+                var list = historyItemGroup.ToList();
+
+                for (int i = 0; i < list.Count - 1; i++)
+                {
+                    var grabbedEvent = list[i];
+                    if (grabbedEvent.EventType != MigrationHistoryEventType.Grabbed) continue;
+                    if (grabbedEvent.Data.GetValueOrDefault("downloadClient") == null || grabbedEvent.Data.GetValueOrDefault("downloadClientId") == null) continue;
+
+                    // Check if it is already associated with a failed/imported event.
+                    int j;
+                    for (j = i + 1; j < list.Count;j++)
+                    {
+                        if (list[j].EventType != MigrationHistoryEventType.DownloadFolderImported &&
+                            list[j].EventType != MigrationHistoryEventType.DownloadFailed)
+                        {
+                            continue;
+                        }
+
+                        if (list[j].Data.ContainsKey("downloadClient") && list[j].Data["downloadClient"] == grabbedEvent.Data["downloadClient"] &&
+                            list[j].Data.ContainsKey("downloadClientId") && list[j].Data["downloadClientId"] == grabbedEvent.Data["downloadClientId"])
+                        {
+                            break;
+                        }
+                    }
+
+                    if (j != list.Count)
+                    {
+                        list.RemoveAt(j);
+                        list.RemoveAt(i--);
+                        continue;
+                    }
+
+                    var importedEvent = list[i + 1];
+                    if (importedEvent.EventType != MigrationHistoryEventType.DownloadFolderImported) continue;
+
+                    var droppedPath = importedEvent.Data.GetValueOrDefault("droppedPath");
+                    if (droppedPath != null && new FileInfo(droppedPath).Directory.Name == grabbedEvent.SourceTitle)
+                    {
+                        historyItemsToAssociate[importedEvent] = grabbedEvent;
+
+                        list.RemoveAt(i + 1);
+                        list.RemoveAt(i--);
+                    }
+                }
+            }
+
+            foreach (var pair in historyItemsToAssociate)
+            {
+                using (IDbCommand updateHistoryCmd = conn.CreateCommand())
+                {
+                    pair.Key.Data["downloadClient"] = pair.Value.Data["downloadClient"];
+                    pair.Key.Data["downloadClientId"] = pair.Value.Data["downloadClientId"];
+
+                    updateHistoryCmd.Transaction = tran;
+                    updateHistoryCmd.CommandText = "UPDATE History SET Data = ? WHERE Id = ?";
+                    updateHistoryCmd.AddParameter(pair.Key.Data.ToJson());
+                    updateHistoryCmd.AddParameter(pair.Key.Id);
+
+                    updateHistoryCmd.ExecuteNonQuery();
+                }
+            }
+            
+            _logger.Info("Updated old History items. {0}/{1} old ImportedEvents were associated with GrabbedEvents.", historyItemsToAssociate.Count, numHistoryItemsNotAssociated);
+        }
+    }
+}

src/NzbDrone.Core/Datastore/Migration/051_rename_download_client_settings.cs

@@ -1,88 +0,0 @@
-using NzbDrone.Core.Datastore.Migration.Framework;
-using FluentMigrator;
-using System.Data;
-using NzbDrone.Common.Serializer;
-using NzbDrone.Core.Download.Clients.UsenetBlackhole;
-using Newtonsoft.Json;
-using System;
-using NzbDrone.Core.Download.Clients.Pneumatic;
-
-namespace NzbDrone.Core.Datastore.Migration
-{
-    [Migration(51)]
-    public class rename_download_client_settings : NzbDroneMigrationBase
-    {
-        protected override void MainDbUpgrade()
-        {
-            Execute.WithConnection(ConvertFolderSettings);
-        }
-
-        private void ConvertFolderSettings(IDbConnection conn, IDbTransaction tran)
-        {
-            using (IDbCommand downloadClientsCmd = conn.CreateCommand())
-            {
-                downloadClientsCmd.Transaction = tran;
-                downloadClientsCmd.CommandText = @"SELECT Value FROM Config WHERE Key = 'downloadedepisodesfolder'";
-                var downloadedEpisodesFolder = downloadClientsCmd.ExecuteScalar() as String;
-
-                downloadClientsCmd.Transaction = tran;
-                downloadClientsCmd.CommandText = @"SELECT Id, Implementation, Settings, ConfigContract FROM DownloadClients WHERE ConfigContract = 'FolderSettings'";
-                using (IDataReader downloadClientReader = downloadClientsCmd.ExecuteReader())
-                {
-                    while (downloadClientReader.Read())
-                    {
-                        var id = downloadClientReader.GetInt32(0);
-                        var implementation = downloadClientReader.GetString(1);
-                        var settings = downloadClientReader.GetString(2);
-                        var configContract = downloadClientReader.GetString(3);
-
-                        var settingsJson = JsonConvert.DeserializeObject(settings) as Newtonsoft.Json.Linq.JObject;
-
-                        if (implementation == "Blackhole")
-                        {
-                            var newSettings = new
-                            {
-                                NzbFolder = settingsJson.Value<String>("folder"),
-                                WatchFolder = downloadedEpisodesFolder
-                            }.ToJson();
-
-                            using (IDbCommand updateCmd = conn.CreateCommand())
-                            {
-                                updateCmd.Transaction = tran;
-                                updateCmd.CommandText = "UPDATE DownloadClients SET Implementation = ?, Settings = ?, ConfigContract = ? WHERE Id = ?";
-                                updateCmd.AddParameter("UsenetBlackhole");
-                                updateCmd.AddParameter(newSettings);
-                                updateCmd.AddParameter("UsenetBlackholeSettings");
-                                updateCmd.AddParameter(id);
-
-                                updateCmd.ExecuteNonQuery();
-                            }
-                        }
-                        else if (implementation == "Pneumatic")
-                        {
-                            var newSettings = new
-                            {
-                                NzbFolder = settingsJson.Value<String>("folder")
-                            }.ToJson();
-
-                            using (IDbCommand updateCmd = conn.CreateCommand())
-                            {
-                                updateCmd.Transaction = tran;
-                                updateCmd.CommandText = "UPDATE DownloadClients SET Settings = ?, ConfigContract = ? WHERE Id = ?";
-                                updateCmd.AddParameter(newSettings);
-                                updateCmd.AddParameter("PneumaticSettings");
-                                updateCmd.AddParameter(id);
-
-                                updateCmd.ExecuteNonQuery();
-                            }
-                        }
-                        else
-                        {
-                            throw new NotSupportedException();
-                        }
-                    }
-                }
-            }
-}
-    }
-}

src/NzbDrone.Core/Datastore/Migration/Framework/NzbDroneMigrationBase.cs

@@ -6,11 +6,11 @@ namespace NzbDrone.Core.Datastore.Migration.Framework
 {
     public abstract class NzbDroneMigrationBase : FluentMigrator.Migration
     {
-        private Logger _logger;
+        protected readonly Logger _logger;
 
         protected NzbDroneMigrationBase()
         {
-            _logger = NzbDroneLogger.GetLogger();
+            _logger = NzbDroneLogger.GetLogger(this);
         }
 
         protected virtual void MainDbUpgrade()

src/NzbDrone.Core/Download/CompletedDownloadService.cs

@@ -30,18 +30,21 @@ namespace NzbDrone.Core.Download
         private readonly IConfigService _configService;
         private readonly IDiskProvider _diskProvider;
         private readonly IDownloadedEpisodesImportService _downloadedEpisodesImportService;
+        private readonly IHistoryService _historyService;
         private readonly Logger _logger;
 
         public CompletedDownloadService(IEventAggregator eventAggregator,
                                      IConfigService configService,
                                      IDiskProvider diskProvider,
                                      IDownloadedEpisodesImportService downloadedEpisodesImportService,
+                                     IHistoryService historyService,
                                      Logger logger)
         {
             _eventAggregator = eventAggregator;
             _configService = configService;
             _diskProvider = diskProvider;
             _downloadedEpisodesImportService = downloadedEpisodesImportService;
+            _historyService = historyService;
             _logger = logger;
         }
 
@@ -112,6 +115,34 @@ namespace NzbDrone.Core.Download
                     }
                     else
                     {
+                        if (grabbedItems.Any())
+                        {
+                            var episodeIds = trackedDownload.DownloadItem.RemoteEpisode.Episodes.Select(v => v.Id).ToList();
+
+                            // Check if we can associate it with a previous drone factory import.
+                            importedItems = importedHistory.Where(v => v.Data.GetValueOrDefault(DownloadTrackingService.DOWNLOAD_CLIENT_ID) == null &&
+                                                                  episodeIds.Contains(v.EpisodeId) && 
+                                                                  v.Data.GetValueOrDefault("droppedPath") != null &&
+                                                                  new FileInfo(v.Data["droppedPath"]).Directory.Name == grabbedItems.First().SourceTitle
+                                                                  ).ToList();
+                            if (importedItems.Count == 1)
+                            {
+                                var importedFile = new FileInfo(importedItems.First().Data["droppedPath"]);
+
+                                if (importedFile.Directory.Name == grabbedItems.First().SourceTitle)
+                                {
+                                    trackedDownload.State = TrackedDownloadState.Imported;
+
+                                    importedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT] = grabbedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT];
+                                    importedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT_ID] = grabbedItems.First().Data[DownloadTrackingService.DOWNLOAD_CLIENT_ID];
+                                    _historyService.UpdateHistoryData(importedItems.First().Id, importedItems.First().Data);
+
+                                    _logger.Debug("Storage path does not exist, but found probable drone factory ImportEvent: " + trackedDownload.DownloadItem.Title);
+                                    return;
+                                }
+                            }
+                        }
+
                         _logger.Debug("Storage path does not exist: " + trackedDownload.DownloadItem.Title);
                         return;
                     }

src/NzbDrone.Core/History/HistoryService.cs

@@ -25,6 +25,7 @@ namespace NzbDrone.Core.History
         History MostRecentForEpisode(int episodeId);
         History Get(int id);
         List<History> FindBySourceTitle(string sourceTitle);
+        void UpdateHistoryData(Int32 historyId, Dictionary<String, String> data);
     }
 
     public class HistoryService : IHistoryService, IHandle<EpisodeGrabbedEvent>, IHandle<EpisodeImportedEvent>, IHandle<DownloadFailedEvent>
@@ -101,6 +102,13 @@ namespace NzbDrone.Core.History
                 .FirstOrDefault();
         }
 
+        public void UpdateHistoryData(Int32 historyId, Dictionary<String, String> data)
+        {
+            var history = _historyRepository.Get(historyId);
+            history.Data = data;
+            _historyRepository.Update(history);
+        }
+
         public void Handle(EpisodeGrabbedEvent message)
         {
             foreach (var episode in message.Episode.Episodes)

src/NzbDrone.Core/MediaFiles/DownloadedEpisodesImportService.cs

@@ -160,7 +160,8 @@ namespace NzbDrone.Core.MediaFiles
                 }
             }
 
-            return ProcessFiles(series, quality, videoFiles);
+            var decisions = _importDecisionMaker.GetImportDecisions(videoFiles.ToList(), series, true, quality);
+            return _importApprovedEpisodes.Import(decisions, true);
         }
 
         private void ProcessVideoFile(string videoFile)
@@ -179,13 +180,8 @@ namespace NzbDrone.Core.MediaFiles
                 return;
             }
 
-            ProcessFiles(series, null, videoFile);
+            var decisions = _importDecisionMaker.GetImportDecisions(new [] { videoFile }.ToList(), series, true, null);
-        }
+            _importApprovedEpisodes.Import(decisions, true);
-
-        private List<ImportDecision> ProcessFiles(Series series, QualityModel quality, params string[] videoFiles)
-        {
-            var decisions = _importDecisionMaker.GetImportDecisions(videoFiles.ToList(), series, true, quality);
-            return _importApprovedEpisodes.Import(decisions, true);
         }
 
         private void ProcessFolder(string path)

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -197,7 +197,7 @@
     <Compile Include="Datastore\Migration\048_add_title_to_scenemappings.cs" />
     <Compile Include="Datastore\Migration\049_fix_dognzb_url.cs" />
     <Compile Include="Datastore\Migration\050_add_hash_to_metadata_files.cs" />
-    <Compile Include="Datastore\Migration\051_rename_download_client_settings.cs" />
+    <Compile Include="Datastore\Migration\051_download_client_import.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationController.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationExtension.cs" />