New: Improved detection of already imported downloads Closes #232pull/1392/head
Mark McDowall
4 years ago
committed by
ta264
parent
e1eb9a0ba7
commit
25b37ace34
@ -0,0 +1,106 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Data;
|
||||
using FluentMigrator;
|
||||
using NzbDrone.Common.Serializer;
|
||||
using NzbDrone.Core.Datastore.Migration.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Datastore.Migration
|
||||
{
|
||||
[Migration(20)]
|
||||
public class AddDownloadHistory : NzbDroneMigrationBase
|
||||
{
|
||||
protected override void MainDbUpgrade()
|
||||
{
|
||||
Create.TableForModel("DownloadHistory")
|
||||
.WithColumn("EventType").AsInt32().NotNullable()
|
||||
.WithColumn("AuthorId").AsInt32().NotNullable()
|
||||
.WithColumn("DownloadId").AsString().NotNullable()
|
||||
.WithColumn("SourceTitle").AsString().NotNullable()
|
||||
.WithColumn("Date").AsDateTime().NotNullable()
|
||||
.WithColumn("Protocol").AsInt32().Nullable()
|
||||
.WithColumn("IndexerId").AsInt32().Nullable()
|
||||
.WithColumn("DownloadClientId").AsInt32().Nullable()
|
||||
.WithColumn("Release").AsString().Nullable()
|
||||
.WithColumn("Data").AsString().Nullable();
|
||||
|
||||
Create.Index().OnTable("DownloadHistory").OnColumn("EventType");
|
||||
Create.Index().OnTable("DownloadHistory").OnColumn("SeriesId");
|
||||
Create.Index().OnTable("DownloadHistory").OnColumn("DownloadId");
|
||||
|
||||
Execute.WithConnection(InitialImportedDownloadHistory);
|
||||
|
||||
Execute.Sql("DELETE From History where EventType = 8;");
|
||||
}
|
||||
|
||||
private static readonly Dictionary<int, int> EventTypeMap = new Dictionary<int, int>()
|
||||
{
|
||||
// EpisodeHistoryType.Grabbed -> DownloadHistoryType.Grabbed
|
||||
{ 1, 1 },
|
||||
|
||||
// EpisodeHistoryType.DownloadFolderImported -> DownloadHistoryType.DownloadImported
|
||||
{ 8, 2 },
|
||||
|
||||
// EpisodeHistoryType.DownloadFailed -> DownloadHistoryType.DownloadFailed
|
||||
{ 4, 3 },
|
||||
|
||||
// EpisodeHistoryType.DownloadIgnored -> DownloadHistoryType.DownloadIgnored
|
||||
{ 10, 4 },
|
||||
|
||||
// EpisodeHistoryType.DownloadImportIncomplete -> DownloadHistoryType.DownloadImportIncomplete
|
||||
{ 7, 6 }
|
||||
};
|
||||
|
||||
private void InitialImportedDownloadHistory(IDbConnection conn, IDbTransaction tran)
|
||||
{
|
||||
using (var cmd = conn.CreateCommand())
|
||||
{
|
||||
cmd.Transaction = tran;
|
||||
cmd.CommandText = "SELECT AuthorId, DownloadId, EventType, SourceTitle, Date, Data FROM History WHERE DownloadId IS NOT NULL AND EventType IN (1, 8, 4, 10, 7) GROUP BY EventType, DownloadId";
|
||||
|
||||
using (var reader = cmd.ExecuteReader())
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
var seriesId = reader.GetInt32(0);
|
||||
var downloadId = reader.GetString(1);
|
||||
var eventType = reader.GetInt32(2);
|
||||
var sourceTitle = reader.GetString(3);
|
||||
var date = reader.GetDateTime(4);
|
||||
var rawData = reader.GetString(5);
|
||||
var data = Json.Deserialize<Dictionary<string, string>>(rawData);
|
||||
|
||||
var downloadHistoryEventType = EventTypeMap[eventType];
|
||||
var protocol = data.ContainsKey("protocol") ? Convert.ToInt32(data["protocol"]) : (int?)null;
|
||||
var downloadHistoryData = new Dictionary<string, string>();
|
||||
|
||||
if (data.ContainsKey("indexer"))
|
||||
{
|
||||
downloadHistoryData.Add("indexer", data["indexer"]);
|
||||
}
|
||||
|
||||
if (data.ContainsKey("downloadClient"))
|
||||
{
|
||||
downloadHistoryData.Add("downloadClient", data["downloadClient"]);
|
||||
}
|
||||
|
||||
using (var updateCmd = conn.CreateCommand())
|
||||
{
|
||||
updateCmd.Transaction = tran;
|
||||
updateCmd.CommandText = @"INSERT INTO DownloadHistory (EventType, AuthorId, DownloadId, SourceTitle, Date, Protocol, Data) VALUES (?, ?, ?, ?, ?, ?, ?)";
|
||||
updateCmd.AddParameter(downloadHistoryEventType);
|
||||
updateCmd.AddParameter(seriesId);
|
||||
updateCmd.AddParameter(downloadId);
|
||||
updateCmd.AddParameter(sourceTitle);
|
||||
updateCmd.AddParameter(date);
|
||||
updateCmd.AddParameter(protocol);
|
||||
updateCmd.AddParameter(downloadHistoryData.ToJson());
|
||||
|
||||
updateCmd.ExecuteNonQuery();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using NzbDrone.Core.Datastore;
|
||||
using NzbDrone.Core.Indexers;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
|
||||
namespace NzbDrone.Core.Download.History
|
||||
{
|
||||
public class DownloadHistory : ModelBase
|
||||
{
|
||||
public DownloadHistoryEventType EventType { get; set; }
|
||||
public int AuthorId { get; set; }
|
||||
public string DownloadId { get; set; }
|
||||
public string SourceTitle { get; set; }
|
||||
public DateTime Date { get; set; }
|
||||
public DownloadProtocol Protocol { get; set; }
|
||||
public int IndexerId { get; set; }
|
||||
public int DownloadClientId { get; set; }
|
||||
public ReleaseInfo Release { get; set; }
|
||||
public Dictionary<string, string> Data { get; set; }
|
||||
public DownloadHistory()
|
||||
{
|
||||
Data = new Dictionary<string, string>();
|
||||
}
|
||||
}
|
||||
|
||||
public enum DownloadHistoryEventType
|
||||
{
|
||||
DownloadGrabbed = 1,
|
||||
DownloadImported = 2,
|
||||
DownloadFailed = 3,
|
||||
DownloadIgnored = 4,
|
||||
FileImported = 5,
|
||||
DownloadImportIncomplete = 6
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using NzbDrone.Core.Datastore;
|
||||
using NzbDrone.Core.Messaging.Events;
|
||||
|
||||
namespace NzbDrone.Core.Download.History
|
||||
{
|
||||
public interface IDownloadHistoryRepository : IBasicRepository<DownloadHistory>
|
||||
{
|
||||
List<DownloadHistory> FindByDownloadId(string downloadId);
|
||||
void DeleteByAuthorId(int authorId);
|
||||
}
|
||||
|
||||
public class DownloadHistoryRepository : BasicRepository<DownloadHistory>, IDownloadHistoryRepository
|
||||
{
|
||||
public DownloadHistoryRepository(IMainDatabase database, IEventAggregator eventAggregator)
|
||||
: base(database, eventAggregator)
|
||||
{
|
||||
}
|
||||
|
||||
public List<DownloadHistory> FindByDownloadId(string downloadId)
|
||||
{
|
||||
return Query(h => h.DownloadId == downloadId)
|
||||
.OrderByDescending(h => h.Date)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public void DeleteByAuthorId(int authorId)
|
||||
{
|
||||
Delete(r => r.AuthorId == authorId);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,240 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Common.Serializer;
|
||||
using NzbDrone.Core.Books.Events;
|
||||
using NzbDrone.Core.History;
|
||||
using NzbDrone.Core.MediaFiles.Events;
|
||||
using NzbDrone.Core.Messaging.Events;
|
||||
|
||||
namespace NzbDrone.Core.Download.History
|
||||
{
|
||||
public interface IDownloadHistoryService
|
||||
{
|
||||
bool DownloadAlreadyImported(string downloadId);
|
||||
DownloadHistory GetLatestDownloadHistoryItem(string downloadId);
|
||||
}
|
||||
|
||||
public class DownloadHistoryService : IDownloadHistoryService,
|
||||
IHandle<BookGrabbedEvent>,
|
||||
IHandle<TrackImportedEvent>,
|
||||
IHandle<BookImportIncompleteEvent>,
|
||||
IHandle<DownloadCompletedEvent>,
|
||||
IHandle<DownloadFailedEvent>,
|
||||
IHandle<DownloadIgnoredEvent>,
|
||||
IHandle<AuthorDeletedEvent>
|
||||
{
|
||||
private readonly IDownloadHistoryRepository _repository;
|
||||
private readonly IHistoryService _historyService;
|
||||
|
||||
public DownloadHistoryService(IDownloadHistoryRepository repository, IHistoryService historyService)
|
||||
{
|
||||
_repository = repository;
|
||||
_historyService = historyService;
|
||||
}
|
||||
|
||||
public bool DownloadAlreadyImported(string downloadId)
|
||||
{
|
||||
var events = _repository.FindByDownloadId(downloadId);
|
||||
|
||||
// Events are ordered by date descending, if a grabbed event comes before an imported event then it was never imported
|
||||
// or grabbed again after importing and should be reprocessed.
|
||||
foreach (var e in events)
|
||||
{
|
||||
if (e.EventType == DownloadHistoryEventType.DownloadGrabbed)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (e.EventType == DownloadHistoryEventType.DownloadImported)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public DownloadHistory GetLatestDownloadHistoryItem(string downloadId)
|
||||
{
|
||||
var events = _repository.FindByDownloadId(downloadId);
|
||||
|
||||
// Events are ordered by date descending. We'll return the most recent expected event.
|
||||
foreach (var e in events)
|
||||
{
|
||||
if (e.EventType == DownloadHistoryEventType.DownloadGrabbed)
|
||||
{
|
||||
return e;
|
||||
}
|
||||
|
||||
if (e.EventType == DownloadHistoryEventType.DownloadImported)
|
||||
{
|
||||
return e;
|
||||
}
|
||||
|
||||
if (e.EventType == DownloadHistoryEventType.DownloadFailed)
|
||||
{
|
||||
return e;
|
||||
}
|
||||
|
||||
if (e.EventType == DownloadHistoryEventType.DownloadImportIncomplete)
|
||||
{
|
||||
return e;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public void Handle(BookGrabbedEvent message)
|
||||
{
|
||||
var history = new DownloadHistory
|
||||
{
|
||||
EventType = DownloadHistoryEventType.DownloadGrabbed,
|
||||
AuthorId = message.Book.Author.Id,
|
||||
DownloadId = message.DownloadId,
|
||||
SourceTitle = message.Book.Release.Title,
|
||||
Date = DateTime.UtcNow,
|
||||
Protocol = message.Book.Release.DownloadProtocol,
|
||||
IndexerId = message.Book.Release.IndexerId,
|
||||
DownloadClientId = message.DownloadClientId,
|
||||
Release = message.Book.Release
|
||||
};
|
||||
|
||||
history.Data.Add("Indexer", message.Book.Release.Indexer);
|
||||
history.Data.Add("DownloadClient", message.DownloadClient);
|
||||
history.Data.Add("DownloadClientName", message.DownloadClientName);
|
||||
history.Data.Add("PreferredWordScore", message.Book.PreferredWordScore.ToString());
|
||||
|
||||
_repository.Insert(history);
|
||||
}
|
||||
|
||||
public void Handle(TrackImportedEvent message)
|
||||
{
|
||||
if (!message.NewDownload)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var downloadId = message.DownloadId;
|
||||
|
||||
// Try to find the downloadId if the user used manual import (from wanted: missing) or the
|
||||
// API to import and downloadId wasn't provided.
|
||||
if (downloadId.IsNullOrWhiteSpace())
|
||||
{
|
||||
downloadId = _historyService.FindDownloadId(message);
|
||||
}
|
||||
|
||||
if (downloadId.IsNullOrWhiteSpace())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var history = new DownloadHistory
|
||||
{
|
||||
EventType = DownloadHistoryEventType.FileImported,
|
||||
AuthorId = message.BookInfo.Author.Id,
|
||||
DownloadId = downloadId,
|
||||
SourceTitle = message.BookInfo.Path,
|
||||
Date = DateTime.UtcNow,
|
||||
Protocol = message.DownloadClientInfo.Protocol,
|
||||
DownloadClientId = message.DownloadClientInfo.Id
|
||||
};
|
||||
|
||||
history.Data.Add("DownloadClient", message.DownloadClientInfo.Type);
|
||||
history.Data.Add("DownloadClientName", message.DownloadClientInfo.Name);
|
||||
history.Data.Add("SourcePath", message.BookInfo.Path);
|
||||
history.Data.Add("DestinationPath", message.ImportedBook.Path);
|
||||
|
||||
_repository.Insert(history);
|
||||
}
|
||||
|
||||
public void Handle(BookImportIncompleteEvent message)
|
||||
{
|
||||
var history = new DownloadHistory
|
||||
{
|
||||
EventType = DownloadHistoryEventType.DownloadImportIncomplete,
|
||||
AuthorId = message.TrackedDownload.RemoteBook?.Author.Id ?? 0,
|
||||
DownloadId = message.TrackedDownload.DownloadItem.DownloadId,
|
||||
SourceTitle = message.TrackedDownload.DownloadItem.OutputPath.ToString(),
|
||||
Date = DateTime.UtcNow,
|
||||
Protocol = message.TrackedDownload.Protocol,
|
||||
DownloadClientId = message.TrackedDownload.DownloadClient
|
||||
};
|
||||
|
||||
history.Data.Add("DownloadClient", message.TrackedDownload.DownloadItem.DownloadClientInfo.Type);
|
||||
history.Data.Add("DownloadClientName", message.TrackedDownload.DownloadItem.DownloadClientInfo.Name);
|
||||
history.Data.Add("StatusMessages", message.TrackedDownload.StatusMessages.ToJson());
|
||||
|
||||
_repository.Insert(history);
|
||||
}
|
||||
|
||||
public void Handle(DownloadCompletedEvent message)
|
||||
{
|
||||
var history = new DownloadHistory
|
||||
{
|
||||
EventType = DownloadHistoryEventType.DownloadImported,
|
||||
AuthorId = message.TrackedDownload.RemoteBook.Author.Id,
|
||||
DownloadId = message.TrackedDownload.DownloadItem.DownloadId,
|
||||
SourceTitle = message.TrackedDownload.DownloadItem.OutputPath.ToString(),
|
||||
Date = DateTime.UtcNow,
|
||||
Protocol = message.TrackedDownload.Protocol,
|
||||
DownloadClientId = message.TrackedDownload.DownloadClient
|
||||
};
|
||||
|
||||
history.Data.Add("DownloadClient", message.TrackedDownload.DownloadItem.DownloadClientInfo.Type);
|
||||
history.Data.Add("DownloadClientName", message.TrackedDownload.DownloadItem.DownloadClientInfo.Name);
|
||||
|
||||
_repository.Insert(history);
|
||||
}
|
||||
|
||||
public void Handle(DownloadFailedEvent message)
|
||||
{
|
||||
// Don't track failed download for an unknown download
|
||||
if (message.TrackedDownload == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var history = new DownloadHistory
|
||||
{
|
||||
EventType = DownloadHistoryEventType.DownloadFailed,
|
||||
AuthorId = message.AuthorId,
|
||||
DownloadId = message.DownloadId,
|
||||
SourceTitle = message.SourceTitle,
|
||||
Date = DateTime.UtcNow,
|
||||
Protocol = message.TrackedDownload.Protocol,
|
||||
DownloadClientId = message.TrackedDownload.DownloadClient
|
||||
};
|
||||
|
||||
history.Data.Add("DownloadClient", message.TrackedDownload.DownloadItem.DownloadClientInfo.Type);
|
||||
history.Data.Add("DownloadClientName", message.TrackedDownload.DownloadItem.DownloadClientInfo.Name);
|
||||
|
||||
_repository.Insert(history);
|
||||
}
|
||||
|
||||
public void Handle(DownloadIgnoredEvent message)
|
||||
{
|
||||
var history = new DownloadHistory
|
||||
{
|
||||
EventType = DownloadHistoryEventType.DownloadIgnored,
|
||||
AuthorId = message.AuthorId,
|
||||
DownloadId = message.DownloadId,
|
||||
SourceTitle = message.SourceTitle,
|
||||
Date = DateTime.UtcNow,
|
||||
Protocol = message.DownloadClientInfo.Protocol,
|
||||
DownloadClientId = message.DownloadClientInfo.Id
|
||||
};
|
||||
|
||||
history.Data.Add("DownloadClient", message.DownloadClientInfo.Type);
|
||||
history.Data.Add("DownloadClientName", message.DownloadClientInfo.Name);
|
||||
|
||||
_repository.Insert(history);
|
||||
}
|
||||
|
||||
public void Handle(AuthorDeletedEvent message)
|
||||
{
|
||||
_repository.DeleteByAuthorId(message.Author.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue