New: Improved detection of already imported downloads Closes #232pull/1392/head
Mark McDowall
4 years ago
committed by
ta264
parent
e1eb9a0ba7
commit
25b37ace34
@ -0,0 +1,106 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Data;
|
||||||
|
using FluentMigrator;
|
||||||
|
using NzbDrone.Common.Serializer;
|
||||||
|
using NzbDrone.Core.Datastore.Migration.Framework;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Datastore.Migration
|
||||||
|
{
|
||||||
|
[Migration(20)]
|
||||||
|
public class AddDownloadHistory : NzbDroneMigrationBase
|
||||||
|
{
|
||||||
|
protected override void MainDbUpgrade()
|
||||||
|
{
|
||||||
|
Create.TableForModel("DownloadHistory")
|
||||||
|
.WithColumn("EventType").AsInt32().NotNullable()
|
||||||
|
.WithColumn("AuthorId").AsInt32().NotNullable()
|
||||||
|
.WithColumn("DownloadId").AsString().NotNullable()
|
||||||
|
.WithColumn("SourceTitle").AsString().NotNullable()
|
||||||
|
.WithColumn("Date").AsDateTime().NotNullable()
|
||||||
|
.WithColumn("Protocol").AsInt32().Nullable()
|
||||||
|
.WithColumn("IndexerId").AsInt32().Nullable()
|
||||||
|
.WithColumn("DownloadClientId").AsInt32().Nullable()
|
||||||
|
.WithColumn("Release").AsString().Nullable()
|
||||||
|
.WithColumn("Data").AsString().Nullable();
|
||||||
|
|
||||||
|
Create.Index().OnTable("DownloadHistory").OnColumn("EventType");
|
||||||
|
Create.Index().OnTable("DownloadHistory").OnColumn("SeriesId");
|
||||||
|
Create.Index().OnTable("DownloadHistory").OnColumn("DownloadId");
|
||||||
|
|
||||||
|
Execute.WithConnection(InitialImportedDownloadHistory);
|
||||||
|
|
||||||
|
Execute.Sql("DELETE From History where EventType = 8;");
|
||||||
|
}
|
||||||
|
|
||||||
|
private static readonly Dictionary<int, int> EventTypeMap = new Dictionary<int, int>()
|
||||||
|
{
|
||||||
|
// EpisodeHistoryType.Grabbed -> DownloadHistoryType.Grabbed
|
||||||
|
{ 1, 1 },
|
||||||
|
|
||||||
|
// EpisodeHistoryType.DownloadFolderImported -> DownloadHistoryType.DownloadImported
|
||||||
|
{ 8, 2 },
|
||||||
|
|
||||||
|
// EpisodeHistoryType.DownloadFailed -> DownloadHistoryType.DownloadFailed
|
||||||
|
{ 4, 3 },
|
||||||
|
|
||||||
|
// EpisodeHistoryType.DownloadIgnored -> DownloadHistoryType.DownloadIgnored
|
||||||
|
{ 10, 4 },
|
||||||
|
|
||||||
|
// EpisodeHistoryType.DownloadImportIncomplete -> DownloadHistoryType.DownloadImportIncomplete
|
||||||
|
{ 7, 6 }
|
||||||
|
};
|
||||||
|
|
||||||
|
private void InitialImportedDownloadHistory(IDbConnection conn, IDbTransaction tran)
|
||||||
|
{
|
||||||
|
using (var cmd = conn.CreateCommand())
|
||||||
|
{
|
||||||
|
cmd.Transaction = tran;
|
||||||
|
cmd.CommandText = "SELECT AuthorId, DownloadId, EventType, SourceTitle, Date, Data FROM History WHERE DownloadId IS NOT NULL AND EventType IN (1, 8, 4, 10, 7) GROUP BY EventType, DownloadId";
|
||||||
|
|
||||||
|
using (var reader = cmd.ExecuteReader())
|
||||||
|
{
|
||||||
|
while (reader.Read())
|
||||||
|
{
|
||||||
|
var seriesId = reader.GetInt32(0);
|
||||||
|
var downloadId = reader.GetString(1);
|
||||||
|
var eventType = reader.GetInt32(2);
|
||||||
|
var sourceTitle = reader.GetString(3);
|
||||||
|
var date = reader.GetDateTime(4);
|
||||||
|
var rawData = reader.GetString(5);
|
||||||
|
var data = Json.Deserialize<Dictionary<string, string>>(rawData);
|
||||||
|
|
||||||
|
var downloadHistoryEventType = EventTypeMap[eventType];
|
||||||
|
var protocol = data.ContainsKey("protocol") ? Convert.ToInt32(data["protocol"]) : (int?)null;
|
||||||
|
var downloadHistoryData = new Dictionary<string, string>();
|
||||||
|
|
||||||
|
if (data.ContainsKey("indexer"))
|
||||||
|
{
|
||||||
|
downloadHistoryData.Add("indexer", data["indexer"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (data.ContainsKey("downloadClient"))
|
||||||
|
{
|
||||||
|
downloadHistoryData.Add("downloadClient", data["downloadClient"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
using (var updateCmd = conn.CreateCommand())
|
||||||
|
{
|
||||||
|
updateCmd.Transaction = tran;
|
||||||
|
updateCmd.CommandText = @"INSERT INTO DownloadHistory (EventType, AuthorId, DownloadId, SourceTitle, Date, Protocol, Data) VALUES (?, ?, ?, ?, ?, ?, ?)";
|
||||||
|
updateCmd.AddParameter(downloadHistoryEventType);
|
||||||
|
updateCmd.AddParameter(seriesId);
|
||||||
|
updateCmd.AddParameter(downloadId);
|
||||||
|
updateCmd.AddParameter(sourceTitle);
|
||||||
|
updateCmd.AddParameter(date);
|
||||||
|
updateCmd.AddParameter(protocol);
|
||||||
|
updateCmd.AddParameter(downloadHistoryData.ToJson());
|
||||||
|
|
||||||
|
updateCmd.ExecuteNonQuery();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,36 @@
|
|||||||
|
using System;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using NzbDrone.Core.Datastore;
|
||||||
|
using NzbDrone.Core.Indexers;
|
||||||
|
using NzbDrone.Core.Parser.Model;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Download.History
|
||||||
|
{
|
||||||
|
public class DownloadHistory : ModelBase
|
||||||
|
{
|
||||||
|
public DownloadHistoryEventType EventType { get; set; }
|
||||||
|
public int AuthorId { get; set; }
|
||||||
|
public string DownloadId { get; set; }
|
||||||
|
public string SourceTitle { get; set; }
|
||||||
|
public DateTime Date { get; set; }
|
||||||
|
public DownloadProtocol Protocol { get; set; }
|
||||||
|
public int IndexerId { get; set; }
|
||||||
|
public int DownloadClientId { get; set; }
|
||||||
|
public ReleaseInfo Release { get; set; }
|
||||||
|
public Dictionary<string, string> Data { get; set; }
|
||||||
|
public DownloadHistory()
|
||||||
|
{
|
||||||
|
Data = new Dictionary<string, string>();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public enum DownloadHistoryEventType
|
||||||
|
{
|
||||||
|
DownloadGrabbed = 1,
|
||||||
|
DownloadImported = 2,
|
||||||
|
DownloadFailed = 3,
|
||||||
|
DownloadIgnored = 4,
|
||||||
|
FileImported = 5,
|
||||||
|
DownloadImportIncomplete = 6
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,33 @@
|
|||||||
|
using System.Collections.Generic;
|
||||||
|
using System.Linq;
|
||||||
|
using NzbDrone.Core.Datastore;
|
||||||
|
using NzbDrone.Core.Messaging.Events;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Download.History
|
||||||
|
{
|
||||||
|
public interface IDownloadHistoryRepository : IBasicRepository<DownloadHistory>
|
||||||
|
{
|
||||||
|
List<DownloadHistory> FindByDownloadId(string downloadId);
|
||||||
|
void DeleteByAuthorId(int authorId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class DownloadHistoryRepository : BasicRepository<DownloadHistory>, IDownloadHistoryRepository
|
||||||
|
{
|
||||||
|
public DownloadHistoryRepository(IMainDatabase database, IEventAggregator eventAggregator)
|
||||||
|
: base(database, eventAggregator)
|
||||||
|
{
|
||||||
|
}
|
||||||
|
|
||||||
|
public List<DownloadHistory> FindByDownloadId(string downloadId)
|
||||||
|
{
|
||||||
|
return Query(h => h.DownloadId == downloadId)
|
||||||
|
.OrderByDescending(h => h.Date)
|
||||||
|
.ToList();
|
||||||
|
}
|
||||||
|
|
||||||
|
public void DeleteByAuthorId(int authorId)
|
||||||
|
{
|
||||||
|
Delete(r => r.AuthorId == authorId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,240 @@
|
|||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using NzbDrone.Common.Extensions;
|
||||||
|
using NzbDrone.Common.Serializer;
|
||||||
|
using NzbDrone.Core.Books.Events;
|
||||||
|
using NzbDrone.Core.History;
|
||||||
|
using NzbDrone.Core.MediaFiles.Events;
|
||||||
|
using NzbDrone.Core.Messaging.Events;
|
||||||
|
|
||||||
|
namespace NzbDrone.Core.Download.History
|
||||||
|
{
|
||||||
|
public interface IDownloadHistoryService
|
||||||
|
{
|
||||||
|
bool DownloadAlreadyImported(string downloadId);
|
||||||
|
DownloadHistory GetLatestDownloadHistoryItem(string downloadId);
|
||||||
|
}
|
||||||
|
|
||||||
|
public class DownloadHistoryService : IDownloadHistoryService,
|
||||||
|
IHandle<BookGrabbedEvent>,
|
||||||
|
IHandle<TrackImportedEvent>,
|
||||||
|
IHandle<BookImportIncompleteEvent>,
|
||||||
|
IHandle<DownloadCompletedEvent>,
|
||||||
|
IHandle<DownloadFailedEvent>,
|
||||||
|
IHandle<DownloadIgnoredEvent>,
|
||||||
|
IHandle<AuthorDeletedEvent>
|
||||||
|
{
|
||||||
|
private readonly IDownloadHistoryRepository _repository;
|
||||||
|
private readonly IHistoryService _historyService;
|
||||||
|
|
||||||
|
public DownloadHistoryService(IDownloadHistoryRepository repository, IHistoryService historyService)
|
||||||
|
{
|
||||||
|
_repository = repository;
|
||||||
|
_historyService = historyService;
|
||||||
|
}
|
||||||
|
|
||||||
|
public bool DownloadAlreadyImported(string downloadId)
|
||||||
|
{
|
||||||
|
var events = _repository.FindByDownloadId(downloadId);
|
||||||
|
|
||||||
|
// Events are ordered by date descending, if a grabbed event comes before an imported event then it was never imported
|
||||||
|
// or grabbed again after importing and should be reprocessed.
|
||||||
|
foreach (var e in events)
|
||||||
|
{
|
||||||
|
if (e.EventType == DownloadHistoryEventType.DownloadGrabbed)
|
||||||
|
{
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (e.EventType == DownloadHistoryEventType.DownloadImported)
|
||||||
|
{
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
public DownloadHistory GetLatestDownloadHistoryItem(string downloadId)
|
||||||
|
{
|
||||||
|
var events = _repository.FindByDownloadId(downloadId);
|
||||||
|
|
||||||
|
// Events are ordered by date descending. We'll return the most recent expected event.
|
||||||
|
foreach (var e in events)
|
||||||
|
{
|
||||||
|
if (e.EventType == DownloadHistoryEventType.DownloadGrabbed)
|
||||||
|
{
|
||||||
|
return e;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (e.EventType == DownloadHistoryEventType.DownloadImported)
|
||||||
|
{
|
||||||
|
return e;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (e.EventType == DownloadHistoryEventType.DownloadFailed)
|
||||||
|
{
|
||||||
|
return e;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (e.EventType == DownloadHistoryEventType.DownloadImportIncomplete)
|
||||||
|
{
|
||||||
|
return e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Handle(BookGrabbedEvent message)
|
||||||
|
{
|
||||||
|
var history = new DownloadHistory
|
||||||
|
{
|
||||||
|
EventType = DownloadHistoryEventType.DownloadGrabbed,
|
||||||
|
AuthorId = message.Book.Author.Id,
|
||||||
|
DownloadId = message.DownloadId,
|
||||||
|
SourceTitle = message.Book.Release.Title,
|
||||||
|
Date = DateTime.UtcNow,
|
||||||
|
Protocol = message.Book.Release.DownloadProtocol,
|
||||||
|
IndexerId = message.Book.Release.IndexerId,
|
||||||
|
DownloadClientId = message.DownloadClientId,
|
||||||
|
Release = message.Book.Release
|
||||||
|
};
|
||||||
|
|
||||||
|
history.Data.Add("Indexer", message.Book.Release.Indexer);
|
||||||
|
history.Data.Add("DownloadClient", message.DownloadClient);
|
||||||
|
history.Data.Add("DownloadClientName", message.DownloadClientName);
|
||||||
|
history.Data.Add("PreferredWordScore", message.Book.PreferredWordScore.ToString());
|
||||||
|
|
||||||
|
_repository.Insert(history);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Handle(TrackImportedEvent message)
|
||||||
|
{
|
||||||
|
if (!message.NewDownload)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var downloadId = message.DownloadId;
|
||||||
|
|
||||||
|
// Try to find the downloadId if the user used manual import (from wanted: missing) or the
|
||||||
|
// API to import and downloadId wasn't provided.
|
||||||
|
if (downloadId.IsNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
downloadId = _historyService.FindDownloadId(message);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (downloadId.IsNullOrWhiteSpace())
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var history = new DownloadHistory
|
||||||
|
{
|
||||||
|
EventType = DownloadHistoryEventType.FileImported,
|
||||||
|
AuthorId = message.BookInfo.Author.Id,
|
||||||
|
DownloadId = downloadId,
|
||||||
|
SourceTitle = message.BookInfo.Path,
|
||||||
|
Date = DateTime.UtcNow,
|
||||||
|
Protocol = message.DownloadClientInfo.Protocol,
|
||||||
|
DownloadClientId = message.DownloadClientInfo.Id
|
||||||
|
};
|
||||||
|
|
||||||
|
history.Data.Add("DownloadClient", message.DownloadClientInfo.Type);
|
||||||
|
history.Data.Add("DownloadClientName", message.DownloadClientInfo.Name);
|
||||||
|
history.Data.Add("SourcePath", message.BookInfo.Path);
|
||||||
|
history.Data.Add("DestinationPath", message.ImportedBook.Path);
|
||||||
|
|
||||||
|
_repository.Insert(history);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Handle(BookImportIncompleteEvent message)
|
||||||
|
{
|
||||||
|
var history = new DownloadHistory
|
||||||
|
{
|
||||||
|
EventType = DownloadHistoryEventType.DownloadImportIncomplete,
|
||||||
|
AuthorId = message.TrackedDownload.RemoteBook?.Author.Id ?? 0,
|
||||||
|
DownloadId = message.TrackedDownload.DownloadItem.DownloadId,
|
||||||
|
SourceTitle = message.TrackedDownload.DownloadItem.OutputPath.ToString(),
|
||||||
|
Date = DateTime.UtcNow,
|
||||||
|
Protocol = message.TrackedDownload.Protocol,
|
||||||
|
DownloadClientId = message.TrackedDownload.DownloadClient
|
||||||
|
};
|
||||||
|
|
||||||
|
history.Data.Add("DownloadClient", message.TrackedDownload.DownloadItem.DownloadClientInfo.Type);
|
||||||
|
history.Data.Add("DownloadClientName", message.TrackedDownload.DownloadItem.DownloadClientInfo.Name);
|
||||||
|
history.Data.Add("StatusMessages", message.TrackedDownload.StatusMessages.ToJson());
|
||||||
|
|
||||||
|
_repository.Insert(history);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Handle(DownloadCompletedEvent message)
|
||||||
|
{
|
||||||
|
var history = new DownloadHistory
|
||||||
|
{
|
||||||
|
EventType = DownloadHistoryEventType.DownloadImported,
|
||||||
|
AuthorId = message.TrackedDownload.RemoteBook.Author.Id,
|
||||||
|
DownloadId = message.TrackedDownload.DownloadItem.DownloadId,
|
||||||
|
SourceTitle = message.TrackedDownload.DownloadItem.OutputPath.ToString(),
|
||||||
|
Date = DateTime.UtcNow,
|
||||||
|
Protocol = message.TrackedDownload.Protocol,
|
||||||
|
DownloadClientId = message.TrackedDownload.DownloadClient
|
||||||
|
};
|
||||||
|
|
||||||
|
history.Data.Add("DownloadClient", message.TrackedDownload.DownloadItem.DownloadClientInfo.Type);
|
||||||
|
history.Data.Add("DownloadClientName", message.TrackedDownload.DownloadItem.DownloadClientInfo.Name);
|
||||||
|
|
||||||
|
_repository.Insert(history);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Handle(DownloadFailedEvent message)
|
||||||
|
{
|
||||||
|
// Don't track failed download for an unknown download
|
||||||
|
if (message.TrackedDownload == null)
|
||||||
|
{
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
var history = new DownloadHistory
|
||||||
|
{
|
||||||
|
EventType = DownloadHistoryEventType.DownloadFailed,
|
||||||
|
AuthorId = message.AuthorId,
|
||||||
|
DownloadId = message.DownloadId,
|
||||||
|
SourceTitle = message.SourceTitle,
|
||||||
|
Date = DateTime.UtcNow,
|
||||||
|
Protocol = message.TrackedDownload.Protocol,
|
||||||
|
DownloadClientId = message.TrackedDownload.DownloadClient
|
||||||
|
};
|
||||||
|
|
||||||
|
history.Data.Add("DownloadClient", message.TrackedDownload.DownloadItem.DownloadClientInfo.Type);
|
||||||
|
history.Data.Add("DownloadClientName", message.TrackedDownload.DownloadItem.DownloadClientInfo.Name);
|
||||||
|
|
||||||
|
_repository.Insert(history);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Handle(DownloadIgnoredEvent message)
|
||||||
|
{
|
||||||
|
var history = new DownloadHistory
|
||||||
|
{
|
||||||
|
EventType = DownloadHistoryEventType.DownloadIgnored,
|
||||||
|
AuthorId = message.AuthorId,
|
||||||
|
DownloadId = message.DownloadId,
|
||||||
|
SourceTitle = message.SourceTitle,
|
||||||
|
Date = DateTime.UtcNow,
|
||||||
|
Protocol = message.DownloadClientInfo.Protocol,
|
||||||
|
DownloadClientId = message.DownloadClientInfo.Id
|
||||||
|
};
|
||||||
|
|
||||||
|
history.Data.Add("DownloadClient", message.DownloadClientInfo.Type);
|
||||||
|
history.Data.Add("DownloadClientName", message.DownloadClientInfo.Name);
|
||||||
|
|
||||||
|
_repository.Insert(history);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void Handle(AuthorDeletedEvent message)
|
||||||
|
{
|
||||||
|
_repository.DeleteByAuthorId(message.Author.Id);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
Reference in new issue