Blacklisting torrents and using more info to evaluate matches

New: Blacklisting torrents manually
New: Details on why a release was blacklisted in the UI
New: Blacklist matching take into account indexer, size, date and name

src/NzbDrone.Api/Blacklist/BlacklistResource.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using NzbDrone.Api.REST;
 using NzbDrone.Core.Qualities;
 using NzbDrone.Api.Series;
+using NzbDrone.Core.Indexers;
 
 namespace NzbDrone.Api.Blacklist
 {
@@ -13,6 +14,9 @@ namespace NzbDrone.Api.Blacklist
         public string SourceTitle { get; set; }
         public QualityModel Quality { get; set; }
         public DateTime Date { get; set; }
+        public DownloadProtocol Protocol { get; set; }
+        public string Indexer { get; set; }
+        public string Message { get; set; }
 
         public SeriesResource Series { get; set; }
     }

src/NzbDrone.Api/History/HistoryResource.cs

@@ -18,7 +18,6 @@ namespace NzbDrone.Api.History
         public Boolean QualityCutoffNotMet { get; set; }
         public DateTime Date { get; set; }
         public string Indexer { get; set; }
-        public string NzbInfoUrl { get; set; }
         public string ReleaseGroup { get; set; }
         public string DownloadId { get; set; }
 

src/NzbDrone.Core.Test/Blacklisting/BlacklistRepositoryFixture.cs

@@ -47,7 +47,7 @@ namespace NzbDrone.Core.Test.Blacklisting
         {
             Subject.Insert(_blacklist);
 
-            Subject.Blacklisted(_blacklist.SeriesId, _blacklist.SourceTitle.ToUpperInvariant()).Should().HaveCount(1);
+            Subject.BlacklistedByTitle(_blacklist.SeriesId, _blacklist.SourceTitle.ToUpperInvariant()).Should().HaveCount(1);
         }
     }
 }

src/NzbDrone.Core.Test/Blacklisting/BlacklistServiceFixture.cs

@@ -28,9 +28,12 @@ namespace NzbDrone.Core.Test.Blacklisting
                      };
 
             _event.Data.Add("publishedDate", DateTime.UtcNow.ToString("s") + "Z");
+            _event.Data.Add("size", "1000");
+            _event.Data.Add("indexer", "nzbs.org");
+            _event.Data.Add("protocol", "1");
+            _event.Data.Add("message", "Marked as failed");
         }
 
-
         [Test]
         public void should_add_to_repository()
         {
@@ -39,5 +42,17 @@ namespace NzbDrone.Core.Test.Blacklisting
             Mocker.GetMock<IBlacklistRepository>()
                 .Verify(v => v.Insert(It.Is<Blacklist>(b => b.EpisodeIds == _event.EpisodeIds)), Times.Once());
         }
+
+        [Test]
+        public void should_add_to_repository_missing_size_and_protocol()
+        {
+            Subject.Handle(_event);
+
+            _event.Data.Remove("size");
+            _event.Data.Remove("protocol");
+
+            Mocker.GetMock<IBlacklistRepository>()
+                .Verify(v => v.Insert(It.Is<Blacklist>(b => b.EpisodeIds == _event.EpisodeIds)), Times.Once());
+        }
     }
 }

src/NzbDrone.Core/Blacklisting/Blacklist.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Qualities;
 using NzbDrone.Core.Tv;
 
@@ -8,12 +9,17 @@ namespace NzbDrone.Core.Blacklisting
 {
     public class Blacklist : ModelBase
     {
-        public Int32 SeriesId { get; set; }
+        public int SeriesId { get; set; }
         public Series Series { get; set; }
-        public List<Int32> EpisodeIds { get; set; }
+        public List<int> EpisodeIds { get; set; }
-        public String SourceTitle { get; set; }
+        public string SourceTitle { get; set; }
         public QualityModel Quality { get; set; }
         public DateTime Date { get; set; }
         public DateTime? PublishedDate { get; set; }
+        public long? Size { get; set; }
+        public DownloadProtocol Protocol { get; set; }
+        public string Indexer { get; set; }
+        public string Message { get; set; }
+        public string TorrentInfoHash { get; set; }
     }
 }

src/NzbDrone.Core/Blacklisting/BlacklistRepository.cs

@@ -8,7 +8,8 @@ namespace NzbDrone.Core.Blacklisting
 {
     public interface IBlacklistRepository : IBasicRepository<Blacklist>
     {
-        List<Blacklist> Blacklisted(int seriesId, string sourceTitle);
+        List<Blacklist> BlacklistedByTitle(int seriesId, string sourceTitle);
+        List<Blacklist> BlacklistedByTorrentInfoHash(int seriesId, string torrentInfoHash);
         List<Blacklist> BlacklistedBySeries(int seriesId);
     }
 
@@ -19,12 +20,18 @@ namespace NzbDrone.Core.Blacklisting
         {
         }
 
-        public List<Blacklist> Blacklisted(int seriesId, string sourceTitle)
+        public List<Blacklist> BlacklistedByTitle(int seriesId, string sourceTitle)
         {
             return Query.Where(e => e.SeriesId == seriesId)
                         .AndWhere(e => e.SourceTitle.Contains(sourceTitle));
         }
 
+        public List<Blacklist> BlacklistedByTorrentInfoHash(int seriesId, string torrentInfoHash)
+        {
+            return Query.Where(e => e.SeriesId == seriesId)
+                        .AndWhere(e => e.TorrentInfoHash.Contains(torrentInfoHash));
+        }
+
         public List<Blacklist> BlacklistedBySeries(int seriesId)
         {
             return Query.Where(b => b.SeriesId == seriesId);

src/NzbDrone.Core/Blacklisting/BlacklistService.cs

@@ -3,20 +3,22 @@ using System.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Download;
+using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Messaging.Commands;
 using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Tv.Events;
 
 namespace NzbDrone.Core.Blacklisting
 {
     public interface IBlacklistService
     {
-        bool Blacklisted(int seriesId, string sourceTitle, DateTime publishedDate);
+        bool Blacklisted(int seriesId, ReleaseInfo release);
         PagingSpec<Blacklist> Paged(PagingSpec<Blacklist> pagingSpec);
         void Delete(int id);
     }
-
     public class BlacklistService : IBlacklistService,
+
                                     IExecute<ClearBlacklistCommand>,
                                     IHandle<DownloadFailedEvent>,
                                     IHandleAsync<SeriesDeletedEvent>
@@ -28,11 +30,29 @@ namespace NzbDrone.Core.Blacklisting
             _blacklistRepository = blacklistRepository;
         }
 
-        public bool Blacklisted(int seriesId, string sourceTitle, DateTime publishedDate)
+        public bool Blacklisted(int seriesId, ReleaseInfo release)
         {
-            var blacklisted = _blacklistRepository.Blacklisted(seriesId, sourceTitle);
+            var blacklistedByTitle = _blacklistRepository.BlacklistedByTitle(seriesId, release.Title);
+            
+            if (release.DownloadProtocol == DownloadProtocol.Torrent)
+            {
+                var torrentInfo = release as TorrentInfo;
+
+                if (torrentInfo == null) return false;
+
+                if (torrentInfo.InfoHash.IsNullOrWhiteSpace())
+                {
+                    return blacklistedByTitle.Where(b => b.Protocol == DownloadProtocol.Torrent)
+                                             .Any(b => SameTorrent(b, torrentInfo));
+                }
+
+                var blacklistedByTorrentInfohash = _blacklistRepository.BlacklistedByTitle(seriesId, torrentInfo.InfoHash);
+
+                return blacklistedByTorrentInfohash.Any(b => SameTorrent(b, torrentInfo));
+            }
 
-            return blacklisted.Any(item => HasSamePublishedDate(item, publishedDate));
+            return blacklistedByTitle.Where(b => b.Protocol == DownloadProtocol.Usenet)
+                                     .Any(b => SameNzb(b, release));
         }
 
         public PagingSpec<Blacklist> Paged(PagingSpec<Blacklist> pagingSpec)
@@ -45,12 +65,58 @@ namespace NzbDrone.Core.Blacklisting
             _blacklistRepository.Delete(id);
         }
 
-        private static bool HasSamePublishedDate(Blacklist item, DateTime publishedDate)
+        private bool SameNzb(Blacklist item, ReleaseInfo release)
+        {
+            if (item.PublishedDate == release.PublishDate)
+            {
+                return true;
+            }
+
+            if (!HasSameIndexer(item, release.Indexer) &&
+                HasSamePublishedDate(item, release.PublishDate) &&
+                HasSameSize(item, release.Size))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        private bool SameTorrent(Blacklist item, TorrentInfo release)
+        {
+            if (release.InfoHash.IsNotNullOrWhiteSpace())
+            {
+                return release.InfoHash.Equals(item.TorrentInfoHash);
+            }
+
+            return item.Indexer.Equals(release.Indexer, StringComparison.InvariantCultureIgnoreCase);
+        }
+
+        private bool HasSameIndexer(Blacklist item, string indexer)
+        {
+            if (item.Indexer.IsNullOrWhiteSpace())
+            {
+                return true;
+            }
+
+            return item.Indexer.Equals(indexer, StringComparison.InvariantCultureIgnoreCase);
+        }
+
+        private bool HasSamePublishedDate(Blacklist item, DateTime publishedDate)
         {
             if (!item.PublishedDate.HasValue) return true;
 
-            return item.PublishedDate.Value.AddDays(-2) <= publishedDate &&
+            return item.PublishedDate.Value.AddMinutes(-2) <= publishedDate &&
-                   item.PublishedDate.Value.AddDays(2) >= publishedDate;
+                   item.PublishedDate.Value.AddMinutes(2) >= publishedDate;
+        }
+
+        private bool HasSameSize(Blacklist item, long size)
+        {
+            if (!item.Size.HasValue) return true;
+
+            var difference = Math.Abs(item.Size.Value - size);
+
+            return difference <= 2.Megabytes();
         }
 
         public void Execute(ClearBlacklistCommand message)
@@ -67,7 +133,12 @@ namespace NzbDrone.Core.Blacklisting
                                 SourceTitle = message.SourceTitle,
                                 Quality = message.Quality,
                                 Date = DateTime.UtcNow,
-                                PublishedDate = DateTime.Parse(message.Data.GetValueOrDefault("publishedDate"))
+                                PublishedDate = DateTime.Parse(message.Data.GetValueOrDefault("publishedDate")),
+                                Size = Int64.Parse(message.Data.GetValueOrDefault("size", "0")),
+                                Indexer = message.Data.GetValueOrDefault("indexer"),
+                                Protocol = (DownloadProtocol)Convert.ToInt32(message.Data.GetValueOrDefault("protocol")),
+                                Message = message.Message,
+                                TorrentInfoHash = message.Data.GetValueOrDefault("torrentInfoHash")
                             };
 
             _blacklistRepository.Insert(blacklist);

src/NzbDrone.Core/Datastore/Migration/083_additonal_blacklist_columns.cs

@@ -0,0 +1,22 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(83)]
+    public class additonal_blacklist_columns : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Alter.Table("Blacklist").AddColumn("Size").AsInt64().Nullable();
+            Alter.Table("Blacklist").AddColumn("Protocol").AsInt32().Nullable();
+            Alter.Table("Blacklist").AddColumn("Indexer").AsString().Nullable();
+            Alter.Table("Blacklist").AddColumn("Message").AsString().Nullable();
+            Alter.Table("Blacklist").AddColumn("TorrentInfoHash").AsString().Nullable();
+
+            Update.Table("Blacklist")
+                  .Set(new { Protocol = 1 })
+                  .AllRows();
+        }
+    }
+}

src/NzbDrone.Core/DecisionEngine/Specifications/BlacklistSpecification.cs

@@ -20,14 +20,8 @@ namespace NzbDrone.Core.DecisionEngine.Specifications
         public RejectionType Type { get { return RejectionType.Permanent; } }
 
         public Decision IsSatisfiedBy(RemoteEpisode subject, SearchCriteriaBase searchCriteria)
-        {
+        {          
-            if (subject.Release.DownloadProtocol == DownloadProtocol.Torrent)
+            if (_blacklistService.Blacklisted(subject.Series.Id, subject.Release))
-            {
-                return Decision.Accept();
-            }
-
-
-            if (_blacklistService.Blacklisted(subject.Series.Id, subject.Release.Title, subject.Release.PublishDate))
             {
                 _logger.Debug("{0} is blacklisted, rejecting.", subject.Release.Title);
                 return Decision.Reject("Release is blacklisted");

src/NzbDrone.Core/History/HistoryService.cs

@@ -6,9 +6,11 @@ using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Download;
+using NzbDrone.Core.Indexers;
 using NzbDrone.Core.MediaFiles;
 using NzbDrone.Core.MediaFiles.Events;
 using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.Profiles;
 using NzbDrone.Core.Qualities;
 
@@ -78,6 +80,51 @@ namespace NzbDrone.Core.History
                 .FirstOrDefault();
         }
 
+        private string FindDownloadId(EpisodeImportedEvent trackedDownload)
+        {
+            _logger.Debug("Trying to find downloadId for {0} from history", trackedDownload.ImportedEpisode.Path);
+
+            var episodeIds = trackedDownload.EpisodeInfo.Episodes.Select(c => c.Id).ToList();
+
+            var allHistory = _historyRepository.FindDownloadHistory(trackedDownload.EpisodeInfo.Series.Id, trackedDownload.ImportedEpisode.Quality);
+
+
+            //Find download related items for these episdoes
+            var episodesHistory = allHistory.Where(h => episodeIds.Contains(h.EpisodeId)).ToList();
+
+            var processedDownloadId = episodesHistory
+                .Where(c => c.EventType != HistoryEventType.Grabbed && c.DownloadId != null)
+                .Select(c => c.DownloadId);
+
+            var stillDownloading = episodesHistory.Where(c => c.EventType == HistoryEventType.Grabbed && !processedDownloadId.Contains(c.DownloadId)).ToList();
+
+            string downloadId = null;
+
+            if (stillDownloading.Any())
+            {
+                foreach (var matchingHistory in trackedDownload.EpisodeInfo.Episodes.Select(e => stillDownloading.Where(c => c.EpisodeId == e.Id).ToList()))
+                {
+                    if (matchingHistory.Count != 1)
+                    {
+                        return null;
+                    }
+
+                    var newDownloadId = matchingHistory.Single().DownloadId;
+
+                    if (downloadId == null || downloadId == newDownloadId)
+                    {
+                        downloadId = newDownloadId;
+                    }
+                    else
+                    {
+                        return null;
+                    }
+                }
+            }
+
+            return downloadId;
+        }
+
         public void Handle(EpisodeGrabbedEvent message)
         {
             foreach (var episode in message.Episode.Episodes)
@@ -101,12 +148,24 @@ namespace NzbDrone.Core.History
                 history.Data.Add("AgeMinutes", message.Episode.Release.AgeMinutes.ToString());
                 history.Data.Add("PublishedDate", message.Episode.Release.PublishDate.ToString("s") + "Z");
                 history.Data.Add("DownloadClient", message.DownloadClient);
+                history.Data.Add("Size", message.Episode.Release.Size.ToString());
+                history.Data.Add("DownloadUrl", message.Episode.Release.DownloadUrl);
+                history.Data.Add("Guid", message.Episode.Release.Guid);
+                history.Data.Add("TvRageId", message.Episode.Release.TvRageId.ToString());
+                history.Data.Add("Protocol", ((int)message.Episode.Release.DownloadProtocol).ToString());
 
                 if (!message.Episode.ParsedEpisodeInfo.ReleaseHash.IsNullOrWhiteSpace())
                 {
                     history.Data.Add("ReleaseHash", message.Episode.ParsedEpisodeInfo.ReleaseHash);
                 }
 
+                var torrentRelease = message.Episode.Release as TorrentInfo;
+
+                if (torrentRelease != null)
+                {
+                    history.Data.Add("TorrentInfoHash", torrentRelease.InfoHash);
+                }
+
                 _historyRepository.Insert(history);
             }
         }
@@ -148,52 +207,6 @@ namespace NzbDrone.Core.History
             }
         }
 
-
-        private string FindDownloadId(EpisodeImportedEvent trackedDownload)
-        {
-            _logger.Debug("Trying to find downloadId for {0} from history", trackedDownload.ImportedEpisode.Path);
-
-            var episodeIds = trackedDownload.EpisodeInfo.Episodes.Select(c => c.Id).ToList();
-
-            var allHistory = _historyRepository.FindDownloadHistory(trackedDownload.EpisodeInfo.Series.Id, trackedDownload.ImportedEpisode.Quality);
-
-
-            //Find download related items for these episdoes
-            var episodesHistory = allHistory.Where(h => episodeIds.Contains(h.EpisodeId)).ToList();
-
-            var processedDownloadId = episodesHistory
-                .Where(c => c.EventType != HistoryEventType.Grabbed && c.DownloadId != null)
-                .Select(c => c.DownloadId);
-
-            var stillDownloading = episodesHistory.Where(c => c.EventType == HistoryEventType.Grabbed && !processedDownloadId.Contains(c.DownloadId)).ToList();
-
-            string downloadId = null;
-
-            if (stillDownloading.Any())
-            {
-                foreach (var matchingHistory in trackedDownload.EpisodeInfo.Episodes.Select(e => stillDownloading.Where(c => c.EpisodeId == e.Id).ToList()))
-                {
-                    if (matchingHistory.Count != 1)
-                    {
-                        return null;
-                    }
-
-                    var newDownloadId = matchingHistory.Single().DownloadId;
-
-                    if (downloadId == null || downloadId == newDownloadId)
-                    {
-                        downloadId = newDownloadId;
-                    }
-                    else
-                    {
-                        return null;
-                    }
-                }
-            }
-
-            return downloadId;
-        }
-
         public void Handle(DownloadFailedEvent message)
         {
             foreach (var episodeId in message.EpisodeIds)

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -254,6 +254,7 @@
     <Compile Include="Datastore\Migration\081_move_dot_prefix_to_transmission_category.cs" />
     <Compile Include="Datastore\Migration\079_dedupe_tags.cs" />
     <Compile Include="Datastore\Migration\070_delay_profile.cs" />
+    <Compile Include="Datastore\Migration\083_additonal_blacklist_columns.cs" />
     <Compile Include="Datastore\Migration\082_add_fanzub_settings.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationContext.cs" />
     <Compile Include="Datastore\Migration\Framework\MigrationController.cs" />

src/UI/Activity/Blacklist/BlacklistActionsCell.js

@@ -1,19 +1,27 @@
+var vent = require('vent');
 var NzbDroneCell = require('../../Cells/NzbDroneCell');
+var BlacklistDetailsLayout = require('./Details/BlacklistDetailsLayout');
 
 module.exports = NzbDroneCell.extend({
-    className : 'blacklist-controls-cell',
+    className : 'blacklist-actions-cell',
 
     events : {
-        'click' : '_delete'
+        'click .x-details' : '_details',
+        'click .x-delete'  : '_delete'
     },
 
     render : function() {
         this.$el.empty();
-        this.$el.html('<i class="icon-sonarr-delete"></i>');
+        this.$el.html('<i class="icon-sonarr-info x-details"></i>' +
+                      '<i class="icon-sonarr-delete x-delete"></i>');
 
         return this;
     },
 
+    _details : function() {
+        vent.trigger(vent.Commands.OpenModalCommand, new BlacklistDetailsLayout({ model : this.model }));
+    },
+
     _delete : function() {
         this.model.destroy();
     }

src/UI/Activity/Blacklist/Details/BlacklistDetailsLayout.js

@@ -0,0 +1,14 @@
+var Marionette = require('marionette');
+var BlacklistDetailsView = require('./BlacklistDetailsView');
+
+module.exports = Marionette.Layout.extend({
+    template : 'Activity/Blacklist/Details/BlacklistDetailsLayoutTemplate',
+
+    regions : {
+        bodyRegion : '.modal-body'
+    },
+
+    onShow : function() {
+        this.bodyRegion.show(new BlacklistDetailsView({ model : this.model }));
+    }
+});

src/UI/Activity/Blacklist/Details/BlacklistDetailsLayoutTemplate.hbs

@@ -0,0 +1,18 @@
+<div class="modal-content">
+    <div class="history-detail-modal">
+        <div class="modal-header">
+            <button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
+
+            <h3>
+                Blacklisted
+            </h3>
+
+        </div>
+        <div class="modal-body">
+
+        </div>
+        <div class="modal-footer">
+            <button class="btn" data-dismiss="modal">close</button>
+        </div>
+    </div>
+</div>

src/UI/Activity/Blacklist/Details/BlacklistDetailsView.js

@@ -0,0 +1,5 @@
+var Marionette = require('marionette');
+
+module.exports = Marionette.ItemView.extend({
+    template : 'Activity/Blacklist/Details/BlacklistDetailsViewTemplate'
+});

src/UI/Activity/Blacklist/Details/BlacklistDetailsViewTemplate.hbs

@@ -0,0 +1,23 @@
+<dl class="dl-horizontal info">
+
+    <dt>Name:</dt>
+    <dd>{{sourceTitle}}</dd>
+
+    {{#if protocol}}
+    {{#unless_eq protocol compare="unknown"}}
+    <dt>Protocol:</dt>
+    <dd>{{protocol}}</dd>
+    {{/unless_eq}}
+    {{/if}}
+
+    {{#if indexer}}
+    <dt>Indexer:</dt>
+    <dd>{{indexer}}</dd>
+    {{/if}}
+
+
+    {{#if message}}
+    <dt>Message:</dt>
+    <dd>{{message}}</dd>
+    {{/if}}
+</dl>

src/UI/Cells/cells.less

@@ -236,3 +236,15 @@ td.delete-episode-file-cell {
 .age-cell {
   cursor : default;
 }
+
+.blacklist-actions-cell {
+  min-width  : 55px;
+  width      : 55px;
+  text-align : right !important;
+
+  i {
+    .clickable();
+    margin-left  : 2px;
+    margin-right : 2px;
+  }
+}