From ded757422793f05bcf3a9af0338c3d5ed09b48a5 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 17 Dec 2017 18:34:58 -0500 Subject: [PATCH] Fix Headphones Download Auth, Add Special Parsing (#152) * Add Secondary Backup Parsing Method if Regex Fails * Add BasicAuth info to release for nzb download handling --- .../HeadphonesTests/HeadphonesFixture.cs | 5 +- .../DecisionEngine/DownloadDecisionMaker.cs | 6 ++ .../TrackedDownloadService.cs | 23 ++++++- .../Download/UsenetClientBase.cs | 11 +++- .../History/HistoryRepository.cs | 4 +- .../Indexers/Headphones/Headphones.cs | 5 +- .../Headphones/HeadphonesRssParser.cs | 23 +++++++ src/NzbDrone.Core/Indexers/RssParser.cs | 6 ++ src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs | 5 +- src/NzbDrone.Core/Parser/Parser.cs | 65 +++++++++++++++++++ 11 files changed, 147 insertions(+), 7 deletions(-) create mode 100644 src/NzbDrone.Core/Indexers/Headphones/HeadphonesRssParser.cs diff --git a/src/NzbDrone.Core.Test/IndexerTests/HeadphonesTests/HeadphonesFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/HeadphonesTests/HeadphonesFixture.cs index a577b0a5c..f1b30ad42 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/HeadphonesTests/HeadphonesFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/HeadphonesTests/HeadphonesFixture.cs @@ -24,7 +24,9 @@ namespace NzbDrone.Core.Test.IndexerTests.HeadphonesTests Name = "Headphones VIP", Settings = new HeadphonesSettings() { - Categories = new int[] { 3000 } + Categories = new int[] { 3000 }, + Username = "user", + Password = "pass" } }; @@ -53,6 +55,7 @@ namespace NzbDrone.Core.Test.IndexerTests.HeadphonesTests releaseInfo.Title.Should().Be("Lady Gaga Born This Way 2CD FLAC 2011 WRE"); releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Usenet); releaseInfo.DownloadUrl.Should().Be("https://indexer.codeshy.com/api?t=g&guid=123456&apikey=123456789"); + releaseInfo.BasicAuthString.Should().Be("dXNlcjpwYXNz"); releaseInfo.Indexer.Should().Be(Subject.Definition.Name); releaseInfo.PublishDate.Should().Be(DateTime.Parse("2013/06/02 08:58:54")); releaseInfo.Size.Should().Be(917347414); diff --git a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs index eda5bd493..f26f26662 100644 --- a/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs +++ b/src/NzbDrone.Core/DecisionEngine/DownloadDecisionMaker.cs @@ -62,6 +62,12 @@ namespace NzbDrone.Core.DecisionEngine { var parsedAlbumInfo = Parser.Parser.ParseAlbumTitle(report.Title); + if (parsedAlbumInfo == null && searchCriteria != null) + { + parsedAlbumInfo = Parser.Parser.ParseAlbumTitleWithSearchCriteria(report.Title, + searchCriteria.Artist, searchCriteria.Albums); + } + if (parsedAlbumInfo != null) { // TODO: Artist Data Augment without calling to parse title again diff --git a/src/NzbDrone.Core/Download/TrackedDownloads/TrackedDownloadService.cs b/src/NzbDrone.Core/Download/TrackedDownloads/TrackedDownloadService.cs index 434b67166..a8a64c1f8 100644 --- a/src/NzbDrone.Core/Download/TrackedDownloads/TrackedDownloadService.cs +++ b/src/NzbDrone.Core/Download/TrackedDownloads/TrackedDownloadService.cs @@ -6,6 +6,7 @@ using NzbDrone.Common.Cache; using NzbDrone.Common.Extensions; using NzbDrone.Core.History; using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Music; using NzbDrone.Core.Parser; namespace NzbDrone.Core.Download.TrackedDownloads @@ -116,11 +117,31 @@ namespace NzbDrone.Core.Download.TrackedDownloads { // Try parsing the original source title and if that fails, try parsing it as a special // TODO: Pass the TVDB ID and TVRage IDs in as well so we have a better chance for finding the item + var historyArtist = firstHistoryItem.Artist; + var historyAlbums = new List { firstHistoryItem.Album }; + parsedAlbumInfo = Parser.Parser.ParseAlbumTitle(firstHistoryItem.SourceTitle); if (parsedAlbumInfo != null) { - trackedDownload.RemoteAlbum = _parsingService.Map(parsedAlbumInfo, firstHistoryItem.ArtistId, historyItems.Where(v => v.EventType == HistoryEventType.Grabbed).Select(h => h.AlbumId).Distinct()); + trackedDownload.RemoteAlbum = _parsingService.Map(parsedAlbumInfo, + firstHistoryItem.ArtistId, + historyItems.Where(v => v.EventType == HistoryEventType.Grabbed).Select(h => h.AlbumId) + .Distinct()); + } + else + { + parsedAlbumInfo = + Parser.Parser.ParseAlbumTitleWithSearchCriteria(firstHistoryItem.SourceTitle, + historyArtist, historyAlbums); + + if (parsedAlbumInfo != null) + { + trackedDownload.RemoteAlbum = _parsingService.Map(parsedAlbumInfo, + firstHistoryItem.ArtistId, + historyItems.Where(v => v.EventType == HistoryEventType.Grabbed).Select(h => h.AlbumId) + .Distinct()); + } } } } diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs index 9b1057048..3e74f2fbb 100644 --- a/src/NzbDrone.Core/Download/UsenetClientBase.cs +++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs @@ -8,6 +8,7 @@ using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Configuration; using NLog; +using NzbDrone.Common.Extensions; using NzbDrone.Core.RemotePathMappings; namespace NzbDrone.Core.Download @@ -43,7 +44,15 @@ namespace NzbDrone.Core.Download try { - nzbData = _httpClient.Get(new HttpRequest(url)).ResponseData; + var nzbDataRequest = new HttpRequest(url); + + // TODO: Look into moving download request handling to indexer + if (remoteAlbum.Release.BasicAuthString.IsNotNullOrWhiteSpace()) + { + nzbDataRequest.Headers.Set("Authorization", "Basic " + remoteAlbum.Release.BasicAuthString); + } + + nzbData = _httpClient.Get(nzbDataRequest).ResponseData; _logger.Debug("Downloaded nzb for release '{0}' finished ({1} bytes from {2})", remoteAlbum.Release.Title, nzbData.Length, url); } diff --git a/src/NzbDrone.Core/History/HistoryRepository.cs b/src/NzbDrone.Core/History/HistoryRepository.cs index 9f0a066ce..1cdaa7986 100644 --- a/src/NzbDrone.Core/History/HistoryRepository.cs +++ b/src/NzbDrone.Core/History/HistoryRepository.cs @@ -47,7 +47,9 @@ namespace NzbDrone.Core.History public List FindByDownloadId(string downloadId) { - return Query.Where(h => h.DownloadId == downloadId); + return Query.Join(JoinType.Left, h => h.Artist, (h, a) => h.ArtistId == a.Id) + .Join(JoinType.Left, h => h.Album, (h, r) => h.AlbumId == r.Id) + .Where(h => h.DownloadId == downloadId); } public List GetByArtist(int artistId, HistoryEventType? eventType) diff --git a/src/NzbDrone.Core/Indexers/Headphones/Headphones.cs b/src/NzbDrone.Core/Indexers/Headphones/Headphones.cs index e4b993dd7..14b5abab8 100644 --- a/src/NzbDrone.Core/Indexers/Headphones/Headphones.cs +++ b/src/NzbDrone.Core/Indexers/Headphones/Headphones.cs @@ -32,7 +32,10 @@ namespace NzbDrone.Core.Indexers.Headphones public override IParseIndexerResponse GetParser() { - return new NewznabRssParser(); + return new HeadphonesRssParser + { + Settings = Settings + }; } public Headphones(IHeadphonesCapabilitiesProvider capabilitiesProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger) diff --git a/src/NzbDrone.Core/Indexers/Headphones/HeadphonesRssParser.cs b/src/NzbDrone.Core/Indexers/Headphones/HeadphonesRssParser.cs new file mode 100644 index 000000000..5825d68a7 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Headphones/HeadphonesRssParser.cs @@ -0,0 +1,23 @@ +using System; +using System.Text; +using NzbDrone.Core.Indexers.Newznab; + +namespace NzbDrone.Core.Indexers.Headphones +{ + public class HeadphonesRssParser : NewznabRssParser + { + public const string ns = "{http://www.newznab.com/DTD/2010/feeds/attributes/}"; + public HeadphonesSettings Settings { get; set; } + + public HeadphonesRssParser() + { + PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes; + UseEnclosureUrl = true; + } + + protected override string GetBasicAuth() + { + return Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes($"{Settings.Username}:{Settings.Password}")); ; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index a77563236..8e4b93d19 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -156,6 +156,7 @@ namespace NzbDrone.Core.Indexers releaseInfo.Title = GetTitle(item); releaseInfo.PublishDate = GetPublishDate(item); releaseInfo.DownloadUrl = GetDownloadUrl(item); + releaseInfo.BasicAuthString = GetBasicAuth(); releaseInfo.InfoUrl = GetInfoUrl(item); releaseInfo.CommentUrl = GetCommentUrl(item); @@ -198,6 +199,11 @@ namespace NzbDrone.Core.Indexers return XElementExtensions.ParseDate(dateString); } + protected virtual string GetBasicAuth() + { + return null; + } + protected virtual string GetDownloadUrl(XElement item) { if (UseEnclosureUrl) diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 690a78315..7f556015e 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -526,6 +526,7 @@ + diff --git a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs index 6514724c2..1bd53ae1f 100644 --- a/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs +++ b/src/NzbDrone.Core/Parser/Model/ReleaseInfo.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Text; using NzbDrone.Core.Indexers; @@ -10,6 +10,7 @@ namespace NzbDrone.Core.Parser.Model public string Title { get; set; } public long Size { get; set; } public string DownloadUrl { get; set; } + public string BasicAuthString { get; set; } public string InfoUrl { get; set; } public string CommentUrl { get; set; } public int IndexerId { get; set; } @@ -87,4 +88,4 @@ namespace NzbDrone.Core.Parser.Model } } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Parser/Parser.cs b/src/NzbDrone.Core/Parser/Parser.cs index 099521f64..924c1f231 100644 --- a/src/NzbDrone.Core/Parser/Parser.cs +++ b/src/NzbDrone.Core/Parser/Parser.cs @@ -7,6 +7,7 @@ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Instrumentation; using NzbDrone.Core.MediaFiles.MediaInfo; +using NzbDrone.Core.Music; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Languages; using TagLib; @@ -319,6 +320,70 @@ namespace NzbDrone.Core.Parser return null; } + public static ParsedAlbumInfo ParseAlbumTitleWithSearchCriteria(string title, Artist artist, List album) + { + try + { + if (!ValidateBeforeParsing(title)) return null; + + Logger.Debug("Parsing string '{0}'", title); + + if (ReversedTitleRegex.IsMatch(title)) + { + var titleWithoutExtension = RemoveFileExtension(title).ToCharArray(); + Array.Reverse(titleWithoutExtension); + + title = new string (titleWithoutExtension) + title.Substring(titleWithoutExtension.Length); + + Logger.Debug("Reversed name detected. Converted to '{0}'", title); + } + + var releaseTitle = RemoveFileExtension(title); + + var simpleTitle = SimpleTitleRegex.Replace(releaseTitle, string.Empty); + + simpleTitle = WebsitePrefixRegex.Replace(simpleTitle, string.Empty); + + simpleTitle = CleanTorrentSuffixRegex.Replace(simpleTitle, string.Empty); + + var result = new ParsedAlbumInfo(); + + if (simpleTitle.ToLowerInvariant().Contains(artist.Name.ToLowerInvariant())) + { + result.ArtistName = artist.Name; + var artistRegex = new Regex(Regex.Escape(artist.Name.ToLowerInvariant())); + var albumReleaseString = artistRegex.Replace(simpleTitle.ToLowerInvariant(), string.Empty, 1); + + var selectedAlbum = album.FirstOrDefault(s => albumReleaseString.Contains(s.Title.ToLowerInvariant())); + + if (selectedAlbum != null) + { + result.AlbumTitle = selectedAlbum.Title; + } + + result.Language = LanguageParser.ParseLanguage(releaseTitle); + Logger.Debug("Language parsed: {0}", result.Language); + + result.Quality = QualityParser.ParseQuality(title, null, 0); + Logger.Debug("Quality parsed: {0}", result.Quality); + + result.ReleaseGroup = ParseReleaseGroup(releaseTitle); + + Logger.Debug("Release Group parsed: {0}", result.ReleaseGroup); + + return result; + } + } + catch (Exception e) + { + if (!title.ToLower().Contains("password") && !title.ToLower().Contains("yenc")) + Logger.Error(e, "An error has occurred while trying to parse {0}", title); + } + + Logger.Debug("Unable to parse {0}", title); + return null; + } + public static ParsedAlbumInfo ParseAlbumTitle(string title) { try