diff --git a/frontend/src/Indexer/Stats/Stats.js b/frontend/src/Indexer/Stats/Stats.js index b788a2a7f..46775f55a 100644 --- a/frontend/src/Indexer/Stats/Stats.js +++ b/frontend/src/Indexer/Stats/Stats.js @@ -1,6 +1,7 @@ import PropTypes from 'prop-types'; import React from 'react'; import BarChart from 'Components/Chart/BarChart'; +import DoughnutChart from 'Components/Chart/DoughnutChart'; import StackedBarChart from 'Components/Chart/StackedBarChart'; import LoadingIndicator from 'Components/Loading/LoadingIndicator'; import PageContent from 'Components/Page/PageContent'; @@ -87,6 +88,36 @@ function getUserAgentQueryData(indexerStats) { return data; } +function getHostGrabsData(indexerStats) { + const data = indexerStats.map((indexer) => { + return { + label: indexer.host ? indexer.host : 'Other', + value: indexer.numberOfGrabs + }; + }); + + data.sort((a, b) => { + return b.value - a.value; + }); + + return data; +} + +function getHostQueryData(indexerStats) { + const data = indexerStats.map((indexer) => { + return { + label: indexer.host ? indexer.host : 'Other', + value: indexer.numberOfQueries + }; + }); + + data.sort((a, b) => { + return b.value - a.value; + }); + + return data; +} + function Stats(props) { const { item, @@ -148,6 +179,20 @@ function Stats(props) { horizontal={true} /> +
+ +
+
+ +
} diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index c25b5ab9e..662add61b 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -17,9 +17,9 @@ namespace NzbDrone.Core.Download { public interface IDownloadService { - void SendReportToClient(ReleaseInfo release, bool redirect); - Task DownloadReport(string link, int indexerId, string source, string title); - void RecordRedirect(string link, int indexerId, string source, string title); + void SendReportToClient(ReleaseInfo release, string source, string host, bool redirect); + Task DownloadReport(string link, int indexerId, string source, string host, string title); + void RecordRedirect(string link, int indexerId, string source, string host, string title); } public class DownloadService : IDownloadService @@ -49,7 +49,7 @@ namespace NzbDrone.Core.Download _logger = logger; } - public void SendReportToClient(ReleaseInfo release, bool redirect) + public void SendReportToClient(ReleaseInfo release, string source, string host, bool redirect) { var downloadTitle = release.Title; var downloadClient = _downloadClientProvider.GetDownloadClient(release.DownloadProtocol); @@ -79,13 +79,13 @@ namespace NzbDrone.Core.Download catch (ReleaseUnavailableException) { _logger.Trace("Release {0} no longer available on indexer.", release); - _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, release.Source, release.Title, redirect)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, source, host, release.Title, redirect)); throw; } catch (DownloadClientRejectedReleaseException) { _logger.Trace("Release {0} rejected by download client, possible duplicate.", release); - _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, release.Source, release.Title, redirect)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, source, host, release.Title, redirect)); throw; } catch (ReleaseDownloadException ex) @@ -100,17 +100,17 @@ namespace NzbDrone.Core.Download _indexerStatusService.RecordFailure(release.IndexerId); } - _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, release.Source, release.Title, redirect)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, false, source, host, release.Title, redirect)); throw; } _logger.ProgressInfo("Report sent to {0}. {1}", downloadClient.Definition.Name, downloadTitle); - _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, true, release.Source, release.Title, redirect)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(release.IndexerId, true, source, host, release.Title, redirect)); } - public async Task DownloadReport(string link, int indexerId, string source, string title) + public async Task DownloadReport(string link, int indexerId, string source, string host, string title) { var url = new Uri(link); @@ -133,7 +133,7 @@ namespace NzbDrone.Core.Download catch (ReleaseUnavailableException) { _logger.Trace("Release {0} no longer available on indexer.", link); - _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, title)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, host, title)); throw; } catch (ReleaseDownloadException ex) @@ -148,17 +148,17 @@ namespace NzbDrone.Core.Download _indexerStatusService.RecordFailure(indexerId); } - _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, title)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, host, title)); throw; } - _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, title)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, host, title)); return downloadedBytes; } - public void RecordRedirect(string link, int indexerId, string source, string title) + public void RecordRedirect(string link, int indexerId, string source, string host, string title) { - _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, true, source, title, true)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, true, source, host, title, true)); } } } diff --git a/src/NzbDrone.Core/History/HistoryService.cs b/src/NzbDrone.Core/History/HistoryService.cs index a0753ee94..ab94ed2a3 100644 --- a/src/NzbDrone.Core/History/HistoryService.cs +++ b/src/NzbDrone.Core/History/HistoryService.cs @@ -152,6 +152,7 @@ namespace NzbDrone.Core.History history.Data.Add("QueryType", message.Query.SearchType ?? string.Empty); history.Data.Add("Categories", string.Join(",", message.Query.Categories) ?? string.Empty); history.Data.Add("Source", message.Query.Source ?? string.Empty); + history.Data.Add("Host", message.Query.Host ?? string.Empty); history.Data.Add("QueryResults", message.Results.HasValue ? message.Results.ToString() : null); _historyRepository.Insert(history); @@ -168,6 +169,7 @@ namespace NzbDrone.Core.History }; history.Data.Add("Source", message.Source ?? string.Empty); + history.Data.Add("Host", message.Host ?? string.Empty); history.Data.Add("GrabMethod", message.Redirect ? "Redirect" : "Proxy"); history.Data.Add("Title", message.Title); diff --git a/src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs b/src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs index 7153c0d7d..49e4060db 100644 --- a/src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs +++ b/src/NzbDrone.Core/IndexerSearch/Definitions/SearchCriteriaBase.cs @@ -19,6 +19,7 @@ namespace NzbDrone.Core.IndexerSearch.Definitions public int? Limit { get; set; } public int? Offset { get; set; } public string Source { get; set; } + public string Host { get; set; } public override string ToString() { diff --git a/src/NzbDrone.Core/IndexerSearch/NewznabRequest.cs b/src/NzbDrone.Core/IndexerSearch/NewznabRequest.cs index 595364f85..cb66ef969 100644 --- a/src/NzbDrone.Core/IndexerSearch/NewznabRequest.cs +++ b/src/NzbDrone.Core/IndexerSearch/NewznabRequest.cs @@ -26,6 +26,7 @@ namespace NzbDrone.Core.IndexerSearch public string title { get; set; } public string configured { get; set; } public string source { get; set; } + public string host { get; set; } public string server { get; set; } } } diff --git a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs index 9992e6cc5..ed101fae4 100644 --- a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs @@ -128,6 +128,7 @@ namespace NzbDrone.Core.IndexerSearch spec.Limit = query.limit; spec.Offset = query.offset; spec.Source = query.source; + spec.Host = query.host; spec.IndexerIds = indexerIds; diff --git a/src/NzbDrone.Core/IndexerStats/IndexerStatistics.cs b/src/NzbDrone.Core/IndexerStats/IndexerStatistics.cs index a8b65af07..b1e58a722 100644 --- a/src/NzbDrone.Core/IndexerStats/IndexerStatistics.cs +++ b/src/NzbDrone.Core/IndexerStats/IndexerStatistics.cs @@ -11,6 +11,10 @@ namespace NzbDrone.Core.IndexerStats public int NumberOfGrabs { get; set; } public int NumberOfRssQueries { get; set; } public int NumberOfAuthQueries { get; set; } + public int NumberOfFailedQueries { get; set; } + public int NumberOfFailedGrabs { get; set; } + public int NumberOfFailedRssQueries { get; set; } + public int NumberOfFailedAuthQueries { get; set; } } public class UserAgentStatistics : ResultSet @@ -18,7 +22,6 @@ namespace NzbDrone.Core.IndexerStats public string UserAgent { get; set; } public int NumberOfQueries { get; set; } public int NumberOfGrabs { get; set; } - public int NumberOfRssQueries { get; set; } } public class HostStatistics : ResultSet @@ -26,6 +29,5 @@ namespace NzbDrone.Core.IndexerStats public string Host { get; set; } public int NumberOfQueries { get; set; } public int NumberOfGrabs { get; set; } - public int NumberOfRssQueries { get; set; } } } diff --git a/src/NzbDrone.Core/IndexerStats/IndexerStatisticsRepository.cs b/src/NzbDrone.Core/IndexerStats/IndexerStatisticsRepository.cs index 35aa6b658..0a11c0e16 100644 --- a/src/NzbDrone.Core/IndexerStats/IndexerStatisticsRepository.cs +++ b/src/NzbDrone.Core/IndexerStats/IndexerStatisticsRepository.cs @@ -11,6 +11,7 @@ namespace NzbDrone.Core.IndexerStats { List IndexerStatistics(); List UserAgentStatistics(); + List HostStatistics(); } public class IndexerStatisticsRepository : IIndexerStatisticsRepository @@ -36,6 +37,12 @@ namespace NzbDrone.Core.IndexerStats return UserAgentQuery(UserAgentBuilder()); } + public List HostStatistics() + { + var time = DateTime.UtcNow; + return HostQuery(HostBuilder()); + } + private List Query(SqlBuilder builder) { var sql = builder.AddTemplate(_selectTemplate).LogQuery(); @@ -56,22 +63,41 @@ namespace NzbDrone.Core.IndexerStats } } + private List HostQuery(SqlBuilder builder) + { + var sql = builder.AddTemplate(_selectTemplate).LogQuery(); + + using (var conn = _database.OpenConnection()) + { + return conn.Query(sql.RawSql, sql.Parameters).ToList(); + } + } + private SqlBuilder IndexerBuilder() => new SqlBuilder() .Select(@"Indexers.Id AS IndexerId, Indexers.Name AS IndexerName, SUM(CASE WHEN EventType == 2 then 1 else 0 end) AS NumberOfQueries, + SUM(CASE WHEN EventType == 2 AND Successful == 0 then 1 else 0 end) AS NumberOfFailedQueries, SUM(CASE WHEN EventType == 3 then 1 else 0 end) AS NumberOfRssQueries, + SUM(CASE WHEN EventType == 3 AND Successful == 0 then 1 else 0 end) AS NumberOfFailedRssQueries, SUM(CASE WHEN EventType == 4 then 1 else 0 end) AS NumberOfAuthQueries, + SUM(CASE WHEN EventType == 4 AND Successful == 0 then 1 else 0 end) AS NumberOfFailedAuthQueries, SUM(CASE WHEN EventType == 1 then 1 else 0 end) AS NumberOfGrabs, + SUM(CASE WHEN EventType == 1 AND Successful == 0 then 1 else 0 end) AS NumberOfFailedGrabs, AVG(json_extract(History.Data,'$.elapsedTime')) AS AverageResponseTime") .Join((t, r) => t.IndexerId == r.Id) .GroupBy(x => x.Id); private SqlBuilder UserAgentBuilder() => new SqlBuilder() .Select(@"json_extract(History.Data,'$.source') AS UserAgent, - SUM(CASE WHEN EventType == 2 then 1 else 0 end) AS NumberOfQueries, - SUM(CASE WHEN EventType == 1 then 1 else 0 end) AS NumberOfGrabs, - SUM(CASE WHEN EventType == 3 then 1 else 0 end) AS NumberOfRssQueries") + SUM(CASE WHEN EventType == 2 OR EventType == 3 then 1 else 0 end) AS NumberOfQueries, + SUM(CASE WHEN EventType == 1 then 1 else 0 end) AS NumberOfGrabs") .GroupBy("UserAgent"); + + private SqlBuilder HostBuilder() => new SqlBuilder() + .Select(@"json_extract(History.Data,'$.host') AS Host, + SUM(CASE WHEN EventType == 2 OR EventType == 3 then 1 else 0 end) AS NumberOfQueries, + SUM(CASE WHEN EventType == 1 then 1 else 0 end) AS NumberOfGrabs") + .GroupBy("Host"); } } diff --git a/src/NzbDrone.Core/IndexerStats/IndexerStatisticsService.cs b/src/NzbDrone.Core/IndexerStats/IndexerStatisticsService.cs index bb7594a19..0e3eb370f 100644 --- a/src/NzbDrone.Core/IndexerStats/IndexerStatisticsService.cs +++ b/src/NzbDrone.Core/IndexerStats/IndexerStatisticsService.cs @@ -7,6 +7,7 @@ namespace NzbDrone.Core.IndexerStats { List IndexerStatistics(); List UserAgentStatistics(); + List HostStatistics(); } public class IndexerStatisticsService : IIndexerStatisticsService @@ -20,16 +21,23 @@ namespace NzbDrone.Core.IndexerStats public List IndexerStatistics() { - var seasonStatistics = _indexerStatisticsRepository.IndexerStatistics(); + var indexerStatistics = _indexerStatisticsRepository.IndexerStatistics(); - return seasonStatistics.ToList(); + return indexerStatistics.ToList(); } public List UserAgentStatistics() { - var seasonStatistics = _indexerStatisticsRepository.UserAgentStatistics(); + var userAgentStatistics = _indexerStatisticsRepository.UserAgentStatistics(); - return seasonStatistics.ToList(); + return userAgentStatistics.ToList(); + } + + public List HostStatistics() + { + var hostStatistics = _indexerStatisticsRepository.HostStatistics(); + + return hostStatistics.ToList(); } } } diff --git a/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs b/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs index 3ac88ea28..26f9572e2 100644 --- a/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs +++ b/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs @@ -7,14 +7,16 @@ namespace NzbDrone.Core.Indexers.Events public int IndexerId { get; set; } public bool Successful { get; set; } public string Source { get; set; } + public string Host { get; set; } public string Title { get; set; } public bool Redirect { get; set; } - public IndexerDownloadEvent(int indexerId, bool successful, string source, string title, bool redirect = false) + public IndexerDownloadEvent(int indexerId, bool successful, string source, string host, string title, bool redirect = false) { IndexerId = indexerId; Successful = successful; Source = source; + Host = host; Title = title; Redirect = redirect; } diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerStatsController.cs b/src/Prowlarr.Api.V1/Indexers/IndexerStatsController.cs index 065054c7b..10c5519f7 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerStatsController.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerStatsController.cs @@ -21,6 +21,7 @@ namespace Prowlarr.Api.V1.Indexers { Indexers = _indexerStatisticsService.IndexerStatistics(), UserAgents = _indexerStatisticsService.UserAgentStatistics(), + Hosts = _indexerStatisticsService.HostStatistics() }; return indexerResource; diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerStatsResource.cs b/src/Prowlarr.Api.V1/Indexers/IndexerStatsResource.cs index 21d92ab7e..fcb0e5301 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerStatsResource.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerStatsResource.cs @@ -8,5 +8,6 @@ namespace Prowlarr.Api.V1.Indexers { public List Indexers { get; set; } public List UserAgents { get; set; } + public List Hosts { get; set; } } } diff --git a/src/Prowlarr.Api.V1/Indexers/NewznabController.cs b/src/Prowlarr.Api.V1/Indexers/NewznabController.cs index f984c702b..d76dacdbd 100644 --- a/src/Prowlarr.Api.V1/Indexers/NewznabController.cs +++ b/src/Prowlarr.Api.V1/Indexers/NewznabController.cs @@ -6,10 +6,10 @@ using System.Threading.Tasks; using Microsoft.AspNetCore.Cors; using Microsoft.AspNetCore.Mvc; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch; -using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using Prowlarr.Http.Extensions; using Prowlarr.Http.REST; @@ -44,6 +44,7 @@ namespace NzbDrone.Api.V1.Indexers var requestType = request.t; request.source = UserAgentParser.ParseSource(Request.Headers["User-Agent"]); request.server = Request.GetServerUrl(); + request.host = Request.GetHostName(); if (requestType.IsNullOrWhiteSpace()) { @@ -107,18 +108,19 @@ namespace NzbDrone.Api.V1.Indexers } var source = UserAgentParser.ParseSource(Request.Headers["User-Agent"]); + var host = Request.GetHostName(); var unprotectedlLink = "https://superbits.org/api/v1/torrents/download/797354"; // If Indexer is set to download via Redirect then just redirect to the link if (indexer.SupportsRedirect && indexerDef.Redirect) { - _downloadService.RecordRedirect(unprotectedlLink, id, source, file); + _downloadService.RecordRedirect(unprotectedlLink, id, source, host, file); return RedirectPermanent(unprotectedlLink); } var downloadBytes = Array.Empty(); - downloadBytes = await _downloadService.DownloadReport(unprotectedlLink, id, source, file); + downloadBytes = await _downloadService.DownloadReport(unprotectedlLink, id, source, host, file); // handle magnet URLs if (downloadBytes.Length >= 7 diff --git a/src/Prowlarr.Api.V1/Search/SearchController.cs b/src/Prowlarr.Api.V1/Search/SearchController.cs index d0a6c2956..3af6ba4ac 100644 --- a/src/Prowlarr.Api.V1/Search/SearchController.cs +++ b/src/Prowlarr.Api.V1/Search/SearchController.cs @@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Mvc; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; using NzbDrone.Core.Download; using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers; @@ -58,10 +59,12 @@ namespace Prowlarr.Api.V1.Search var releaseInfo = _remoteReleaseCache.Find(GetCacheKey(release)); var indexerDef = _indexerFactory.Get(release.IndexerId); + var source = UserAgentParser.ParseSource(Request.Headers["User-Agent"]); + var host = Request.GetHostName(); try { - _downloadService.SendReportToClient(releaseInfo, indexerDef.Redirect); + _downloadService.SendReportToClient(releaseInfo, source, host, indexerDef.Redirect); } catch (ReleaseDownloadException ex) { diff --git a/src/Prowlarr.Http/Extensions/RequestExtensions.cs b/src/Prowlarr.Http/Extensions/RequestExtensions.cs index f911849f3..e5fd88d7c 100644 --- a/src/Prowlarr.Http/Extensions/RequestExtensions.cs +++ b/src/Prowlarr.Http/Extensions/RequestExtensions.cs @@ -161,6 +161,15 @@ namespace Prowlarr.Http.Extensions return remoteAddress; } + public static string GetHostName(this HttpRequest request) + { + string ip = request.GetRemoteIP(); + IPAddress myIP = IPAddress.Parse(ip); + IPHostEntry getIPHost = Dns.GetHostEntry(myIP); + List compName = getIPHost.HostName.ToString().Split('.').ToList(); + return compName.First(); + } + public static string GetServerUrl(this HttpRequest request) { var scheme = request.Scheme;