Cleanse Log Exception Data (#83)

src/NzbDrone.Common/Extensions/ExceptionExtensions.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace NzbDrone.Common.Extensions
+{
+    public static class ExceptionExtensions
+    {
+        public static T WithData<T>(this T ex, string key, string value) where T : Exception
+        {
+            ex.AddData(key, value);
+
+            return ex;
+        }
+        public static T WithData<T>(this T ex, string key, int value) where T : Exception
+        {
+            ex.AddData(key, value.ToString());
+
+            return ex;
+        }
+
+        public static T WithData<T>(this T ex, string key, Http.HttpUri value) where T : Exception
+        {
+            ex.AddData(key, value.ToString());
+
+            return ex;
+        }
+
+
+        public static T WithData<T>(this T ex, Http.HttpResponse response, int maxSampleLength = 512) where T : Exception
+        {
+            if (response == null || response.Content == null) return ex;
+
+            var contentSample = response.Content.Substring(0, Math.Min(response.Content.Length, 512));
+
+            if (response.Headers != null)
+            {
+                ex.AddData("ContentType", response.Headers.ContentType ?? string.Empty);
+            }
+            ex.AddData("ContentLength", response.Content.Length.ToString());
+            ex.AddData("ContentSample", contentSample);
+
+            return ex;
+        }
+
+
+        private static void AddData(this Exception ex, string key, string value)
+        {
+            if (value.IsNullOrWhiteSpace()) return;
+
+            ex.Data[key] = value;
+        }
+    }
+}

src/NzbDrone.Common/Instrumentation/CleansingJsonVisitor.cs

@@ -0,0 +1,47 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+using NzbDrone.Common.Serializer;
+
+namespace NzbDrone.Common.Instrumentation
+{
+    public class CleansingJsonVisitor : JsonVisitor
+    {
+        public override void Visit(JArray json)
+        {
+            for (var i = 0; i < json.Count; i++)
+            {
+                if (json[i].Type == JTokenType.String)
+                {
+                    var text = json[i].Value<string>();
+                    json[i] = new JValue(CleanseLogMessage.Cleanse(text));
+                }
+            }
+            foreach (JToken token in json)
+            {
+                Visit(token);
+            }
+        }
+
+        public override void Visit(JProperty property)
+        {
+            if (property.Value.Type == JTokenType.String)
+            {
+                property.Value = CleanseValue(property.Value as JValue);
+            }
+            else
+            {
+                base.Visit(property);
+            }
+        }
+
+        private JValue CleanseValue(JValue value)
+        {
+            var text = value.Value<string>();
+            var cleansed = CleanseLogMessage.Cleanse(text);
+            return new JValue(cleansed);
+        }
+    }
+}

src/NzbDrone.Common/Instrumentation/NzbDroneLogger.cs

@@ -40,9 +40,7 @@ namespace NzbDrone.Common.Instrumentation
                 RegisterDebugger();
             }
 
-            // Need to register for Sentry service (sonarr)
+            RegisterSentry(updateApp);
-            // Exceptron is being considered by Radarr
-            // RegisterSentry(updateApp);
 
             if (updateApp)
             {
@@ -89,22 +87,21 @@ namespace NzbDrone.Common.Instrumentation
 
         private static void RegisterSentry(bool updateClient)
         {
-            // TODO Enable above when we recieve sentry service account.
 
             string dsn;
 
             if (updateClient)
             {
                 dsn = RuntimeInfo.IsProduction
-                    ? "https://b85aa82c65b84b0e99e3b7c281438357:392b5bc007974147a922c5d841c47cf9@sentry.lidarr.audio/11"
+                    ? "https://bbb13f4547294da1bcd52069420aaa5d:950541e562cf43c594fe2dcfaf4c3271@sentry.io/209545"
-                    : "https://6168f0946aba4e60ac23e469ac08eac5:bd59e8454ccc454ea27a90cff1f814ca@sentry.lidarr.audio/9";
+                    : "https://bbb13f4547294da1bcd52069420aaa5d:950541e562cf43c594fe2dcfaf4c3271@sentry.io/209545";
 
             }
             else
             {
                 dsn = RuntimeInfo.IsProduction
-                    ? "https://3e8a38b1a4df4de8b0453a724f5a1139:5a708dd75c724b32ae5128b6a895650f@sentry.lidarr.audio/8"
+                    ? "https://bbb13f4547294da1bcd52069420aaa5d:950541e562cf43c594fe2dcfaf4c3271@sentry.io/209545"
-                    : "https://4ee3580e01d8407c96a7430fbc953512:5f2d07227a0b4fde99dea07041a3ff93@sentry.lidarr.audio/10";
+                    : "https://bbb13f4547294da1bcd52069420aaa5d:950541e562cf43c594fe2dcfaf4c3271@sentry.io/209545";
             }
 
             var target = new SentryTarget(dsn)

src/NzbDrone.Common/Instrumentation/Sentry/LidarrJsonPacketFactory.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using SharpRaven.Data;
 
@@ -6,7 +6,14 @@ namespace NzbDrone.Common.Instrumentation.Sentry
 {
     public class LidarrJsonPacketFactory : IJsonPacketFactory
     {
-        private static string ShortenPath(string path)
+        private readonly SentryPacketCleanser _cleanser;
+ 
+        public LidarrJsonPacketFactory()
+        {
+             _cleanser = new SentryPacketCleanser();
+        }
+
+    private static string ShortenPath(string path)
         {
 
             if (string.IsNullOrWhiteSpace(path))
@@ -37,7 +44,10 @@ namespace NzbDrone.Common.Instrumentation.Sentry
                         frame.Filename = ShortenPath(frame.Filename);
                     }
                 }
+
+                _cleanser.CleansePacket(packet);
             }
+
             catch (Exception)
             {
 
@@ -46,7 +56,6 @@ namespace NzbDrone.Common.Instrumentation.Sentry
             return packet;
         }
 
-
         [Obsolete]
         public JsonPacket Create(string project, SentryMessage message, ErrorLevel level = ErrorLevel.Info, IDictionary<string, string> tags = null,
             string[] fingerprint = null, object extra = null)

src/NzbDrone.Common/Instrumentation/Sentry/SentryPacketCleanser.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+
+namespace NzbDrone.Common.Instrumentation.Sentry
+{
+    public class SentryPacketCleanser
+    {
+        public void CleansePacket(LidarrSentryPacket packet)
+        {
+            packet.Message = CleanseLogMessage.Cleanse(packet.Message);
+
+            if (packet.Fingerprint != null)
+            {
+                for (var i = 0; i < packet.Fingerprint.Length; i++)
+                {
+                    packet.Fingerprint[i] = CleanseLogMessage.Cleanse(packet.Fingerprint[i]);
+                }
+            }
+
+            if (packet.Extra != null)
+            {
+                var target = JObject.FromObject(packet.Extra);
+                new CleansingJsonVisitor().Visit(target);
+                packet.Extra = target;
+            }
+        }
+    }
+}

src/NzbDrone.Common/NzbDrone.Common.csproj

@@ -136,6 +136,7 @@
     <Compile Include="Extensions\Base64Extensions.cs" />
     <Compile Include="Extensions\DateTimeExtensions.cs" />
     <Compile Include="Crypto\HashConverter.cs" />
+    <Compile Include="Extensions\ExceptionExtensions.cs" />
     <Compile Include="Extensions\Int64Extensions.cs" />
     <Compile Include="Extensions\ObjectExtensions.cs" />
     <Compile Include="Extensions\StreamExtensions.cs" />
@@ -175,6 +176,7 @@
     <Compile Include="Extensions\IEnumerableExtensions.cs" />
     <Compile Include="Http\UserAgentBuilder.cs" />
     <Compile Include="Instrumentation\CleanseLogMessage.cs" />
+    <Compile Include="Instrumentation\CleansingJsonVisitor.cs" />
     <Compile Include="Instrumentation\Extensions\SentryLoggerExtensions.cs" />
     <Compile Include="Instrumentation\Extensions\LoggerExtensions.cs" />
     <Compile Include="Instrumentation\GlobalExceptionHandlers.cs" />
@@ -182,10 +184,11 @@
     <Compile Include="Instrumentation\NzbDroneFileTarget.cs" />
     <Compile Include="Instrumentation\NzbDroneLogger.cs" />
     <Compile Include="Instrumentation\Sentry\SentryDebounce.cs" />
+    <Compile Include="Instrumentation\Sentry\SentryPacketCleanser.cs" />
     <Compile Include="Instrumentation\Sentry\SentryTarget.cs" />
     <Compile Include="Instrumentation\Sentry\MachineNameUserFactory.cs" />
-    <Compile Include="Instrumentation\Sentry\SonarrJsonPacketFactory.cs" />
+    <Compile Include="Instrumentation\Sentry\LidarrJsonPacketFactory.cs" />
-    <Compile Include="Instrumentation\Sentry\SonarrSentryPacket.cs" />
+    <Compile Include="Instrumentation\Sentry\LidarrSentryPacket.cs" />
     <Compile Include="Instrumentation\VersionLayoutRenderer.cs" />
     <Compile Include="Extensions\LevenstheinExtensions.cs" />
     <Compile Include="Messaging\IEvent.cs" />
@@ -206,6 +209,7 @@
     <Compile Include="Serializer\HttpUriConverter.cs" />
     <Compile Include="Serializer\IntConverter.cs" />
     <Compile Include="Serializer\Json.cs" />
+    <Compile Include="Serializer\JsonVisitor.cs" />
     <Compile Include="ServiceFactory.cs" />
     <Compile Include="ServiceProvider.cs" />
     <Compile Include="Extensions\StringExtensions.cs" />

src/NzbDrone.Common/Serializer/JsonVisitor.cs

@@ -0,0 +1,95 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+
+namespace NzbDrone.Common.Serializer
+{
+
+    public class JsonVisitor
+    {
+        protected void Dispatch(JToken json)
+        {
+            switch (json.Type)
+            {
+                case JTokenType.Object:
+                    Visit(json as JObject);
+                    break;
+
+                case JTokenType.Array:
+                    Visit(json as JArray);
+                    break;
+
+                case JTokenType.Raw:
+                    Visit(json as JRaw);
+                    break;
+
+                case JTokenType.Constructor:
+                    Visit(json as JConstructor);
+                    break;
+
+                case JTokenType.Property:
+                    Visit(json as JProperty);
+                    break;
+
+                case JTokenType.Comment:
+                case JTokenType.Integer:
+                case JTokenType.Float:
+                case JTokenType.String:
+                case JTokenType.Boolean:
+                case JTokenType.Null:
+                case JTokenType.Undefined:
+                case JTokenType.Date:
+                case JTokenType.Bytes:
+                case JTokenType.Guid:
+                case JTokenType.Uri:
+                case JTokenType.TimeSpan:
+                    Visit(json as JValue);
+                    break;
+
+                default:
+                    break;
+            }
+        }
+
+        public virtual void Visit(JToken json)
+        {
+            Dispatch(json);
+        }
+
+        public virtual void Visit(JContainer json)
+        {
+            Dispatch(json);
+        }
+
+        public virtual void Visit(JArray json)
+        {
+            foreach (JToken token in json)
+            {
+                Visit(token);
+            }
+        }
+        public virtual void Visit(JConstructor json)
+        {
+        }
+
+        public virtual void Visit(JObject json)
+        {
+            foreach (JProperty property in json.Properties())
+            {
+                Visit(property);
+            }
+        }
+
+        public virtual void Visit(JProperty property)
+        {
+            Visit(property.Value);
+        }
+
+        public virtual void Visit(JValue value)
+        {
+
+        }
+    }
+}

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -212,6 +212,7 @@ namespace NzbDrone.Core.Indexers
             catch (CloudFlareCaptchaException ex)
             {
                 _indexerStatusService.RecordFailure(Definition.Id);
+                ex.WithData("FeedUrl", url);
                 if (ex.IsExpired)
                 {
                     _logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in indexer settings.", this);
@@ -226,11 +227,11 @@ namespace NzbDrone.Core.Indexers
                 _indexerStatusService.RecordFailure(Definition.Id);
                 _logger.Warn(ex, "{0}", url);
             }
-            catch (Exception feedEx)
+            catch (Exception ex)
             {
                 _indexerStatusService.RecordFailure(Definition.Id);
-                feedEx.Data.Add("FeedUrl", url);
+                ex.WithData("FeedUrl", url);
-                _logger.Error(feedEx, "An error occurred while processing feed. {0}", url);
+                _logger.Error(ex, "An error occurred while processing feed. {0}", url);
             }
 
             return CleanupReleases(releases);

src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Net;
 using System.Xml;
@@ -69,6 +69,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             catch (XmlException ex)
             {
                 _logger.Debug(ex, "Failed to parse newznab api capabilities for {0}.", indexerSettings.Url);
+                ex.WithData(response);
                 throw;
             }
             catch (Exception ex)

src/NzbDrone.Core/Indexers/RssParser.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
@@ -65,12 +65,14 @@ namespace NzbDrone.Core.Indexers
                 }
                 catch (UnsupportedFeedException itemEx)
                 {
-                    itemEx.Data.Add("Item", item.Title());
+                    itemEx.WithData("FeedUrl", indexerResponse.Request.Url);
+                    itemEx.WithData("ItemTitle", item.Title());
                     throw;
                 }
                 catch (Exception itemEx)
                 {
-                    itemEx.Data.Add("Item", item.Title());
+                    itemEx.WithData("FeedUrl", indexerResponse.Request.Url);
+                    itemEx.WithData("ItemTitle", item.Title());
                     _logger.Error(itemEx, "An error occurred while processing feed item from {0}", indexerResponse.Request.Url);
                 }
             }
@@ -95,8 +97,7 @@ namespace NzbDrone.Core.Indexers
                 var contentSample = indexerResponse.Content.Substring(0, Math.Min(indexerResponse.Content.Length, 512));
                 _logger.Debug("Truncated response content (originally {0} characters): {1}", indexerResponse.Content.Length, contentSample);
 
-                ex.Data.Add("ContentLength", indexerResponse.Content.Length);
+                ex.WithData(indexerResponse.HttpResponse);
-                ex.Data.Add("ContentSample", contentSample);
 
                 throw;
             }

src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Linq;
 using System.IO;
 using System.Text.RegularExpressions;
@@ -40,22 +40,30 @@ namespace NzbDrone.Core.Indexers.TorrentRss
         {
             _logger.Debug("Evaluating TorrentRss feed '{0}'", indexerSettings.BaseUrl);
 
-            var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = indexerSettings };
-            var request = requestGenerator.GetRecentRequests().GetAllTiers().First().First();
-
-            HttpResponse httpResponse = null;
             try
             {
-                httpResponse = _httpClient.Execute(request.HttpRequest);
+                var requestGenerator = new TorrentRssIndexerRequestGenerator { Settings = indexerSettings };
+                var request = requestGenerator.GetRecentRequests().GetAllTiers().First().First();
+
+                HttpResponse httpResponse = null;
+                try
+                {
+                    httpResponse = _httpClient.Execute(request.HttpRequest);
+                }
+                catch (Exception ex)
+                {
+                    _logger.Warn(ex, string.Format("Unable to connect to indexer {0}: {1}", request.Url, ex.Message));
+                    return null;
+                }
+
+                var indexerResponse = new IndexerResponse(request, httpResponse);
+                return GetParserSettings(indexerResponse, indexerSettings);
             }
             catch (Exception ex)
             {
-                _logger.Warn(ex, string.Format("Unable to connect to indexer {0}: {1}", request.Url, ex.Message));
+                ex.WithData("FeedUrl", indexerSettings.BaseUrl);
-                return null;
+                throw;
             }
-
-            var indexerResponse = new IndexerResponse(request, httpResponse);
-            return GetParserSettings(indexerResponse, indexerSettings);
         }
 
         private TorrentRssIndexerParserSettings GetParserSettings(IndexerResponse response, TorrentRssIndexerSettings indexerSettings)