From 6739310ba8a76981aac878823d4729e0c8f58d17 Mon Sep 17 00:00:00 2001
From: Mark McDowall <mark@mcdowall.ca>
Date: Sun, 6 Mar 2022 21:26:13 -0800
Subject: [PATCH] Fixed: Delay health check notifications on startup

Closes #7145

(cherry picked from commit 07f0db477a91b39c1f4b884775c08a55ada487cf)
---
 .../HealthCheck/HealthCheckFailedEvent.cs     |  4 ++-
 .../HealthCheck/HealthCheckService.cs         | 33 ++++++++++++++++++-
 .../Notifications/NotificationService.cs      |  7 ++++
 3 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/src/NzbDrone.Core/HealthCheck/HealthCheckFailedEvent.cs b/src/NzbDrone.Core/HealthCheck/HealthCheckFailedEvent.cs
index 1fc2af1a9..8abb156a5 100644
--- a/src/NzbDrone.Core/HealthCheck/HealthCheckFailedEvent.cs
+++ b/src/NzbDrone.Core/HealthCheck/HealthCheckFailedEvent.cs
@@ -5,10 +5,12 @@ namespace NzbDrone.Core.HealthCheck
     public class HealthCheckFailedEvent : IEvent
     {
         public HealthCheck HealthCheck { get; private set; }
+        public bool IsInStartupGraceperiod { get; private set; }
 
-        public HealthCheckFailedEvent(HealthCheck healthCheck)
+        public HealthCheckFailedEvent(HealthCheck healthCheck, bool isInStartupGraceperiod)
         {
             HealthCheck = healthCheck;
+            IsInStartupGraceperiod = isInStartupGraceperiod;
         }
     }
 }
diff --git a/src/NzbDrone.Core/HealthCheck/HealthCheckService.cs b/src/NzbDrone.Core/HealthCheck/HealthCheckService.cs
index 679044083..da1a5a678 100644
--- a/src/NzbDrone.Core/HealthCheck/HealthCheckService.cs
+++ b/src/NzbDrone.Core/HealthCheck/HealthCheckService.cs
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Linq;
 using NLog;
 using NzbDrone.Common.Cache;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Messaging;
 using NzbDrone.Common.Reflection;
 using NzbDrone.Core.Lifecycle;
@@ -21,6 +22,7 @@ namespace NzbDrone.Core.HealthCheck
                                       IHandleAsync<ApplicationStartedEvent>,
                                       IHandleAsync<IEvent>
     {
+        private readonly DateTime _startupGracePeriodEndTime;
         private readonly IProvideHealthCheck[] _healthChecks;
         private readonly IProvideHealthCheck[] _startupHealthChecks;
         private readonly IProvideHealthCheck[] _scheduledHealthChecks;
@@ -32,10 +34,14 @@ namespace NzbDrone.Core.HealthCheck
 
         private readonly ICached<HealthCheck> _healthCheckResults;
 
+        private bool _hasRunHealthChecksAfterGracePeriod = false;
+        private bool _isRunningHealthChecksAfterGracePeriod = false;
+
         public HealthCheckService(IEnumerable<IProvideHealthCheck> healthChecks,
                                   IServerSideNotificationService serverSideNotificationService,
                                   IEventAggregator eventAggregator,
                                   ICacheManager cacheManager,
+                                  IRuntimeInfo runtimeInfo,
                                   Logger logger)
         {
             _healthChecks = healthChecks.ToArray();
@@ -49,6 +55,7 @@ namespace NzbDrone.Core.HealthCheck
             _startupHealthChecks = _healthChecks.Where(v => v.CheckOnStartup).ToArray();
             _scheduledHealthChecks = _healthChecks.Where(v => v.CheckOnSchedule).ToArray();
             _eventDrivenHealthChecks = GetEventDrivenHealthChecks();
+            _startupGracePeriodEndTime = runtimeInfo.StartTime + TimeSpan.FromMinutes(15);
         }
 
         public List<HealthCheck> Results()
@@ -87,7 +94,7 @@ namespace NzbDrone.Core.HealthCheck
                 {
                     if (_healthCheckResults.Find(result.Source.Name) == null)
                     {
-                        _eventAggregator.PublishEvent(new HealthCheckFailedEvent(result));
+                        _eventAggregator.PublishEvent(new HealthCheckFailedEvent(result, !_hasRunHealthChecksAfterGracePeriod));
                     }
 
                     _healthCheckResults.Set(result.Source.Name, result);
@@ -121,6 +128,30 @@ namespace NzbDrone.Core.HealthCheck
                 return;
             }
 
+            // If we haven't previously re-run health checks after startup grace period run startup checks again and track so they aren't run again.
+            // Return early after re-running checks to avoid triggering checks multiple times.
+            if (!_hasRunHealthChecksAfterGracePeriod && !_isRunningHealthChecksAfterGracePeriod && DateTime.UtcNow > _startupGracePeriodEndTime)
+            {
+                _isRunningHealthChecksAfterGracePeriod = true;
+
+                PerformHealthCheck(_startupHealthChecks);
+
+                // Update after running health checks so new failure notifications aren't sent 2x.
+                _hasRunHealthChecksAfterGracePeriod = true;
+
+                // Explicitly notify for any failed checks since existing failed results would not have sent events.
+                var results = _healthCheckResults.Values.ToList();
+
+                foreach (var result in results)
+                {
+                    _eventAggregator.PublishEvent(new HealthCheckFailedEvent(result, false));
+                }
+
+                _isRunningHealthChecksAfterGracePeriod = false;
+
+                return;
+            }
+
             IEventDrivenHealthCheck[] checks;
             if (!_eventDrivenHealthChecks.TryGetValue(message.GetType(), out checks))
             {
diff --git a/src/NzbDrone.Core/Notifications/NotificationService.cs b/src/NzbDrone.Core/Notifications/NotificationService.cs
index b43835ce3..621590c5b 100644
--- a/src/NzbDrone.Core/Notifications/NotificationService.cs
+++ b/src/NzbDrone.Core/Notifications/NotificationService.cs
@@ -245,6 +245,13 @@ namespace NzbDrone.Core.Notifications
 
         public void Handle(HealthCheckFailedEvent message)
         {
+            // Don't send health check notifications during the start up grace period,
+            // once that duration expires they they'll be retested and fired off if necessary.
+            if (message.IsInStartupGraceperiod)
+            {
+                return;
+            }
+
             foreach (var notification in _notificationFactory.OnHealthIssueEnabled())
             {
                 try