Simplify Healthcheck.Persistence suicide probe (#283)

petabridge · Jul 12, 2024 · deac018 · deac018
1 parent 851fd8b
commit deac018
Show file tree

Hide file tree

Showing 8 changed files with 284 additions and 546 deletions.
diff --git a/src/Akka.HealthCheck.Hosting.Web/Probes/AkkaPersistenceLivenessProbe.cs b/src/Akka.HealthCheck.Hosting.Web/Probes/AkkaPersistenceLivenessProbe.cs
@@ -6,6 +6,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
 using Akka.Actor;
@@ -50,30 +51,18 @@ public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context
                     cancellationToken: cancellationToken);
                 return status.Status switch
                 {
-                    AkkaHealthStatus.Healthy => HealthCheckResult.Healthy(Healthy, new Dictionary<string, object>
+                    AkkaHealthStatus.Healthy => HealthCheckResult.Healthy(JoinString(Healthy, status.StatusMessage), new Dictionary<string, object>
                         {
-                            ["journal-recovered"] = status.JournalRecovered,
-                            ["snapshot-recovered"] = status.SnapshotRecovered,
-                            ["journal-persisted"] = status.JournalPersisted,
-                            ["snapshot-persisted"] = status.SnapshotSaved,
                             ["message"] = status.StatusMessage
                         }),
-                    AkkaHealthStatus.Unhealthy => HealthCheckResult.Unhealthy(UnHealthy, status.Failures,
+                    AkkaHealthStatus.Unhealthy => HealthCheckResult.Unhealthy(JoinString(UnHealthy, status.StatusMessage), status.Failure,
                         new Dictionary<string, object>
                         {
-                            ["journal-recovered"] = status.JournalRecovered,
-                            ["snapshot-recovered"] = status.SnapshotRecovered,
-                            ["journal-persisted"] = status.JournalPersisted,
-                            ["snapshot-persisted"] = status.SnapshotSaved,
                             ["message"] = status.StatusMessage
                         }),
-                    _ => HealthCheckResult.Degraded(Degraded, status.Failures,
+                    _ => HealthCheckResult.Degraded(JoinString(Degraded, status.StatusMessage), status.Failure,
                         new Dictionary<string, object>
                         {
-                            ["journal-recovered"] = status.JournalRecovered,
-                            ["snapshot-recovered"] = status.SnapshotRecovered,
-                            ["journal-persisted"] = status.JournalPersisted,
-                            ["snapshot-persisted"] = status.SnapshotSaved,
                             ["message"] = status.StatusMessage
                         })
                 };
@@ -83,5 +72,9 @@ public async Task<HealthCheckResult> CheckHealthAsync(HealthCheckContext context
                 return HealthCheckResult.Unhealthy(Exception, e, new Dictionary<string, object> { ["message"] = Exception });
             }
         }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private static string JoinString(string prefix, string? message)
+            => message is null ? prefix : $"prefix: {message}";
     }
 }
diff --git a/...heck.Persistence.Tests/AkkaPersistenceLivenessProbeNotAvailableDueToSnapshotStoreSpecs.cs b/...heck.Persistence.Tests/AkkaPersistenceLivenessProbeNotAvailableDueToSnapshotStoreSpecs.cs
@@ -30,7 +30,9 @@ public void AkkaPersistenceLivenessProbeProvidert_Should_Report_Akka_Persistance
 
             var ProbActor = Sys.ActorOf(Props.Create(() => new AkkaPersistenceLivenessProbe(true, 250.Milliseconds(), 3.Seconds())));
             ProbActor.Tell(new SubscribeToLiveness(TestActor));
-            ExpectMsg<LivenessStatus>().IsLive.Should().BeFalse("System should not be live");
+            var firstResult = ExpectMsg<LivenessStatus>();
+            firstResult.Status.Should().Be(AkkaHealthStatus.Degraded, "Initial status should be degraded, not unhealthy");
+            firstResult.IsLive.Should().BeTrue("Degraded should still report as live");
             ExpectMsg<LivenessStatus>(TimeSpan.FromMinutes(1)).IsLive.Should().BeFalse("System should not be live");
 
         }

diff --git a/...ealthCheck.Persistence.Tests/AkkaPersistenceLivenessProbeNotAvailableduetoJournalSpecs.cs b/...ealthCheck.Persistence.Tests/AkkaPersistenceLivenessProbeNotAvailableduetoJournalSpecs.cs
@@ -29,7 +29,9 @@ public void AkkaPersistenceLivenessProbeProvidert_Should_Report_Akka_Persistance
 
             var ProbActor = Sys.ActorOf(Props.Create(() => new AkkaPersistenceLivenessProbe(true, 250.Milliseconds(), 3.Seconds())));
             ProbActor.Tell(new SubscribeToLiveness(TestActor));
-            ExpectMsg<LivenessStatus>().IsLive.Should().BeFalse("System should not be live");
+            var firstResult = ExpectMsg<LivenessStatus>();
+            firstResult.Status.Should().Be(AkkaHealthStatus.Degraded, "Initial status should be degraded, not unhealthy");
+            firstResult.IsLive.Should().BeTrue("Degraded should still report as live");
             ExpectMsg<LivenessStatus>(TimeSpan.FromMinutes(1)).IsLive.Should().BeFalse("System should not be live");
 
         }

diff --git a/src/Akka.HealthCheck.Persistence.Tests/AkkaPersistenceLivenessProbeSubscriptionTest.cs b/src/Akka.HealthCheck.Persistence.Tests/AkkaPersistenceLivenessProbeSubscriptionTest.cs
@@ -30,7 +30,9 @@ public void AkkaPersistenceLivenessProbe_Should_Handle_Subscriptions_In_Any_Stat
         {
             var ProbActor = Sys.ActorOf(Props.Create(() => new AkkaPersistenceLivenessProbe(true, 250.Milliseconds(), 3.Seconds())));
             ProbActor.Tell(new SubscribeToLiveness(TestActor));
-            ExpectMsg<LivenessStatus>().IsLive.Should().BeFalse();
+            var firstResult = ExpectMsg<LivenessStatus>();
+            firstResult.Status.Should().Be(AkkaHealthStatus.Degraded, "Initial status should be degraded, not unhealthy");
+            firstResult.IsLive.Should().BeTrue("Degraded should still report as live");
             AwaitAssert(() => ExpectMsg<LivenessStatus>().IsLive.Should().BeTrue(),TimeSpan.FromSeconds(10));
 
             var probe = CreateTestProbe();

diff --git a/src/Akka.HealthCheck.Persistence.Tests/LivenessProbeTimeoutSpec.cs b/src/Akka.HealthCheck.Persistence.Tests/LivenessProbeTimeoutSpec.cs
@@ -23,17 +23,6 @@ public LivenessProbeTimeoutSpec(ITestOutputHelper output) : base(nameof(Liveness
     {
     }
 
-    [Fact(DisplayName = "AkkaPersistenceLivenessProbe should time out if SaveSnapshot does not respond")]
-    public async Task SaveSnapshotTimeoutTest()
-    {
-        using var cts = new CancellationTokenSource();
-        var delay = new SnapshotInterceptors.CancelableDelay(30.Minutes(), SnapshotInterceptors.Noop.Instance, cts.Token);
-
-        await WithSnapshotSave(
-            save => save.SetInterceptorAsync(delay),
-            () => TestTimeout(cts));
-    }
-
     [Fact(DisplayName = "AkkaPersistenceLivenessProbe should time out if snapshot recovery does not respond")]
     public async Task SnapshotLoadTimeoutTest()
     {
@@ -45,17 +34,6 @@ await WithSnapshotLoad(
             () => TestTimeout(cts));
     }
 
-    [Fact(DisplayName = "AkkaPersistenceLivenessProbe should time out if journal Persist does not respond")]
-    public async Task JournalPersistTimeoutTest()
-    {
-        using var cts = new CancellationTokenSource();
-        var delay = new JournalInterceptors.CancelableDelay(30.Minutes(), JournalInterceptors.Noop.Instance, cts.Token);
-
-        await WithJournalWrite(
-            save => save.SetInterceptorAsync(delay),
-            () => TestTimeout(cts));
-    }
-
     [Fact(DisplayName = "AkkaPersistenceLivenessProbe should time out if journal recovery does not respond")]
     public async Task JournalRecoveryTimeoutTest()
     {
@@ -72,11 +50,12 @@ private async Task TestTimeout(CancellationTokenSource cts)
         var probeActor = Sys.ActorOf(Props.Create(() => new AkkaPersistenceLivenessProbe(true, 250.Milliseconds(), 500.Milliseconds())));
         probeActor.Tell(new SubscribeToLiveness(TestActor));
         var status = ExpectMsg<LivenessStatus>();
-        status.IsLive.Should().BeFalse();
+        status.Status.Should().Be(AkkaHealthStatus.Degraded);
+        status.IsLive.Should().BeTrue();
         status.StatusMessage.Should().StartWith("Warming up probe.");
 
         var timeoutStatusObj = await FishForMessageAsync(
-            msg => msg is LivenessStatus stat && !stat.StatusMessage.StartsWith("Warming up probe."), 
+            msg => msg is LivenessStatus stat && !stat.StatusMessage.StartsWith("Warming up probe.") && !stat.StatusMessage.StartsWith("Persistence warmup complete"), 
             6.Seconds());
 
         var timeoutStatus = (LivenessStatus)timeoutStatusObj;