Split Brain Resolver update (#4819)

* Don't add MemberExited as unreachable in SBR, (migrated from akka/akka#29988)

* Makes the SBR lease name configurable (migrated from akka/akka#30049)

* api approval

src/core/Akka.API.Tests/CoreAPISpec.ApproveCluster.approved.txt

@@ -398,9 +398,12 @@ namespace Akka.Cluster.SBR
     public sealed class LeaseMajoritySettings
     {
         public LeaseMajoritySettings(string leaseImplementation, System.TimeSpan acquireLeaseDelayForMinority, string role) { }
+        public LeaseMajoritySettings(string leaseImplementation, System.TimeSpan acquireLeaseDelayForMinority, string role, string leaseName) { }
         public System.TimeSpan AcquireLeaseDelayForMinority { get; }
         public string LeaseImplementation { get; }
+        public string LeaseName { get; }
         public string Role { get; }
+        public string SafeLeaseName(string systemName) { }
     }
     public class SplitBrainResolverProvider : Akka.Cluster.IDowningProvider
     {

src/core/Akka.Cluster.Tests/SBR/LeaseMajoritySpec.cs

@@ -0,0 +1,52 @@
+using Akka.Cluster.Configuration;
+using Akka.Cluster.SBR;
+using Akka.Configuration;
+using Akka.TestKit;
+using FluentAssertions;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Akka.Cluster.Tests.SBR
+{
+    public class LeaseMajoritySpec : AkkaSpec
+    {
+        private readonly Config _default;
+        private readonly Config _blank;
+        private readonly Config _named;
+
+        public LeaseMajoritySpec(ITestOutputHelper output)
+            : base(output)
+        {
+            _default = ConfigurationFactory.ParseString(@"
+                akka.cluster.split-brain-resolver.lease-majority.lease-implementation = ""akka.coordination.lease.kubernetes""
+                ")
+                .WithFallback(ClusterConfigFactory.Default());
+
+            _blank = ConfigurationFactory.ParseString(@"
+                akka.cluster.split-brain-resolver.lease-majority {
+                    lease-name = "" ""
+                }").WithFallback(_default);
+
+            _named = ConfigurationFactory.ParseString(@"
+                akka.cluster.split-brain-resolver.lease-majority {
+                    lease-name = ""shopping-cart-akka-sbr""
+                }").WithFallback(_default);
+        }
+
+        [Fact]
+        public void Split_Brain_Resolver_Lease_Majority_provider_must_read_the_configured_name()
+        {
+            new SplitBrainResolverSettings(_default).LeaseMajoritySettings.LeaseName.Should().BeNull();
+            new SplitBrainResolverSettings(_blank).LeaseMajoritySettings.LeaseName.Should().BeNull();
+            new SplitBrainResolverSettings(_named).LeaseMajoritySettings.LeaseName.Should().Be("shopping-cart-akka-sbr");
+        }
+
+        [Fact]
+        public void Split_Brain_Resolver_Lease_Majority_provider_must_use_a_safe_name()
+        {
+            new SplitBrainResolverSettings(_default).LeaseMajoritySettings.SafeLeaseName("sysName").Should().Be("sysName-akka-sbr");
+            new SplitBrainResolverSettings(_blank).LeaseMajoritySettings.SafeLeaseName("sysName").Should().Be("sysName-akka-sbr");
+            new SplitBrainResolverSettings(_named).LeaseMajoritySettings.SafeLeaseName("sysName").Should().Be("shopping-cart-akka-sbr");
+        }
+    }
+}

src/core/Akka.Cluster/Configuration/Cluster.conf

@@ -95,13 +95,13 @@ akka {
     #
     # DEFAULT: by default the app-version will default to the entry assembly's version,
     # i.e. the assembly of the executable running `Program.cs`
-    # 
+    #
     # Values can be "assembly-version" or a version string as defined above, i.e.
     # app-version = "1.0.0"
     # app-version = "1.1-beta1"
     # app-version = "1"
     # app-version = "1.1"
-    app-version = assembly-version 
+    app-version = assembly-version
 
 	# Run the coordinated shutdown from phase 'cluster-shutdown' when the cluster
     # is shutdown for other reasons than when leaving, e.g. when downing. This
@@ -386,6 +386,10 @@ akka.cluster.split-brain-resolver.keep-oldest {
 akka.cluster.split-brain-resolver.lease-majority {
   lease-implementation = ""
 
+  # The recommended format for the lease name is "<service-name>-akka-sbr".
+  # When lease-name is not defined, the name will be set to "<actor-system-name>-akka-sbr"
+  lease-name = ""
+
   # This delay is used on the minority side before trying to acquire the lease,
   # as an best effort to try to keep the majority side.
   acquire-lease-delay-for-minority = 2s

src/core/Akka.Cluster/SBR/DowningStrategy.cs

@@ -129,7 +129,13 @@ public override int GetHashCode()
 
     internal class ReverseDownIndirectlyConnected : IDecision
     {
+        public static readonly ReverseDownIndirectlyConnected Instance = new ReverseDownIndirectlyConnected();
+
         public bool IsIndirectlyConnected => true;
+
+        private ReverseDownIndirectlyConnected()
+        {
+        }
     }
 
     internal abstract class DowningStrategy
@@ -424,9 +430,9 @@ public IDecision ReverseDecision(IDecision decision)
                 case DownAll _:
                     return DownAll.Instance;
                 case DownIndirectlyConnected _:
-                    return new ReverseDownIndirectlyConnected();
+                    return ReverseDownIndirectlyConnected.Instance;
                 case AcquireLeaseAndDownIndirectlyConnected _:
-                    return new ReverseDownIndirectlyConnected();
+                    return ReverseDownIndirectlyConnected.Instance;
                 case ReverseDownIndirectlyConnected _:
                     return DownIndirectlyConnected.Instance;
             }

src/core/Akka.Cluster/SBR/SplitBrainResolver.cs

@@ -249,15 +249,15 @@ protected override bool Receive(object message)
                 case MemberLeft ch:
                     Leaving(ch.Member);
                     return true;
+                case MemberExited ch:
+                    Exited(ch.Member);
+                    return true;
                 case UnreachableMember ch:
                     UnreachableMember(ch.Member);
                     return true;
                 case MemberDowned ch:
                     UnreachableMember(ch.Member);
                     return true;
-                case MemberExited ch:
-                    UnreachableMember(ch.Member);
-                    return true;
                 case ReachableMember ch:
                     ReachableMember(ch.Member);
                     return true;
@@ -508,10 +508,10 @@ ImmutableHashSet<UniqueAddress> nodesToDown
                 : "";
 
             Log.Warning(
-                $"SBR took decision {decision} and is downing [{string.Join(", ", nodesToDown.Select(i => i.Address))}]{(downMyself ? " including myself, " : "")}, " +
-                $"[{Strategy.Unreachable.Count}] unreachable of [{Strategy.Members.Count}] members" +
-                indirectlyConnectedLogMessage +
-                $", full reachability status: {Strategy.Reachability}");
+              $"SBR took decision {decision} and is downing [{string.Join(", ", nodesToDown.Select(i => i.Address))}]{(downMyself ? " including myself, " : "")}, " +
+                  $"[{Strategy.Unreachable.Count}] unreachable of [{Strategy.Members.Count}] members" +
+                  indirectlyConnectedLogMessage +
+                  $", full reachability status: [{Strategy.Reachability}]");
         }
 
         public void UnreachableMember(Member m)
@@ -599,6 +599,15 @@ public void Leaving(Member m)
             MutateMemberInfo(false, () => { Strategy.Add(m); });
         }
 
+        public void Exited(Member m)
+        {
+            Log.Debug("SBR exited [{0}]", m);
+            MutateMemberInfo(resetStable: true, () =>
+            {
+                Strategy.Add(m);
+            });
+        }
+
         public void AddJoining(Member m)
         {
             Log.Debug("SBR add Joining/WeaklyUp [{0}]", m);

src/core/Akka.Cluster/SBR/SplitBrainResolverProvider.cs

@@ -69,8 +69,10 @@ public Props DowningActorProps
                     case SplitBrainResolverSettings.LeaseMajorityName:
                         var lms = settings.LeaseMajoritySettings;
                         var leaseOwnerName = cluster.SelfUniqueAddress.Address.HostPort();
-                        var lease = LeaseProvider.Get(system).GetLease($"{system.Name}-akka-sbr",
-                            lms.LeaseImplementation, leaseOwnerName);
+
+                        var leaseName = lms.SafeLeaseName(system.Name);
+                        var lease = LeaseProvider.Get(system).GetLease(leaseName, lms.LeaseImplementation, leaseOwnerName);
+
                         strategy = new LeaseMajority(lms.Role, lease, lms.AcquireLeaseDelayForMinority);
                         break;
                     default:

src/core/Akka.Cluster/SBR/SplitBrainResolverSettings.cs

@@ -113,7 +113,11 @@ string Role(Config c)
 
                 var acquireLeaseDelayForMinority = c.GetTimeSpan("acquire-lease-delay-for-minority");
 
-                return new LeaseMajoritySettings(leaseImplementation, acquireLeaseDelayForMinority, Role(c));
+                var leaseName = c.GetString("lease-name").Trim();
+                if (string.IsNullOrEmpty(leaseName))
+                    leaseName = null;
+
+                return new LeaseMajoritySettings(leaseImplementation, acquireLeaseDelayForMinority, Role(c), leaseName);
             });
         }
 
@@ -161,16 +165,29 @@ public KeepOldestSettings(bool downIfAlone, string role)
     public sealed class LeaseMajoritySettings
     {
         public LeaseMajoritySettings(string leaseImplementation, TimeSpan acquireLeaseDelayForMinority, string role)
+            : this(leaseImplementation, acquireLeaseDelayForMinority, role, null)
+        {
+        }
+
+        public LeaseMajoritySettings(string leaseImplementation, TimeSpan acquireLeaseDelayForMinority, string role, string leaseName)
         {
             LeaseImplementation = leaseImplementation;
             AcquireLeaseDelayForMinority = acquireLeaseDelayForMinority;
             Role = role;
+            LeaseName = leaseName;
+        }
+
+        public string SafeLeaseName(string systemName)
+        {
+            return LeaseName ?? $"{systemName}-akka-sbr";
         }
 
         public string LeaseImplementation { get; }
 
         public TimeSpan AcquireLeaseDelayForMinority { get; }
 
         public string Role { get; }
+
+        public string LeaseName { get; }
     }
 }