diff --git a/src/Polly.Shared/CircuitBreaker/AdvancedCircuitController.cs b/src/Polly.Shared/CircuitBreaker/AdvancedCircuitController.cs index bd2c70697a1..5e180b99375 100644 --- a/src/Polly.Shared/CircuitBreaker/AdvancedCircuitController.cs +++ b/src/Polly.Shared/CircuitBreaker/AdvancedCircuitController.cs @@ -30,6 +30,14 @@ Action onHalfOpen _minimumThroughput = minimumThroughput; } + public override HealthCount HealthCount + { + get + { + return _metrics.GetHealthCount_NeedsLock(); + } + } + public override void OnCircuitReset(Context context) { using (TimedLock.Lock(_lock)) diff --git a/src/Polly.Shared/CircuitBreaker/CircuitBreakerPolicy.cs b/src/Polly.Shared/CircuitBreaker/CircuitBreakerPolicy.cs index 9ed12a1d16b..390f3e59ae5 100644 --- a/src/Polly.Shared/CircuitBreaker/CircuitBreakerPolicy.cs +++ b/src/Polly.Shared/CircuitBreaker/CircuitBreakerPolicy.cs @@ -28,6 +28,14 @@ public CircuitState CircuitState get { return _breakerController.CircuitState; } } + /// + /// Gets the state of the underlying circuit. + /// + public IHealthCount HealthCount + { + get { return _breakerController.HealthCount; } + } + /// /// Gets the last exception handled by the circuit-breaker. /// diff --git a/src/Polly.Shared/CircuitBreaker/CircuitStateController.cs b/src/Polly.Shared/CircuitBreaker/CircuitStateController.cs index 72cecbcc231..fc2983bbd93 100644 --- a/src/Polly.Shared/CircuitBreaker/CircuitStateController.cs +++ b/src/Polly.Shared/CircuitBreaker/CircuitStateController.cs @@ -15,9 +15,9 @@ internal abstract class CircuitStateController : ICircuitController, TimeSpan, Context> onBreak, - Action onReset, + TimeSpan durationOfBreak, + Action, TimeSpan, Context> onBreak, + Action onReset, Action onHalfOpen) { _durationOfBreak = durationOfBreak; @@ -147,6 +147,8 @@ public void OnActionPreExecute() public abstract void OnActionFailure(DelegateResult outcome, Context context); public abstract void OnCircuitReset(Context context); + + public abstract HealthCount HealthCount { get; } } } diff --git a/src/Polly.Shared/CircuitBreaker/ConsecutiveCountCircuitController.cs b/src/Polly.Shared/CircuitBreaker/ConsecutiveCountCircuitController.cs index 2006c27b031..3c4273e3466 100644 --- a/src/Polly.Shared/CircuitBreaker/ConsecutiveCountCircuitController.cs +++ b/src/Polly.Shared/CircuitBreaker/ConsecutiveCountCircuitController.cs @@ -19,6 +19,14 @@ Action onHalfOpen _exceptionsAllowedBeforeBreaking = exceptionsAllowedBeforeBreaking; } + public override HealthCount HealthCount + { + get + { + return new HealthCount() { Failures = _count }; + } + } + public override void OnCircuitReset(Context context) { using (TimedLock.Lock(_lock)) diff --git a/src/Polly.Shared/CircuitBreaker/HealthCount.cs b/src/Polly.Shared/CircuitBreaker/HealthCount.cs index e5112ca4a1b..20a9802f10f 100644 --- a/src/Polly.Shared/CircuitBreaker/HealthCount.cs +++ b/src/Polly.Shared/CircuitBreaker/HealthCount.cs @@ -1,6 +1,6 @@ namespace Polly.CircuitBreaker { - internal class HealthCount + internal class HealthCount : IHealthCount { public int Successes { get; set; } diff --git a/src/Polly.Shared/CircuitBreaker/ICircuitController.cs b/src/Polly.Shared/CircuitBreaker/ICircuitController.cs index a24e1364593..02063243444 100644 --- a/src/Polly.Shared/CircuitBreaker/ICircuitController.cs +++ b/src/Polly.Shared/CircuitBreaker/ICircuitController.cs @@ -5,6 +5,7 @@ namespace Polly.CircuitBreaker internal interface ICircuitController { CircuitState CircuitState { get; } + HealthCount HealthCount { get; } Exception LastException { get; } TResult LastHandledResult { get; } void Isolate(); diff --git a/src/Polly.Shared/CircuitBreaker/IHealthCount.cs b/src/Polly.Shared/CircuitBreaker/IHealthCount.cs new file mode 100644 index 00000000000..9b5c47697c4 --- /dev/null +++ b/src/Polly.Shared/CircuitBreaker/IHealthCount.cs @@ -0,0 +1,28 @@ +namespace Polly.CircuitBreaker +{ + /// + /// Store and report health metrics + /// + public interface IHealthCount + { + /// + /// Success count + /// + int Successes { get; } + + /// + /// Failure count + /// + int Failures { get; } + + /// + /// Total count (probably success + failure) + /// + int Total { get; } + + /// + /// Start time for metric collection + /// + long StartedAt { get; } + } +} diff --git a/src/Polly.Shared/CollectMetricsSyntax.cs b/src/Polly.Shared/CollectMetricsSyntax.cs new file mode 100644 index 00000000000..5219888468d --- /dev/null +++ b/src/Polly.Shared/CollectMetricsSyntax.cs @@ -0,0 +1,27 @@ +using System; +using System.Linq; +using Polly.CircuitBreaker; +using Polly.Utilities; +using Polly.Shared; + +namespace Polly +{ + /// + /// Fluent API for adding a policy to metrics. + /// + public static class CollectMetricsSyntax + { + /// + /// The policy builder. + /// Adds policy to metrics collection + /// + /// The policy name. + /// The policy instance. + public static CircuitBreakerPolicy CollectMetrics(this CircuitBreakerPolicy policyBuilder, string name) + { + CollectedPolicies.Add(name, policyBuilder); + + return policyBuilder; + } + } +} diff --git a/src/Polly.Shared/CollectedPolicies.cs b/src/Polly.Shared/CollectedPolicies.cs new file mode 100644 index 00000000000..18cbc75a153 --- /dev/null +++ b/src/Polly.Shared/CollectedPolicies.cs @@ -0,0 +1,36 @@ +using Polly.CircuitBreaker; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Polly.Shared +{ + /// + /// Static storage for the current collection of policies to report metrics on. + /// + public class CollectedPolicies + { + private static WeakDictionary _policies; + static CollectedPolicies() { + _policies = new WeakDictionary(); + } + + /// + /// Return the current list of policies + /// + public static IEnumerable> All + { + get { return _policies.All(); } + } + + /// + /// Add a policy to the collection + /// + /// Unique name for this policy + /// The policy to record + public static void Add(string name, CircuitBreakerPolicy policy) + { + _policies.Add(name, policy); + } + } +} diff --git a/src/Polly.Shared/HystrixCommand.cs b/src/Polly.Shared/HystrixCommand.cs new file mode 100644 index 00000000000..c4b0d03078e --- /dev/null +++ b/src/Polly.Shared/HystrixCommand.cs @@ -0,0 +1,71 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Polly.Shared +{ + /// + /// Data class for output to be consumed by Hystrix Dashboard (https://github.com/Netflix/Hystrix/tree/master/hystrix-dashboard) + /// + public class HystrixCommand + { + private static readonly DateTime Jan1St1970 = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + public HystrixCommand() + { + currentTime = (long)((DateTime.UtcNow - Jan1St1970).TotalMilliseconds); + rollingCountTimeout = -1; + } + +#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member + public string type { get { return "HystrixCommand"; } } + public string name { get; set; } + public string group { get; set; } + public long currentTime { get; set; } + public bool isCircuitBreakerOpen { get; set; } + public int errorPercentage { get; set; } + public int errorCount { get; set; } + public int requestCount { get; set; } + public int rollingCountBadRequests { get; set; } + public int rollingCountCollapsedRequests { get; set; } + public int rollingCountEmit { get; set; } + public int rollingCountExceptionsThrown { get; set; } + public int rollingCountFailure { get; set; } + public int rollingCountFallbackEmit { get; set; } + public int rollingCountFallbackFailure { get; set; } + public int rollingCountFallbackMissing { get; set; } + public int rollingCountFallbackRejection { get; set; } + public int rollingCountFallbackSuccess { get; set; } + public int rollingCountResponsesFromCache { get; set; } + public int rollingCountSemaphoreRejected { get; set; } + public int rollingCountShortCircuited { get; set; } + public int rollingCountSuccess { get; set; } + public int rollingCountThreadPoolRejected { get; set; } + public int rollingCountTimeout { get; set; } + public int currentConcurrentExecutionCount { get; set; } + public int rollingMaxConcurrentExecutionCount { get; set; } + public int latencyExecute_mean { get; set; } + public Dictionary latencyExecute { get; set; } + public int latencyTotal_mean { get; set; } + public Dictionary latencyTotal { get; set; } + public int propertyValue_circuitBreakerRequestVolumeThreshold { get; set; } + public int propertyValue_circuitBreakerSleepWindowInMilliseconds { get; set; } + public int propertyValue_circuitBreakerErrorThresholdPercentage { get; set; } + public bool propertyValue_circuitBreakerForceOpen { get; set; } + public bool propertyValue_circuitBreakerForceClosed { get; set; } + public bool propertyValue_circuitBreakerEnabled { get; set; } + public string propertyValue_executionIsolationStrategy { get; set; } + public int propertyValue_executionIsolationThreadTimeoutInMilliseconds { get; set; } + public int propertyValue_executionTimeoutInMilliseconds { get; set; } + public bool propertyValue_executionIsolationThreadInterruptOnTimeout { get; set; } + public object propertyValue_executionIsolationThreadPoolKeyOverride { get; set; } + public int propertyValue_executionIsolationSemaphoreMaxConcurrentRequests { get; set; } + public int propertyValue_fallbackIsolationSemaphoreMaxConcurrentRequests { get; set; } + public int propertyValue_metricsRollingStatisticalWindowInMilliseconds { get; set; } + public bool propertyValue_requestCacheEnabled { get; set; } + public bool propertyValue_requestLogEnabled { get; set; } + public int reportingHosts { get; set; } + public string threadPool { get; set; } +#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member + } +} diff --git a/src/Polly.Shared/MetricStream.cs b/src/Polly.Shared/MetricStream.cs new file mode 100644 index 00000000000..5a651a7237e --- /dev/null +++ b/src/Polly.Shared/MetricStream.cs @@ -0,0 +1,38 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Polly.Shared +{ + /// Find and return the current metrics for the system (in Hystrix format) + public class MetricStream + { + /// an infinite stream of all metrics + /// The method will be called between each set of outputs to slow the stream down. We suggest "() => Thread.Sleep(1000)" + public static IEnumerable All(Action sleepFunc) + { + while (true) + { + foreach (var policy in CollectedPolicies.All) + { + yield return new HystrixCommand + { + rollingCountSuccess = policy.Value.HealthCount.Successes, + rollingCountFailure = policy.Value.HealthCount.Failures, + isCircuitBreakerOpen = (policy.Value.CircuitState != CircuitBreaker.CircuitState.Closed), + name = policy.Key, + group = "Group", + latencyExecute = new Dictionary() { { "0", 0 }, { "25", 0 }, { "50", 0 }, { "75", 0 }, { "90", 0 }, { "95", 0 }, { "99", 0 }, { "99.5", 0 }, { "100", 0 } }, + latencyTotal = new Dictionary() { { "0", 0 }, { "25", 0 }, { "50", 0 }, { "75", 0 }, { "90", 0 }, { "95", 0 }, { "99", 0 }, { "99.5", 0 }, { "100", 0 } }, + propertyValue_executionIsolationStrategy = "THREAD", + threadPool = "ThreadPool" + }; + } + + // yield return new HystrixThreadPool { type = "HystrixThreadPool", name = "Order" }; + + sleepFunc.Invoke(); + } + } + } +} diff --git a/src/Polly.Shared/Polly.Shared.projitems b/src/Polly.Shared/Polly.Shared.projitems index eecde2511a8..763fe6cd7d4 100644 --- a/src/Polly.Shared/Polly.Shared.projitems +++ b/src/Polly.Shared/Polly.Shared.projitems @@ -9,6 +9,9 @@ Polly.Shared + + + @@ -35,6 +38,8 @@ + + @@ -66,5 +71,6 @@ + \ No newline at end of file diff --git a/src/Polly.Shared/WeakDictionary.cs b/src/Polly.Shared/WeakDictionary.cs new file mode 100644 index 00000000000..d33dd65c13d --- /dev/null +++ b/src/Polly.Shared/WeakDictionary.cs @@ -0,0 +1,39 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Runtime.CompilerServices; + +namespace Polly.Shared +{ + internal class WeakDictionary + { + private HashSet> list; + + public WeakDictionary() + { + list = new HashSet>(); + } + + public void Add(K k, V v) + { + list.Add(new KeyValuePair(k, new WeakReference(v))); + } + + public IEnumerable> All() + { + var cloneList = new HashSet>(list); + foreach (var pair in cloneList) + { + object value = pair.Value.Target; + if (value != null) + { + yield return new KeyValuePair(pair.Key, (V)value); + } + else + { + list.Remove(pair); + } + } + } + } +} diff --git a/src/Polly.SharedSpecs/AdvancedCircuitBreakerSpecs.cs b/src/Polly.SharedSpecs/AdvancedCircuitBreakerSpecs.cs index 008d6ec67d1..dbccca87b7c 100644 --- a/src/Polly.SharedSpecs/AdvancedCircuitBreakerSpecs.cs +++ b/src/Polly.SharedSpecs/AdvancedCircuitBreakerSpecs.cs @@ -151,6 +151,16 @@ public void Should_initialise_to_closed_state() breaker.CircuitState.Should().Be(CircuitState.Closed); } + [Fact] + public void Should_return_health_count() + { + CircuitBreakerPolicy breaker = Policy + .Handle() + .AdvancedCircuitBreaker(0.5, TimeSpan.FromSeconds(10), 4, TimeSpan.FromSeconds(30)); + + breaker.HealthCount.StartedAt.Should().BeGreaterThan(0); + } + #endregion #region Circuit-breaker threshold-to-break tests diff --git a/src/Polly.SharedSpecs/CollectMetricsSpec.cs b/src/Polly.SharedSpecs/CollectMetricsSpec.cs new file mode 100644 index 00000000000..b0fb3c430aa --- /dev/null +++ b/src/Polly.SharedSpecs/CollectMetricsSpec.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using FluentAssertions; +using Polly.CircuitBreaker; +using Polly.Specs.Helpers; +using Polly.Utilities; +using Xunit; +using Polly.Shared; +using System.Linq; + +namespace Polly.Specs +{ + public class CollectMetricsSpecs : IDisposable + { + [Fact] + public void Should_be_able_to_be_called_on_advanced_circuit_breaker() + { + CircuitBreakerPolicy policy = Policy + .Handle() + .AdvancedCircuitBreaker(0.5, TimeSpan.FromSeconds(10), 4, TimeSpan.MaxValue) + .CollectMetrics("BreakerTest"); + } + + [Fact] + public void Should_be_able_to_enumerate_collected_metric_policies() + { + CircuitBreakerPolicy policyMetrics1 = Policy + .Handle() + .AdvancedCircuitBreaker(0.5, TimeSpan.FromSeconds(10), 4, TimeSpan.MaxValue) + .CollectMetrics("Test 1"); + + CircuitBreakerPolicy policyNoMetrics = Policy + .Handle() + .AdvancedCircuitBreaker(0.5, TimeSpan.FromSeconds(10), 4, TimeSpan.MaxValue); + + CircuitBreakerPolicy policyMetrics2 = Policy + .Handle() + .AdvancedCircuitBreaker(0.5, TimeSpan.FromSeconds(10), 4, TimeSpan.MaxValue) + .CollectMetrics("Test 3"); + + var actual = CollectedPolicies.All.Select(x => x.Value); + Assert.Contains(policyMetrics1, actual); + Assert.DoesNotContain(policyNoMetrics, actual); + Assert.Contains(policyMetrics2, actual); + } + + public void Dispose() + { + } + } +} diff --git a/src/Polly.SharedSpecs/Polly.SharedSpecs.projitems b/src/Polly.SharedSpecs/Polly.SharedSpecs.projitems index e6ee96cc463..f7c8f12033d 100644 --- a/src/Polly.SharedSpecs/Polly.SharedSpecs.projitems +++ b/src/Polly.SharedSpecs/Polly.SharedSpecs.projitems @@ -9,6 +9,8 @@ Polly.SharedSpecs + + diff --git a/src/Polly.SharedSpecs/WeakDictionarySpecs.cs b/src/Polly.SharedSpecs/WeakDictionarySpecs.cs new file mode 100644 index 00000000000..5369d81e9f1 --- /dev/null +++ b/src/Polly.SharedSpecs/WeakDictionarySpecs.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Xunit; + +namespace Polly.Specs +{ + public class WeakDictionarySpecs + { + [Fact] + public void Enumerable() + { + var dict = new Shared.WeakDictionary(); + var fred = new object(); + dict.Add("fred", fred); + + var count = 0; + foreach(var entry in dict.All()) + { + Assert.Equal(entry.Key, "fred"); + count++; + } + Assert.Equal(count, 1); + } + + [Fact] + public void DoesNotHaveAStrongReference() + { + var dict = new Shared.WeakDictionary(); + var fred = new object(); + dict.Add("fred", fred); + + Assert.Equal(dict.All().Count(), 1); + + fred = null; + GC.Collect(); + + var list = dict.All().ToArray(); + Assert.Equal(list.Count(), 0); + } + + [Fact] + public void CanReAddKeys() + { + var dict = new Shared.WeakDictionary(); + var fred = new object(); + dict.Add("fred", fred); + Assert.Equal(dict.All().Count(), 1); + + fred = null; + GC.Collect(); + Assert.Equal(dict.All().Count(), 0); + + fred = new object(); + dict.Add("fred", fred); + Assert.Equal(dict.All().Count(), 1); + } + + [Fact] + public void CanHaveMultipleKeyValues() + { + var dict = new Shared.WeakDictionary(); + var obj = new object(); + dict.Add("fred", obj); + dict.Add("jane", obj); + dict.Add("jim", obj); + + var actual = dict.All().Select(pair => pair.Key); + Assert.Contains("fred", actual); + Assert.Contains("jane", actual); + Assert.Contains("jim", actual); + Assert.Equal(3, actual.Count()); + } + } +}