diff --git a/hystrix-core/build.gradle b/hystrix-core/build.gradle index e05c2b550..8f71a2db7 100644 --- a/hystrix-core/build.gradle +++ b/hystrix-core/build.gradle @@ -5,10 +5,10 @@ dependencies { compile 'com.netflix.archaius:archaius-core:0.4.1' compile 'io.reactivex:rxjava:1.0.10' compile 'org.slf4j:slf4j-api:1.7.0' - compile 'org.hdrhistogram:HdrHistogram:2.1.4' testCompile 'junit:junit-dep:4.10' } + javadoc { // the exclude isn't working, nor is there a subPackages options as docs suggest there should be // we do not want the com.netflix.hystrix.util package include diff --git a/hystrix-core/src/jmh/java/com/netflix/hystrix/perf/RollingPercentilePerfTest.java b/hystrix-core/src/jmh/java/com/netflix/hystrix/perf/RollingPercentilePerfTest.java index e90256ab9..48f9aba6f 100644 --- a/hystrix-core/src/jmh/java/com/netflix/hystrix/perf/RollingPercentilePerfTest.java +++ b/hystrix-core/src/jmh/java/com/netflix/hystrix/perf/RollingPercentilePerfTest.java @@ -18,139 +18,140 @@ import java.util.concurrent.TimeUnit; public class RollingPercentilePerfTest { - @State(Scope.Thread) - public static class PercentileState { - HystrixRollingPercentile percentile; - - @Param({"true", "false"}) - public boolean percentileEnabled; - - @Setup(Level.Iteration) - public void setUp() { - percentile = new HystrixRollingPercentile( - HystrixProperty.Factory.asProperty(100), - HystrixProperty.Factory.asProperty(10), - HystrixProperty.Factory.asProperty(percentileEnabled)); - } - } - - @State(Scope.Thread) - public static class LatencyState { - final Random r = new Random(); - - int latency; - - @Setup(Level.Invocation) - public void setUp() { - latency = r.nextInt(100); - } - } - - @Benchmark - @BenchmarkMode({Mode.Throughput}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public HystrixRollingPercentile writeOnly(PercentileState percentileState, LatencyState latencyState) { - percentileState.percentile.addValue(latencyState.latency); - return percentileState.percentile; - } - - @Benchmark - @BenchmarkMode({Mode.Throughput}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public int readOnly(PercentileState percentileState) { - HystrixRollingPercentile percentile = percentileState.percentile; - return percentile.getMean() + - percentile.getPercentile(10) + - percentile.getPercentile(25) + - percentile.getPercentile(50) + - percentile.getPercentile(75) + - percentile.getPercentile(90) + - percentile.getPercentile(95) + - percentile.getPercentile(99) + - percentile.getPercentile(99.5); - } - - @Benchmark - @Group("writeHeavy") - @GroupThreads(7) - @BenchmarkMode({Mode.Throughput}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public HystrixRollingPercentile writeHeavyLatencyAdd(PercentileState percentileState, LatencyState latencyState) { - percentileState.percentile.addValue(latencyState.latency); - return percentileState.percentile; - } - - @Benchmark - @Group("writeHeavy") - @GroupThreads(1) - @BenchmarkMode({Mode.Throughput}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public int writeHeavyReadMetrics(PercentileState percentileState) { - HystrixRollingPercentile percentile = percentileState.percentile; - return percentile.getMean() + - percentile.getPercentile(10) + - percentile.getPercentile(25) + - percentile.getPercentile(50) + - percentile.getPercentile(75) + - percentile.getPercentile(90) + - percentile.getPercentile(95) + - percentile.getPercentile(99) + - percentile.getPercentile(99.5); - } - - @Benchmark - @Group("evenSplit") - @GroupThreads(4) - @BenchmarkMode({Mode.Throughput}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public HystrixRollingPercentile evenSplitLatencyAdd(PercentileState percentileState, LatencyState latencyState) { - percentileState.percentile.addValue(latencyState.latency); - return percentileState.percentile; - } - - @Benchmark - @Group("evenSplit") - @GroupThreads(4) - @BenchmarkMode({Mode.Throughput}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public int evenSplitReadMetrics(PercentileState percentileState) { - HystrixRollingPercentile percentile = percentileState.percentile; - return percentile.getMean() + - percentile.getPercentile(10) + - percentile.getPercentile(25) + - percentile.getPercentile(50) + - percentile.getPercentile(75) + - percentile.getPercentile(90) + - percentile.getPercentile(95) + - percentile.getPercentile(99) + - percentile.getPercentile(99.5); - } - - @Benchmark - @Group("readHeavy") - @GroupThreads(1) - @BenchmarkMode({Mode.Throughput}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public HystrixRollingPercentile readHeavyLatencyAdd(PercentileState percentileState, LatencyState latencyState) { - percentileState.percentile.addValue(latencyState.latency); - return percentileState.percentile; - } - - @Benchmark - @Group("readHeavy") - @GroupThreads(7) - @BenchmarkMode({Mode.Throughput}) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public int readHeavyReadMetrics(PercentileState percentileState) { - HystrixRollingPercentile percentile = percentileState.percentile; - return percentile.getMean() + - percentile.getPercentile(10) + - percentile.getPercentile(25) + - percentile.getPercentile(50) + - percentile.getPercentile(75) + - percentile.getPercentile(90) + - percentile.getPercentile(95) + - percentile.getPercentile(99) + - percentile.getPercentile(99.5); - } + @State(Scope.Thread) + public static class PercentileState { + HystrixRollingPercentile percentile; + + @Param({"true", "false"}) + public boolean percentileEnabled; + + @Setup(Level.Iteration) + public void setUp() { + percentile = new HystrixRollingPercentile( + HystrixProperty.Factory.asProperty(100), + HystrixProperty.Factory.asProperty(10), + HystrixProperty.Factory.asProperty(1000), + HystrixProperty.Factory.asProperty(percentileEnabled)); + } + } + + @State(Scope.Thread) + public static class LatencyState { + final Random r = new Random(); + + int latency; + + @Setup(Level.Invocation) + public void setUp() { + latency = r.nextInt(100); + } + } + + @Benchmark + @BenchmarkMode({Mode.Throughput}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public HystrixRollingPercentile writeOnly(PercentileState percentileState, LatencyState latencyState) { + percentileState.percentile.addValue(latencyState.latency); + return percentileState.percentile; + } + + @Benchmark + @BenchmarkMode({Mode.Throughput}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int readOnly(PercentileState percentileState) { + HystrixRollingPercentile percentile = percentileState.percentile; + return percentile.getMean() + + percentile.getPercentile(10) + + percentile.getPercentile(25) + + percentile.getPercentile(50) + + percentile.getPercentile(75) + + percentile.getPercentile(90) + + percentile.getPercentile(95) + + percentile.getPercentile(99) + + percentile.getPercentile(99.5); + } + + @Benchmark + @Group("writeHeavy") + @GroupThreads(7) + @BenchmarkMode({Mode.Throughput}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public HystrixRollingPercentile writeHeavyLatencyAdd(PercentileState percentileState, LatencyState latencyState) { + percentileState.percentile.addValue(latencyState.latency); + return percentileState.percentile; + } + + @Benchmark + @Group("writeHeavy") + @GroupThreads(1) + @BenchmarkMode({Mode.Throughput}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int writeHeavyReadMetrics(PercentileState percentileState) { + HystrixRollingPercentile percentile = percentileState.percentile; + return percentile.getMean() + + percentile.getPercentile(10) + + percentile.getPercentile(25) + + percentile.getPercentile(50) + + percentile.getPercentile(75) + + percentile.getPercentile(90) + + percentile.getPercentile(95) + + percentile.getPercentile(99) + + percentile.getPercentile(99.5); + } + + @Benchmark + @Group("evenSplit") + @GroupThreads(4) + @BenchmarkMode({Mode.Throughput}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public HystrixRollingPercentile evenSplitLatencyAdd(PercentileState percentileState, LatencyState latencyState) { + percentileState.percentile.addValue(latencyState.latency); + return percentileState.percentile; + } + + @Benchmark + @Group("evenSplit") + @GroupThreads(4) + @BenchmarkMode({Mode.Throughput}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int evenSplitReadMetrics(PercentileState percentileState) { + HystrixRollingPercentile percentile = percentileState.percentile; + return percentile.getMean() + + percentile.getPercentile(10) + + percentile.getPercentile(25) + + percentile.getPercentile(50) + + percentile.getPercentile(75) + + percentile.getPercentile(90) + + percentile.getPercentile(95) + + percentile.getPercentile(99) + + percentile.getPercentile(99.5); + } + + @Benchmark + @Group("readHeavy") + @GroupThreads(1) + @BenchmarkMode({Mode.Throughput}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public HystrixRollingPercentile readHeavyLatencyAdd(PercentileState percentileState, LatencyState latencyState) { + percentileState.percentile.addValue(latencyState.latency); + return percentileState.percentile; + } + + @Benchmark + @Group("readHeavy") + @GroupThreads(7) + @BenchmarkMode({Mode.Throughput}) + @OutputTimeUnit(TimeUnit.MILLISECONDS) + public int readHeavyReadMetrics(PercentileState percentileState) { + HystrixRollingPercentile percentile = percentileState.percentile; + return percentile.getMean() + + percentile.getPercentile(10) + + percentile.getPercentile(25) + + percentile.getPercentile(50) + + percentile.getPercentile(75) + + percentile.getPercentile(90) + + percentile.getPercentile(95) + + percentile.getPercentile(99) + + percentile.getPercentile(99.5); + } } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapserMetrics.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapserMetrics.java index 31b61b8bf..3ded36764 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapserMetrics.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCollapserMetrics.java @@ -92,8 +92,8 @@ public static Collection getInstances() { this.key = key; this.properties = properties; - this.percentileBatchSize = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileEnabled()); - this.percentileShardSize = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileEnabled()); + this.percentileBatchSize = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileBucketSize(), properties.metricsRollingPercentileEnabled()); + this.percentileShardSize = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileBucketSize(), properties.metricsRollingPercentileEnabled()); } /** diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java index 5bae43294..22829db7e 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/HystrixCommandMetrics.java @@ -142,8 +142,8 @@ public static Collection getInstances() { this.group = commandGroup; this.threadPoolKey = threadPoolKey; this.properties = properties; - this.percentileExecution = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileEnabled()); - this.percentileTotal = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileEnabled()); + this.percentileExecution = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileBucketSize(), properties.metricsRollingPercentileEnabled()); + this.percentileTotal = new HystrixRollingPercentile(properties.metricsRollingPercentileWindowInMilliseconds(), properties.metricsRollingPercentileWindowBuckets(), properties.metricsRollingPercentileBucketSize(), properties.metricsRollingPercentileEnabled()); this.eventNotifier = eventNotifier; } diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingNumber.java b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingNumber.java index fea94ad15..c9bb58d76 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingNumber.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingNumber.java @@ -19,7 +19,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; -import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.locks.ReentrantLock; @@ -93,7 +92,7 @@ private HystrixRollingNumber(Time time, HystrixProperty timeInMilliseco * HystrixRollingNumberEvent defining which counter to increment */ public void increment(HystrixRollingNumberEvent type) { - getCurrentBucket().increment(type); + getCurrentBucket().getAdder(type).increment(); } /** @@ -107,7 +106,7 @@ public void increment(HystrixRollingNumberEvent type) { * long value to be added to the current bucket */ public void add(HystrixRollingNumberEvent type, long value) { - getCurrentBucket().add(type, value); + getCurrentBucket().getAdder(type).add(value); } /** @@ -119,7 +118,7 @@ public void add(HystrixRollingNumberEvent type, long value) { * @param value long value to be given to the max updater */ public void updateRollingMax(HystrixRollingNumberEvent type, long value) { - getCurrentBucket().updateMax(type, value); + getCurrentBucket().getMaxUpdater(type).update(value); } /** @@ -172,7 +171,7 @@ public long getRollingSum(HystrixRollingNumberEvent type) { long sum = 0; for (Bucket b : buckets) { - sum += b.get(type); + sum += b.getAdder(type).sum(); } return sum; } @@ -218,7 +217,11 @@ public long[] getValues(HystrixRollingNumberEvent type) { long values[] = new long[bucketArray.length]; int i = 0; for (Bucket bucket : bucketArray) { - values[i++] = bucket.get(type); + if (type.isCounter()) { + values[i++] = bucket.getAdder(type).sum(); + } else if (type.isMaxUpdater()) { + values[i++] = bucket.getMaxUpdater(type).max(); + } } return values; } @@ -358,54 +361,59 @@ public long getCurrentTimeInMillis() { */ /* package */static class Bucket { final long windowStart; - final AtomicLong[] values; + final LongAdder[] adderForCounterType; + final LongMaxUpdater[] updaterForCounterType; Bucket(long startTime) { this.windowStart = startTime; /* - * Put both sums and maxes in single array. Each HystrixRollingNumberEvent is one or the other + * We support both LongAdder and LongMaxUpdater in a bucket but don't want the memory allocation + * of all types for each so we only allocate the objects if the HystrixRollingNumberEvent matches + * the correct type - though we still have the allocation of empty arrays to the given length + * as we want to keep using the type.ordinal() value for fast random access. */ - values = new AtomicLong[HystrixRollingNumberEvent.values().length]; + // initialize the array of LongAdders + adderForCounterType = new LongAdder[HystrixRollingNumberEvent.values().length]; + for (HystrixRollingNumberEvent type : HystrixRollingNumberEvent.values()) { + if (type.isCounter()) { + adderForCounterType[type.ordinal()] = new LongAdder(); + } + } - for (HystrixRollingNumberEvent type: HystrixRollingNumberEvent.values()) { - values[type.ordinal()] = new AtomicLong(0L); + updaterForCounterType = new LongMaxUpdater[HystrixRollingNumberEvent.values().length]; + for (HystrixRollingNumberEvent type : HystrixRollingNumberEvent.values()) { + if (type.isMaxUpdater()) { + updaterForCounterType[type.ordinal()] = new LongMaxUpdater(); + // initialize to 0 otherwise it is Long.MIN_VALUE + updaterForCounterType[type.ordinal()].update(0); + } } } long get(HystrixRollingNumberEvent type) { - return values[type.ordinal()].get(); - } - - /* package for testing */ void increment(HystrixRollingNumberEvent type) { - add(type, 1); - } - - /* package for testing */ void add(HystrixRollingNumberEvent type, long value) { if (type.isCounter()) { - values[type.ordinal()].addAndGet(value); - return; + return adderForCounterType[type.ordinal()].sum(); } if (type.isMaxUpdater()) { - throw new IllegalStateException("Cannot increment a max-updater"); + return updaterForCounterType[type.ordinal()].max(); } throw new IllegalStateException("Unknown type of event: " + type.name()); } - /* package for testing */ void updateMax(HystrixRollingNumberEvent type, long value) { - if (type.isMaxUpdater()) { - long currentMax = values[type.ordinal()].get(); - if (value > currentMax) { - values[type.ordinal()].set(value); - } - return; - } - if (type.isCounter()) { - throw new IllegalStateException("Cannot update max on a counter"); + LongAdder getAdder(HystrixRollingNumberEvent type) { + if (!type.isCounter()) { + throw new IllegalStateException("Type is not a Counter: " + type.name()); } + return adderForCounterType[type.ordinal()]; + } - throw new IllegalStateException("Unknown type of event: " + type.name()); + LongMaxUpdater getMaxUpdater(HystrixRollingNumberEvent type) { + if (!type.isMaxUpdater()) { + throw new IllegalStateException("Type is not a MaxUpdater: " + type.name()); + } + return updaterForCounterType[type.ordinal()]; } } @@ -414,44 +422,71 @@ long get(HystrixRollingNumberEvent type) { * Cumulative counters (from start of JVM) from each Type */ /* package */static class CumulativeSum { - AtomicLong[] cumulativeValues; + final LongAdder[] adderForCounterType; + final LongMaxUpdater[] updaterForCounterType; CumulativeSum() { /* - * Put both sums and maxes in single array. Each HystrixRollingNumberEvent is one or the other + * We support both LongAdder and LongMaxUpdater in a bucket but don't want the memory allocation + * of all types for each so we only allocate the objects if the HystrixRollingNumberEvent matches + * the correct type - though we still have the allocation of empty arrays to the given length + * as we want to keep using the type.ordinal() value for fast random access. */ - cumulativeValues = new AtomicLong[HystrixRollingNumberEvent.values().length]; - for (HystrixRollingNumberEvent type: HystrixRollingNumberEvent.values()) { - cumulativeValues[type.ordinal()] = new AtomicLong(0L); + // initialize the array of LongAdders + adderForCounterType = new LongAdder[HystrixRollingNumberEvent.values().length]; + for (HystrixRollingNumberEvent type : HystrixRollingNumberEvent.values()) { + if (type.isCounter()) { + adderForCounterType[type.ordinal()] = new LongAdder(); + } + } + + updaterForCounterType = new LongMaxUpdater[HystrixRollingNumberEvent.values().length]; + for (HystrixRollingNumberEvent type : HystrixRollingNumberEvent.values()) { + if (type.isMaxUpdater()) { + updaterForCounterType[type.ordinal()] = new LongMaxUpdater(); + // initialize to 0 otherwise it is Long.MIN_VALUE + updaterForCounterType[type.ordinal()].update(0); + } } } public void addBucket(Bucket lastBucket) { for (HystrixRollingNumberEvent type : HystrixRollingNumberEvent.values()) { if (type.isCounter()) { - cumulativeValues[type.ordinal()].addAndGet(lastBucket.get(type)); + getAdder(type).add(lastBucket.getAdder(type).sum()); } if (type.isMaxUpdater()) { - long addedBucketMax = lastBucket.get(type); - AtomicLong currentBucketMax = cumulativeValues[type.ordinal()]; - if (addedBucketMax > currentBucketMax.get()) { - cumulativeValues[type.ordinal()].set(addedBucketMax); - } + getMaxUpdater(type).update(lastBucket.getMaxUpdater(type).max()); } } } long get(HystrixRollingNumberEvent type) { if (type.isCounter()) { - return cumulativeValues[type.ordinal()].get(); + return adderForCounterType[type.ordinal()].sum(); } if (type.isMaxUpdater()) { - return cumulativeValues[type.ordinal()].get(); + return updaterForCounterType[type.ordinal()].max(); } throw new IllegalStateException("Unknown type of event: " + type.name()); } + + LongAdder getAdder(HystrixRollingNumberEvent type) { + if (!type.isCounter()) { + throw new IllegalStateException("Type is not a Counter: " + type.name()); + } + return adderForCounterType[type.ordinal()]; + } + + LongMaxUpdater getMaxUpdater(HystrixRollingNumberEvent type) { + if (!type.isMaxUpdater()) { + throw new IllegalStateException("Type is not a MaxUpdater: " + type.name()); + } + return updaterForCounterType[type.ordinal()]; + } + } /** diff --git a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java index 151f2e90a..6a2f898be 100644 --- a/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java +++ b/hystrix-core/src/main/java/com/netflix/hystrix/util/HystrixRollingPercentile.java @@ -19,13 +19,12 @@ import java.util.Arrays; import java.util.Collections; import java.util.Iterator; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicIntegerArray; import java.util.concurrent.atomic.AtomicReference; import java.util.concurrent.atomic.AtomicReferenceArray; import java.util.concurrent.locks.ReentrantLock; -import org.HdrHistogram.Histogram; -import org.HdrHistogram.IntCountsHistogram; -import org.HdrHistogram.Recorder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,16 +49,15 @@ public class HystrixRollingPercentile { private static final Time ACTUAL_TIME = new ActualTime(); private final Time time; /* package for testing */ final BucketCircularArray buckets; - private final int timeInMilliseconds; - private final int numberOfBuckets; - private final boolean enabled; - - private final int bucketSizeInMilliseconds; + private final HystrixProperty timeInMilliseconds; + private final HystrixProperty numberOfBuckets; + private final HystrixProperty bucketDataLength; + private final HystrixProperty enabled; /* * This will get flipped each time a new bucket is created. */ - /* package for testing */ volatile PercentileSnapshot currentPercentileSnapshot = new PercentileSnapshot(); + /* package for testing */ volatile PercentileSnapshot currentPercentileSnapshot = new PercentileSnapshot(0); /** * @@ -71,36 +69,32 @@ public class HystrixRollingPercentile { * {@code HystrixProperty} for number of buckets that the time window should be divided into *

* Example: 12 for 5 second buckets in a 1 minute window + * @param bucketDataLength + * {@code HystrixProperty} for number of values stored in each bucket + *

+ * Example: 1000 to store a max of 1000 values in each 5 second bucket * @param enabled * {@code HystrixProperty} whether data should be tracked and percentiles calculated. *

* If 'false' methods will do nothing. */ - public HystrixRollingPercentile(HystrixProperty timeInMilliseconds, HystrixProperty numberOfBuckets, HystrixProperty enabled) { - this(ACTUAL_TIME, timeInMilliseconds.get(), numberOfBuckets.get(), enabled.get()); - - } - /* - * @deprecated Use {@link HystrixRollingPercentile(HystrixProperty, HystrixProperty, HystrixProperty)} instead - */ - @Deprecated public HystrixRollingPercentile(HystrixProperty timeInMilliseconds, HystrixProperty numberOfBuckets, HystrixProperty bucketDataLength, HystrixProperty enabled) { - this(ACTUAL_TIME, timeInMilliseconds.get(), numberOfBuckets.get(), enabled.get()); + this(ACTUAL_TIME, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); + } - /* package for testing */ HystrixRollingPercentile(Time time, int timeInMilliseconds, int numberOfBuckets, boolean enabled) { + /* package for testing */ HystrixRollingPercentile(Time time, HystrixProperty timeInMilliseconds, HystrixProperty numberOfBuckets, HystrixProperty bucketDataLength, HystrixProperty enabled) { this.time = time; this.timeInMilliseconds = timeInMilliseconds; this.numberOfBuckets = numberOfBuckets; + this.bucketDataLength = bucketDataLength; this.enabled = enabled; - if (this.timeInMilliseconds % this.numberOfBuckets != 0) { + if (this.timeInMilliseconds.get() % this.numberOfBuckets.get() != 0) { throw new IllegalArgumentException("The timeInMilliseconds must divide equally into numberOfBuckets. For example 1000/10 is ok, 1000/11 is not."); } - this.bucketSizeInMilliseconds = timeInMilliseconds / numberOfBuckets; - - buckets = new BucketCircularArray(this.numberOfBuckets); + buckets = new BucketCircularArray(this.numberOfBuckets.get()); } /** @@ -111,12 +105,12 @@ public HystrixRollingPercentile(HystrixProperty timeInMilliseconds, Hys */ public void addValue(int... value) { /* no-op if disabled */ - if (!enabled) + if (!enabled.get()) return; for (int v : value) { try { - getCurrentBucket().bucketData.addValue(v); + getCurrentBucket().data.addValue(v); } catch (Exception e) { logger.error("Failed to add value: " + v, e); } @@ -136,7 +130,7 @@ public void addValue(int... value) { */ public int getPercentile(double percentile) { /* no-op if disabled */ - if (!enabled) + if (!enabled.get()) return -1; // force logic to move buckets forward in case other requests aren't making it happen @@ -152,7 +146,7 @@ public int getPercentile(double percentile) { */ public int getMean() { /* no-op if disabled */ - if (!enabled) + if (!enabled.get()) return -1; // force logic to move buckets forward in case other requests aren't making it happen @@ -172,6 +166,10 @@ private PercentileSnapshot getCurrentPercentileSnapshot() { return currentPercentileSnapshot; } + private int getBucketSizeInMilliseconds() { + return timeInMilliseconds.get() / numberOfBuckets.get(); + } + private ReentrantLock newBucketLock = new ReentrantLock(); private Bucket getCurrentBucket() { @@ -185,7 +183,7 @@ private Bucket getCurrentBucket() { * NOTE: This is thread-safe because it's accessing 'buckets' which is a LinkedBlockingDeque */ Bucket currentBucket = buckets.peekLast(); - if (currentBucket != null && currentTime < currentBucket.windowStart + bucketSizeInMilliseconds) { + if (currentBucket != null && currentTime < currentBucket.windowStart + getBucketSizeInMilliseconds()) { // if we're within the bucket 'window of time' return the current one // NOTE: We do not worry if we are BEFORE the window in a weird case of where thread scheduling causes that to occur, // we'll just use the latest as long as we're not AFTER the window @@ -220,21 +218,21 @@ private Bucket getCurrentBucket() { try { if (buckets.peekLast() == null) { // the list is empty so create the first bucket - Bucket newBucket = new Bucket(currentTime); + Bucket newBucket = new Bucket(currentTime, bucketDataLength.get()); buckets.addLast(newBucket); return newBucket; } else { // We go into a loop so that it will create as many buckets as needed to catch up to the current time // as we want the buckets complete even if we don't have transactions during a period of time. - for (int i = 0; i < numberOfBuckets; i++) { + for (int i = 0; i < numberOfBuckets.get(); i++) { // we have at least 1 bucket so retrieve it Bucket lastBucket = buckets.peekLast(); - if (currentTime < lastBucket.windowStart + bucketSizeInMilliseconds) { + if (currentTime < lastBucket.windowStart + getBucketSizeInMilliseconds()) { // if we're within the bucket 'window of time' return the current one // NOTE: We do not worry if we are BEFORE the window in a weird case of where thread scheduling causes that to occur, // we'll just use the latest as long as we're not AFTER the window return lastBucket; - } else if (currentTime - (lastBucket.windowStart + bucketSizeInMilliseconds) > timeInMilliseconds) { + } else if (currentTime - (lastBucket.windowStart + getBucketSizeInMilliseconds()) > timeInMilliseconds.get()) { // the time passed is greater than the entire rolling counter so we want to clear it all and start from scratch reset(); // recursively call getCurrentBucket which will create a new bucket and return it @@ -242,7 +240,7 @@ private Bucket getCurrentBucket() { } else { // we're past the window so we need to create a new bucket Bucket[] allBuckets = buckets.getArray(); // create a new bucket and add it as the new 'last' (once this is done other threads will start using it on subsequent retrievals) - buckets.addLast(new Bucket(lastBucket.windowStart + bucketSizeInMilliseconds)); + buckets.addLast(new Bucket(lastBucket.windowStart + getBucketSizeInMilliseconds(), bucketDataLength.get())); // we created a new bucket so let's re-generate the PercentileSnapshot (not including the new bucket) currentPercentileSnapshot = new PercentileSnapshot(allBuckets); } @@ -276,80 +274,91 @@ private Bucket getCurrentBucket() { */ public void reset() { /* no-op if disabled */ - if (!enabled) + if (!enabled.get()) return; // clear buckets so we start over again buckets.clear(); } - /*package-private*/ static class PercentileBucketData { - final Recorder recorder; - final AtomicReference stableHistogram = new AtomicReference(null); + /* package-private for testing */ static class PercentileBucketData { + private final int length; + private final AtomicIntegerArray list; + private final AtomicInteger index = new AtomicInteger(); - public PercentileBucketData() { - this.recorder = new Recorder(4); + public PercentileBucketData(int dataLength) { + this.length = dataLength; + this.list = new AtomicIntegerArray(dataLength); } public void addValue(int... latency) { - for (int l: latency) { - recorder.recordValue(l); + for (int l : latency) { + /* We just wrap around the beginning and over-write if we go past 'dataLength' as that will effectively cause us to "sample" the most recent data */ + list.set(index.getAndIncrement() % length, l); + // TODO Alternative to AtomicInteger? The getAndIncrement may be a source of contention on high throughput circuits on large multi-core systems. + // LongAdder isn't suited to this as it is not consistent. Perhaps a different data structure that doesn't need indexed adds? + // A threadlocal data storage that only aggregates when fetched would be ideal. Similar to LongAdder except for accumulating lists of data. } } + + public int length() { + if (index.get() > list.length()) { + return list.length(); + } else { + return index.get(); + } + } + } /** * @NotThreadSafe */ /* package for testing */ static class PercentileSnapshot { - /* package-private*/ final IntCountsHistogram aggregateHistogram; - private final long count; - private final int mean; - private final int p0; - private final int p5; - private final int p10; - private final int p25; - private final int p50; - private final int p75; - private final int p90; - private final int p95; - private final int p99; - private final int p995; - private final int p999; - private final int p100; - - - /* package for testing */ PercentileSnapshot() { - this(new Bucket[0]); - } + private final int[] data; + private final int length; + private int mean; - /* package for testing */ PercentileSnapshot(long startTime, int... data) { - this(new Bucket[]{new Bucket(startTime, data)}); + /* package for testing */ PercentileSnapshot(Bucket[] buckets) { + int lengthFromBuckets = 0; + // we need to calculate it dynamically as it could have been changed by properties (rare, but possible) + // also this way we capture the actual index size rather than the max so size the int[] to only what we need + for (Bucket bd : buckets) { + lengthFromBuckets += bd.data.length; + } + data = new int[lengthFromBuckets]; + int index = 0; + int sum = 0; + for (Bucket bd : buckets) { + PercentileBucketData pbd = bd.data; + int length = pbd.length(); + for (int i = 0; i < length; i++) { + int v = pbd.list.get(i); + this.data[index++] = v; + sum += v; + } + } + this.length = index; + if (this.length == 0) { + this.mean = 0; + } else { + this.mean = sum / this.length; + } + + Arrays.sort(this.data, 0, length); } - /* package for testing */ PercentileSnapshot(Bucket[] buckets) { - aggregateHistogram = new IntCountsHistogram(4); - for (Bucket bucket: buckets) { - PercentileBucketData bucketData = bucket.bucketData; - //if stable snapshot not already generated, generate it now - bucketData.stableHistogram.compareAndSet(null, bucketData.recorder.getIntervalHistogram()); - aggregateHistogram.add(bucket.bucketData.stableHistogram.get()); + /* package for testing */ PercentileSnapshot(int... data) { + this.data = data; + this.length = data.length; + + int sum = 0; + for (int v : data) { + sum += v; } + this.mean = sum / this.length; - count = aggregateHistogram.getTotalCount(); - mean = (int) aggregateHistogram.getMean(); - p0 = (int) aggregateHistogram.getValueAtPercentile(0); - p5 = (int) aggregateHistogram.getValueAtPercentile(5); - p10 = (int) aggregateHistogram.getValueAtPercentile(10); - p25 = (int) aggregateHistogram.getValueAtPercentile(25); - p50 = (int) aggregateHistogram.getValueAtPercentile(50); - p75 = (int) aggregateHistogram.getValueAtPercentile(75); - p90 = (int) aggregateHistogram.getValueAtPercentile(90); - p95 = (int) aggregateHistogram.getValueAtPercentile(95); - p99 = (int) aggregateHistogram.getValueAtPercentile(99); - p995 = (int) aggregateHistogram.getValueAtPercentile(99.5); - p999 = (int) aggregateHistogram.getValueAtPercentile(99.9); - p100 = (int) aggregateHistogram.getValueAtPercentile(100); + Arrays.sort(this.data, 0, length); } /* package for testing */ int getMean() { @@ -360,31 +369,48 @@ public void addValue(int... latency) { * Provides percentile computation. */ public int getPercentile(double percentile) { - if (count == 0) { + if (length == 0) { + return 0; + } + return computePercentile(percentile); + } + + /** + * @see Percentile (Wikipedia) + * @see Percentile + * + * @param percent percentile of data desired + * @return data at the asked-for percentile. Interpolation is used if exactness is not possible + */ + private int computePercentile(double percent) { + // Some just-in-case edge cases + if (length <= 0) { return 0; + } else if (percent <= 0.0) { + return data[0]; + } else if (percent >= 100.0) { + return data[length - 1]; } - int permyriad = (int) (percentile * 100); - switch(permyriad) { - case 0 : return p0; - case 500 : return p5; - case 1000: return p10; - case 2500: return p25; - case 5000: return p50; - case 7500: return p75; - case 9000: return p90; - case 9500: return p95; - case 9900: return p99; - case 9950: return p995; - case 9990: return p999; - case 10000: return p100; - default: return getArbitraryPercentile(percentile); + // ranking (http://en.wikipedia.org/wiki/Percentile#Alternative_methods) + double rank = (percent / 100.0) * length; + + // linear interpolation between closest ranks + int iLow = (int) Math.floor(rank); + int iHigh = (int) Math.ceil(rank); + assert 0 <= iLow && iLow <= rank && rank <= iHigh && iHigh <= length; + assert (iHigh - iLow) <= 1; + if (iHigh >= length) { + // Another edge case + return data[length - 1]; + } else if (iLow == iHigh) { + return data[iLow]; + } else { + // Interpolate between the two bounding values + return (int) (data[iLow] + (rank - iLow) * (data[iHigh] - data[iLow])); } } - private synchronized int getArbitraryPercentile(double percentile) { - return (int) aggregateHistogram.getValueAtPercentile(percentile); - } } /** @@ -568,19 +594,13 @@ private Bucket[] getArray() { */ /* package for testing */ static class Bucket { final long windowStart; - final PercentileBucketData bucketData; + final PercentileBucketData data; - Bucket(long startTime) { + Bucket(long startTime, int bucketDataLength) { this.windowStart = startTime; - this.bucketData = new PercentileBucketData(); + this.data = new PercentileBucketData(bucketDataLength); } - public Bucket(long startTime, int[] data) { - this.windowStart = startTime; - - this.bucketData = new PercentileBucketData(); - bucketData.addValue(data); - } } /* package for testing */ static interface Time { diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingNumberTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingNumberTest.java index b4711c033..5cc815c58 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingNumberTest.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingNumberTest.java @@ -134,9 +134,9 @@ public void testIncrementInSingleBucket() { assertEquals(1, counter.buckets.size()); // the count should be 4 - assertEquals(4, counter.buckets.getLast().get(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(2, counter.buckets.getLast().get(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, counter.buckets.getLast().get(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(4, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SUCCESS).sum()); + assertEquals(2, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.FAILURE).sum()); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); } catch (Exception e) { e.printStackTrace(); @@ -157,7 +157,7 @@ public void testTimeout() { assertEquals(1, counter.buckets.size()); // the count should be 1 - assertEquals(1, counter.buckets.getLast().get(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); // sleep to get to a new bucket @@ -170,7 +170,7 @@ public void testTimeout() { assertEquals(4, counter.buckets.size()); // the counts of the last bucket - assertEquals(1, counter.buckets.getLast().get(HystrixRollingNumberEvent.TIMEOUT)); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); // the total counts assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.TIMEOUT)); @@ -194,7 +194,7 @@ public void testShortCircuited() { assertEquals(1, counter.buckets.size()); // the count should be 1 - assertEquals(1, counter.buckets.getLast().get(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); assertEquals(1, counter.getRollingSum(HystrixRollingNumberEvent.SHORT_CIRCUITED)); // sleep to get to a new bucket @@ -207,7 +207,7 @@ public void testShortCircuited() { assertEquals(4, counter.buckets.size()); // the counts of the last bucket - assertEquals(1, counter.buckets.getLast().get(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); // the total counts assertEquals(2, counter.getRollingSum(HystrixRollingNumberEvent.SHORT_CIRCUITED)); @@ -250,7 +250,7 @@ private void testCounterType(HystrixRollingNumberEvent type) { assertEquals(1, counter.buckets.size()); // the count should be 1 - assertEquals(1, counter.buckets.getLast().get(type)); + assertEquals(1, counter.buckets.getLast().getAdder(type).sum()); assertEquals(1, counter.getRollingSum(type)); // sleep to get to a new bucket @@ -263,7 +263,7 @@ private void testCounterType(HystrixRollingNumberEvent type) { assertEquals(4, counter.buckets.size()); // the counts of the last bucket - assertEquals(1, counter.buckets.getLast().get(type)); + assertEquals(1, counter.buckets.getLast().getAdder(type).sum()); // the total counts assertEquals(2, counter.getRollingSum(type)); @@ -307,10 +307,10 @@ public void testIncrementInMultipleBuckets() { assertEquals(4, counter.buckets.size()); // the counts of the last bucket - assertEquals(2, counter.buckets.getLast().get(HystrixRollingNumberEvent.SUCCESS)); - assertEquals(3, counter.buckets.getLast().get(HystrixRollingNumberEvent.FAILURE)); - assertEquals(1, counter.buckets.getLast().get(HystrixRollingNumberEvent.TIMEOUT)); - assertEquals(1, counter.buckets.getLast().get(HystrixRollingNumberEvent.SHORT_CIRCUITED)); + assertEquals(2, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SUCCESS).sum()); + assertEquals(3, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.FAILURE).sum()); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.TIMEOUT).sum()); + assertEquals(1, counter.buckets.getLast().getAdder(HystrixRollingNumberEvent.SHORT_CIRCUITED).sum()); // the total counts assertEquals(6, counter.getRollingSum(HystrixRollingNumberEvent.SUCCESS)); @@ -395,7 +395,7 @@ public void testUpdateMax1() { assertEquals(1, counter.buckets.size()); // the count should be 10 - assertEquals(10, counter.buckets.getLast().get(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); + assertEquals(10, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); assertEquals(10, counter.getRollingMaxValue(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); // sleep to get to a new bucket @@ -408,7 +408,7 @@ public void testUpdateMax1() { assertEquals(4, counter.buckets.size()); // the max - assertEquals(20, counter.buckets.getLast().get(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); + assertEquals(20, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); // counts per bucket long values[] = counter.getValues(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE); @@ -438,7 +438,7 @@ public void testUpdateMax2() { assertEquals(1, counter.buckets.size()); // the count should be 30 - assertEquals(30, counter.buckets.getLast().get(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); + assertEquals(30, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); assertEquals(30, counter.getRollingMaxValue(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); // sleep to get to a new bucket @@ -452,7 +452,7 @@ public void testUpdateMax2() { assertEquals(4, counter.buckets.size()); // the count - assertEquals(50, counter.buckets.getLast().get(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); + assertEquals(50, counter.buckets.getLast().getMaxUpdater(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE).max()); assertEquals(50, counter.getValueOfLatestBucket(HystrixRollingNumberEvent.THREAD_MAX_ACTIVE)); // values per bucket diff --git a/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingPercentileTest.java b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingPercentileTest.java index d34be0579..964981c5e 100644 --- a/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingPercentileTest.java +++ b/hystrix-core/src/test/java/com/netflix/hystrix/util/HystrixRollingPercentileTest.java @@ -37,9 +37,10 @@ public class HystrixRollingPercentileTest { - private static final int timeInMilliseconds = 60000; - private static final int numberOfBuckets = 12; // 12 buckets at 5000ms each - private static final boolean enabled = true; + private static final HystrixProperty timeInMilliseconds = HystrixProperty.Factory.asProperty(60000); + private static final HystrixProperty numberOfBuckets = HystrixProperty.Factory.asProperty(12); // 12 buckets at 5000ms each + private static final HystrixProperty bucketDataLength = HystrixProperty.Factory.asProperty(1000); + private static final HystrixProperty enabled = HystrixProperty.Factory.asProperty(true); private static ExecutorService threadPool; @@ -61,7 +62,7 @@ public static void tearDown() { @Test public void testRolling() { MockedTime time = new MockedTime(); - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, enabled); + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); p.addValue(1000); p.addValue(1000); p.addValue(1000); @@ -103,7 +104,7 @@ public void testRolling() { time.increment(6000); // the rolling version should have the same data as creating a snapshot like this - PercentileSnapshot ps = new PercentileSnapshot(System.currentTimeMillis(), 1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600); + PercentileSnapshot ps = new PercentileSnapshot(1000, 1000, 1000, 2000, 1000, 500, 200, 200, 1600, 200, 1600, 1600); assertEquals(ps.getPercentile(0.15), p.getPercentile(0.15)); assertEquals(ps.getPercentile(0.50), p.getPercentile(0.50)); @@ -124,7 +125,7 @@ public void testRolling() { @Test public void testValueIsZeroAfterRollingWindowPassesAndNoTraffic() { MockedTime time = new MockedTime(); - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, enabled); + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); p.addValue(1000); p.addValue(1000); p.addValue(1000); @@ -142,11 +143,7 @@ public void testValueIsZeroAfterRollingWindowPassesAndNoTraffic() { assertEquals(1, p.buckets.size()); // a bucket has been created so we have a new percentile - assertEquals(1000, p.getPercentile(0)); - assertEquals(1000, p.getPercentile(25)); - assertEquals(1000, p.getPercentile(50)); - assertEquals(2000, p.getPercentile(75)); - assertEquals(4000, p.getPercentile(100)); + assertEquals(1500, p.getPercentile(50)); // let 1 minute pass time.increment(60000); @@ -160,18 +157,11 @@ public void testSampleDataOverTime1() { System.out.println("\n\n***************************** testSampleDataOverTime1 \n"); MockedTime time = new MockedTime(); - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, enabled); + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); int previousTime = 0; - - int maxSoFar = -1; - for (int i = 0; i < SampleDataHolder1.data.length; i++) { int timeInMillisecondsSinceStart = SampleDataHolder1.data[i][0]; int latency = SampleDataHolder1.data[i][1]; - if (latency > maxSoFar) { - System.out.println("New MAX latency : " + latency); - maxSoFar = latency; - } time.increment(timeInMillisecondsSinceStart - previousTime); previousTime = timeInMillisecondsSinceStart; p.addValue(latency); @@ -188,8 +178,6 @@ public void testSampleDataOverTime1() { System.out.println("Median: " + p.getPercentile(50)); System.out.println("Median: " + p.getPercentile(50)); - System.out.println("MAX : " + p.currentPercentileSnapshot.aggregateHistogram.getMaxValue()); - /* * In a loop as a use case was found where very different values were calculated in subsequent requests. */ @@ -209,7 +197,7 @@ public void testSampleDataOverTime2() { System.out.println("\n\n***************************** testSampleDataOverTime2 \n"); MockedTime time = new MockedTime(); int previousTime = 0; - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, enabled); + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, enabled); for (int i = 0; i < SampleDataHolder2.data.length; i++) { int timeInMillisecondsSinceStart = SampleDataHolder2.data[i][0]; int latency = SampleDataHolder2.data[i][1]; @@ -225,8 +213,6 @@ public void testSampleDataOverTime2() { System.out.println("99.5th: " + p.getPercentile(99.5)); System.out.println("99.99: " + p.getPercentile(99.99)); - System.out.println("MAX : " + p.currentPercentileSnapshot.aggregateHistogram.getMaxValue()); - if (p.getPercentile(50) > 90 || p.getPercentile(50) < 50) { fail("We expect around 60-70 but got: " + p.getPercentile(50)); } @@ -237,36 +223,61 @@ public void testSampleDataOverTime2() { } public PercentileSnapshot getPercentileForValues(int... values) { - return new PercentileSnapshot(System.currentTimeMillis(), values); + return new PercentileSnapshot(values); } @Test public void testPercentileAlgorithm_Median1() { - PercentileSnapshot list = new PercentileSnapshot(System.currentTimeMillis(), 100, 100, 100, 100, 200, 200, 200, 300, 300, 300, 300); + PercentileSnapshot list = new PercentileSnapshot(100, 100, 100, 100, 200, 200, 200, 300, 300, 300, 300); Assert.assertEquals(200, list.getPercentile(50)); } @Test public void testPercentileAlgorithm_Median2() { - PercentileSnapshot list = new PercentileSnapshot(System.currentTimeMillis(), 100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 500); + PercentileSnapshot list = new PercentileSnapshot(100, 100, 100, 100, 100, 100, 100, 100, 100, 100, 500); Assert.assertEquals(100, list.getPercentile(50)); } @Test public void testPercentileAlgorithm_Median3() { - PercentileSnapshot list = new PercentileSnapshot(System.currentTimeMillis(), 50, 75, 100, 125, 160, 170, 180, 200, 210, 300, 500); - Assert.assertEquals(170, list.getPercentile(50)); + PercentileSnapshot list = new PercentileSnapshot(50, 75, 100, 125, 160, 170, 180, 200, 210, 300, 500); + // list.addValue(50); // 1 + // list.addValue(75); // 2 + // list.addValue(100); // 3 + // list.addValue(125); // 4 + // list.addValue(160); // 5 + // list.addValue(170); // 6 + // list.addValue(180); // 7 + // list.addValue(200); // 8 + // list.addValue(210); // 9 + // list.addValue(300); // 10 + // list.addValue(500); // 11 + + Assert.assertEquals(175, list.getPercentile(50)); } @Test public void testPercentileAlgorithm_Median4() { - PercentileSnapshot list = new PercentileSnapshot(System.currentTimeMillis(), 300, 75, 125, 500, 100, 160, 180, 200, 210, 50, 170); - Assert.assertEquals(170, list.getPercentile(50)); + PercentileSnapshot list = new PercentileSnapshot(300, 75, 125, 500, 100, 160, 180, 200, 210, 50, 170); + // unsorted so it is expected to sort it for us + // list.addValue(300); // 10 + // list.addValue(75); // 2 + // list.addValue(125); // 4 + // list.addValue(500); // 11 + // list.addValue(100); // 3 + // list.addValue(160); // 5 + // list.addValue(180); // 7 + // list.addValue(200); // 8 + // list.addValue(210); // 9 + // list.addValue(50); // 1 + // list.addValue(170); // 6 + + Assert.assertEquals(175, list.getPercentile(50)); } @Test public void testPercentileAlgorithm_Extremes() { - PercentileSnapshot p = new PercentileSnapshot(System.currentTimeMillis(), 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 800, 768, 657, 700, 867); + PercentileSnapshot p = new PercentileSnapshot(2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 800, 768, 657, 700, 867); System.out.println("0.01: " + p.getPercentile(0.01)); System.out.println("10th: " + p.getPercentile(10)); @@ -290,10 +301,8 @@ public void testPercentileAlgorithm_Extremes() { @Test public void testPercentileAlgorithm_HighPercentile() { PercentileSnapshot p = getPercentileForValues(1, 2, 3); - Assert.assertEquals(1, p.getPercentile(0)); Assert.assertEquals(2, p.getPercentile(50)); - Assert.assertEquals(2, p.getPercentile(75)); - Assert.assertEquals(3, p.getPercentile(100)); + Assert.assertEquals(3, p.getPercentile(75)); } @Test @@ -306,17 +315,23 @@ public void testPercentileAlgorithm_LowPercentile() { @Test public void testPercentileAlgorithm_Percentiles() { PercentileSnapshot p = getPercentileForValues(10, 30, 20, 40); - Assert.assertEquals(10, p.getPercentile(30), 1.0e-5); - Assert.assertEquals(10, p.getPercentile(25), 1.0e-5); - Assert.assertEquals(30, p.getPercentile(75), 1.0e-5); - Assert.assertEquals(20, p.getPercentile(50), 1.0e-5); - Assert.assertEquals(40, p.getPercentile(90), 1.0e-5); + Assert.assertEquals(22, p.getPercentile(30), 1.0e-5); + Assert.assertEquals(20, p.getPercentile(25), 1.0e-5); + Assert.assertEquals(40, p.getPercentile(75), 1.0e-5); + Assert.assertEquals(30, p.getPercentile(50), 1.0e-5); // invalid percentiles Assert.assertEquals(10, p.getPercentile(-1)); Assert.assertEquals(40, p.getPercentile(101)); } + @Test + public void testPercentileAlgorithm_NISTExample() { + PercentileSnapshot p = getPercentileForValues(951772, 951567, 951937, 951959, 951442, 950610, 951591, 951195, 951772, 950925, 951990, 951682); + Assert.assertEquals(951983, p.getPercentile(90)); + Assert.assertEquals(951990, p.getPercentile(100)); + } + /** * This code should work without throwing exceptions but the data returned will all be -1 since the rolling percentile is disabled. */ @@ -324,7 +339,7 @@ public void testPercentileAlgorithm_Percentiles() { public void testDoesNothingWhenDisabled() { MockedTime time = new MockedTime(); int previousTime = 0; - HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, false); + HystrixRollingPercentile p = new HystrixRollingPercentile(time, timeInMilliseconds, numberOfBuckets, bucketDataLength, HystrixProperty.Factory.asProperty(false)); for (int i = 0; i < SampleDataHolder2.data.length; i++) { int timeInMillisecondsSinceStart = SampleDataHolder2.data[i][0]; int latency = SampleDataHolder2.data[i][1]; @@ -338,25 +353,10 @@ public void testDoesNothingWhenDisabled() { assertEquals(-1, p.getMean()); } - private static class MockedTime implements Time { - - private AtomicInteger time = new AtomicInteger(0); - - @Override - public long getCurrentTimeInMillis() { - return time.get(); - } - - public void increment(int millis) { - time.addAndGet(millis); - } - - } - @Test public void testThreadSafety() { final MockedTime time = new MockedTime(); - final HystrixRollingPercentile p = new HystrixRollingPercentile(time, 100, 25, true); + final HystrixRollingPercentile p = new HystrixRollingPercentile(time, HystrixProperty.Factory.asProperty(100), HystrixProperty.Factory.asProperty(25), HystrixProperty.Factory.asProperty(1000), HystrixProperty.Factory.asProperty(true)); final int NUM_THREADS = 1000; final int NUM_ITERATIONS = 1000000; @@ -370,7 +370,7 @@ public void testThreadSafety() { Future metricsPoller = threadPool.submit(new Runnable() { @Override public void run() { - while(!Thread.currentThread().isInterrupted()) { + while (!Thread.currentThread().isInterrupted()) { aggregateMetrics.addAndGet(p.getMean() + p.getPercentile(10) + p.getPercentile(50) + p.getPercentile(90)); //System.out.println("AGGREGATE : " + p.getPercentile(10) + " : " + p.getPercentile(50) + " : " + p.getPercentile(90)); } @@ -408,10 +408,10 @@ public void run() { @Test public void testWriteThreadSafety() { final MockedTime time = new MockedTime(); - final HystrixRollingPercentile p = new HystrixRollingPercentile(time, 100, 25, true); + final HystrixRollingPercentile p = new HystrixRollingPercentile(time, HystrixProperty.Factory.asProperty(100), HystrixProperty.Factory.asProperty(25), HystrixProperty.Factory.asProperty(1000), HystrixProperty.Factory.asProperty(true)); - final int NUM_THREADS = 1000; - final int NUM_ITERATIONS = 1000000; + final int NUM_THREADS = 10; + final int NUM_ITERATIONS = 1000; final CountDownLatch latch = new CountDownLatch(NUM_THREADS); @@ -435,7 +435,7 @@ public void run() { try { latch.await(100, TimeUnit.SECONDS); - assertEquals(added.get(), p.buckets.peekLast().bucketData.recorder.getIntervalHistogram().getTotalCount()); + assertEquals(added.get(), p.buckets.peekLast().data.length()); } catch (InterruptedException ex) { fail("Timeout on all threads writing percentiles"); } @@ -448,6 +448,22 @@ public void testThreadSafetyMulti() { } } + + private static class MockedTime implements Time { + + private AtomicInteger time = new AtomicInteger(0); + + @Override + public long getCurrentTimeInMillis() { + return time.get(); + } + + public void increment(int millis) { + time.addAndGet(millis); + } + + } + /* sub-class to avoid 65k limit of a single class */ private static class SampleDataHolder1 { /*