From 7116c5e75e80d345161a27b2c7fd67b0a0f0e1f4 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Fri, 4 Feb 2022 17:31:44 +0100 Subject: [PATCH 01/47] sampler and exporter implementations for consistent sampling --- consistent-sampling/build.gradle.kts | 13 + ...ntReservoirSamplingBatchSpanProcessor.java | 677 ++++++++++ ...voirSamplingBatchSpanProcessorBuilder.java | 158 +++ .../samplers/ConsistentAlwaysOffSampler.java | 23 + .../samplers/ConsistentAlwaysOnSampler.java | 22 + .../ConsistentComposedAndSampler.java | 57 + .../samplers/ConsistentComposedOrSampler.java | 65 + .../ConsistentParentBasedSampler.java | 62 + .../ConsistentProbabilityBasedSampler.java | 71 ++ .../ConsistentRateLimitingSampler.java | 137 ++ .../contrib/samplers/ConsistentSampler.java | 212 ++++ .../contrib/state/OtelTraceState.java | 290 +++++ .../contrib/util/DefaultRandomGenerator.java | 58 + .../contrib/util/RandomGenerator.java | 89 ++ .../contrib/util/RandomUtil.java | 109 ++ ...servoirSamplingBatchSpanProcessorTest.java | 1117 +++++++++++++++++ ...ConsistentProbabilityBasedSamplerTest.java | 87 ++ .../ConsistentRateLimitingSamplerTest.java | 202 +++ .../samplers/ConsistentSamplerTest.java | 72 ++ .../contrib/state/OtelTraceStateTest.java | 74 ++ .../contrib/util/RandomUtilTest.java | 78 ++ .../opentelemetry/contrib/util/TestUtil.java | 83 ++ settings.gradle.kts | 1 + 23 files changed, 3757 insertions(+) create mode 100644 consistent-sampling/build.gradle.kts create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessor.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorBuilder.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOnSampler.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java create mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java create mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java create mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java create mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java create mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java create mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java create mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/util/RandomUtilTest.java create mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java diff --git a/consistent-sampling/build.gradle.kts b/consistent-sampling/build.gradle.kts new file mode 100644 index 000000000..03626a177 --- /dev/null +++ b/consistent-sampling/build.gradle.kts @@ -0,0 +1,13 @@ +plugins { + id("otel.java-conventions") + id("otel.publish-conventions") +} + +description = "Sampler and exporter implementations for consistent sampling" + +dependencies { + api("io.opentelemetry:opentelemetry-sdk") + testImplementation("com.google.guava:guava:31.0.1-jre") + testImplementation("org.hipparchus:hipparchus-core:2.0") + testImplementation("org.hipparchus:hipparchus-stat:2.0") +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessor.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessor.java new file mode 100644 index 000000000..0da4ea687 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessor.java @@ -0,0 +1,677 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.export; + +import io.opentelemetry.api.common.AttributeKey; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.metrics.LongCounter; +import io.opentelemetry.api.metrics.Meter; +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.contrib.state.OtelTraceState; +import io.opentelemetry.contrib.util.RandomGenerator; +import io.opentelemetry.contrib.util.RandomUtil; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.internal.DaemonThreadFactory; +import io.opentelemetry.sdk.trace.ReadWriteSpan; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.data.DelegatingSpanData; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; +import java.util.PriorityQueue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +/** + * Implementation of the {@link SpanProcessor} that batches spans exported by the SDK then pushes + * them to the exporter pipeline. + * + *

All spans reported by the SDK implementation are first added to a synchronized queue (with a + * {@code maxQueueSize} maximum size, if queue is full spans are dropped). Spans are exported either + * when there are {@code maxExportBatchSize} pending spans or {@code scheduleDelayNanos} has passed + * since the last export finished. + */ +public final class ConsistentReservoirSamplingBatchSpanProcessor implements SpanProcessor { + + private static final String WORKER_THREAD_NAME = + ConsistentReservoirSamplingBatchSpanProcessor.class.getSimpleName() + "_WorkerThread"; + private static final AttributeKey SPAN_PROCESSOR_TYPE_LABEL = + AttributeKey.stringKey("spanProcessorType"); + private static final AttributeKey SPAN_PROCESSOR_DROPPED_LABEL = + AttributeKey.booleanKey("dropped"); + private static final String SPAN_PROCESSOR_TYPE_VALUE = + ConsistentReservoirSamplingBatchSpanProcessor.class.getSimpleName(); + + private final Worker worker; + private final AtomicBoolean isShutdown = new AtomicBoolean(false); + + /** + * Returns a new Builder for {@link ConsistentReservoirSamplingBatchSpanProcessor}. + * + * @param spanExporter the {@code SpanExporter} to where the Spans are pushed. + * @return a new {@link ConsistentReservoirSamplingBatchSpanProcessor}. + * @throws NullPointerException if the {@code spanExporter} is {@code null}. + */ + public static ConsistentReservoirSamplingBatchSpanProcessorBuilder builder( + SpanExporter spanExporter) { + return new ConsistentReservoirSamplingBatchSpanProcessorBuilder(spanExporter); + } + + private static final class ReadableSpanWithPriority { + + private final ReadableSpan readableSpan; + private int pval; + private final int rval; + private long priority; + + public static ReadableSpanWithPriority create( + ReadableSpan readableSpan, RandomGenerator threadSafeRandomGenerator) { + String otelTraceStateString = + readableSpan.getSpanContext().getTraceState().get(OtelTraceState.TRACE_STATE_KEY); + OtelTraceState otelTraceState = OtelTraceState.parse(otelTraceStateString); + int pval; + int rval; + long priority = threadSafeRandomGenerator.nextLong(); + if (otelTraceState.hasValidR()) { + rval = otelTraceState.getR(); + } else { + rval = + Math.min( + threadSafeRandomGenerator.numberOfLeadingZerosOfRandomLong(), + OtelTraceState.getMaxR()); + } + + if (otelTraceState.hasValidP()) { + pval = otelTraceState.getP(); + } else { + // if the p-value is not defined assume it is zero, + // which corresponds to an adjusted count of 1 + pval = 0; + } + + return new ReadableSpanWithPriority(readableSpan, pval, rval, priority); + } + + private ReadableSpanWithPriority(ReadableSpan readableSpan, int pval, int rval, long priority) { + this.readableSpan = readableSpan; + this.pval = pval; + this.rval = rval; + this.priority = priority; + } + + private ReadableSpan getReadableSpan() { + return readableSpan; + } + + private int getP() { + return pval; + } + + private void setP(int pval) { + this.pval = pval; + } + + private int getR() { + return rval; + } + + // returns true if this span survived down sampling + private boolean downSample(RandomGenerator threadSafeRandomGenerator) { + pval += 1; + if (pval > rval) { + return false; + } + priority = threadSafeRandomGenerator.nextLong(); + return true; + } + + private static int comparePthenPriority( + ReadableSpanWithPriority s1, ReadableSpanWithPriority s2) { + int compareP = Integer.compare(s1.pval, s2.pval); + if (compareP != 0) { + return compareP; + } + return Long.compare(s1.priority, s2.priority); + } + + private static int compareRthenPriority( + ReadableSpanWithPriority s1, ReadableSpanWithPriority s2) { + int compareR = Integer.compare(s1.rval, s2.rval); + if (compareR != 0) { + return compareR; + } + return Long.compare(s1.priority, s2.priority); + } + } + + private interface Reservoir { + void add(ReadableSpanWithPriority readableSpanWithPriority); + + List getResult(); + + boolean isEmpty(); + } + + /** + * Reservoir sampling buffer that collects a fixed number of spans. + * + *

Consistent sampling requires that spans are sampled only if r-value >= p-value, where + * p-value describes which sampling rate from the discrete set of possible sampling rates is + * applied. Consistent sampling allows to choose the sampling rate (the p-value) individually for + * every span. Therefore, the number of sampled spans can be reduced by increasing the p-value of + * spans, such that spans for which r-value < p-value get discarded. To reduce the number of + * sampled spans one can therefore apply the following procedure until the desired number of spans + * are left: + * + *

1) Randomly choose a span among the spans with smallest p-values + * + *

2) Increment its p-value by 1 + * + *

3) Discard the span, if r-value < p-value + * + *

4) continue with 1) + * + *

By always incrementing one of the smallest p-values, this approach tries to balance the + * sampling rates (p-values). Balanced sampling rates are better for estimation (compare VarOpt + * sampling, see https://arxiv.org/abs/0803.0473). + * + *

This reservoir sampling approach implements the described procedure in a streaming fashion. + * In order to ensure that spans have fair chances regardless of processing order, a uniform + * random number (priority) is associated with its p-value. When choosing a span among the spans + * with smallest p-value, we take that with the smallest priority. + */ + private static final class Reservoir1 implements Reservoir { + private final int reservoirSize; + private final PriorityQueue queue; + private final RandomGenerator threadSafeRandomGenerator; + + public Reservoir1(int reservoirSize, RandomGenerator threadSafeRandomGenerator) { + if (reservoirSize < 1) { + throw new IllegalArgumentException(); + } + this.reservoirSize = reservoirSize; + this.queue = + new PriorityQueue<>(reservoirSize, ReadableSpanWithPriority::comparePthenPriority); + this.threadSafeRandomGenerator = threadSafeRandomGenerator; + } + + @Override + public void add(ReadableSpanWithPriority readableSpanWithPriority) { + if (queue.size() < reservoirSize) { + queue.add(readableSpanWithPriority); + return; + } + + do { + ReadableSpanWithPriority head = queue.peek(); + if (ReadableSpanWithPriority.comparePthenPriority(readableSpanWithPriority, head) > 0) { + queue.remove(); + queue.add(readableSpanWithPriority); + readableSpanWithPriority = head; + } + } while (readableSpanWithPriority.downSample(threadSafeRandomGenerator)); + } + + @Override + public List getResult() { + List result = new ArrayList<>(queue.size()); + for (ReadableSpanWithPriority readableSpanWithPriority : queue) { + SpanData spanData = readableSpanWithPriority.getReadableSpan().toSpanData(); + SpanContext spanContext = spanData.getSpanContext(); + TraceState traceState = spanContext.getTraceState(); + String otelTraceStateString = traceState.get(OtelTraceState.TRACE_STATE_KEY); + OtelTraceState otelTraceState = OtelTraceState.parse(otelTraceStateString); + if ((!otelTraceState.hasValidR() && readableSpanWithPriority.getP() > 0) + || (otelTraceState.hasValidR() + && readableSpanWithPriority.getP() != otelTraceState.getP())) { + otelTraceState.setP(readableSpanWithPriority.getP()); + spanData = updateSpanDataWithOtelTraceState(spanData, otelTraceState); + } + result.add(spanData); + } + return result; + } + + @Override + public boolean isEmpty() { + return queue.isEmpty(); + } + } + + /** + * This reservoir implementation is (almost) statistically equivalent to {@link Reservoir1}. + * + *

It uses a priority queue where the minimum is the span with the smallet r-value. In this way + * the add-operation is more efficient, and has a worst case time complexity of O(log n) where n + * denotes the reservoir size. + */ + private static final class Reservoir2 implements Reservoir { + private final int reservoirSize; + private int maxDiscardedRValue = 0; + private long numberOfDiscardedSpansWithMaxDiscardedRValue = 0; + private final PriorityQueue queue; + private final RandomGenerator threadSafeRandomGenerator; + + public Reservoir2(int reservoirSize, RandomGenerator threadSafeRandomGenerator) { + if (reservoirSize < 1) { + throw new IllegalArgumentException(); + } + this.reservoirSize = reservoirSize; + this.queue = + new PriorityQueue<>(reservoirSize, ReadableSpanWithPriority::compareRthenPriority); + this.threadSafeRandomGenerator = threadSafeRandomGenerator; + } + + @Override + public void add(ReadableSpanWithPriority readableSpanWithPriority) { + + if (queue.size() < reservoirSize) { + queue.add(readableSpanWithPriority); + return; + } + + ReadableSpanWithPriority head = queue.peek(); + if (ReadableSpanWithPriority.compareRthenPriority(readableSpanWithPriority, head) > 0) { + queue.remove(); + queue.add(readableSpanWithPriority); + readableSpanWithPriority = head; + } + if (readableSpanWithPriority.getR() > maxDiscardedRValue) { + maxDiscardedRValue = readableSpanWithPriority.getR(); + numberOfDiscardedSpansWithMaxDiscardedRValue = 1; + } else if (readableSpanWithPriority.getR() == maxDiscardedRValue) { + numberOfDiscardedSpansWithMaxDiscardedRValue += 1; + } + } + + @Override + public List getResult() { + + if (numberOfDiscardedSpansWithMaxDiscardedRValue == 0) { + return queue.stream().map(x -> x.readableSpan.toSpanData()).collect(Collectors.toList()); + } + + List readableSpansWithPriority = new ArrayList<>(queue.size()); + int numberOfSampledSpansWithMaxDiscardedRValue = 0; + int numSampledSpansWithGreaterRValueAndSmallPValue = 0; + for (ReadableSpanWithPriority readableSpanWithPriority : queue) { + if (readableSpanWithPriority.getR() == maxDiscardedRValue) { + numberOfSampledSpansWithMaxDiscardedRValue += 1; + } else if (readableSpanWithPriority.getP() <= maxDiscardedRValue) { + numSampledSpansWithGreaterRValueAndSmallPValue += 1; + } + readableSpansWithPriority.add(readableSpanWithPriority); + } + + // Z = reservoirSize + // L = maxDiscardedRValue + // R = numberOfDiscardedSpansWithMaxDiscardedRValue + // K = numSampledSpansWithGreaterRValueAndSmallPValue + // X = numberOfSampledSpansWithMaxDiscardedRValue + // + // The sampling approach described above for Reservoir1 can be equivalently performed by + // keeping Z spans with largest r-values (in case of ties with highest priority) and adjusting + // the p-values at the end. We know that the largest r-value among the dropped spans is L and + // that we had to discard exactly R spans with (r-value == L). This implies that their + // corresponding p-values were raised to (L + 1) which finally violated the sampling condition + // (r-value >= p-value). We only raise the p-value of some span if it belongs to the set of + // spans with minimum p-value. Therefore, the minimum p-value must be given by L. To determine + // the p-values of all kept spans, we consider 3 cases: + // + // 1) For all X kept spans with r-value == L the corresponding p-value must also be L. + // Otherwise, the span would have been discarded. There are R spans with (r-value == L) which + // have been discarded. Therefore, among the original (X + R) spans with (r-value == L) we + // have kept X spans. + // + // 2) For spans with (p-value > L) the p-value will not be changed as they do not belong to + // the set of spans with minimal p-values. + // + // 3) For the remaining K spans for which (r-value > L) and (p-value <= L) the p-value needs + // to be adjusted. The new p-value will be either L or (L + 1). When starting to sample the + // first spans with (p-value == L), we have N = R + K + X spans which all have (r-value >= L) + // and (p-value == L). This set can be divided into two sets of spans dependent on whether + // (r-value == L) or (r-value > L). We know that there were (R + X) spans with (r-value == L) + // and K spans with (r-value > L). When randomly selecting a span to increase its p-value, the + // span will only be discarded if the span belongs to the first set (r-value == L). We will + // call such an event "failure". If the selected span belongs to the second set (r-value > L), + // its p-value will be increased by 1 to (L + 1) but the span will not be dropped. The + // sampling procedure will be stopped after R "failures". The number of "successes" follows a + // negative hypergeometric distribution + // (see https://en.wikipedia.org/wiki/Negative_hypergeometric_distribution). + // Therefore, we need to sample a random value from a negative hypergeometric distribution + // with N = R + X + K elements of which K are "successes" and after drawing R "failures", in + // order to determine how many spans out of K will get a p-value equal to (L + 1). The + // expected number is given by R * K / (N - K + 1) = R * K / (R + X + 1). Instead of drawing + // the number from the negative hypergeometric distribution we could also set it to the + // stochastically rounded expected value. This makes this reservoir sampling approach not + // fully equivalent to the approach described above for Reservoir1, but this (probably) leads + // to a smaller variance when it comes to estimation. (TODO: This still has to be verified!) + + double expectedNumPValueIncrements = + numSampledSpansWithGreaterRValueAndSmallPValue + * (numberOfDiscardedSpansWithMaxDiscardedRValue + / (double) + (numberOfDiscardedSpansWithMaxDiscardedRValue + + numberOfSampledSpansWithMaxDiscardedRValue + + 1L)); + int roundedExpectedNumPValueIncrements = + Math.toIntExact( + RandomUtil.roundStochastically( + threadSafeRandomGenerator, expectedNumPValueIncrements)); + + BitSet incrementIndicators = + RandomUtil.generateRandomBitSet( + threadSafeRandomGenerator, + numSampledSpansWithGreaterRValueAndSmallPValue, + roundedExpectedNumPValueIncrements); + + int incrementIndicatorIndex = 0; + List result = new ArrayList<>(queue.size()); + for (ReadableSpanWithPriority readableSpanWithPriority : readableSpansWithPriority) { + if (readableSpanWithPriority.getP() <= maxDiscardedRValue) { + readableSpanWithPriority.setP(maxDiscardedRValue); + if (readableSpanWithPriority.getR() > maxDiscardedRValue) { + if (incrementIndicators.get(incrementIndicatorIndex)) { + readableSpanWithPriority.setP(maxDiscardedRValue + 1); + } + incrementIndicatorIndex += 1; + } + } + + SpanData spanData = readableSpanWithPriority.getReadableSpan().toSpanData(); + SpanContext spanContext = spanData.getSpanContext(); + TraceState traceState = spanContext.getTraceState(); + String otelTraceStateString = traceState.get(OtelTraceState.TRACE_STATE_KEY); + OtelTraceState otelTraceState = OtelTraceState.parse(otelTraceStateString); + if ((!otelTraceState.hasValidR() && readableSpanWithPriority.getP() > 0) + || (otelTraceState.hasValidR() + && readableSpanWithPriority.getP() != otelTraceState.getP())) { + otelTraceState.setP(readableSpanWithPriority.getP()); + spanData = updateSpanDataWithOtelTraceState(spanData, otelTraceState); + } + result.add(spanData); + } + + return result; + } + + @Override + public boolean isEmpty() { + return queue.isEmpty(); + } + } + + private static SpanData updateSpanDataWithOtelTraceState( + SpanData spanData, OtelTraceState otelTraceState) { + SpanContext spanContext = spanData.getSpanContext(); + TraceState traceState = spanContext.getTraceState(); + String updatedOtelTraceStateString = otelTraceState.serialize(); + TraceState updatedTraceState = + traceState.toBuilder() + .put(OtelTraceState.TRACE_STATE_KEY, updatedOtelTraceStateString) + .build(); + SpanContext updatedSpanContext = + SpanContext.create( + spanContext.getTraceId(), + spanContext.getSpanId(), + spanContext.getTraceFlags(), + updatedTraceState); + return new DelegatingSpanData(spanData) { + @Override + public SpanContext getSpanContext() { + return updatedSpanContext; + } + }; + } + + ConsistentReservoirSamplingBatchSpanProcessor( + SpanExporter spanExporter, + MeterProvider meterProvider, + long scheduleDelayNanos, + int reservoirSize, + long exporterTimeoutNanos, + RandomGenerator threadSafeRandomGenerator, + boolean useAlternativeReservoirImplementation) { + this.worker = + new Worker( + spanExporter, + meterProvider, + scheduleDelayNanos, + reservoirSize, + exporterTimeoutNanos, + threadSafeRandomGenerator, + useAlternativeReservoirImplementation); + Thread workerThread = new DaemonThreadFactory(WORKER_THREAD_NAME).newThread(worker); + workerThread.start(); + } + + @Override + public void onStart(Context parentContext, ReadWriteSpan span) {} + + @Override + public boolean isStartRequired() { + return false; + } + + @Override + public void onEnd(ReadableSpan span) { + if (span == null || !span.getSpanContext().isSampled()) { + return; + } + worker.addSpan(span); + } + + @Override + public boolean isEndRequired() { + return true; + } + + @Override + public CompletableResultCode shutdown() { + if (isShutdown.getAndSet(true)) { + return CompletableResultCode.ofSuccess(); + } + return worker.shutdown(); + } + + @Override + public CompletableResultCode forceFlush() { + return worker.forceFlush(); + } + + // Visible for testing + boolean isReservoirEmpty() { + return worker.isReservoirEmpty(); + } + + private static final class Worker implements Runnable { + + private final LongCounter processedSpansCounter; + private final Attributes droppedAttrs; + private final Attributes exportedAttrs; + private final boolean useAlternativeReservoirImplementation; + + private static final Logger logger = Logger.getLogger(Worker.class.getName()); + private final SpanExporter spanExporter; + private final long scheduleDelayNanos; + private final int reservoirSize; + private final long exporterTimeoutNanos; + + private long nextExportTime; + + private final RandomGenerator threadSafeRandomGenerator; + private final Object reservoirLock = new Object(); + private Reservoir reservoir; + private final BlockingQueue signal; + private volatile boolean continueWork = true; + + private static Reservoir createReservoir( + int reservoirSize, + boolean useAlternativeReservoirImplementation, + RandomGenerator threadSafeRandomGenerator) { + if (useAlternativeReservoirImplementation) { + return new Reservoir2(reservoirSize, threadSafeRandomGenerator); + } else { + return new Reservoir1(reservoirSize, threadSafeRandomGenerator); + } + } + + private Worker( + SpanExporter spanExporter, + MeterProvider meterProvider, + long scheduleDelayNanos, + int reservoirSize, + long exporterTimeoutNanos, + RandomGenerator threadSafeRandomGenerator, + boolean useAlternativeReservoirImplementation) { + this.useAlternativeReservoirImplementation = useAlternativeReservoirImplementation; + this.spanExporter = spanExporter; + this.scheduleDelayNanos = scheduleDelayNanos; + this.reservoirSize = reservoirSize; + this.exporterTimeoutNanos = exporterTimeoutNanos; + this.threadSafeRandomGenerator = threadSafeRandomGenerator; + synchronized (reservoirLock) { + this.reservoir = + createReservoir( + reservoirSize, useAlternativeReservoirImplementation, threadSafeRandomGenerator); + } + this.signal = new ArrayBlockingQueue<>(1); + Meter meter = meterProvider.meterBuilder("io.opentelemetry.sdk.trace").build(); + processedSpansCounter = + meter + .counterBuilder("processedSpans") + .setUnit("1") + .setDescription( + "The number of spans processed by the BatchSpanProcessor. " + + "[dropped=true if they were dropped due to high throughput]") + .build(); + droppedAttrs = + Attributes.of( + SPAN_PROCESSOR_TYPE_LABEL, + SPAN_PROCESSOR_TYPE_VALUE, + SPAN_PROCESSOR_DROPPED_LABEL, + true); + exportedAttrs = + Attributes.of( + SPAN_PROCESSOR_TYPE_LABEL, + SPAN_PROCESSOR_TYPE_VALUE, + SPAN_PROCESSOR_DROPPED_LABEL, + false); + } + + private void addSpan(ReadableSpan span) { + ReadableSpanWithPriority readableSpanWithPriority = + ReadableSpanWithPriority.create(span, threadSafeRandomGenerator); + synchronized (reservoirLock) { + reservoir.add(readableSpanWithPriority); + } + processedSpansCounter.add(1, droppedAttrs); + } + + @Override + public void run() { + updateNextExportTime(); + CompletableResultCode completableResultCode = null; + while (continueWork) { + + if (completableResultCode != null || System.nanoTime() >= nextExportTime) { + Reservoir oldReservoir; + Reservoir newReservoir = + createReservoir( + reservoirSize, useAlternativeReservoirImplementation, threadSafeRandomGenerator); + synchronized (reservoirLock) { + oldReservoir = reservoir; + reservoir = newReservoir; + } + exportCurrentBatch(oldReservoir.getResult()); + updateNextExportTime(); + if (completableResultCode != null) { + completableResultCode.succeed(); + } + } + + try { + long pollWaitTime = nextExportTime - System.nanoTime(); + if (pollWaitTime > 0) { + completableResultCode = signal.poll(pollWaitTime, TimeUnit.NANOSECONDS); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + return; + } + } + } + + private void updateNextExportTime() { + nextExportTime = System.nanoTime() + scheduleDelayNanos; + } + + private CompletableResultCode shutdown() { + CompletableResultCode result = new CompletableResultCode(); + + CompletableResultCode flushResult = forceFlush(); + flushResult.whenComplete( + () -> { + continueWork = false; + CompletableResultCode shutdownResult = spanExporter.shutdown(); + shutdownResult.whenComplete( + () -> { + if (!flushResult.isSuccess() || !shutdownResult.isSuccess()) { + result.fail(); + } else { + result.succeed(); + } + }); + }); + + return result; + } + + private CompletableResultCode forceFlush() { + CompletableResultCode flushResult = new CompletableResultCode(); + signal.offer(flushResult); + return flushResult; + } + + private void exportCurrentBatch(List batch) { + if (batch.isEmpty()) { + return; + } + + try { + CompletableResultCode result = spanExporter.export(Collections.unmodifiableList(batch)); + result.join(exporterTimeoutNanos, TimeUnit.NANOSECONDS); + if (result.isSuccess()) { + processedSpansCounter.add(batch.size(), exportedAttrs); + } else { + logger.log(Level.FINE, "Exporter failed"); + } + } catch (RuntimeException e) { + logger.log(Level.WARNING, "Exporter threw an Exception", e); + } finally { + batch.clear(); + } + } + + private boolean isReservoirEmpty() { + synchronized (reservoirLock) { + return reservoir.isEmpty(); + } + } + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorBuilder.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorBuilder.java new file mode 100644 index 000000000..f8b27dbcd --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorBuilder.java @@ -0,0 +1,158 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.export; + +import static io.opentelemetry.api.internal.Utils.checkArgument; +import static java.util.Objects.requireNonNull; + +import io.opentelemetry.api.metrics.MeterProvider; +import io.opentelemetry.contrib.util.RandomGenerator; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import java.time.Duration; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; + +/** Builder class for {@link ConsistentReservoirSamplingBatchSpanProcessorBuilder}. */ +public final class ConsistentReservoirSamplingBatchSpanProcessorBuilder { + + // Visible for testing + static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 5000; + // Visible for testing + static final int DEFAULT_RESERVOIR_SIZE = 2048; + // Visible for testing + static final int DEFAULT_EXPORT_TIMEOUT_MILLIS = 30_000; + + private final SpanExporter spanExporter; + private long scheduleDelayNanos = TimeUnit.MILLISECONDS.toNanos(DEFAULT_SCHEDULE_DELAY_MILLIS); + private int reservoirSize = DEFAULT_RESERVOIR_SIZE; + private long exporterTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(DEFAULT_EXPORT_TIMEOUT_MILLIS); + private MeterProvider meterProvider = MeterProvider.noop(); + private RandomGenerator threadSafeRandomGenerator = () -> ThreadLocalRandom.current().nextLong(); + private boolean useAlternativeReservoirImplementation = false; + + ConsistentReservoirSamplingBatchSpanProcessorBuilder(SpanExporter spanExporter) { + this.spanExporter = requireNonNull(spanExporter, "spanExporter"); + } + + // TODO: Consider to add support for constant Attributes and/or Resource. + + /** + * Sets the delay interval between two consecutive exports. If unset, defaults to {@value + * DEFAULT_SCHEDULE_DELAY_MILLIS}ms. + */ + public ConsistentReservoirSamplingBatchSpanProcessorBuilder setScheduleDelay( + long delay, TimeUnit unit) { + requireNonNull(unit, "unit"); + checkArgument(delay >= 0, "delay must be non-negative"); + scheduleDelayNanos = unit.toNanos(delay); + return this; + } + + /** + * Sets the delay interval between two consecutive exports. If unset, defaults to {@value + * DEFAULT_SCHEDULE_DELAY_MILLIS}ms. + */ + public ConsistentReservoirSamplingBatchSpanProcessorBuilder setScheduleDelay(Duration delay) { + requireNonNull(delay, "delay"); + return setScheduleDelay(delay.toNanos(), TimeUnit.NANOSECONDS); + } + + // Visible for testing + long getScheduleDelayNanos() { + return scheduleDelayNanos; + } + + /** + * Sets the maximum time an export will be allowed to run before being cancelled. If unset, + * defaults to {@value DEFAULT_EXPORT_TIMEOUT_MILLIS}ms. + */ + public ConsistentReservoirSamplingBatchSpanProcessorBuilder setExporterTimeout( + long timeout, TimeUnit unit) { + requireNonNull(unit, "unit"); + checkArgument(timeout >= 0, "timeout must be non-negative"); + exporterTimeoutNanos = unit.toNanos(timeout); + return this; + } + + /** + * Sets the maximum time an export will be allowed to run before being cancelled. If unset, + * defaults to {@value DEFAULT_EXPORT_TIMEOUT_MILLIS}ms. + */ + public ConsistentReservoirSamplingBatchSpanProcessorBuilder setExporterTimeout(Duration timeout) { + requireNonNull(timeout, "timeout"); + return setExporterTimeout(timeout.toNanos(), TimeUnit.NANOSECONDS); + } + + // Visible for testing + long getExporterTimeoutNanos() { + return exporterTimeoutNanos; + } + + /** + * Sets the reservoir size, themaximum number of Spans that can be collected. + * + *

See the ConsistentReservoirSamplingBatchSpanProcessor class description for a high-level + * design description of this class. + * + *

Default value is {@code 2048}. + * + * @param reservoirSize the reservoir size, the maximum number of Spans that are kept + * @return this. + * @see ConsistentReservoirSamplingBatchSpanProcessorBuilder#DEFAULT_RESERVOIR_SIZE + */ + public ConsistentReservoirSamplingBatchSpanProcessorBuilder setReservoirSize(int reservoirSize) { + this.reservoirSize = reservoirSize; + return this; + } + + // Visible for testing + int getReservoirSize() { + return reservoirSize; + } + + /** + * Sets the {@link MeterProvider} to use to collect metrics related to batch export. If not set, + * metrics will not be collected. + */ + public ConsistentReservoirSamplingBatchSpanProcessorBuilder setMeterProvider( + MeterProvider meterProvider) { + requireNonNull(meterProvider, "meterProvider"); + this.meterProvider = meterProvider; + return this; + } + + // Visible for testing + ConsistentReservoirSamplingBatchSpanProcessorBuilder setThreadSafeRandomGenerator( + RandomGenerator threadSafeRandomGenerator) { + this.threadSafeRandomGenerator = threadSafeRandomGenerator; + return this; + } + + // Visible for testing + ConsistentReservoirSamplingBatchSpanProcessorBuilder useAlternativeReservoirImplementation( + boolean useAlternativeReservoirImplementation) { + this.useAlternativeReservoirImplementation = useAlternativeReservoirImplementation; + return this; + } + + /** + * Returns a new {@link ConsistentReservoirSamplingBatchSpanProcessorBuilder} that batches, then + * converts spans to proto and forwards them to the given {@code spanExporter}. + * + * @return a new {@link ConsistentReservoirSamplingBatchSpanProcessorBuilder}. + * @throws NullPointerException if the {@code spanExporter} is {@code null}. + */ + public ConsistentReservoirSamplingBatchSpanProcessor build() { + return new ConsistentReservoirSamplingBatchSpanProcessor( + spanExporter, + meterProvider, + scheduleDelayNanos, + reservoirSize, + exporterTimeoutNanos, + threadSafeRandomGenerator, + useAlternativeReservoirImplementation); + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java new file mode 100644 index 000000000..56b68c8c9 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java @@ -0,0 +1,23 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import io.opentelemetry.contrib.state.OtelTraceState; +import javax.annotation.concurrent.Immutable; + +@Immutable +public final class ConsistentAlwaysOffSampler extends ConsistentSampler { + + @Override + protected int getP(int parentP, boolean isRoot) { + return OtelTraceState.getMaxP(); + } + + @Override + public String getDescription() { + return "ConsistentAlwaysOffSampler"; + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOnSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOnSampler.java new file mode 100644 index 000000000..6ce7a6575 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOnSampler.java @@ -0,0 +1,22 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import javax.annotation.concurrent.Immutable; + +@Immutable +public class ConsistentAlwaysOnSampler extends ConsistentSampler { + + @Override + protected int getP(int parentP, boolean isRoot) { + return 0; + } + + @Override + public String getDescription() { + return "ConsistentAlwaysOnSampler"; + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java new file mode 100644 index 000000000..dedaee713 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java @@ -0,0 +1,57 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import static java.util.Objects.requireNonNull; + +import io.opentelemetry.contrib.state.OtelTraceState; +import javax.annotation.concurrent.Immutable; + +/** + * A consistent sampler composes two consistent samplers. + * + *

This sampler samples if both samplers would sample. + */ +@Immutable +public final class ConsistentComposedAndSampler extends ConsistentSampler { + + private final ConsistentSampler sampler1; + private final ConsistentSampler sampler2; + private final String description; + + public static ConsistentComposedAndSampler create( + ConsistentSampler sampler1, ConsistentSampler sampler2) { + return new ConsistentComposedAndSampler(sampler1, sampler2); + } + + private ConsistentComposedAndSampler(ConsistentSampler sampler1, ConsistentSampler sampler2) { + this.sampler1 = requireNonNull(sampler1); + this.sampler2 = requireNonNull(sampler2); + this.description = + "ConsistentComposedAndSampler{" + + "sampler1=" + + sampler1.getDescription() + + ",sampler2=" + + sampler2.getDescription() + + '}'; + } + + @Override + protected int getP(int parentP, boolean isRoot) { + int p1 = sampler1.getP(parentP, isRoot); + int p2 = sampler2.getP(parentP, isRoot); + if (OtelTraceState.isValidP(p1) && OtelTraceState.isValidP(p2)) { + return Math.max(p1, p2); + } else { + return OtelTraceState.getInvalidP(); + } + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java new file mode 100644 index 000000000..ce869b7c8 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java @@ -0,0 +1,65 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import static java.util.Objects.requireNonNull; + +import io.opentelemetry.contrib.state.OtelTraceState; +import javax.annotation.concurrent.Immutable; + +/** + * A consistent sampler composes two consistent samplers. + * + *

This sampler samples if any of the two samplers would sample. + */ +@Immutable +public final class ConsistentComposedOrSampler extends ConsistentSampler { + + private final ConsistentSampler sampler1; + private final ConsistentSampler sampler2; + private final String description; + + public static ConsistentComposedOrSampler create( + ConsistentSampler sampler1, ConsistentSampler sampler2) { + return new ConsistentComposedOrSampler(sampler1, sampler2); + } + + private ConsistentComposedOrSampler(ConsistentSampler sampler1, ConsistentSampler sampler2) { + this.sampler1 = requireNonNull(sampler1); + this.sampler2 = requireNonNull(sampler2); + this.description = + "ConsistentComposedOrSampler{" + + "sampler1=" + + sampler1.getDescription() + + ",sampler2=" + + sampler2.getDescription() + + '}'; + } + + @Override + protected int getP(int parentP, boolean isRoot) { + int p1 = sampler1.getP(parentP, isRoot); + int p2 = sampler2.getP(parentP, isRoot); + if (OtelTraceState.isValidP(p1)) { + if (OtelTraceState.isValidP(p2)) { + return Math.min(p1, p2); + } else { + return p1; + } + } else { + if (OtelTraceState.isValidP(p2)) { + return p2; + } else { + return OtelTraceState.getInvalidP(); + } + } + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java new file mode 100644 index 000000000..30624a7bb --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java @@ -0,0 +1,62 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import static java.util.Objects.requireNonNull; + +import io.opentelemetry.contrib.util.DefaultRandomGenerator; +import io.opentelemetry.contrib.util.RandomGenerator; +import javax.annotation.concurrent.Immutable; + +/** + * A consistent sampler that makes the same sampling decision as the paren and optionally falls back + * to an alternative consistent sampler, if the parent p-value is invalid (like for root spans). + */ +@Immutable +public final class ConsistentParentBasedSampler extends ConsistentSampler { + + private final ConsistentSampler rootSampler; + + private final String description; + + /** + * Constructs a new consistent parent based sampler using the given root sampler. + * + * @param rootSampler the root sampler + */ + public ConsistentParentBasedSampler(ConsistentSampler rootSampler) { + this(rootSampler, DefaultRandomGenerator.get()); + } + + /** + * Constructs a new consistent parent based sampler using the given root sampler and the given + * thread-safe random generator. + * + * @param rootSampler the root sampler + * @param threadSafeRandomGenerator a thread-safe random generator + */ + public ConsistentParentBasedSampler( + ConsistentSampler rootSampler, RandomGenerator threadSafeRandomGenerator) { + super(threadSafeRandomGenerator); + this.rootSampler = requireNonNull(rootSampler); + this.description = + "ConsistentComposedSampler{rootSampler=" + rootSampler.getDescription() + '}'; + } + + @Override + protected int getP(int parentP, boolean isRoot) { + if (isRoot) { + return rootSampler.getP(parentP, isRoot); + } else { + return parentP; + } + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java new file mode 100644 index 000000000..428329e44 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java @@ -0,0 +1,71 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import io.opentelemetry.contrib.util.DefaultRandomGenerator; +import io.opentelemetry.contrib.util.RandomGenerator; +import javax.annotation.concurrent.Immutable; + +@Immutable +public class ConsistentProbabilityBasedSampler extends ConsistentSampler { + + private final int lowerPValue; + private final int upperPValue; + private final double probabilityToUseLowerPValue; + private final String description; + + /** + * Constructor. + * + * @param samplingProbability the sampling probability + */ + public ConsistentProbabilityBasedSampler(double samplingProbability) { + this(samplingProbability, DefaultRandomGenerator.get()); + } + + /** + * Constructor. + * + * @param samplingProbability the sampling probability + * @param threadSafeRandomGenerator a thread-safe random generator + */ + public ConsistentProbabilityBasedSampler( + double samplingProbability, RandomGenerator threadSafeRandomGenerator) { + super(threadSafeRandomGenerator); + if (samplingProbability < 0.0 || samplingProbability > 1.0) { + throw new IllegalArgumentException("Sampling probability must be in range [0.0, 1.0]!"); + } + this.description = + String.format("ConsistentProbabilityBasedSampler{%.6f}", samplingProbability); + + lowerPValue = getLowerBoundP(samplingProbability); + upperPValue = getUpperBoundP(samplingProbability); + + if (lowerPValue == upperPValue) { + probabilityToUseLowerPValue = 1; + } else { + double upperSamplingProbability = getSamplingProbability(lowerPValue); + double lowerSamplingProbability = getSamplingProbability(upperPValue); + probabilityToUseLowerPValue = + (samplingProbability - lowerSamplingProbability) + / (upperSamplingProbability - lowerSamplingProbability); + } + } + + @Override + protected int getP(int parentP, boolean isRoot) { + if (threadSafeRandomGenerator.nextBoolean(probabilityToUseLowerPValue)) { + return lowerPValue; + } else { + return upperPValue; + } + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java new file mode 100644 index 000000000..06ed6955d --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -0,0 +1,137 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import static java.util.Objects.requireNonNull; + +import io.opentelemetry.api.internal.GuardedBy; +import io.opentelemetry.contrib.util.DefaultRandomGenerator; +import io.opentelemetry.contrib.util.RandomGenerator; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.util.function.LongSupplier; +import javax.annotation.concurrent.Immutable; + +/** + * This consistent {@link Sampler} adjust the sampling probability dynamically to limit the rate of + * sampled spans. + * + *

This sampler uses exponential smoothing to estimate on irregular data (compare Wright, David + * J. "Forecasting data published at irregular time intervals using an extension of Holt's method." + * Management science 32.4 (1986): 499-510.) to estimate the current rate of spans. + */ +@Immutable +public class ConsistentRateLimitingSampler extends ConsistentSampler { + + private final String description; + private final LongSupplier nanoTimeSupplier; + private final double inverseAdaptationTimeNanos; + private final double targetSpansPerNanosLimit; + + @GuardedBy("this") + private double effectiveWindowCount; + + @GuardedBy("this") + private double effectiveWindowNanos; + + @GuardedBy("this") + private long lastNanoTime; + + /** + * Constructor. + * + * @param targetSpansPerSecondLimit the desired spans per second limit + * @param adaptationTimeSeconds the typical time to adapt to a new load (time constant used for + * exponential smoothing) + */ + public ConsistentRateLimitingSampler( + double targetSpansPerSecondLimit, double adaptationTimeSeconds) { + this( + targetSpansPerSecondLimit, + adaptationTimeSeconds, + DefaultRandomGenerator.get(), + System::nanoTime); + } + + /** + * Constructor. + * + * @param targetSpansPerSecondLimit the desired spans per second limit + * @param adaptationTimeSeconds the typical time to adapt to a new load (time constant used for + * exponential smoothing) + * @param threadSafeRandomGenerator a thread-safe random generator + * @param nanoTimeSupplier a supplier for the current nano time + */ + public ConsistentRateLimitingSampler( + double targetSpansPerSecondLimit, + double adaptationTimeSeconds, + RandomGenerator threadSafeRandomGenerator, + LongSupplier nanoTimeSupplier) { + super(threadSafeRandomGenerator); + + if (targetSpansPerSecondLimit < 0.0) { + throw new IllegalArgumentException("Limit for sampled spans per second must be nonnegative!"); + } + if (adaptationTimeSeconds < 0.0) { + throw new IllegalArgumentException("Adaptation rate must be nonnegative!"); + } + this.description = + String.format( + "ConsistentRateLimitingSampler{%.6f, %.6f}", + targetSpansPerSecondLimit, adaptationTimeSeconds); + this.nanoTimeSupplier = requireNonNull(nanoTimeSupplier); + + this.inverseAdaptationTimeNanos = 1e-9 / adaptationTimeSeconds; + this.targetSpansPerNanosLimit = 1e-9 * targetSpansPerSecondLimit; + + synchronized (this) { + this.effectiveWindowCount = 0; + this.effectiveWindowNanos = 0; + this.lastNanoTime = nanoTimeSupplier.getAsLong(); + } + } + + private synchronized double updateAndGetSamplingProbability() { + long currentNanoTime = Math.max(nanoTimeSupplier.getAsLong(), lastNanoTime); + long nanoTimeDelta = currentNanoTime - lastNanoTime; + lastNanoTime = currentNanoTime; + double decayFactor = Math.exp(-nanoTimeDelta * inverseAdaptationTimeNanos); + effectiveWindowCount = effectiveWindowCount * decayFactor + 1; + effectiveWindowNanos = effectiveWindowNanos * decayFactor + nanoTimeDelta; + return Math.min(1., (effectiveWindowNanos * targetSpansPerNanosLimit) / effectiveWindowCount); + } + + @Override + protected int getP(int parentP, boolean isRoot) { + double samplingProbability = updateAndGetSamplingProbability(); + + if (samplingProbability >= 1.) { + return 0; + } + + int lowerPValue = getLowerBoundP(samplingProbability); + int upperPValue = getUpperBoundP(samplingProbability); + + if (lowerPValue == upperPValue) { + return lowerPValue; + } + + double upperSamplingRate = getSamplingProbability(lowerPValue); + double lowerSamplingRate = getSamplingProbability(upperPValue); + double probabilityToUseLowerPValue = + (samplingProbability - lowerSamplingRate) / (upperSamplingRate - lowerSamplingRate); + + if (threadSafeRandomGenerator.nextBoolean(probabilityToUseLowerPValue)) { + return lowerPValue; + } else { + return upperPValue; + } + } + + @Override + public String getDescription() { + return description; + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java new file mode 100644 index 000000000..bb8edf011 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -0,0 +1,212 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import static java.util.Objects.requireNonNull; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.contrib.state.OtelTraceState; +import io.opentelemetry.contrib.util.DefaultRandomGenerator; +import io.opentelemetry.contrib.util.RandomGenerator; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingDecision; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.List; + +/** Abstract base class for consistent samplers. */ +abstract class ConsistentSampler implements Sampler { + + protected final RandomGenerator threadSafeRandomGenerator; + + protected ConsistentSampler(RandomGenerator threadSafeRandomGenerator) { + this.threadSafeRandomGenerator = requireNonNull(threadSafeRandomGenerator); + } + + protected ConsistentSampler() { + this.threadSafeRandomGenerator = DefaultRandomGenerator.get(); + } + + @Override + public final SamplingResult shouldSample( + Context parentContext, + String traceId, + String name, + SpanKind spanKind, + Attributes attributes, + List parentLinks) { + + Span parentSpan = Span.fromContext(parentContext); + SpanContext parentSpanContext = parentSpan.getSpanContext(); + boolean isRoot = !parentSpanContext.isValid(); + boolean isParentSampled = parentSpanContext.isSampled(); + + TraceState parentTraceState = parentSpanContext.getTraceState(); + String otelTraceStateString = parentTraceState.get(OtelTraceState.TRACE_STATE_KEY); + OtelTraceState otelTraceState = OtelTraceState.parse(otelTraceStateString); + + if (!otelTraceState.hasValidR()) { + otelTraceState.invalidateP(); + } + // Invariant checking: unset p-value when p-value, r-value, and isParentSampled are inconsistent + if (otelTraceState.hasValidR() && otelTraceState.hasValidP()) { + if ((((otelTraceState.getP() <= otelTraceState.getR()) == isParentSampled) + || (isParentSampled && (otelTraceState.getP() == OtelTraceState.getMaxP()))) + == false) { + otelTraceState.invalidateP(); + } + } + + // generate new r-value if not available + if (!otelTraceState.hasValidR()) { + otelTraceState.setR( + Math.min( + threadSafeRandomGenerator.numberOfLeadingZerosOfRandomLong(), + OtelTraceState.getMaxR())); + } + + // determine and set new p-value that is used for the sampling decision + int newP = getP(otelTraceState.getP(), isRoot); + otelTraceState.setP(newP); + + // determine sampling decision + boolean isSampled; + if (otelTraceState.hasValidP()) { + isSampled = (otelTraceState.getP() <= otelTraceState.getR()); + } else { + // if new p-value is invalid, respect sampling decision of parent + isSampled = isParentSampled; + } + SamplingDecision samplingDecision = + isSampled ? SamplingDecision.RECORD_AND_SAMPLE : SamplingDecision.DROP; + + String newOtTraceState = otelTraceState.serialize(); + + return new SamplingResult() { + + @Override + public SamplingDecision getDecision() { + return samplingDecision; + } + + @Override + public Attributes getAttributes() { + return Attributes.empty(); + } + + @Override + public TraceState getUpdatedTraceState(TraceState parentTraceState) { + return parentTraceState.toBuilder() + .put(OtelTraceState.TRACE_STATE_KEY, newOtTraceState) + .build(); + } + }; + } + + /** + * Returns the p-value that is used for the sampling decision. + * + *

The returned p-value is translated into corresponding sampling probabilities as given in the + * following: + * + *

p-value = 0 => sampling probability = 1 + * + *

p-value = 1 => sampling probability = 1/2 + * + *

p-value = 2 => sampling probability = 1/4 + * + *

... + * + *

p-value = (z-2) => sampling probability = 1/2^(z-2) + * + *

p-value = (z-1) => sampling probability = 1/2^(z-1) + * + *

p-value = z => sampling probability = 0 + * + *

Here z denotes OtelTraceState.getMaxP(). + * + *

Any other p-values have no meaning and will lead to inconsistent sampling decisions. The + * parent sampled flag will define the sampling decision in this case. + * + *

NOTE: In future, further information like span attributes could be also added as arguments + * such that the sampling probability could be made dependent on those extra arguments. However, + * in any case the returned p-value must not depend directly or indirectly on the r-value. In + * particular this means that the parent sampled flag must not be used for the calculation of the + * p-value as the sampled flag depends itself on the r-value. + * + * @param parentP is the p-value (if known) that was used for a consistent sampling decision by + * the parent + * @param isRoot is true for the root span + * @return this Builder + */ + protected abstract int getP(int parentP, boolean isRoot); + + /** + * Returns the sampling probability for a given p-value. + * + * @param p the p-value + * @return the sampling probability in the range [0,1] or Double.Nan if the p-value is invalid + */ + protected static double getSamplingProbability(int p) { + if (OtelTraceState.isValidP(p)) { + if (p == OtelTraceState.getMaxP()) { + return 0.; + } else { + return Double.longBitsToDouble((0x3FFL - p) << 52); + } + } else { + return Double.NaN; + } + } + + private static final double SMALLEST_POSITIVE_SAMPLING_PROBABILITY = + getSamplingProbability(OtelTraceState.getMaxP() - 1); + + /** + * Returns the largest p-value for which {@code getSamplingProbability(p) >= samplingProbability}. + * + * @param samplingProbability the sampling probability + * @return the p-value + */ + protected static int getLowerBoundP(double samplingProbability) { + if (!(samplingProbability >= 0. && samplingProbability <= 1.)) { + throw new IllegalArgumentException(); + } + if (samplingProbability <= SMALLEST_POSITIVE_SAMPLING_PROBABILITY) { + return OtelTraceState.getMaxP() - (samplingProbability > 0. ? 1 : 0); + } else { + long longSamplingProbability = Double.doubleToRawLongBits(samplingProbability); + long mantissa = longSamplingProbability & 0x000FFFFFFFFFFFFFL; + long exponent = longSamplingProbability >>> 52; + return (int) (0x3FFL - exponent) - (mantissa != 0 ? 1 : 0); + } + } + + /** + * Returns the smallest p-value for which {@code getSamplingProbability(p) <= + * samplingProbability}. + * + * @param samplingProbability the sampling probability + * @return the p-value + */ + protected static int getUpperBoundP(double samplingProbability) { + if (!(samplingProbability >= 0. && samplingProbability <= 1.)) { + throw new IllegalArgumentException(); + } + if (samplingProbability <= SMALLEST_POSITIVE_SAMPLING_PROBABILITY) { + return OtelTraceState.getMaxP(); + } else { + long longSamplingProbability = Double.doubleToRawLongBits(samplingProbability); + long exponent = longSamplingProbability >>> 52; + return (int) (0x3FFL - exponent); + } + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java new file mode 100644 index 000000000..19522fadf --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -0,0 +1,290 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.state; + +import java.util.ArrayList; +import java.util.List; +import javax.annotation.Nullable; + +public final class OtelTraceState { + + public static final String TRACE_STATE_KEY = "ot"; + + private static final char P_SUBKEY = 'p'; + private static final char R_SUBKEY = 'r'; + private static final int MAX_P = 63; + private static final int MAX_R = 62; + private static final int INVALID_P = -1; + private static final int INVALID_R = -1; + private static final int TRACE_STATE_SIZE_LIMIT = 256; + + private int rval; // valid in the interval [0, MAX_R] + private int pval; // valid in the interval [0, MAX_P] + + @Nullable private final List otherKeyValuePairs; + + private OtelTraceState(int rvalue, int pvalue, @Nullable List otherKeyValuePairs) { + this.rval = rvalue; + this.pval = pvalue; + this.otherKeyValuePairs = otherKeyValuePairs; + } + + private OtelTraceState() { + this.rval = INVALID_R; + this.pval = INVALID_P; + this.otherKeyValuePairs = null; + } + + public boolean hasValidR() { + return isValidR(rval); + } + + public boolean hasValidP() { + return isValidP(pval); + } + + public void invalidateP() { + pval = INVALID_P; + } + + public void invalidateR() { + rval = INVALID_R; + } + + /** + * Sets a new p-value. + * + *

If the given p-value is invalid, the current p-value is invalidated. + * + * @param pval the new p-value + */ + public void setP(int pval) { + if (isValidP(pval)) { + this.pval = pval; + } else { + invalidateP(); + } + } + + /** + * Sets a new r-value. + * + *

If the given r-value is invalid, the current r-value is invalidated. + * + * @param rval the new r-value + */ + public void setR(int rval) { + if (isValidR(rval)) { + this.rval = rval; + } else { + invalidateR(); + } + } + + /** + * Returns a string representing this state. + * + * @return a string + */ + public String serialize() { + StringBuilder sb = new StringBuilder(); + if (hasValidP()) { + sb.append("p:").append(pval); + } + if (hasValidR()) { + if (sb.length() > 0) { + sb.append(';'); + } + sb.append("r:").append(rval); + } + if (otherKeyValuePairs != null) { + for (String s : otherKeyValuePairs) { + int ex = sb.length(); + if (ex != 0) { + ex += 1; + } + if (ex + s.length() > TRACE_STATE_SIZE_LIMIT) { + break; + } + if (sb.length() > 0) { + sb.append(';'); + } + sb.append(s); + } + } + return sb.toString(); + } + + private static boolean isValueByte(char r) { + if (isLowerCaseAlphaNum(r)) { + return true; + } + if (isUpperCaseAlpha(r)) { + return true; + } + return r == '.' || r == '_' || r == '-'; + } + + private static boolean isLowerCaseAlphaNum(char r) { + return isLowerCaseAlpha(r) || isLowerCaseNum(r); + } + + private static boolean isLowerCaseNum(char r) { + return r >= '0' && r <= '9'; + } + + private static boolean isLowerCaseAlpha(char r) { + return r >= 'a' && r <= 'z'; + } + + private static boolean isUpperCaseAlpha(char r) { + return r >= 'A' && r <= 'Z'; + } + + private static int parseOneOrTwoDigitNumber( + String ts, int from, int to, int twoDigitMaxValue, int invalidValue) { + if (to - from == 1) { + char c = ts.charAt(from); + if (isLowerCaseNum(c)) { + return c - '0'; + } + } else if (to - from == 2) { + char c1 = ts.charAt(from); + char c2 = ts.charAt(from + 1); + if (isLowerCaseNum(c1) && isLowerCaseNum(c2)) { + int v = (c1 - '0') * 10 + (c2 - '0'); + if (v <= twoDigitMaxValue) { + return v; + } + } + } + return invalidValue; + } + + public static boolean isValidR(int v) { + return 0 <= v && v <= MAX_R; + } + + public static boolean isValidP(int v) { + return 0 <= v && v <= MAX_P; + } + + /** + * Parses the OtelTraceState from a given string. + * + *

If the string cannot be successfully parsed. A new OtelTraceState is returned + * + * @param ts the string + * @return the parsed OtelTraceState or a new empty OtelTraceState in case of parsing errors + */ + public static OtelTraceState parse(@Nullable String ts) { + List otherKeyValuePairs = null; + int p = INVALID_P; + int r = INVALID_R; + // boolean error = false; + + if (ts == null || ts.isEmpty()) { + return new OtelTraceState(); + } + + if (ts.length() > TRACE_STATE_SIZE_LIMIT) { + // error = true; + return new OtelTraceState(); + } + + int tsStartPos = 0; + int len = ts.length(); + + while (true) { + int eqPos = tsStartPos; + for (; eqPos < len; eqPos++) { + char c = ts.charAt(eqPos); + if (!isLowerCaseAlpha(c) && (!isLowerCaseNum(c) || eqPos == tsStartPos)) { + break; + } + } + if (eqPos == tsStartPos || eqPos == len || ts.charAt(eqPos) != ':') { + // error = true; + return new OtelTraceState(); + } + + int sepPos = eqPos + 1; + for (; sepPos < len; sepPos++) { + if (isValueByte(ts.charAt(sepPos))) { + continue; + } + break; + } + + if (eqPos - tsStartPos == 1 && ts.charAt(tsStartPos) == P_SUBKEY) { + p = parseOneOrTwoDigitNumber(ts, eqPos + 1, sepPos, MAX_P, INVALID_P); + } else if (eqPos - tsStartPos == 1 && ts.charAt(tsStartPos) == R_SUBKEY) { + r = parseOneOrTwoDigitNumber(ts, eqPos + 1, sepPos, MAX_R, INVALID_R); + } else { + if (otherKeyValuePairs == null) { + otherKeyValuePairs = new ArrayList<>(); + } + otherKeyValuePairs.add(ts.substring(tsStartPos, sepPos)); + } + + if (sepPos < len && ts.charAt(sepPos) != ';') { + // error = true; + return new OtelTraceState(); + } + + if (sepPos == len) { + break; + } + + tsStartPos = sepPos + 1; + + // test for a trailing ; + if (tsStartPos == len) { + return new OtelTraceState(); + } + } + + return new OtelTraceState(r, p, otherKeyValuePairs); + } + + public int getR() { + return rval; + } + + public int getP() { + return pval; + } + + public static int getMaxP() { + return MAX_P; + } + + public static int getMaxR() { + return MAX_R; + } + + /** + * Returns an r-value that is guaranteed to be invalid. + * + *

{@code isValidR(getInvalidR())} will always return true. + * + * @return an invalid r-value + */ + public static int getInvalidR() { + return INVALID_R; + } + + /** + * Returns a p-value that is guaranteed to be invalid. + * + *

{@code isValidP(getInvalidP())} will always return true. + * + * @return an invalid p-value + */ + public static int getInvalidP() { + return INVALID_P; + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java new file mode 100644 index 000000000..8dbcf4ea5 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java @@ -0,0 +1,58 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.util; + +import java.util.concurrent.ThreadLocalRandom; + +public class DefaultRandomGenerator implements RandomGenerator { + + private static final class ThreadLocalData { + private long randomBits = 0; + private int bitCount = 0; + + boolean nextRandomBit() { + if ((bitCount & 0x3F) == 0) { + randomBits = ThreadLocalRandom.current().nextLong(); + } + boolean randomBit = ((randomBits >>> bitCount) & 1L) != 0L; + bitCount += 1; + return randomBit; + } + } + + private static final ThreadLocal THREAD_LOCAL_DATA = + new ThreadLocal() { + @Override + protected ThreadLocalData initialValue() { + return new ThreadLocalData(); + } + }; + + private static final DefaultRandomGenerator INSTANCE = new DefaultRandomGenerator(); + + private DefaultRandomGenerator() {} + + public static RandomGenerator get() { + return INSTANCE; + } + + @Override + public long nextLong() { + return ThreadLocalRandom.current().nextLong(); + } + + @Override + public boolean nextBoolean(double probability) { + return RandomUtil.generateRandomBoolean( + () -> THREAD_LOCAL_DATA.get().nextRandomBit(), probability); + } + + @Override + public int numberOfLeadingZerosOfRandomLong() { + return RandomUtil.numberOfLeadingZerosOfRandomLong( + () -> THREAD_LOCAL_DATA.get().nextRandomBit()); + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java new file mode 100644 index 000000000..114188ca6 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java @@ -0,0 +1,89 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.util; + +/** A random generator. */ +public interface RandomGenerator { + + /** + * Returns a pseudorandomly chosen {@code long} value. + * + * @return a pseudorandomly chosen {@code long} value + */ + long nextLong(); + + /** + * Returns a pseudorandomly chosen {@code int} value between zero (inclusive) and the specified + * bound (exclusive). + * + *

The implementation is based on Daniel Lemire's algorithm as described in "Fast random + * integer generation in an interval." ACM Transactions on Modeling and Computer Simulation + * (TOMACS) 29.1 (2019): 3. + * + * @param bound the upper bound (exclusive) for the returned value. Must be positive. + * @return a pseudorandomly chosen {@code int} value between zero (inclusive) and the bound + * (exclusive) + * @throws IllegalArgumentException if {@code bound} is not positive + */ + default int nextInt(int bound) { + if (bound <= 0) { + throw new IllegalArgumentException(); + } + long x = nextLong() >>> 33; // use only 31 random bits + long m = x * bound; + int l = (int) m & 0x7FFFFFFF; + if (l < bound) { + int t = (-bound & 0x7FFFFFFF) % bound; + while (l < t) { + x = nextLong() >>> 33; // use only 31 random bits + m = x * bound; + l = (int) m & 0x7FFFFFFF; + } + } + return (int) (m >>> 31); + } + + /** + * Returns a pseudorandomly chosen {@code boolean} value where the probability of returning {@code + * true} is predefined. + * + * @param probability the probability of returning {@code true} + * @return a random {@code boolean} + */ + default boolean nextBoolean(double probability) { + long randomBits = 0; + int bitCounter = 0; + while (true) { + if (probability <= 0) { + return false; + } + if (probability >= 1) { + return true; + } + boolean b = probability > 0.5; + if ((bitCounter & 0x3f) == 0) { + randomBits = nextLong(); + } + if (((randomBits >>> bitCounter) & 1L) == 1L) { + return b; + } + bitCounter += 1; + probability += probability; + if (b) { + probability -= 1; + } + } + } + + /** + * Returns the number of leading zeros of a uniform random 64-bit integer. + * + * @return the number of leading zeros + */ + default int numberOfLeadingZerosOfRandomLong() { + return Long.numberOfLeadingZeros(nextLong()); + } +} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java new file mode 100644 index 000000000..c7cd47629 --- /dev/null +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java @@ -0,0 +1,109 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.util; + +import java.util.BitSet; +import java.util.function.BooleanSupplier; + +public final class RandomUtil { + + private RandomUtil() {} + + /** + * Returns a pseudorandomly chosen {@code boolean} value where the probability of returning {@code + * true} is predefined. + * + * @param randomBooleanSupplier a random boolean supplier + * @param probability the probability of returning {@code true} + * @return a random {@code boolean} + */ + public static boolean generateRandomBoolean( + BooleanSupplier randomBooleanSupplier, double probability) { + while (true) { + if (probability <= 0) { + return false; + } + if (probability >= 1) { + return true; + } + boolean b = probability > 0.5; + if (randomBooleanSupplier.getAsBoolean()) { + return b; + } + probability += probability; + if (b) { + probability -= 1; + } + } + } + + /** + * Stochastically rounds the given floating-point value. + * + *

see https://en.wikipedia.org/wiki/Rounding#Stochastic_rounding + * + * @param randomGenerator a random generator + * @param x the value to be rounded + * @return the rounded value + */ + public static long roundStochastically(RandomGenerator randomGenerator, double x) { + long i = (long) Math.floor(x); + if (randomGenerator.nextBoolean(x - i)) { + return i + 1; + } else { + return i; + } + } + + /** + * Returns the number of leading zeros of a uniform random 64-bit integer. + * + * @param randomBooleanSupplier a random boolean supplier + * @return the truncated geometrically distributed random value + */ + public static int numberOfLeadingZerosOfRandomLong(BooleanSupplier randomBooleanSupplier) { + int count = 0; + while (count < Long.SIZE && randomBooleanSupplier.getAsBoolean()) { + count += 1; + } + return count; + } + + /** + * Generates a random bit set where a given number of 1-bits are randomly set. + * + * @param randomGenerator the random generator + * @param numBits the total number of bits + * @param numOneBits the number of 1-bits + * @return a random bit set + * @throws IllegalArgumentException if {@code 0 <= numOneBits <= numBits} is violated + */ + public static BitSet generateRandomBitSet( + RandomGenerator randomGenerator, int numBits, int numOneBits) { + + if (numOneBits < 0 || numOneBits > numBits) { + throw new IllegalArgumentException(); + } + + BitSet result = new BitSet(numBits); + int numZeroBits = numBits - numOneBits; + + // based on Fisher-Yates shuffling + for (int i = Math.max(numZeroBits, numOneBits); i < numBits; ++i) { + int j = randomGenerator.nextInt(i + 1); + if (result.get(j)) { + result.set(i); + } else { + result.set(j); + } + } + if (numZeroBits < numOneBits) { + result.flip(0, numBits); + } + + return result; + } +} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java new file mode 100644 index 000000000..5151bc520 --- /dev/null +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java @@ -0,0 +1,1117 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.export; + +import static io.opentelemetry.contrib.util.TestUtil.verifyObservedPvaluesUsingGtest; +import static java.util.stream.Collectors.toList; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatCode; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; +import static org.awaitility.Awaitility.await; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.argThat; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.when; + +import io.opentelemetry.api.internal.GuardedBy; +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.Tracer; +import io.opentelemetry.contrib.samplers.ConsistentAlwaysOnSampler; +import io.opentelemetry.contrib.samplers.ConsistentProbabilityBasedSampler; +import io.opentelemetry.contrib.state.OtelTraceState; +import io.opentelemetry.contrib.util.RandomGenerator; +import io.opentelemetry.sdk.common.CompletableResultCode; +import io.opentelemetry.sdk.trace.ReadableSpan; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.data.SpanData; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.EnumSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.SplittableRandom; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.IntStream; +import java.util.stream.LongStream; +import javax.annotation.Nullable; +import org.assertj.core.api.Assertions; +import org.assertj.core.data.Percentage; +import org.hipparchus.distribution.discrete.BinomialDistribution; +import org.hipparchus.stat.descriptive.StatisticalSummary; +import org.hipparchus.stat.descriptive.StreamingStatistics; +import org.hipparchus.stat.inference.GTest; +import org.hipparchus.stat.inference.TTest; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentMatchers; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.mockito.junit.jupiter.MockitoSettings; +import org.mockito.quality.Strictness; + +@SuppressWarnings("PreferJavaTimeOverload") +@ExtendWith(MockitoExtension.class) +@MockitoSettings(strictness = Strictness.LENIENT) +class ConsistentReservoirSamplingBatchSpanProcessorTest { + + private static final String SPAN_NAME_1 = "MySpanName/1"; + private static final String SPAN_NAME_2 = "MySpanName/2"; + private static final long MAX_SCHEDULE_DELAY_MILLIS = 500; + + @Nullable private SdkTracerProvider sdkTracerProvider; + private final BlockingSpanExporter blockingSpanExporter = new BlockingSpanExporter(); + + @Mock private Sampler mockSampler; + @Mock private SpanExporter mockSpanExporter; + + @BeforeEach + void setUp() { + when(mockSpanExporter.shutdown()).thenReturn(CompletableResultCode.ofSuccess()); + } + + @AfterEach + void cleanup() { + if (sdkTracerProvider != null) { + sdkTracerProvider.shutdown(); + } + } + + @Nullable + private ReadableSpan createEndedSpan(String spanName) { + Tracer tracer = sdkTracerProvider.get(getClass().getName()); + Span span = tracer.spanBuilder(spanName).startSpan(); + span.end(); + if (span instanceof ReadableSpan) { + return (ReadableSpan) span; + } else { + return null; + } + } + + @Test + void configTest_EmptyOptions() { + ConsistentReservoirSamplingBatchSpanProcessorBuilder config = + ConsistentReservoirSamplingBatchSpanProcessor.builder( + new WaitingSpanExporter(0, CompletableResultCode.ofSuccess())); + Assertions.assertThat(config.getScheduleDelayNanos()) + .isEqualTo( + TimeUnit.MILLISECONDS.toNanos( + ConsistentReservoirSamplingBatchSpanProcessorBuilder + .DEFAULT_SCHEDULE_DELAY_MILLIS)); + Assertions.assertThat(config.getReservoirSize()) + .isEqualTo(ConsistentReservoirSamplingBatchSpanProcessorBuilder.DEFAULT_RESERVOIR_SIZE); + Assertions.assertThat(config.getExporterTimeoutNanos()) + .isEqualTo( + TimeUnit.MILLISECONDS.toNanos( + ConsistentReservoirSamplingBatchSpanProcessorBuilder + .DEFAULT_EXPORT_TIMEOUT_MILLIS)); + } + + @Test + void invalidConfig() { + assertThatThrownBy( + () -> + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) + .setScheduleDelay(-1, TimeUnit.MILLISECONDS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("delay must be non-negative"); + assertThatThrownBy( + () -> + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) + .setScheduleDelay(1, null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("unit"); + assertThatThrownBy( + () -> + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) + .setScheduleDelay(null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("delay"); + assertThatThrownBy( + () -> + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) + .setExporterTimeout(-1, TimeUnit.MILLISECONDS)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("timeout must be non-negative"); + assertThatThrownBy( + () -> + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) + .setExporterTimeout(1, null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("unit"); + assertThatThrownBy( + () -> + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) + .setExporterTimeout(null)) + .isInstanceOf(NullPointerException.class) + .hasMessage("timeout"); + } + + @Test + void startEndRequirements() { + ConsistentReservoirSamplingBatchSpanProcessor spansProcessor = + ConsistentReservoirSamplingBatchSpanProcessor.builder( + new WaitingSpanExporter(0, CompletableResultCode.ofSuccess())) + .build(); + Assertions.assertThat(spansProcessor.isStartRequired()).isFalse(); + Assertions.assertThat(spansProcessor.isEndRequired()).isTrue(); + } + + @Test + void exportDifferentSampledSpans() { + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(2, CompletableResultCode.ofSuccess()); + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) + .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) + .build()) + .build(); + + ReadableSpan span1 = createEndedSpan(SPAN_NAME_1); + ReadableSpan span2 = createEndedSpan(SPAN_NAME_2); + List exported = waitingSpanExporter.waitForExport(); + Assertions.assertThat(exported) + .containsExactlyInAnyOrder(span1.toSpanData(), span2.toSpanData()); + } + + @Test + void exportMoreSpansThanTheBufferSize() { + CompletableSpanExporter spanExporter = new CompletableSpanExporter(); + + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + ConsistentReservoirSamplingBatchSpanProcessor.builder(spanExporter) + .setReservoirSize(6) + .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) + .build()) + .build(); + + ReadableSpan span1 = createEndedSpan(SPAN_NAME_1); + ReadableSpan span2 = createEndedSpan(SPAN_NAME_1); + ReadableSpan span3 = createEndedSpan(SPAN_NAME_1); + ReadableSpan span4 = createEndedSpan(SPAN_NAME_1); + ReadableSpan span5 = createEndedSpan(SPAN_NAME_1); + ReadableSpan span6 = createEndedSpan(SPAN_NAME_1); + + spanExporter.succeed(); + + await() + .untilAsserted( + () -> + Assertions.assertThat(spanExporter.getExported()) + .containsExactlyInAnyOrder( + span1.toSpanData(), + span2.toSpanData(), + span3.toSpanData(), + span4.toSpanData(), + span5.toSpanData(), + span6.toSpanData())); + } + + @Test + void forceExport() { + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(100, CompletableResultCode.ofSuccess(), 1); + ConsistentReservoirSamplingBatchSpanProcessor batchSpanProcessor = + ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) + // .setReservoirSize(10_000) + // Force flush should send all spans, make sure the number of spans we check here is + // not divisible by the batch size. + .setReservoirSize(49) + .setScheduleDelay(10, TimeUnit.SECONDS) + .build(); + + sdkTracerProvider = SdkTracerProvider.builder().addSpanProcessor(batchSpanProcessor).build(); + for (int i = 0; i < 100; i++) { + createEndedSpan("notExported"); + } + + batchSpanProcessor.forceFlush().join(10, TimeUnit.SECONDS); + List exported = waitingSpanExporter.getExported(); + Assertions.assertThat(exported).isNotNull(); + Assertions.assertThat(exported.size()).isEqualTo(49); + } + + @Test + void exportSpansToMultipleServices() { + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(2, CompletableResultCode.ofSuccess()); + WaitingSpanExporter waitingSpanExporter2 = + new WaitingSpanExporter(2, CompletableResultCode.ofSuccess()); + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + ConsistentReservoirSamplingBatchSpanProcessor.builder( + SpanExporter.composite( + Arrays.asList(waitingSpanExporter, waitingSpanExporter2))) + .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) + .build()) + .build(); + + ReadableSpan span1 = createEndedSpan(SPAN_NAME_1); + ReadableSpan span2 = createEndedSpan(SPAN_NAME_2); + List exported1 = waitingSpanExporter.waitForExport(); + List exported2 = waitingSpanExporter2.waitForExport(); + Assertions.assertThat(exported1) + .containsExactlyInAnyOrder(span1.toSpanData(), span2.toSpanData()); + Assertions.assertThat(exported2) + .containsExactlyInAnyOrder(span1.toSpanData(), span2.toSpanData()); + } + + @Test + void exportMoreSpansThanTheMaximumLimit() { + int maxQueuedSpans = 8; + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(maxQueuedSpans, CompletableResultCode.ofSuccess()); + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + ConsistentReservoirSamplingBatchSpanProcessor.builder( + SpanExporter.composite( + Arrays.asList(blockingSpanExporter, waitingSpanExporter))) + .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) + .setReservoirSize(maxQueuedSpans) + .build()) + .build(); + + List spansToExport = new ArrayList<>(maxQueuedSpans + 1); + // Wait to block the worker thread in the BatchSampledSpansProcessor. This ensures that no items + // can be removed from the queue. Need to add a span to trigger the export otherwise the + // pipeline is never called. + spansToExport.add(createEndedSpan("blocking_span").toSpanData()); + blockingSpanExporter.waitUntilIsBlocked(); + + for (int i = 0; i < maxQueuedSpans; i++) { + // First export maxQueuedSpans, the worker thread is blocked so all items should be queued. + spansToExport.add(createEndedSpan("span_1_" + i).toSpanData()); + } + + // TODO: assertThat(spanExporter.getReferencedSpans()).isEqualTo(maxQueuedSpans); + + // Now we should start dropping. + for (int i = 0; i < 7; i++) { + createEndedSpan("span_2_" + i); + // TODO: assertThat(getDroppedSpans()).isEqualTo(i + 1); + } + + // TODO: assertThat(getReferencedSpans()).isEqualTo(maxQueuedSpans); + + // Release the blocking exporter + blockingSpanExporter.unblock(); + + // While we wait for maxQueuedSpans we ensure that the queue is also empty after this. + List exported = waitingSpanExporter.waitForExport(); + Assertions.assertThat(exported).isNotNull(); + Assertions.assertThat(exported).hasSize(maxQueuedSpans + 1); + // assertThat(exported).containsExactlyInAnyOrderElementsOf(spansToExport); + exported.clear(); + spansToExport.clear(); + + waitingSpanExporter.reset(); + // We cannot compare with maxReferencedSpans here because the worker thread may get + // unscheduled immediately after exporting, but before updating the pushed spans, if that is + // the case at most bufferSize spans will miss. + // TODO: assertThat(getPushedSpans()).isAtLeast((long) maxQueuedSpans - maxBatchSize); + + for (int i = 0; i < maxQueuedSpans; i++) { + spansToExport.add(createEndedSpan("span_3_" + i).toSpanData()); + // No more dropped spans. + // TODO: assertThat(getDroppedSpans()).isEqualTo(7); + } + + exported = waitingSpanExporter.waitForExport(); + Assertions.assertThat(exported).isNotNull(); + Assertions.assertThat(exported).containsExactlyInAnyOrderElementsOf(spansToExport); + } + + @Test + void ignoresNullSpans() { + ConsistentReservoirSamplingBatchSpanProcessor processor = + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter).build(); + try { + assertThatCode( + () -> { + processor.onStart(null, null); + processor.onEnd(null); + }) + .doesNotThrowAnyException(); + } finally { + processor.shutdown(); + } + } + + @Test + void exporterThrowsException() { + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(1, CompletableResultCode.ofSuccess()); + doThrow(new IllegalArgumentException("No export for you.")) + .when(mockSpanExporter) + .export(ArgumentMatchers.anyList()); + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + ConsistentReservoirSamplingBatchSpanProcessor.builder( + SpanExporter.composite( + Arrays.asList(mockSpanExporter, waitingSpanExporter))) + .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) + .build()) + .build(); + ReadableSpan span1 = createEndedSpan(SPAN_NAME_1); + List exported = waitingSpanExporter.waitForExport(); + Assertions.assertThat(exported).containsExactly(span1.toSpanData()); + waitingSpanExporter.reset(); + // Continue to export after the exception was received. + ReadableSpan span2 = createEndedSpan(SPAN_NAME_2); + exported = waitingSpanExporter.waitForExport(); + Assertions.assertThat(exported).containsExactly(span2.toSpanData()); + } + + @Test + @Timeout(5) + public void continuesIfExporterTimesOut() throws InterruptedException { + int exporterTimeoutMillis = 10; + ConsistentReservoirSamplingBatchSpanProcessor bsp = + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) + .setExporterTimeout(exporterTimeoutMillis, TimeUnit.MILLISECONDS) + .setScheduleDelay(1, TimeUnit.MILLISECONDS) + .setReservoirSize(1) + .build(); + sdkTracerProvider = SdkTracerProvider.builder().addSpanProcessor(bsp).build(); + + CountDownLatch exported = new CountDownLatch(1); + // We return a result we never complete, meaning it will timeout. + when(mockSpanExporter.export( + argThat( + spans -> { + Assertions.assertThat(spans) + .anySatisfy( + span -> Assertions.assertThat(span.getName()).isEqualTo(SPAN_NAME_1)); + exported.countDown(); + return true; + }))) + .thenReturn(new CompletableResultCode()); + createEndedSpan(SPAN_NAME_1); + exported.await(); + // Timed out so the span was dropped. + await().untilAsserted(() -> Assertions.assertThat(bsp.isReservoirEmpty()).isTrue()); + + // Still processing new spans. + CountDownLatch exportedAgain = new CountDownLatch(1); + reset(mockSpanExporter); + when(mockSpanExporter.shutdown()).thenReturn(CompletableResultCode.ofSuccess()); + when(mockSpanExporter.export( + argThat( + spans -> { + Assertions.assertThat(spans) + .anySatisfy( + span -> Assertions.assertThat(span.getName()).isEqualTo(SPAN_NAME_2)); + exportedAgain.countDown(); + return true; + }))) + .thenReturn(CompletableResultCode.ofSuccess()); + createEndedSpan(SPAN_NAME_2); + exported.await(); + await().untilAsserted(() -> Assertions.assertThat(bsp.isReservoirEmpty()).isTrue()); + } + + @Test + void exportNotSampledSpans() { + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(1, CompletableResultCode.ofSuccess()); + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) + .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) + .build()) + .setSampler(mockSampler) + .build(); + + when(mockSampler.shouldSample(any(), any(), any(), any(), any(), anyList())) + .thenReturn(SamplingResult.drop()); + sdkTracerProvider.get("test").spanBuilder(SPAN_NAME_1).startSpan().end(); + sdkTracerProvider.get("test").spanBuilder(SPAN_NAME_2).startSpan().end(); + when(mockSampler.shouldSample(any(), any(), any(), any(), any(), anyList())) + .thenReturn(SamplingResult.recordAndSample()); + ReadableSpan span = createEndedSpan(SPAN_NAME_2); + // Spans are recorded and exported in the same order as they are ended, we test that a non + // sampled span is not exported by creating and ending a sampled span after a non sampled span + // and checking that the first exported span is the sampled span (the non sampled did not get + // exported). + List exported = waitingSpanExporter.waitForExport(); + // Need to check this because otherwise the variable span1 is unused, other option is to not + // have a span1 variable. + Assertions.assertThat(exported).containsExactly(span.toSpanData()); + } + + @Test + void exportNotSampledSpans_recordOnly() { + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(1, CompletableResultCode.ofSuccess()); + + when(mockSampler.shouldSample(any(), any(), any(), any(), any(), anyList())) + .thenReturn(SamplingResult.recordOnly()); + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) + .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) + .build()) + .setSampler(mockSampler) + .build(); + + createEndedSpan(SPAN_NAME_1); + when(mockSampler.shouldSample(any(), any(), any(), any(), any(), anyList())) + .thenReturn(SamplingResult.recordAndSample()); + ReadableSpan span = createEndedSpan(SPAN_NAME_2); + + // Spans are recorded and exported in the same order as they are ended, we test that a non + // exported span is not exported by creating and ending a sampled span after a non sampled span + // and checking that the first exported span is the sampled span (the non sampled did not get + // exported). + List exported = waitingSpanExporter.waitForExport(); + // Need to check this because otherwise the variable span1 is unused, other option is to not + // have a span1 variable. + Assertions.assertThat(exported).containsExactly(span.toSpanData()); + } + + @Test + @Timeout(10) + void shutdownFlushes() { + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(1, CompletableResultCode.ofSuccess()); + // Set the export delay to large value, in order to confirm the #flush() below works + + sdkTracerProvider = + SdkTracerProvider.builder() + .addSpanProcessor( + ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) + .setScheduleDelay(10, TimeUnit.SECONDS) + .build()) + .build(); + + ReadableSpan span2 = createEndedSpan(SPAN_NAME_2); + + // Force a shutdown, which forces processing of all remaining spans. + sdkTracerProvider.shutdown().join(10, TimeUnit.SECONDS); + + List exported = waitingSpanExporter.getExported(); + Assertions.assertThat(exported).containsExactly(span2.toSpanData()); + Assertions.assertThat(waitingSpanExporter.shutDownCalled.get()).isTrue(); + } + + @Test + void shutdownPropagatesSuccess() { + ConsistentReservoirSamplingBatchSpanProcessor processor = + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter).build(); + CompletableResultCode result = processor.shutdown(); + result.join(1, TimeUnit.SECONDS); + Assertions.assertThat(result.isSuccess()).isTrue(); + } + + @Test + void shutdownPropagatesFailure() { + when(mockSpanExporter.shutdown()).thenReturn(CompletableResultCode.ofFailure()); + ConsistentReservoirSamplingBatchSpanProcessor processor = + ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter).build(); + CompletableResultCode result = processor.shutdown(); + result.join(1, TimeUnit.SECONDS); + Assertions.assertThat(result.isSuccess()).isFalse(); + } + + private static final class BlockingSpanExporter implements SpanExporter { + + final Object monitor = new Object(); + + private enum State { + WAIT_TO_BLOCK, + BLOCKED, + UNBLOCKED + } + + @GuardedBy("monitor") + State state = State.WAIT_TO_BLOCK; + + @Override + public CompletableResultCode export(Collection spanDataList) { + synchronized (monitor) { + while (state != State.UNBLOCKED) { + try { + state = State.BLOCKED; + // Some threads may wait for Blocked State. + monitor.notifyAll(); + monitor.wait(); + } catch (InterruptedException e) { + // Do nothing + } + } + } + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + private void waitUntilIsBlocked() { + synchronized (monitor) { + while (state != State.BLOCKED) { + try { + monitor.wait(); + } catch (InterruptedException e) { + // Do nothing + } + } + } + } + + @Override + public CompletableResultCode shutdown() { + // Do nothing; + return CompletableResultCode.ofSuccess(); + } + + private void unblock() { + synchronized (monitor) { + state = State.UNBLOCKED; + monitor.notifyAll(); + } + } + } + + private static class CompletableSpanExporter implements SpanExporter { + + private final List results = new ArrayList<>(); + + private final List exported = new ArrayList<>(); + + private volatile boolean succeeded; + + List getExported() { + return exported; + } + + void succeed() { + succeeded = true; + results.forEach(CompletableResultCode::succeed); + } + + @Override + public CompletableResultCode export(Collection spans) { + exported.addAll(spans); + if (succeeded) { + return CompletableResultCode.ofSuccess(); + } + CompletableResultCode result = new CompletableResultCode(); + results.add(result); + return result; + } + + @Override + public CompletableResultCode flush() { + if (succeeded) { + return CompletableResultCode.ofSuccess(); + } else { + return CompletableResultCode.ofFailure(); + } + } + + @Override + public CompletableResultCode shutdown() { + return flush(); + } + } + + static class WaitingSpanExporter implements SpanExporter { + + private final List spanDataList = new ArrayList<>(); + private final int numberToWaitFor; + private final CompletableResultCode exportResultCode; + private CountDownLatch countDownLatch; + private int timeout = 10; + private final AtomicBoolean shutDownCalled = new AtomicBoolean(false); + + WaitingSpanExporter(int numberToWaitFor, CompletableResultCode exportResultCode) { + countDownLatch = new CountDownLatch(numberToWaitFor); + this.numberToWaitFor = numberToWaitFor; + this.exportResultCode = exportResultCode; + } + + WaitingSpanExporter(int numberToWaitFor, CompletableResultCode exportResultCode, int timeout) { + this(numberToWaitFor, exportResultCode); + this.timeout = timeout; + } + + List getExported() { + List result = new ArrayList<>(spanDataList); + spanDataList.clear(); + return result; + } + + /** + * Waits until we received numberOfSpans spans to export. Returns the list of exported {@link + * SpanData} objects, otherwise {@code null} if the current thread is interrupted. + * + * @return the list of exported {@link SpanData} objects, otherwise {@code null} if the current + * thread is interrupted. + */ + @Nullable + List waitForExport() { + try { + countDownLatch.await(timeout, TimeUnit.SECONDS); + } catch (InterruptedException e) { + // Preserve the interruption status as per guidance. + Thread.currentThread().interrupt(); + return null; + } + return getExported(); + } + + @Override + public CompletableResultCode export(Collection spans) { + this.spanDataList.addAll(spans); + for (int i = 0; i < spans.size(); i++) { + countDownLatch.countDown(); + } + return exportResultCode; + } + + @Override + public CompletableResultCode flush() { + return CompletableResultCode.ofSuccess(); + } + + @Override + public CompletableResultCode shutdown() { + shutDownCalled.set(true); + return CompletableResultCode.ofSuccess(); + } + + public void reset() { + this.countDownLatch = new CountDownLatch(numberToWaitFor); + } + } + + @Test + void exportDifferentConsistentlySampledSpans() { + int reservoirSize = 10; + int numberOfSpans = 100; + + WaitingSpanExporter waitingSpanExporter = + new WaitingSpanExporter(reservoirSize, CompletableResultCode.ofSuccess()); + + ConsistentReservoirSamplingBatchSpanProcessor spanProcessor = + ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) + .setReservoirSize(reservoirSize) + .setScheduleDelay(10, TimeUnit.SECONDS) + .setReservoirSize(reservoirSize) + .build(); + + sdkTracerProvider = + SdkTracerProvider.builder() + .setSampler(new ConsistentAlwaysOnSampler()) + .addSpanProcessor(spanProcessor) + .build(); + + List spans = + IntStream.range(0, numberOfSpans) + .mapToObj(i -> createEndedSpan("MySpanName/" + i)) + .collect(toList()); + + Assertions.assertThat(spans).hasSize(numberOfSpans); + + spanProcessor.forceFlush().join(10, TimeUnit.SECONDS); + + List exported = waitingSpanExporter.waitForExport(); + Assertions.assertThat(exported).hasSize(reservoirSize); + } + + private enum Tests { + VERIFY_MEAN, + VERIFY_PVALUE_DISTRIBUTION, + VERIFY_ORDER_INDEPENDENCE + } + + private void testConsistentSampling( + boolean useAlternativeReservoirImplementation, + long seed, + int numCycles, + int numberOfSpans, + int reservoirSize, + double samplingProbability, + EnumSet tests) { + + SplittableRandom rng1 = new SplittableRandom(seed); + SplittableRandom rng2 = rng1.split(); + RandomGenerator threadSafeRandomGenerator1 = + () -> { + synchronized (rng1) { + return rng1.nextLong(); + } + }; + RandomGenerator threadSafeRandomGenerator2 = + () -> { + synchronized (rng2) { + return rng1.nextLong(); + } + }; + + WaitingSpanExporter spanExporter = + new WaitingSpanExporter(0, CompletableResultCode.ofSuccess()); + + ConsistentReservoirSamplingBatchSpanProcessor spanProcessor = + ConsistentReservoirSamplingBatchSpanProcessor.builder(spanExporter) + .setReservoirSize(reservoirSize) + .setThreadSafeRandomGenerator(threadSafeRandomGenerator1) + .setScheduleDelay(1000, TimeUnit.SECONDS) + .setReservoirSize(reservoirSize) + .useAlternativeReservoirImplementation(useAlternativeReservoirImplementation) + .build(); + + sdkTracerProvider = + SdkTracerProvider.builder() + .setSampler( + new ConsistentProbabilityBasedSampler( + samplingProbability, threadSafeRandomGenerator2)) + .addSpanProcessor(spanProcessor) + .build(); + + Map observedPvalues = new HashMap<>(); + Map spanNameCounts = new HashMap<>(); + + double[] totalAdjustedCounts = new double[numCycles]; + + for (int k = 0; k < numCycles; ++k) { + String prefixSpanName = "MySpanName/" + k + "/"; + List spans = + LongStream.range(0, numberOfSpans) + .mapToObj(i -> createEndedSpan(prefixSpanName + i)) + .filter(Objects::nonNull) + .collect(toList()); + + if (samplingProbability >= 1.) { + Assertions.assertThat(spans).hasSize(numberOfSpans); + } + + spanProcessor.forceFlush().join(1000, TimeUnit.SECONDS); + + List exported = spanExporter.getExported(); + Assertions.assertThat(exported).hasSize(Math.min(reservoirSize, spans.size())); + + double totalAdjustedCount = 0; + for (SpanData spanData : exported) { + String traceStateString = + spanData.getSpanContext().getTraceState().get(OtelTraceState.TRACE_STATE_KEY); + OtelTraceState traceState = OtelTraceState.parse(traceStateString); + assertTrue(traceState.hasValidR()); + assertTrue(traceState.hasValidP()); + observedPvalues.merge(traceState.getP(), 1L, Long::sum); + totalAdjustedCount += Math.pow(2., traceState.getP()); + spanNameCounts.merge(spanData.getName().split("/")[2], 1L, Long::sum); + } + totalAdjustedCounts[k] = totalAdjustedCount; + } + + long totalNumberOfSpans = numberOfSpans * (long) numCycles; + if (numCycles == 1) { + Assertions.assertThat(observedPvalues).hasSizeLessThanOrEqualTo(2); + } + if (tests.contains(Tests.VERIFY_MEAN)) { + Assertions.assertThat(reservoirSize) + .isGreaterThanOrEqualTo( + 100); // require a lower limit on the reservoir size, to justify the assumption of the + // t-test that values are normally distributed + + Assertions.assertThat( + new TTest().tTest(totalNumberOfSpans / (double) numCycles, totalAdjustedCounts)) + .isGreaterThan(0.01); + } + if (tests.contains(Tests.VERIFY_PVALUE_DISTRIBUTION)) { + Assertions.assertThat(observedPvalues) + .hasSizeLessThanOrEqualTo(2); // test does not work for more than 2 different p-values + + // The expected number of sampled spans is binomially distributed with the given sampling + // probability. However, due to the reservoir sampling buffer the maximum number of sampled + // spans is given by the reservoir size. The effective sampling rate is therefore given by + // sum_{i=0}^n p^i*(1-p)^{n-i}*min(i,k) (n choose i) + // where p denotes the sampling rate, n is the total number of original spans, and k denotes + // the reservoir size + double p1 = + new BinomialDistribution(numberOfSpans - 1, samplingProbability) + .cumulativeProbability(reservoirSize - 1); + double p2 = + new BinomialDistribution(numberOfSpans, samplingProbability) + .cumulativeProbability(reservoirSize); + Assertions.assertThat(p1).isLessThanOrEqualTo(p2); + + double effectiveSamplingProbability = + samplingProbability * p1 + (reservoirSize / (double) numberOfSpans) * (1. - p2); + verifyObservedPvaluesUsingGtest( + totalNumberOfSpans, observedPvalues, effectiveSamplingProbability); + } + if (tests.contains(Tests.VERIFY_ORDER_INDEPENDENCE)) { + Assertions.assertThat(spanNameCounts.size()).isEqualTo(numberOfSpans); + long[] observed = spanNameCounts.values().stream().mapToLong(x -> x).toArray(); + double[] expected = new double[numberOfSpans]; + Arrays.fill(expected, 1.); + Assertions.assertThat(new GTest().gTest(expected, observed)).isGreaterThan(0.01); + } + } + + private void testConsistentSampling(boolean useAlternativeReservoirImplementation) { + testConsistentSampling( + useAlternativeReservoirImplementation, + 0x34e7052af91d5355L, + 10000, + 1000, + 100, + 1., + EnumSet.of(Tests.VERIFY_MEAN, Tests.VERIFY_ORDER_INDEPENDENCE)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0xcd02a41e10ff273dL, + 10000, + 1000, + 100, + 0.8, + EnumSet.of(Tests.VERIFY_MEAN, Tests.VERIFY_ORDER_INDEPENDENCE)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0x2c3d086534e14407L, + 10000, + 1000, + 100, + 0.1, + EnumSet.of( + Tests.VERIFY_MEAN, Tests.VERIFY_PVALUE_DISTRIBUTION, Tests.VERIFY_ORDER_INDEPENDENCE)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0xd3f8a40433cf0522L, + 10000, + 1000, + 200, + 0.9, + EnumSet.of(Tests.VERIFY_MEAN, Tests.VERIFY_ORDER_INDEPENDENCE)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0xf25638ca67eceadcL, + 10000, + 100, + 100, + 1.0, + EnumSet.of(Tests.VERIFY_MEAN)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0x14c5f8f815618ce2L, + 10000, + 200, + 100, + 1.0, + EnumSet.of(Tests.VERIFY_MEAN, Tests.VERIFY_ORDER_INDEPENDENCE)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0xb6c27f1169e128ddL, + 10000, + 1000, + 200, + 0.2, + EnumSet.of( + Tests.VERIFY_MEAN, Tests.VERIFY_PVALUE_DISTRIBUTION, Tests.VERIFY_ORDER_INDEPENDENCE)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0xab558ff7c5c73c18L, + 1000, + 10000, + 200, + 1., + EnumSet.of( + Tests.VERIFY_MEAN, Tests.VERIFY_PVALUE_DISTRIBUTION, Tests.VERIFY_ORDER_INDEPENDENCE)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0xe53010c4b009a6c0L, + 10000, + 1000, + 2000, + 0.2, + EnumSet.of( + Tests.VERIFY_MEAN, Tests.VERIFY_PVALUE_DISTRIBUTION, Tests.VERIFY_ORDER_INDEPENDENCE)); + testConsistentSampling( + useAlternativeReservoirImplementation, + 0xc41d327fd1a6866aL, + 1000000, + 5, + 4, + 1.0, + EnumSet.of(Tests.VERIFY_ORDER_INDEPENDENCE)); + } + + @Test + void testConsistentSampling() { + testConsistentSampling(false); + } + + @Test + void testConsistentSamplingWithAlternativeReservoirImplementation() { + testConsistentSampling(true); + } + + private StatisticalSummary calculateStatisticalSummary( + boolean useAlternativeReservoirImplementation, + long seed, + int numCycles, + int numberOfSpans, + int reservoirSize, + double samplingProbability) { + + SplittableRandom rng1 = new SplittableRandom(seed); + SplittableRandom rng2 = rng1.split(); + RandomGenerator threadSafeRandomGenerator1 = + () -> { + synchronized (rng1) { + return rng1.nextLong(); + } + }; + RandomGenerator threadSafeRandomGenerator2 = + () -> { + synchronized (rng2) { + return rng1.nextLong(); + } + }; + + WaitingSpanExporter spanExporter = + new WaitingSpanExporter(0, CompletableResultCode.ofSuccess()); + + ConsistentReservoirSamplingBatchSpanProcessor spanProcessor = + ConsistentReservoirSamplingBatchSpanProcessor.builder(spanExporter) + .setReservoirSize(reservoirSize) + .setThreadSafeRandomGenerator(threadSafeRandomGenerator1) + .setScheduleDelay(1000, TimeUnit.SECONDS) + .setReservoirSize(reservoirSize) + .useAlternativeReservoirImplementation(useAlternativeReservoirImplementation) + .build(); + + sdkTracerProvider = + SdkTracerProvider.builder() + .setSampler( + new ConsistentProbabilityBasedSampler( + samplingProbability, threadSafeRandomGenerator2)) + .addSpanProcessor(spanProcessor) + .build(); + + StreamingStatistics streamingStatistics = new StreamingStatistics(); + + for (int k = 0; k < numCycles; ++k) { + String prefixSpanName = "MySpanName/" + k + "/"; + List spans = + LongStream.range(0, numberOfSpans) + .mapToObj(i -> createEndedSpan(prefixSpanName + i)) + .filter(Objects::nonNull) + .collect(toList()); + + if (samplingProbability >= 1.) { + Assertions.assertThat(spans).hasSize(numberOfSpans); + } + + spanProcessor.forceFlush().join(1000, TimeUnit.SECONDS); + + List exported = spanExporter.getExported(); + Assertions.assertThat(exported).hasSize(Math.min(reservoirSize, spans.size())); + + double totalAdjustedCount = 0; + for (SpanData spanData : exported) { + String traceStateString = + spanData.getSpanContext().getTraceState().get(OtelTraceState.TRACE_STATE_KEY); + OtelTraceState traceState = OtelTraceState.parse(traceStateString); + assertTrue(traceState.hasValidR()); + assertTrue(traceState.hasValidP()); + totalAdjustedCount += Math.pow(2., traceState.getP()); + } + streamingStatistics.accept(totalAdjustedCount); + } + return streamingStatistics; + } + + @Test + void testVarianceDifferencesBetweenReservoirImplementations1() { + boolean useAlternativeReservoirImplementationFalse = false; + boolean useAlternativeReservoirImplementationTrue = true; + + StatisticalSummary variant1 = + calculateStatisticalSummary( + useAlternativeReservoirImplementationFalse, 0x225de9590067d658L, 10000, 100, 50, 0.8); + + StatisticalSummary variant2 = + calculateStatisticalSummary( + useAlternativeReservoirImplementationTrue, 0x23b418ed68d668L, 10000, 100, 50, 0.8); + + Assertions.assertThat(variant1.getMean()).isCloseTo(100, Percentage.withPercentage(1)); + Assertions.assertThat(variant2.getMean()).isCloseTo(100, Percentage.withPercentage(1)); + + Assertions.assertThat(variant1.getVariance()) + .isCloseTo(111.17153690368997, Percentage.withPercentage(0.01)); + Assertions.assertThat(variant2.getVariance()) + .isCloseTo(95.65024978497851, Percentage.withPercentage(0.01)); + } + + @Test + void testVarianceDifferencesBetweenReservoirImplementations2() { + boolean useAlternativeReservoirImplementationFalse = false; + boolean useAlternativeReservoirImplementationTrue = true; + + StatisticalSummary variant1 = + calculateStatisticalSummary( + useAlternativeReservoirImplementationFalse, 0x7f33baf84d59df65L, 100000, 10, 4, 0.9); + + StatisticalSummary variant2 = + calculateStatisticalSummary( + useAlternativeReservoirImplementationTrue, 0xdb3bf0109e0a4b43L, 100000, 10, 4, 0.9); + + Assertions.assertThat(variant1.getMean()).isCloseTo(10, Percentage.withPercentage(2)); + Assertions.assertThat(variant2.getMean()).isCloseTo(10, Percentage.withPercentage(2)); + Assertions.assertThat(variant1.getVariance()) + .isCloseTo(22.617777121371127, Percentage.withPercentage(0.01)); + Assertions.assertThat(variant2.getVariance()) + .isCloseTo(19.465000847508666, Percentage.withPercentage(0.01)); + } + + @Test + void testVarianceDifferencesBetweenReservoirImplementations3() { + boolean useAlternativeReservoirImplementationFalse = false; + boolean useAlternativeReservoirImplementationTrue = true; + + StatisticalSummary variant1 = + calculateStatisticalSummary( + useAlternativeReservoirImplementationFalse, 0x72d7312adac9c84dL, 10000, 1000, 700, 1); + + StatisticalSummary variant2 = + calculateStatisticalSummary( + useAlternativeReservoirImplementationTrue, 0x7ea0c32d80a319d0L, 10000, 1000, 700, 1.); + + Assertions.assertThat(variant1.getMean()).isCloseTo(1000, Percentage.withPercentage(1)); + Assertions.assertThat(variant2.getMean()).isCloseTo(1000, Percentage.withPercentage(1)); + Assertions.assertThat(variant1.getVariance()) + .isCloseTo(594.9523749875004, Percentage.withPercentage(0.01)); + Assertions.assertThat(variant2.getVariance()) + .isCloseTo(362.2863018701875, Percentage.withPercentage(0.01)); + } +} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java new file mode 100644 index 000000000..a2120788d --- /dev/null +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java @@ -0,0 +1,87 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import static io.opentelemetry.contrib.util.TestUtil.verifyObservedPvaluesUsingGtest; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.api.trace.TraceState; +import io.opentelemetry.context.Context; +import io.opentelemetry.contrib.state.OtelTraceState; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.sdk.trace.samplers.SamplingDecision; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.SplittableRandom; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class ConsistentProbabilityBasedSamplerTest { + + private Context parentContext; + private String traceId; + private String name; + private SpanKind spanKind; + private Attributes attributes; + private List parentLinks; + + @BeforeEach + public void init() { + + parentContext = Context.root(); + traceId = "0123456789abcdef0123456789abcdef"; + name = "name"; + spanKind = SpanKind.SERVER; + attributes = Attributes.empty(); + parentLinks = Collections.emptyList(); + } + + private void test(SplittableRandom rng, double samplingProbability) { + int numSpans = 1000000; + + Sampler sampler = new ConsistentProbabilityBasedSampler(samplingProbability, rng::nextLong); + + Map observedPvalues = new HashMap<>(); + for (long i = 0; i < numSpans; ++i) { + SamplingResult samplingResult = + sampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + if (samplingResult.getDecision() == SamplingDecision.RECORD_AND_SAMPLE) { + String traceStateString = + samplingResult + .getUpdatedTraceState(TraceState.getDefault()) + .get(OtelTraceState.TRACE_STATE_KEY); + OtelTraceState traceState = OtelTraceState.parse(traceStateString); + assertTrue(traceState.hasValidR()); + assertTrue(traceState.hasValidP()); + observedPvalues.merge(traceState.getP(), 1L, Long::sum); + } + } + verifyObservedPvaluesUsingGtest(numSpans, observedPvalues, samplingProbability); + } + + @Test + public void test() { + + // fix seed to get reproducible results + SplittableRandom random = new SplittableRandom(0); + + test(random, 1.); + test(random, 0.5); + test(random, 0.25); + test(random, 0.125); + test(random, 0.0); + test(random, 0.45); + test(random, 0.2); + test(random, 0.13); + test(random, 0.05); + } +} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java new file mode 100644 index 000000000..d6b9b94fe --- /dev/null +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java @@ -0,0 +1,202 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import static org.assertj.core.api.Assertions.assertThat; + +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.trace.SpanKind; +import io.opentelemetry.context.Context; +import io.opentelemetry.sdk.trace.data.LinkData; +import io.opentelemetry.sdk.trace.samplers.SamplingDecision; +import io.opentelemetry.sdk.trace.samplers.SamplingResult; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.SplittableRandom; +import java.util.concurrent.TimeUnit; +import java.util.function.LongSupplier; +import org.assertj.core.data.Percentage; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class ConsistentRateLimitingSamplerTest { + + private long[] nanoTime; + private LongSupplier nanoTimeSupplier; + private Context parentContext; + private String traceId; + private String name; + private SpanKind spanKind; + private Attributes attributes; + private List parentLinks; + + @BeforeEach + void init() { + nanoTime = new long[] {0L}; + nanoTimeSupplier = () -> nanoTime[0]; + parentContext = Context.root(); + traceId = "0123456789abcdef0123456789abcdef"; + name = "name"; + spanKind = SpanKind.SERVER; + attributes = Attributes.empty(); + parentLinks = Collections.emptyList(); + } + + private void advanceTime(long nanosIncrement) { + nanoTime[0] += nanosIncrement; + } + + private long getCurrentTimeNanos() { + return nanoTime[0]; + } + + @Test + void testConstantRate() { + + double targetSpansPerSecondLimit = 1000; + double adaptationTimeSeconds = 5; + SplittableRandom random = new SplittableRandom(0L); + + ConsistentRateLimitingSampler sampler = + new ConsistentRateLimitingSampler( + targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); + + long nanosBetweenSpans = TimeUnit.MICROSECONDS.toNanos(100); + int numSpans = 1000000; + + List spanSampledNanos = new ArrayList<>(); + + for (int i = 0; i < numSpans; ++i) { + advanceTime(nanosBetweenSpans); + SamplingResult samplingResult = + sampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + if (SamplingDecision.RECORD_AND_SAMPLE.equals(samplingResult.getDecision())) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + + long numSampledSpansInLast5Seconds = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(95) && x <= TimeUnit.SECONDS.toNanos(100)) + .count(); + + assertThat(numSampledSpansInLast5Seconds / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + } + + @Test + void testRateIncrease() { + + double targetSpansPerSecondLimit = 1000; + double adaptationTimeSeconds = 5; + SplittableRandom random = new SplittableRandom(0L); + + ConsistentRateLimitingSampler sampler = + new ConsistentRateLimitingSampler( + targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); + + long nanosBetweenSpans1 = TimeUnit.MICROSECONDS.toNanos(100); + long nanosBetweenSpans2 = TimeUnit.MICROSECONDS.toNanos(10); + int numSpans1 = 500000; + int numSpans2 = 5000000; + + List spanSampledNanos = new ArrayList<>(); + + for (int i = 0; i < numSpans1; ++i) { + advanceTime(nanosBetweenSpans1); + SamplingResult samplingResult = + sampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + if (SamplingDecision.RECORD_AND_SAMPLE.equals(samplingResult.getDecision())) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + for (int i = 0; i < numSpans2; ++i) { + advanceTime(nanosBetweenSpans2); + SamplingResult samplingResult = + sampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + if (SamplingDecision.RECORD_AND_SAMPLE.equals(samplingResult.getDecision())) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + + long numSampledSpansWithin5SecondsBeforeChange = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(45) && x <= TimeUnit.SECONDS.toNanos(50)) + .count(); + long numSampledSpansWithin5SecondsAfterChange = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(50) && x <= TimeUnit.SECONDS.toNanos(55)) + .count(); + long numSampledSpansInLast5Seconds = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(95) && x <= TimeUnit.SECONDS.toNanos(100)) + .count(); + + assertThat(numSampledSpansWithin5SecondsBeforeChange / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + assertThat(numSampledSpansWithin5SecondsAfterChange / 5.) + .isGreaterThan(2. * targetSpansPerSecondLimit); + assertThat(numSampledSpansInLast5Seconds / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + } + + @Test + void testRateDecrease() { + + double targetSpansPerSecondLimit = 1000; + double adaptationTimeSeconds = 5; + SplittableRandom random = new SplittableRandom(0L); + + ConsistentRateLimitingSampler sampler = + new ConsistentRateLimitingSampler( + targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); + + long nanosBetweenSpans1 = TimeUnit.MICROSECONDS.toNanos(10); + long nanosBetweenSpans2 = TimeUnit.MICROSECONDS.toNanos(100); + int numSpans1 = 5000000; + int numSpans2 = 500000; + + List spanSampledNanos = new ArrayList<>(); + + for (int i = 0; i < numSpans1; ++i) { + advanceTime(nanosBetweenSpans1); + SamplingResult samplingResult = + sampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + if (SamplingDecision.RECORD_AND_SAMPLE.equals(samplingResult.getDecision())) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + for (int i = 0; i < numSpans2; ++i) { + advanceTime(nanosBetweenSpans2); + SamplingResult samplingResult = + sampler.shouldSample(parentContext, traceId, name, spanKind, attributes, parentLinks); + if (SamplingDecision.RECORD_AND_SAMPLE.equals(samplingResult.getDecision())) { + spanSampledNanos.add(getCurrentTimeNanos()); + } + } + + long numSampledSpansWithin5SecondsBeforeChange = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(45) && x <= TimeUnit.SECONDS.toNanos(50)) + .count(); + long numSampledSpansWithin5SecondsAfterChange = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(50) && x <= TimeUnit.SECONDS.toNanos(55)) + .count(); + long numSampledSpansInLast5Seconds = + spanSampledNanos.stream() + .filter(x -> x > TimeUnit.SECONDS.toNanos(95) && x <= TimeUnit.SECONDS.toNanos(100)) + .count(); + + assertThat(numSampledSpansWithin5SecondsBeforeChange / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + assertThat(numSampledSpansWithin5SecondsAfterChange / 5.) + .isLessThan(0.5 * targetSpansPerSecondLimit); + assertThat(numSampledSpansInLast5Seconds / 5.) + .isCloseTo(targetSpansPerSecondLimit, Percentage.withPercentage(5)); + } +} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java new file mode 100644 index 000000000..d0adf2f79 --- /dev/null +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java @@ -0,0 +1,72 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.samplers; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.opentelemetry.contrib.state.OtelTraceState; +import java.util.SplittableRandom; +import org.junit.jupiter.api.Test; + +class ConsistentSamplerTest { + + @Test + void testGetSamplingRate() { + assertEquals(Double.NaN, ConsistentSampler.getSamplingProbability(-1)); + for (int i = 0; i < OtelTraceState.getMaxP() - 1; i += 1) { + assertEquals(Math.pow(0.5, i), ConsistentSampler.getSamplingProbability(i)); + } + assertEquals(0., ConsistentSampler.getSamplingProbability(OtelTraceState.getMaxP())); + assertEquals( + Double.NaN, ConsistentSampler.getSamplingProbability(OtelTraceState.getMaxP() + 1)); + } + + @Test + void testGetLowerBoundP() { + assertEquals(0, ConsistentSampler.getLowerBoundP(1.0)); + assertEquals(0, ConsistentSampler.getLowerBoundP(Math.nextDown(1.0))); + for (int i = 1; i < OtelTraceState.getMaxP() - 1; i += 1) { + double samplingProbability = Math.pow(0.5, i); + assertEquals(i, ConsistentSampler.getLowerBoundP(samplingProbability)); + assertEquals(i - 1, ConsistentSampler.getLowerBoundP(Math.nextUp(samplingProbability))); + assertEquals(i, ConsistentSampler.getLowerBoundP(Math.nextDown(samplingProbability))); + } + assertEquals(OtelTraceState.getMaxP() - 1, ConsistentSampler.getLowerBoundP(Double.MIN_NORMAL)); + assertEquals(OtelTraceState.getMaxP() - 1, ConsistentSampler.getLowerBoundP(Double.MIN_VALUE)); + assertEquals(OtelTraceState.getMaxP(), ConsistentSampler.getLowerBoundP(0.0)); + } + + @Test + void testGetUpperBoundP() { + assertEquals(0, ConsistentSampler.getUpperBoundP(1.0)); + assertEquals(1, ConsistentSampler.getUpperBoundP(Math.nextDown(1.0))); + for (int i = 1; i < OtelTraceState.getMaxP() - 1; i += 1) { + double samplingProbability = Math.pow(0.5, i); + assertEquals(i, ConsistentSampler.getUpperBoundP(samplingProbability)); + assertEquals(i, ConsistentSampler.getUpperBoundP(Math.nextUp(samplingProbability))); + assertEquals(i + 1, ConsistentSampler.getUpperBoundP(Math.nextDown(samplingProbability))); + } + assertEquals(OtelTraceState.getMaxP(), ConsistentSampler.getUpperBoundP(Double.MIN_NORMAL)); + assertEquals(OtelTraceState.getMaxP(), ConsistentSampler.getUpperBoundP(Double.MIN_VALUE)); + assertEquals(OtelTraceState.getMaxP(), ConsistentSampler.getUpperBoundP(0.0)); + } + + @Test + void testRandomValues() { + int numCycles = 1000; + SplittableRandom random = new SplittableRandom(0L); + for (int i = 0; i < numCycles; ++i) { + double samplingProbability = Math.exp(-1. / random.nextDouble()); + int pmin = ConsistentSampler.getLowerBoundP(samplingProbability); + int pmax = ConsistentSampler.getUpperBoundP(samplingProbability); + assertThat(ConsistentSampler.getSamplingProbability(pmin)) + .isGreaterThanOrEqualTo(samplingProbability); + assertThat(ConsistentSampler.getSamplingProbability(pmax)) + .isLessThanOrEqualTo(samplingProbability); + } + } +} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java new file mode 100644 index 000000000..ab371f4e3 --- /dev/null +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java @@ -0,0 +1,74 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.state; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +public class OtelTraceStateTest { + + private static String getXString(int len) { + return Stream.generate(() -> "X").limit(len).collect(Collectors.joining()); + } + + @Test + public void test() { + + Assertions.assertEquals("", OtelTraceState.parse("").serialize()); + assertEquals("", OtelTraceState.parse("").serialize()); + + assertEquals("", OtelTraceState.parse("a").serialize()); + assertEquals("", OtelTraceState.parse("#").serialize()); + assertEquals("", OtelTraceState.parse(" ").serialize()); + + assertEquals("p:5", OtelTraceState.parse("p:5").serialize()); + assertEquals("p:63", OtelTraceState.parse("p:63").serialize()); + assertEquals("", OtelTraceState.parse("p:64").serialize()); + assertEquals("", OtelTraceState.parse("p:5;").serialize()); + assertEquals("", OtelTraceState.parse("p:99").serialize()); + assertEquals("", OtelTraceState.parse("p:").serialize()); + assertEquals("", OtelTraceState.parse("p:232").serialize()); + assertEquals("", OtelTraceState.parse("x;p:5").serialize()); + assertEquals("", OtelTraceState.parse("p:5;x").serialize()); + assertEquals("p:5;x:3", OtelTraceState.parse("x:3;p:5").serialize()); + assertEquals("p:5;x:3", OtelTraceState.parse("p:5;x:3").serialize()); + assertEquals("", OtelTraceState.parse("p:5;x:3;").serialize()); + assertEquals( + "p:5;a:" + getXString(246) + ";x:3", + OtelTraceState.parse("a:" + getXString(246) + ";p:5;x:3").serialize()); + assertEquals("", OtelTraceState.parse("a:" + getXString(247) + ";p:5;x:3").serialize()); + + assertEquals("r:5", OtelTraceState.parse("r:5").serialize()); + assertEquals("r:62", OtelTraceState.parse("r:62").serialize()); + assertEquals("", OtelTraceState.parse("r:63").serialize()); + assertEquals("", OtelTraceState.parse("r:5;").serialize()); + assertEquals("", OtelTraceState.parse("r:99").serialize()); + assertEquals("", OtelTraceState.parse("r:").serialize()); + assertEquals("", OtelTraceState.parse("r:232").serialize()); + assertEquals("", OtelTraceState.parse("x;r:5").serialize()); + assertEquals("", OtelTraceState.parse("r:5;x").serialize()); + assertEquals("r:5;x:3", OtelTraceState.parse("x:3;r:5").serialize()); + assertEquals("r:5;x:3", OtelTraceState.parse("r:5;x:3").serialize()); + assertEquals("", OtelTraceState.parse("r:5;x:3;").serialize()); + assertEquals( + "r:5;a:" + getXString(246) + ";x:3", + OtelTraceState.parse("a:" + getXString(246) + ";r:5;x:3").serialize()); + assertEquals("", OtelTraceState.parse("a:" + getXString(247) + ";r:5;x:3").serialize()); + + assertEquals("p:7;r:5", OtelTraceState.parse("r:5;p:7").serialize()); + assertEquals("p:4;r:5", OtelTraceState.parse("r:5;p:4").serialize()); + assertEquals("p:7;r:5", OtelTraceState.parse("r:5;p:7").serialize()); + assertEquals("p:4;r:5", OtelTraceState.parse("r:5;p:4").serialize()); + + assertEquals("r:6", OtelTraceState.parse("r:5;r:6").serialize()); + assertEquals("p:6;r:10", OtelTraceState.parse("p:5;p:6;r:10").serialize()); + assertEquals("", OtelTraceState.parse("p5;p:6;r:10").serialize()); + } +} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/RandomUtilTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/RandomUtilTest.java new file mode 100644 index 000000000..8010c2450 --- /dev/null +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/RandomUtilTest.java @@ -0,0 +1,78 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.util; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import java.util.BitSet; +import java.util.SplittableRandom; +import java.util.stream.DoubleStream; +import org.hipparchus.stat.inference.GTest; +import org.junit.jupiter.api.Test; + +public class RandomUtilTest { + + private static void testGenerateRandomBitSet(long seed, int numBits, int numOneBits) { + + int numCycles = 100000; + + SplittableRandom splittableRandom = new SplittableRandom(seed); + + long[] observed = new long[numBits]; + double[] expected = DoubleStream.generate(() -> 1.).limit(numBits).toArray(); + + for (int i = 0; i < numCycles; ++i) { + BitSet bitSet = + RandomUtil.generateRandomBitSet(splittableRandom::nextLong, numBits, numOneBits); + bitSet.stream().forEach(k -> observed[k] += 1); + assertThat(bitSet.cardinality()).isEqualTo(numOneBits); + } + if (numBits > 1) { + assertThat(new GTest().gTest(expected, observed)).isGreaterThan(0.01); + } else if (numBits == 1) { + assertThat(observed[0]).isEqualTo(numOneBits * numCycles); + } else { + fail("numBits was non-positive!"); + } + } + + @Test + void testGenerateRandomBitSet() { + testGenerateRandomBitSet(0x4a5580b958d52182L, 1, 0); + testGenerateRandomBitSet(0x529dff14b0ce7414L, 1, 1); + testGenerateRandomBitSet(0x2d3f673a9e1da536L, 2, 0); + testGenerateRandomBitSet(0xb9a6735e64361bacL, 2, 1); + testGenerateRandomBitSet(0xb5aafedc7031506fL, 2, 2); + testGenerateRandomBitSet(0xaecabe7698971ee1L, 3, 0); + testGenerateRandomBitSet(0x119ccf35dc52b34dL, 3, 1); + testGenerateRandomBitSet(0xcaf2b7a98f194ce2L, 3, 2); + testGenerateRandomBitSet(0xe28e8cc3d3de0c2aL, 3, 3); + testGenerateRandomBitSet(0xb69989dce9cc8b34L, 4, 0); + testGenerateRandomBitSet(0x6575d4c848c95dc8L, 4, 1); + testGenerateRandomBitSet(0xed0ad0525ad632e9L, 4, 2); + testGenerateRandomBitSet(0x34db9303b405a706L, 4, 3); + testGenerateRandomBitSet(0x8e97972893044140L, 4, 4); + testGenerateRandomBitSet(0x47f966b8f28dac77L, 5, 0); + testGenerateRandomBitSet(0x7996db4a5f1e4680L, 5, 1); + testGenerateRandomBitSet(0x577fcf18bbc0ba30L, 5, 2); + testGenerateRandomBitSet(0x36b1ed999d2986b0L, 5, 3); + testGenerateRandomBitSet(0xa8e099ed958d03bbL, 5, 4); + testGenerateRandomBitSet(0xc2b50bbf3263b414L, 5, 5); + testGenerateRandomBitSet(0x2994550582b091e9L, 6, 0); + testGenerateRandomBitSet(0xd2797c539136f6faL, 6, 1); + testGenerateRandomBitSet(0xf3ffae1d93983fd9L, 6, 2); + testGenerateRandomBitSet(0x281e0f9873455ea6L, 6, 3); + testGenerateRandomBitSet(0x5344c2887e30d621L, 6, 4); + testGenerateRandomBitSet(0xa8f4ed6e3e1cf385L, 6, 5); + testGenerateRandomBitSet(0x6bd0f9f11520ae57L, 6, 6); + + testGenerateRandomBitSet(0x514f52732c193e62L, 1000, 1); + testGenerateRandomBitSet(0xe214063ae29d9802L, 1000, 10); + testGenerateRandomBitSet(0x602fdb45063e7b0fL, 1000, 990); + testGenerateRandomBitSet(0xe0ef0cb214de3ec0L, 1000, 999); + } +} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java new file mode 100644 index 000000000..91b24d86e --- /dev/null +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java @@ -0,0 +1,83 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.contrib.util; + +import static org.assertj.core.api.Assertions.assertThat; + +import com.google.common.collect.ImmutableMap; +import java.util.HashMap; +import java.util.Map; +import org.hipparchus.stat.inference.GTest; + +public final class TestUtil { + + private TestUtil() {} + + public static void verifyObservedPvaluesUsingGtest( + long originalNumberOfSpans, Map observedPvalues, double samplingProbability) { + + Object notSampled = + new Object() { + @Override + public String toString() { + return "NOT SAMPLED"; + } + }; + + ImmutableMap expectedProbabilities; + if (samplingProbability >= 1.) { + expectedProbabilities = ImmutableMap.of(0, 1.); + } else if (samplingProbability <= 0.) { + expectedProbabilities = ImmutableMap.of(notSampled, 1.); + } else { + int exponent = 0; + while (true) { + if (Math.pow(0.5, exponent + 1) < samplingProbability + && Math.pow(0.5, exponent) >= samplingProbability) { + break; + } + exponent += 1; + } + if (samplingProbability == Math.pow(0.5, exponent)) { + expectedProbabilities = + ImmutableMap.of(notSampled, 1 - samplingProbability, exponent, samplingProbability); + } else { + expectedProbabilities = + ImmutableMap.of( + notSampled, + 1 - samplingProbability, + exponent, + 2 * samplingProbability - Math.pow(0.5, exponent), + exponent + 1, + Math.pow(0.5, exponent) - samplingProbability); + } + } + + Map extendedObservedAdjustedCounts = new HashMap<>(observedPvalues); + long numberOfSpansNotSampled = + originalNumberOfSpans - observedPvalues.values().stream().mapToLong(i -> i).sum(); + if (numberOfSpansNotSampled > 0) { + extendedObservedAdjustedCounts.put(notSampled, numberOfSpansNotSampled); + } + + double[] expectedValues = new double[expectedProbabilities.size()]; + long[] observedValues = new long[expectedProbabilities.size()]; + + int counter = 0; + for (Object key : expectedProbabilities.keySet()) { + observedValues[counter] = extendedObservedAdjustedCounts.getOrDefault(key, 0L); + double p = expectedProbabilities.get(key); + expectedValues[counter] = p * originalNumberOfSpans; + counter += 1; + } + + if (expectedProbabilities.size() > 1) { + assertThat(new GTest().gTest(expectedValues, observedValues)).isGreaterThan(0.01); + } else { + assertThat((double) observedValues[0]).isEqualTo(expectedValues[0]); + } + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index fa9d30dd8..9f80afff2 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -39,6 +39,7 @@ rootProject.name = "opentelemetry-java-contrib" include(":all") include(":aws-xray") include(":samplers") +include(":consistent-sampling") include(":dependencyManagement") include(":disruptor-processor") include(":example") From 6b4ba10eb16ba8392945f1e59b852e9b4f6be637 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Mon, 14 Feb 2022 15:20:03 +0100 Subject: [PATCH 02/47] improved dependencies (in particular, removed dependency on guava) --- consistent-sampling/build.gradle.kts | 3 +-- ...servoirSamplingBatchSpanProcessorTest.java | 6 ++--- .../opentelemetry/contrib/util/TestUtil.java | 22 +++++++------------ 3 files changed, 11 insertions(+), 20 deletions(-) diff --git a/consistent-sampling/build.gradle.kts b/consistent-sampling/build.gradle.kts index 03626a177..1ad1faad1 100644 --- a/consistent-sampling/build.gradle.kts +++ b/consistent-sampling/build.gradle.kts @@ -6,8 +6,7 @@ plugins { description = "Sampler and exporter implementations for consistent sampling" dependencies { - api("io.opentelemetry:opentelemetry-sdk") - testImplementation("com.google.guava:guava:31.0.1-jre") + api("io.opentelemetry:opentelemetry-sdk-trace") testImplementation("org.hipparchus:hipparchus-core:2.0") testImplementation("org.hipparchus:hipparchus-stat:2.0") } diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java index 5151bc520..50240583e 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java @@ -46,7 +46,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.IntStream; import java.util.stream.LongStream; -import javax.annotation.Nullable; import org.assertj.core.api.Assertions; import org.assertj.core.data.Percentage; import org.hipparchus.distribution.discrete.BinomialDistribution; @@ -74,7 +73,7 @@ class ConsistentReservoirSamplingBatchSpanProcessorTest { private static final String SPAN_NAME_2 = "MySpanName/2"; private static final long MAX_SCHEDULE_DELAY_MILLIS = 500; - @Nullable private SdkTracerProvider sdkTracerProvider; + private SdkTracerProvider sdkTracerProvider; private final BlockingSpanExporter blockingSpanExporter = new BlockingSpanExporter(); @Mock private Sampler mockSampler; @@ -86,13 +85,13 @@ void setUp() { } @AfterEach + @SuppressWarnings("FieldMissingNullable") void cleanup() { if (sdkTracerProvider != null) { sdkTracerProvider.shutdown(); } } - @Nullable private ReadableSpan createEndedSpan(String spanName) { Tracer tracer = sdkTracerProvider.get(getClass().getName()); Span span = tracer.spanBuilder(spanName).startSpan(); @@ -675,7 +674,6 @@ List getExported() { * @return the list of exported {@link SpanData} objects, otherwise {@code null} if the current * thread is interrupted. */ - @Nullable List waitForExport() { try { countDownLatch.await(timeout, TimeUnit.SECONDS); diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java index 91b24d86e..af196c1e9 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java @@ -7,7 +7,6 @@ import static org.assertj.core.api.Assertions.assertThat; -import com.google.common.collect.ImmutableMap; import java.util.HashMap; import java.util.Map; import org.hipparchus.stat.inference.GTest; @@ -27,11 +26,11 @@ public String toString() { } }; - ImmutableMap expectedProbabilities; + Map expectedProbabilities = new HashMap<>(); if (samplingProbability >= 1.) { - expectedProbabilities = ImmutableMap.of(0, 1.); + expectedProbabilities.put(0, 1.); } else if (samplingProbability <= 0.) { - expectedProbabilities = ImmutableMap.of(notSampled, 1.); + expectedProbabilities.put(notSampled, 1.); } else { int exponent = 0; while (true) { @@ -42,17 +41,12 @@ public String toString() { exponent += 1; } if (samplingProbability == Math.pow(0.5, exponent)) { - expectedProbabilities = - ImmutableMap.of(notSampled, 1 - samplingProbability, exponent, samplingProbability); + expectedProbabilities.put(notSampled, 1 - samplingProbability); + expectedProbabilities.put(exponent, samplingProbability); } else { - expectedProbabilities = - ImmutableMap.of( - notSampled, - 1 - samplingProbability, - exponent, - 2 * samplingProbability - Math.pow(0.5, exponent), - exponent + 1, - Math.pow(0.5, exponent) - samplingProbability); + expectedProbabilities.put(notSampled, 1 - samplingProbability); + expectedProbabilities.put(exponent, 2 * samplingProbability - Math.pow(0.5, exponent)); + expectedProbabilities.put(exponent + 1, Math.pow(0.5, exponent) - samplingProbability); } } From 038f803280e8da234310fd4b7c804cfc0947a59f Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Mon, 14 Feb 2022 15:36:06 +0100 Subject: [PATCH 03/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java Co-authored-by: Anuraag Agrawal --- .../contrib/samplers/ConsistentParentBasedSampler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java index 30624a7bb..a4ab63f57 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java @@ -12,7 +12,7 @@ import javax.annotation.concurrent.Immutable; /** - * A consistent sampler that makes the same sampling decision as the paren and optionally falls back + * A consistent sampler that makes the same sampling decision as the parent and optionally falls back * to an alternative consistent sampler, if the parent p-value is invalid (like for root spans). */ @Immutable From a686b3ac524454272cde3b0e8ae4e00976891829 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Mon, 14 Feb 2022 20:03:22 +0100 Subject: [PATCH 04/47] reverted some changes --- .../contrib/samplers/ConsistentParentBasedSampler.java | 5 +++-- .../ConsistentReservoirSamplingBatchSpanProcessorTest.java | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java index a4ab63f57..64e7cbd1a 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java @@ -12,8 +12,9 @@ import javax.annotation.concurrent.Immutable; /** - * A consistent sampler that makes the same sampling decision as the parent and optionally falls back - * to an alternative consistent sampler, if the parent p-value is invalid (like for root spans). + * A consistent sampler that makes the same sampling decision as the parent and optionally falls + * back to an alternative consistent sampler, if the parent p-value is invalid (like for root + * spans). */ @Immutable public final class ConsistentParentBasedSampler extends ConsistentSampler { diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java index 50240583e..5151bc520 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java @@ -46,6 +46,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.IntStream; import java.util.stream.LongStream; +import javax.annotation.Nullable; import org.assertj.core.api.Assertions; import org.assertj.core.data.Percentage; import org.hipparchus.distribution.discrete.BinomialDistribution; @@ -73,7 +74,7 @@ class ConsistentReservoirSamplingBatchSpanProcessorTest { private static final String SPAN_NAME_2 = "MySpanName/2"; private static final long MAX_SCHEDULE_DELAY_MILLIS = 500; - private SdkTracerProvider sdkTracerProvider; + @Nullable private SdkTracerProvider sdkTracerProvider; private final BlockingSpanExporter blockingSpanExporter = new BlockingSpanExporter(); @Mock private Sampler mockSampler; @@ -85,13 +86,13 @@ void setUp() { } @AfterEach - @SuppressWarnings("FieldMissingNullable") void cleanup() { if (sdkTracerProvider != null) { sdkTracerProvider.shutdown(); } } + @Nullable private ReadableSpan createEndedSpan(String spanName) { Tracer tracer = sdkTracerProvider.get(getClass().getName()); Span span = tracer.spanBuilder(spanName).startSpan(); @@ -674,6 +675,7 @@ List getExported() { * @return the list of exported {@link SpanData} objects, otherwise {@code null} if the current * thread is interrupted. */ + @Nullable List waitForExport() { try { countDownLatch.await(timeout, TimeUnit.SECONDS); From 08bbc4089145787bff29c40b5913801081f00ac0 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 15 Feb 2022 08:32:16 +0100 Subject: [PATCH 05/47] removed wrong immutable annotation --- .../contrib/samplers/ConsistentRateLimitingSampler.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 06ed6955d..19eea02b5 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -12,7 +12,6 @@ import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.samplers.Sampler; import java.util.function.LongSupplier; -import javax.annotation.concurrent.Immutable; /** * This consistent {@link Sampler} adjust the sampling probability dynamically to limit the rate of @@ -22,7 +21,6 @@ * J. "Forecasting data published at irregular time intervals using an extension of Holt's method." * Management science 32.4 (1986): 499-510.) to estimate the current rate of spans. */ -@Immutable public class ConsistentRateLimitingSampler extends ConsistentSampler { private final String description; From 9258ef4b029ba527179a7c1e6f991873849db5b5 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 15 Feb 2022 08:32:48 +0100 Subject: [PATCH 06/47] added javadoc --- .../contrib/samplers/ConsistentProbabilityBasedSampler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java index 428329e44..0d7c3aec3 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java @@ -9,6 +9,7 @@ import io.opentelemetry.contrib.util.RandomGenerator; import javax.annotation.concurrent.Immutable; +/** A consistent sampler that samples with a fixed probability. */ @Immutable public class ConsistentProbabilityBasedSampler extends ConsistentSampler { From 000ffea29389658bab35f838b7cdd185e74bc97d Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 15 Feb 2022 08:33:12 +0100 Subject: [PATCH 07/47] avoid else statements when returning --- .../contrib/samplers/ConsistentComposedOrSampler.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java index ce869b7c8..d286fee67 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java @@ -46,15 +46,13 @@ protected int getP(int parentP, boolean isRoot) { if (OtelTraceState.isValidP(p1)) { if (OtelTraceState.isValidP(p2)) { return Math.min(p1, p2); - } else { - return p1; } + return p1; } else { if (OtelTraceState.isValidP(p2)) { return p2; - } else { - return OtelTraceState.getInvalidP(); } + return OtelTraceState.getInvalidP(); } } From f7911c35c4c502b205493b02e10e5e441d47a00b Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 15 Feb 2022 14:13:43 +0100 Subject: [PATCH 08/47] factory methods for consistent samplers, avoid exposure of implementations --- .../samplers/ConsistentAlwaysOffSampler.java | 10 +- .../samplers/ConsistentAlwaysOnSampler.java | 10 +- .../ConsistentComposedAndSampler.java | 9 +- .../samplers/ConsistentComposedOrSampler.java | 9 +- .../ConsistentParentBasedSampler.java | 6 +- .../ConsistentProbabilityBasedSampler.java | 6 +- .../ConsistentRateLimitingSampler.java | 7 +- .../contrib/samplers/ConsistentSampler.java | 115 +++++++++++++++++- ...servoirSamplingBatchSpanProcessorTest.java | 11 +- ...ConsistentProbabilityBasedSamplerTest.java | 2 +- .../ConsistentRateLimitingSamplerTest.java | 12 +- 11 files changed, 156 insertions(+), 41 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java index 56b68c8c9..ae0af4cbc 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java @@ -9,7 +9,15 @@ import javax.annotation.concurrent.Immutable; @Immutable -public final class ConsistentAlwaysOffSampler extends ConsistentSampler { +final class ConsistentAlwaysOffSampler extends ConsistentSampler { + + private ConsistentAlwaysOffSampler() {} + + private static final ConsistentSampler INSTANCE = new ConsistentAlwaysOffSampler(); + + static ConsistentSampler getInstance() { + return INSTANCE; + } @Override protected int getP(int parentP, boolean isRoot) { diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOnSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOnSampler.java index 6ce7a6575..9ca49bd0d 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOnSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOnSampler.java @@ -8,7 +8,15 @@ import javax.annotation.concurrent.Immutable; @Immutable -public class ConsistentAlwaysOnSampler extends ConsistentSampler { +final class ConsistentAlwaysOnSampler extends ConsistentSampler { + + private ConsistentAlwaysOnSampler() {} + + private static final ConsistentSampler INSTANCE = new ConsistentAlwaysOnSampler(); + + static ConsistentSampler getInstance() { + return INSTANCE; + } @Override protected int getP(int parentP, boolean isRoot) { diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java index dedaee713..003fa187d 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java @@ -16,18 +16,13 @@ *

This sampler samples if both samplers would sample. */ @Immutable -public final class ConsistentComposedAndSampler extends ConsistentSampler { +final class ConsistentComposedAndSampler extends ConsistentSampler { private final ConsistentSampler sampler1; private final ConsistentSampler sampler2; private final String description; - public static ConsistentComposedAndSampler create( - ConsistentSampler sampler1, ConsistentSampler sampler2) { - return new ConsistentComposedAndSampler(sampler1, sampler2); - } - - private ConsistentComposedAndSampler(ConsistentSampler sampler1, ConsistentSampler sampler2) { + ConsistentComposedAndSampler(ConsistentSampler sampler1, ConsistentSampler sampler2) { this.sampler1 = requireNonNull(sampler1); this.sampler2 = requireNonNull(sampler2); this.description = diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java index d286fee67..a3bd4defb 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java @@ -16,18 +16,13 @@ *

This sampler samples if any of the two samplers would sample. */ @Immutable -public final class ConsistentComposedOrSampler extends ConsistentSampler { +final class ConsistentComposedOrSampler extends ConsistentSampler { private final ConsistentSampler sampler1; private final ConsistentSampler sampler2; private final String description; - public static ConsistentComposedOrSampler create( - ConsistentSampler sampler1, ConsistentSampler sampler2) { - return new ConsistentComposedOrSampler(sampler1, sampler2); - } - - private ConsistentComposedOrSampler(ConsistentSampler sampler1, ConsistentSampler sampler2) { + ConsistentComposedOrSampler(ConsistentSampler sampler1, ConsistentSampler sampler2) { this.sampler1 = requireNonNull(sampler1); this.sampler2 = requireNonNull(sampler2); this.description = diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java index 64e7cbd1a..8cf18c85b 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java @@ -17,7 +17,7 @@ * spans). */ @Immutable -public final class ConsistentParentBasedSampler extends ConsistentSampler { +final class ConsistentParentBasedSampler extends ConsistentSampler { private final ConsistentSampler rootSampler; @@ -28,7 +28,7 @@ public final class ConsistentParentBasedSampler extends ConsistentSampler { * * @param rootSampler the root sampler */ - public ConsistentParentBasedSampler(ConsistentSampler rootSampler) { + ConsistentParentBasedSampler(ConsistentSampler rootSampler) { this(rootSampler, DefaultRandomGenerator.get()); } @@ -39,7 +39,7 @@ public ConsistentParentBasedSampler(ConsistentSampler rootSampler) { * @param rootSampler the root sampler * @param threadSafeRandomGenerator a thread-safe random generator */ - public ConsistentParentBasedSampler( + ConsistentParentBasedSampler( ConsistentSampler rootSampler, RandomGenerator threadSafeRandomGenerator) { super(threadSafeRandomGenerator); this.rootSampler = requireNonNull(rootSampler); diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java index 0d7c3aec3..aff3d5cd4 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java @@ -11,7 +11,7 @@ /** A consistent sampler that samples with a fixed probability. */ @Immutable -public class ConsistentProbabilityBasedSampler extends ConsistentSampler { +final class ConsistentProbabilityBasedSampler extends ConsistentSampler { private final int lowerPValue; private final int upperPValue; @@ -23,7 +23,7 @@ public class ConsistentProbabilityBasedSampler extends ConsistentSampler { * * @param samplingProbability the sampling probability */ - public ConsistentProbabilityBasedSampler(double samplingProbability) { + ConsistentProbabilityBasedSampler(double samplingProbability) { this(samplingProbability, DefaultRandomGenerator.get()); } @@ -33,7 +33,7 @@ public ConsistentProbabilityBasedSampler(double samplingProbability) { * @param samplingProbability the sampling probability * @param threadSafeRandomGenerator a thread-safe random generator */ - public ConsistentProbabilityBasedSampler( + ConsistentProbabilityBasedSampler( double samplingProbability, RandomGenerator threadSafeRandomGenerator) { super(threadSafeRandomGenerator); if (samplingProbability < 0.0 || samplingProbability > 1.0) { diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 19eea02b5..00b059aa8 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -21,7 +21,7 @@ * J. "Forecasting data published at irregular time intervals using an extension of Holt's method." * Management science 32.4 (1986): 499-510.) to estimate the current rate of spans. */ -public class ConsistentRateLimitingSampler extends ConsistentSampler { +final class ConsistentRateLimitingSampler extends ConsistentSampler { private final String description; private final LongSupplier nanoTimeSupplier; @@ -44,8 +44,7 @@ public class ConsistentRateLimitingSampler extends ConsistentSampler { * @param adaptationTimeSeconds the typical time to adapt to a new load (time constant used for * exponential smoothing) */ - public ConsistentRateLimitingSampler( - double targetSpansPerSecondLimit, double adaptationTimeSeconds) { + ConsistentRateLimitingSampler(double targetSpansPerSecondLimit, double adaptationTimeSeconds) { this( targetSpansPerSecondLimit, adaptationTimeSeconds, @@ -62,7 +61,7 @@ public ConsistentRateLimitingSampler( * @param threadSafeRandomGenerator a thread-safe random generator * @param nanoTimeSupplier a supplier for the current nano time */ - public ConsistentRateLimitingSampler( + ConsistentRateLimitingSampler( double targetSpansPerSecondLimit, double adaptationTimeSeconds, RandomGenerator threadSafeRandomGenerator, diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index bb8edf011..083ad4dec 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -21,9 +21,122 @@ import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.util.List; +import java.util.function.LongSupplier; +import javax.annotation.Nonnull; /** Abstract base class for consistent samplers. */ -abstract class ConsistentSampler implements Sampler { +public abstract class ConsistentSampler implements Sampler { + + /** + * Returns a {@link ConsistentSampler} that samples all spans. + * + * @return a sampler + */ + public static final ConsistentSampler alwaysOn() { + return ConsistentAlwaysOnSampler.getInstance(); + } + + /** + * Returns a {@link ConsistentSampler} that does not sample any span. + * + * @return a sampler + */ + public static final ConsistentSampler alwaysOff() { + return ConsistentAlwaysOffSampler.getInstance(); + } + + /** + * Returns a {@link ConsistentSampler} that samples each span with a fixed probability. + * + * @param samplingProbability the sampling probability + * @return a sampler + */ + public static final ConsistentSampler probabilityBased(double samplingProbability) { + return new ConsistentProbabilityBasedSampler(samplingProbability); + } + + /** + * Returns a {@link ConsistentSampler} that samples each span with a fixed probability. + * + * @param samplingProbability the sampling probability + * @param threadSafeRandomGenerator a thread-safe random generator + * @return a sampler + */ + public static final ConsistentSampler probabilityBased( + double samplingProbability, RandomGenerator threadSafeRandomGenerator) { + return new ConsistentProbabilityBasedSampler(samplingProbability, threadSafeRandomGenerator); + } + + /** + * Returns a new {@link ConsistentSampler} that respects the sampling decision of the parent span + * or falls-back to the given sampler if it is a root span. + * + * @param rootSampler the root sampler + */ + public static final ConsistentSampler parentBased(@Nonnull ConsistentSampler rootSampler) { + return new ConsistentParentBasedSampler(rootSampler); + } + + /** + * Returns a new {@link ConsistentSampler} that respects the sampling decision of the parent span + * or falls-back to the given sampler if it is a root span. + * + * @param rootSampler the root sampler + * @param threadSafeRandomGenerator a thread-safe random generator + */ + public static final ConsistentSampler parentBased( + ConsistentSampler rootSampler, RandomGenerator threadSafeRandomGenerator) { + return new ConsistentParentBasedSampler(rootSampler, threadSafeRandomGenerator); + } + + /** + * Returns a new {@link ConsistentSampler} that attempts to adjust the sampling probability + * dynamically to meet the target span rate. + * + * @param targetSpansPerSecondLimit the desired spans per second limit + * @param adaptationTimeSeconds the typical time to adapt to a new load (time constant used for + * exponential smoothing) + */ + public static final ConsistentSampler rateLimited( + double targetSpansPerSecondLimit, double adaptationTimeSeconds) { + return new ConsistentRateLimitingSampler(targetSpansPerSecondLimit, adaptationTimeSeconds); + } + + /** + * Returns a new {@link ConsistentSampler} that attempts to adjust the sampling probability + * dynamically to meet the target span rate. + * + * @param targetSpansPerSecondLimit the desired spans per second limit + * @param adaptationTimeSeconds the typical time to adapt to a new load (time constant used for + * exponential smoothing) + * @param threadSafeRandomGenerator a thread-safe random generator + * @param nanoTimeSupplier a supplier for the current nano time + */ + public static final ConsistentSampler rateLimited( + double targetSpansPerSecondLimit, + double adaptationTimeSeconds, + RandomGenerator threadSafeRandomGenerator, + LongSupplier nanoTimeSupplier) { + return new ConsistentRateLimitingSampler( + targetSpansPerSecondLimit, + adaptationTimeSeconds, + threadSafeRandomGenerator, + nanoTimeSupplier); + } + + public ConsistentSampler and(ConsistentSampler otherConsistentSampler) { + if (otherConsistentSampler == this) { + return this; + } + return new ConsistentComposedAndSampler(this, otherConsistentSampler); + } + + public ConsistentSampler or(ConsistentSampler otherConsistentSampler) { + if (otherConsistentSampler == this) { + return this; + } + return new ConsistentComposedOrSampler(this, otherConsistentSampler); + } protected final RandomGenerator threadSafeRandomGenerator; diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java index 5151bc520..dbcad7c1d 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java @@ -21,8 +21,7 @@ import io.opentelemetry.api.internal.GuardedBy; import io.opentelemetry.api.trace.Span; import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.contrib.samplers.ConsistentAlwaysOnSampler; -import io.opentelemetry.contrib.samplers.ConsistentProbabilityBasedSampler; +import io.opentelemetry.contrib.samplers.ConsistentSampler; import io.opentelemetry.contrib.state.OtelTraceState; import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.common.CompletableResultCode; @@ -729,7 +728,7 @@ void exportDifferentConsistentlySampledSpans() { sdkTracerProvider = SdkTracerProvider.builder() - .setSampler(new ConsistentAlwaysOnSampler()) + .setSampler(ConsistentSampler.alwaysOn()) .addSpanProcessor(spanProcessor) .build(); @@ -791,8 +790,7 @@ private void testConsistentSampling( sdkTracerProvider = SdkTracerProvider.builder() .setSampler( - new ConsistentProbabilityBasedSampler( - samplingProbability, threadSafeRandomGenerator2)) + ConsistentSampler.probabilityBased(samplingProbability, threadSafeRandomGenerator2)) .addSpanProcessor(spanProcessor) .build(); @@ -1013,8 +1011,7 @@ private StatisticalSummary calculateStatisticalSummary( sdkTracerProvider = SdkTracerProvider.builder() .setSampler( - new ConsistentProbabilityBasedSampler( - samplingProbability, threadSafeRandomGenerator2)) + ConsistentSampler.probabilityBased(samplingProbability, threadSafeRandomGenerator2)) .addSpanProcessor(spanProcessor) .build(); diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java index a2120788d..03333659d 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java @@ -48,7 +48,7 @@ public void init() { private void test(SplittableRandom rng, double samplingProbability) { int numSpans = 1000000; - Sampler sampler = new ConsistentProbabilityBasedSampler(samplingProbability, rng::nextLong); + Sampler sampler = ConsistentSampler.probabilityBased(samplingProbability, rng::nextLong); Map observedPvalues = new HashMap<>(); for (long i = 0; i < numSpans; ++i) { diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java index d6b9b94fe..6446296b0 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java @@ -61,8 +61,8 @@ void testConstantRate() { double adaptationTimeSeconds = 5; SplittableRandom random = new SplittableRandom(0L); - ConsistentRateLimitingSampler sampler = - new ConsistentRateLimitingSampler( + ConsistentSampler sampler = + ConsistentSampler.rateLimited( targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); long nanosBetweenSpans = TimeUnit.MICROSECONDS.toNanos(100); @@ -95,8 +95,8 @@ void testRateIncrease() { double adaptationTimeSeconds = 5; SplittableRandom random = new SplittableRandom(0L); - ConsistentRateLimitingSampler sampler = - new ConsistentRateLimitingSampler( + ConsistentSampler sampler = + ConsistentSampler.rateLimited( targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); long nanosBetweenSpans1 = TimeUnit.MICROSECONDS.toNanos(100); @@ -151,8 +151,8 @@ void testRateDecrease() { double adaptationTimeSeconds = 5; SplittableRandom random = new SplittableRandom(0L); - ConsistentRateLimitingSampler sampler = - new ConsistentRateLimitingSampler( + ConsistentSampler sampler = + ConsistentSampler.rateLimited( targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); long nanosBetweenSpans1 = TimeUnit.MICROSECONDS.toNanos(10); From b3459f9e79d1a69c17e4cdf7aeb811801b692b6b Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 15 Feb 2022 14:43:22 +0100 Subject: [PATCH 09/47] added javadoc for AND and OR sampler composition --- .../contrib/samplers/ConsistentSampler.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index 083ad4dec..28d30bc2e 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -124,6 +124,21 @@ public static final ConsistentSampler rateLimited( nanoTimeSupplier); } + /** + * Returns a {@link ConsistentSampler} that samples a span if both this and the other given + * consistent sampler would sample the span. + * + *

If the other consistent sampler is the same as this, this consistent sampler will be + * returned. + * + *

The returned sampler takes care of setting the trace state correctly, which would not happen + * if the {@link #shouldSample(Context, String, String, SpanKind, Attributes, List)} method was + * called for each sampler individually. Also, the combined sampler is more efficient than + * evaluating the two samplers individually and combining both results afterwards. + * + * @param otherConsistentSampler the other consistent sampler + * @return the composed consistent sampler + */ public ConsistentSampler and(ConsistentSampler otherConsistentSampler) { if (otherConsistentSampler == this) { return this; @@ -131,6 +146,21 @@ public ConsistentSampler and(ConsistentSampler otherConsistentSampler) { return new ConsistentComposedAndSampler(this, otherConsistentSampler); } + /** + * Returns a {@link ConsistentSampler} that samples a span if either this or the other given + * consistent sampler would sample the span. + * + *

If the other consistent sampler is the same as this, this consistent sampler will be + * returned. + * + *

The returned sampler takes care of setting the trace state correctly, which would not happen + * if the {@link #shouldSample(Context, String, String, SpanKind, Attributes, List)} method was + * called for each sampler individually. Also, the combined sampler is more efficient than + * evaluating the two samplers individually and combining both results afterwards. + * + * @param otherConsistentSampler the other consistent sampler + * @return the composed consistent sampler + */ public ConsistentSampler or(ConsistentSampler otherConsistentSampler) { if (otherConsistentSampler == this) { return this; From 185f579d21277497317bd066dde28577ce05f67f Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 15 Feb 2022 20:47:19 +0100 Subject: [PATCH 10/47] replaced use of synchronized by atomic reference --- .../ConsistentRateLimitingSampler.java | 58 +++++++++++-------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 00b059aa8..3080971bf 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -7,11 +7,12 @@ import static java.util.Objects.requireNonNull; -import io.opentelemetry.api.internal.GuardedBy; import io.opentelemetry.contrib.util.DefaultRandomGenerator; import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.samplers.Sampler; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; +import javax.annotation.concurrent.Immutable; /** * This consistent {@link Sampler} adjust the sampling probability dynamically to limit the rate of @@ -23,19 +24,24 @@ */ final class ConsistentRateLimitingSampler extends ConsistentSampler { + @Immutable + private static final class State { + private final double effectiveWindowCount; + private final double effectiveWindowNanos; + private final long lastNanoTime; + + public State(double effectiveWindowCount, double effectiveWindowNanos, long lastNanoTime) { + this.effectiveWindowCount = effectiveWindowCount; + this.effectiveWindowNanos = effectiveWindowNanos; + this.lastNanoTime = lastNanoTime; + } + } + private final String description; private final LongSupplier nanoTimeSupplier; private final double inverseAdaptationTimeNanos; private final double targetSpansPerNanosLimit; - - @GuardedBy("this") - private double effectiveWindowCount; - - @GuardedBy("this") - private double effectiveWindowNanos; - - @GuardedBy("this") - private long lastNanoTime; + private final AtomicReference state; /** * Constructor. @@ -83,26 +89,32 @@ final class ConsistentRateLimitingSampler extends ConsistentSampler { this.inverseAdaptationTimeNanos = 1e-9 / adaptationTimeSeconds; this.targetSpansPerNanosLimit = 1e-9 * targetSpansPerSecondLimit; - synchronized (this) { - this.effectiveWindowCount = 0; - this.effectiveWindowNanos = 0; - this.lastNanoTime = nanoTimeSupplier.getAsLong(); - } + this.state = new AtomicReference<>(new State(0, 0, nanoTimeSupplier.getAsLong())); } - private synchronized double updateAndGetSamplingProbability() { - long currentNanoTime = Math.max(nanoTimeSupplier.getAsLong(), lastNanoTime); - long nanoTimeDelta = currentNanoTime - lastNanoTime; - lastNanoTime = currentNanoTime; + private State updateState(State oldState, long currentNanoTime) { + if (currentNanoTime <= oldState.lastNanoTime) { + return new State( + oldState.effectiveWindowCount + 1, oldState.effectiveWindowNanos, oldState.lastNanoTime); + } + long nanoTimeDelta = currentNanoTime - oldState.lastNanoTime; double decayFactor = Math.exp(-nanoTimeDelta * inverseAdaptationTimeNanos); - effectiveWindowCount = effectiveWindowCount * decayFactor + 1; - effectiveWindowNanos = effectiveWindowNanos * decayFactor + nanoTimeDelta; - return Math.min(1., (effectiveWindowNanos * targetSpansPerNanosLimit) / effectiveWindowCount); + double currentEffectiveWindowCount = oldState.effectiveWindowCount * decayFactor + 1; + double currentEffectiveWindowNanos = + oldState.effectiveWindowNanos * decayFactor + nanoTimeDelta; + return new State(currentEffectiveWindowCount, currentEffectiveWindowNanos, currentNanoTime); } @Override protected int getP(int parentP, boolean isRoot) { - double samplingProbability = updateAndGetSamplingProbability(); + long currentNanoTime = nanoTimeSupplier.getAsLong(); + State currentState = state.updateAndGet(s -> updateState(s, currentNanoTime)); + + double samplingProbability = + Math.min( + 1., + (currentState.effectiveWindowNanos * targetSpansPerNanosLimit) + / currentState.effectiveWindowCount); if (samplingProbability >= 1.) { return 0; From abb7e7188d2fc6e2079b781009edc7b047708b5a Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 15 Feb 2022 20:53:41 +0100 Subject: [PATCH 11/47] simplified thread local initialization --- .../opentelemetry/contrib/util/DefaultRandomGenerator.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java index 8dbcf4ea5..4985473a5 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java @@ -24,12 +24,7 @@ boolean nextRandomBit() { } private static final ThreadLocal THREAD_LOCAL_DATA = - new ThreadLocal() { - @Override - protected ThreadLocalData initialValue() { - return new ThreadLocalData(); - } - }; + ThreadLocal.withInitial(ThreadLocalData::new); private static final DefaultRandomGenerator INSTANCE = new DefaultRandomGenerator(); From dd3df4ed697e968ee822228a92ad5aba58c32a6a Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Wed, 16 Feb 2022 16:12:11 +0100 Subject: [PATCH 12/47] removed consistent reservoir sampling --- ...ntReservoirSamplingBatchSpanProcessor.java | 677 ---------- ...voirSamplingBatchSpanProcessorBuilder.java | 158 --- ...servoirSamplingBatchSpanProcessorTest.java | 1114 ----------------- 3 files changed, 1949 deletions(-) delete mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessor.java delete mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorBuilder.java delete mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessor.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessor.java deleted file mode 100644 index 0da4ea687..000000000 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessor.java +++ /dev/null @@ -1,677 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.contrib.export; - -import io.opentelemetry.api.common.AttributeKey; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.metrics.LongCounter; -import io.opentelemetry.api.metrics.Meter; -import io.opentelemetry.api.metrics.MeterProvider; -import io.opentelemetry.api.trace.SpanContext; -import io.opentelemetry.api.trace.TraceState; -import io.opentelemetry.context.Context; -import io.opentelemetry.contrib.state.OtelTraceState; -import io.opentelemetry.contrib.util.RandomGenerator; -import io.opentelemetry.contrib.util.RandomUtil; -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.internal.DaemonThreadFactory; -import io.opentelemetry.sdk.trace.ReadWriteSpan; -import io.opentelemetry.sdk.trace.ReadableSpan; -import io.opentelemetry.sdk.trace.SpanProcessor; -import io.opentelemetry.sdk.trace.data.DelegatingSpanData; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.export.SpanExporter; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.Collections; -import java.util.List; -import java.util.PriorityQueue; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Implementation of the {@link SpanProcessor} that batches spans exported by the SDK then pushes - * them to the exporter pipeline. - * - *

All spans reported by the SDK implementation are first added to a synchronized queue (with a - * {@code maxQueueSize} maximum size, if queue is full spans are dropped). Spans are exported either - * when there are {@code maxExportBatchSize} pending spans or {@code scheduleDelayNanos} has passed - * since the last export finished. - */ -public final class ConsistentReservoirSamplingBatchSpanProcessor implements SpanProcessor { - - private static final String WORKER_THREAD_NAME = - ConsistentReservoirSamplingBatchSpanProcessor.class.getSimpleName() + "_WorkerThread"; - private static final AttributeKey SPAN_PROCESSOR_TYPE_LABEL = - AttributeKey.stringKey("spanProcessorType"); - private static final AttributeKey SPAN_PROCESSOR_DROPPED_LABEL = - AttributeKey.booleanKey("dropped"); - private static final String SPAN_PROCESSOR_TYPE_VALUE = - ConsistentReservoirSamplingBatchSpanProcessor.class.getSimpleName(); - - private final Worker worker; - private final AtomicBoolean isShutdown = new AtomicBoolean(false); - - /** - * Returns a new Builder for {@link ConsistentReservoirSamplingBatchSpanProcessor}. - * - * @param spanExporter the {@code SpanExporter} to where the Spans are pushed. - * @return a new {@link ConsistentReservoirSamplingBatchSpanProcessor}. - * @throws NullPointerException if the {@code spanExporter} is {@code null}. - */ - public static ConsistentReservoirSamplingBatchSpanProcessorBuilder builder( - SpanExporter spanExporter) { - return new ConsistentReservoirSamplingBatchSpanProcessorBuilder(spanExporter); - } - - private static final class ReadableSpanWithPriority { - - private final ReadableSpan readableSpan; - private int pval; - private final int rval; - private long priority; - - public static ReadableSpanWithPriority create( - ReadableSpan readableSpan, RandomGenerator threadSafeRandomGenerator) { - String otelTraceStateString = - readableSpan.getSpanContext().getTraceState().get(OtelTraceState.TRACE_STATE_KEY); - OtelTraceState otelTraceState = OtelTraceState.parse(otelTraceStateString); - int pval; - int rval; - long priority = threadSafeRandomGenerator.nextLong(); - if (otelTraceState.hasValidR()) { - rval = otelTraceState.getR(); - } else { - rval = - Math.min( - threadSafeRandomGenerator.numberOfLeadingZerosOfRandomLong(), - OtelTraceState.getMaxR()); - } - - if (otelTraceState.hasValidP()) { - pval = otelTraceState.getP(); - } else { - // if the p-value is not defined assume it is zero, - // which corresponds to an adjusted count of 1 - pval = 0; - } - - return new ReadableSpanWithPriority(readableSpan, pval, rval, priority); - } - - private ReadableSpanWithPriority(ReadableSpan readableSpan, int pval, int rval, long priority) { - this.readableSpan = readableSpan; - this.pval = pval; - this.rval = rval; - this.priority = priority; - } - - private ReadableSpan getReadableSpan() { - return readableSpan; - } - - private int getP() { - return pval; - } - - private void setP(int pval) { - this.pval = pval; - } - - private int getR() { - return rval; - } - - // returns true if this span survived down sampling - private boolean downSample(RandomGenerator threadSafeRandomGenerator) { - pval += 1; - if (pval > rval) { - return false; - } - priority = threadSafeRandomGenerator.nextLong(); - return true; - } - - private static int comparePthenPriority( - ReadableSpanWithPriority s1, ReadableSpanWithPriority s2) { - int compareP = Integer.compare(s1.pval, s2.pval); - if (compareP != 0) { - return compareP; - } - return Long.compare(s1.priority, s2.priority); - } - - private static int compareRthenPriority( - ReadableSpanWithPriority s1, ReadableSpanWithPriority s2) { - int compareR = Integer.compare(s1.rval, s2.rval); - if (compareR != 0) { - return compareR; - } - return Long.compare(s1.priority, s2.priority); - } - } - - private interface Reservoir { - void add(ReadableSpanWithPriority readableSpanWithPriority); - - List getResult(); - - boolean isEmpty(); - } - - /** - * Reservoir sampling buffer that collects a fixed number of spans. - * - *

Consistent sampling requires that spans are sampled only if r-value >= p-value, where - * p-value describes which sampling rate from the discrete set of possible sampling rates is - * applied. Consistent sampling allows to choose the sampling rate (the p-value) individually for - * every span. Therefore, the number of sampled spans can be reduced by increasing the p-value of - * spans, such that spans for which r-value < p-value get discarded. To reduce the number of - * sampled spans one can therefore apply the following procedure until the desired number of spans - * are left: - * - *

1) Randomly choose a span among the spans with smallest p-values - * - *

2) Increment its p-value by 1 - * - *

3) Discard the span, if r-value < p-value - * - *

4) continue with 1) - * - *

By always incrementing one of the smallest p-values, this approach tries to balance the - * sampling rates (p-values). Balanced sampling rates are better for estimation (compare VarOpt - * sampling, see https://arxiv.org/abs/0803.0473). - * - *

This reservoir sampling approach implements the described procedure in a streaming fashion. - * In order to ensure that spans have fair chances regardless of processing order, a uniform - * random number (priority) is associated with its p-value. When choosing a span among the spans - * with smallest p-value, we take that with the smallest priority. - */ - private static final class Reservoir1 implements Reservoir { - private final int reservoirSize; - private final PriorityQueue queue; - private final RandomGenerator threadSafeRandomGenerator; - - public Reservoir1(int reservoirSize, RandomGenerator threadSafeRandomGenerator) { - if (reservoirSize < 1) { - throw new IllegalArgumentException(); - } - this.reservoirSize = reservoirSize; - this.queue = - new PriorityQueue<>(reservoirSize, ReadableSpanWithPriority::comparePthenPriority); - this.threadSafeRandomGenerator = threadSafeRandomGenerator; - } - - @Override - public void add(ReadableSpanWithPriority readableSpanWithPriority) { - if (queue.size() < reservoirSize) { - queue.add(readableSpanWithPriority); - return; - } - - do { - ReadableSpanWithPriority head = queue.peek(); - if (ReadableSpanWithPriority.comparePthenPriority(readableSpanWithPriority, head) > 0) { - queue.remove(); - queue.add(readableSpanWithPriority); - readableSpanWithPriority = head; - } - } while (readableSpanWithPriority.downSample(threadSafeRandomGenerator)); - } - - @Override - public List getResult() { - List result = new ArrayList<>(queue.size()); - for (ReadableSpanWithPriority readableSpanWithPriority : queue) { - SpanData spanData = readableSpanWithPriority.getReadableSpan().toSpanData(); - SpanContext spanContext = spanData.getSpanContext(); - TraceState traceState = spanContext.getTraceState(); - String otelTraceStateString = traceState.get(OtelTraceState.TRACE_STATE_KEY); - OtelTraceState otelTraceState = OtelTraceState.parse(otelTraceStateString); - if ((!otelTraceState.hasValidR() && readableSpanWithPriority.getP() > 0) - || (otelTraceState.hasValidR() - && readableSpanWithPriority.getP() != otelTraceState.getP())) { - otelTraceState.setP(readableSpanWithPriority.getP()); - spanData = updateSpanDataWithOtelTraceState(spanData, otelTraceState); - } - result.add(spanData); - } - return result; - } - - @Override - public boolean isEmpty() { - return queue.isEmpty(); - } - } - - /** - * This reservoir implementation is (almost) statistically equivalent to {@link Reservoir1}. - * - *

It uses a priority queue where the minimum is the span with the smallet r-value. In this way - * the add-operation is more efficient, and has a worst case time complexity of O(log n) where n - * denotes the reservoir size. - */ - private static final class Reservoir2 implements Reservoir { - private final int reservoirSize; - private int maxDiscardedRValue = 0; - private long numberOfDiscardedSpansWithMaxDiscardedRValue = 0; - private final PriorityQueue queue; - private final RandomGenerator threadSafeRandomGenerator; - - public Reservoir2(int reservoirSize, RandomGenerator threadSafeRandomGenerator) { - if (reservoirSize < 1) { - throw new IllegalArgumentException(); - } - this.reservoirSize = reservoirSize; - this.queue = - new PriorityQueue<>(reservoirSize, ReadableSpanWithPriority::compareRthenPriority); - this.threadSafeRandomGenerator = threadSafeRandomGenerator; - } - - @Override - public void add(ReadableSpanWithPriority readableSpanWithPriority) { - - if (queue.size() < reservoirSize) { - queue.add(readableSpanWithPriority); - return; - } - - ReadableSpanWithPriority head = queue.peek(); - if (ReadableSpanWithPriority.compareRthenPriority(readableSpanWithPriority, head) > 0) { - queue.remove(); - queue.add(readableSpanWithPriority); - readableSpanWithPriority = head; - } - if (readableSpanWithPriority.getR() > maxDiscardedRValue) { - maxDiscardedRValue = readableSpanWithPriority.getR(); - numberOfDiscardedSpansWithMaxDiscardedRValue = 1; - } else if (readableSpanWithPriority.getR() == maxDiscardedRValue) { - numberOfDiscardedSpansWithMaxDiscardedRValue += 1; - } - } - - @Override - public List getResult() { - - if (numberOfDiscardedSpansWithMaxDiscardedRValue == 0) { - return queue.stream().map(x -> x.readableSpan.toSpanData()).collect(Collectors.toList()); - } - - List readableSpansWithPriority = new ArrayList<>(queue.size()); - int numberOfSampledSpansWithMaxDiscardedRValue = 0; - int numSampledSpansWithGreaterRValueAndSmallPValue = 0; - for (ReadableSpanWithPriority readableSpanWithPriority : queue) { - if (readableSpanWithPriority.getR() == maxDiscardedRValue) { - numberOfSampledSpansWithMaxDiscardedRValue += 1; - } else if (readableSpanWithPriority.getP() <= maxDiscardedRValue) { - numSampledSpansWithGreaterRValueAndSmallPValue += 1; - } - readableSpansWithPriority.add(readableSpanWithPriority); - } - - // Z = reservoirSize - // L = maxDiscardedRValue - // R = numberOfDiscardedSpansWithMaxDiscardedRValue - // K = numSampledSpansWithGreaterRValueAndSmallPValue - // X = numberOfSampledSpansWithMaxDiscardedRValue - // - // The sampling approach described above for Reservoir1 can be equivalently performed by - // keeping Z spans with largest r-values (in case of ties with highest priority) and adjusting - // the p-values at the end. We know that the largest r-value among the dropped spans is L and - // that we had to discard exactly R spans with (r-value == L). This implies that their - // corresponding p-values were raised to (L + 1) which finally violated the sampling condition - // (r-value >= p-value). We only raise the p-value of some span if it belongs to the set of - // spans with minimum p-value. Therefore, the minimum p-value must be given by L. To determine - // the p-values of all kept spans, we consider 3 cases: - // - // 1) For all X kept spans with r-value == L the corresponding p-value must also be L. - // Otherwise, the span would have been discarded. There are R spans with (r-value == L) which - // have been discarded. Therefore, among the original (X + R) spans with (r-value == L) we - // have kept X spans. - // - // 2) For spans with (p-value > L) the p-value will not be changed as they do not belong to - // the set of spans with minimal p-values. - // - // 3) For the remaining K spans for which (r-value > L) and (p-value <= L) the p-value needs - // to be adjusted. The new p-value will be either L or (L + 1). When starting to sample the - // first spans with (p-value == L), we have N = R + K + X spans which all have (r-value >= L) - // and (p-value == L). This set can be divided into two sets of spans dependent on whether - // (r-value == L) or (r-value > L). We know that there were (R + X) spans with (r-value == L) - // and K spans with (r-value > L). When randomly selecting a span to increase its p-value, the - // span will only be discarded if the span belongs to the first set (r-value == L). We will - // call such an event "failure". If the selected span belongs to the second set (r-value > L), - // its p-value will be increased by 1 to (L + 1) but the span will not be dropped. The - // sampling procedure will be stopped after R "failures". The number of "successes" follows a - // negative hypergeometric distribution - // (see https://en.wikipedia.org/wiki/Negative_hypergeometric_distribution). - // Therefore, we need to sample a random value from a negative hypergeometric distribution - // with N = R + X + K elements of which K are "successes" and after drawing R "failures", in - // order to determine how many spans out of K will get a p-value equal to (L + 1). The - // expected number is given by R * K / (N - K + 1) = R * K / (R + X + 1). Instead of drawing - // the number from the negative hypergeometric distribution we could also set it to the - // stochastically rounded expected value. This makes this reservoir sampling approach not - // fully equivalent to the approach described above for Reservoir1, but this (probably) leads - // to a smaller variance when it comes to estimation. (TODO: This still has to be verified!) - - double expectedNumPValueIncrements = - numSampledSpansWithGreaterRValueAndSmallPValue - * (numberOfDiscardedSpansWithMaxDiscardedRValue - / (double) - (numberOfDiscardedSpansWithMaxDiscardedRValue - + numberOfSampledSpansWithMaxDiscardedRValue - + 1L)); - int roundedExpectedNumPValueIncrements = - Math.toIntExact( - RandomUtil.roundStochastically( - threadSafeRandomGenerator, expectedNumPValueIncrements)); - - BitSet incrementIndicators = - RandomUtil.generateRandomBitSet( - threadSafeRandomGenerator, - numSampledSpansWithGreaterRValueAndSmallPValue, - roundedExpectedNumPValueIncrements); - - int incrementIndicatorIndex = 0; - List result = new ArrayList<>(queue.size()); - for (ReadableSpanWithPriority readableSpanWithPriority : readableSpansWithPriority) { - if (readableSpanWithPriority.getP() <= maxDiscardedRValue) { - readableSpanWithPriority.setP(maxDiscardedRValue); - if (readableSpanWithPriority.getR() > maxDiscardedRValue) { - if (incrementIndicators.get(incrementIndicatorIndex)) { - readableSpanWithPriority.setP(maxDiscardedRValue + 1); - } - incrementIndicatorIndex += 1; - } - } - - SpanData spanData = readableSpanWithPriority.getReadableSpan().toSpanData(); - SpanContext spanContext = spanData.getSpanContext(); - TraceState traceState = spanContext.getTraceState(); - String otelTraceStateString = traceState.get(OtelTraceState.TRACE_STATE_KEY); - OtelTraceState otelTraceState = OtelTraceState.parse(otelTraceStateString); - if ((!otelTraceState.hasValidR() && readableSpanWithPriority.getP() > 0) - || (otelTraceState.hasValidR() - && readableSpanWithPriority.getP() != otelTraceState.getP())) { - otelTraceState.setP(readableSpanWithPriority.getP()); - spanData = updateSpanDataWithOtelTraceState(spanData, otelTraceState); - } - result.add(spanData); - } - - return result; - } - - @Override - public boolean isEmpty() { - return queue.isEmpty(); - } - } - - private static SpanData updateSpanDataWithOtelTraceState( - SpanData spanData, OtelTraceState otelTraceState) { - SpanContext spanContext = spanData.getSpanContext(); - TraceState traceState = spanContext.getTraceState(); - String updatedOtelTraceStateString = otelTraceState.serialize(); - TraceState updatedTraceState = - traceState.toBuilder() - .put(OtelTraceState.TRACE_STATE_KEY, updatedOtelTraceStateString) - .build(); - SpanContext updatedSpanContext = - SpanContext.create( - spanContext.getTraceId(), - spanContext.getSpanId(), - spanContext.getTraceFlags(), - updatedTraceState); - return new DelegatingSpanData(spanData) { - @Override - public SpanContext getSpanContext() { - return updatedSpanContext; - } - }; - } - - ConsistentReservoirSamplingBatchSpanProcessor( - SpanExporter spanExporter, - MeterProvider meterProvider, - long scheduleDelayNanos, - int reservoirSize, - long exporterTimeoutNanos, - RandomGenerator threadSafeRandomGenerator, - boolean useAlternativeReservoirImplementation) { - this.worker = - new Worker( - spanExporter, - meterProvider, - scheduleDelayNanos, - reservoirSize, - exporterTimeoutNanos, - threadSafeRandomGenerator, - useAlternativeReservoirImplementation); - Thread workerThread = new DaemonThreadFactory(WORKER_THREAD_NAME).newThread(worker); - workerThread.start(); - } - - @Override - public void onStart(Context parentContext, ReadWriteSpan span) {} - - @Override - public boolean isStartRequired() { - return false; - } - - @Override - public void onEnd(ReadableSpan span) { - if (span == null || !span.getSpanContext().isSampled()) { - return; - } - worker.addSpan(span); - } - - @Override - public boolean isEndRequired() { - return true; - } - - @Override - public CompletableResultCode shutdown() { - if (isShutdown.getAndSet(true)) { - return CompletableResultCode.ofSuccess(); - } - return worker.shutdown(); - } - - @Override - public CompletableResultCode forceFlush() { - return worker.forceFlush(); - } - - // Visible for testing - boolean isReservoirEmpty() { - return worker.isReservoirEmpty(); - } - - private static final class Worker implements Runnable { - - private final LongCounter processedSpansCounter; - private final Attributes droppedAttrs; - private final Attributes exportedAttrs; - private final boolean useAlternativeReservoirImplementation; - - private static final Logger logger = Logger.getLogger(Worker.class.getName()); - private final SpanExporter spanExporter; - private final long scheduleDelayNanos; - private final int reservoirSize; - private final long exporterTimeoutNanos; - - private long nextExportTime; - - private final RandomGenerator threadSafeRandomGenerator; - private final Object reservoirLock = new Object(); - private Reservoir reservoir; - private final BlockingQueue signal; - private volatile boolean continueWork = true; - - private static Reservoir createReservoir( - int reservoirSize, - boolean useAlternativeReservoirImplementation, - RandomGenerator threadSafeRandomGenerator) { - if (useAlternativeReservoirImplementation) { - return new Reservoir2(reservoirSize, threadSafeRandomGenerator); - } else { - return new Reservoir1(reservoirSize, threadSafeRandomGenerator); - } - } - - private Worker( - SpanExporter spanExporter, - MeterProvider meterProvider, - long scheduleDelayNanos, - int reservoirSize, - long exporterTimeoutNanos, - RandomGenerator threadSafeRandomGenerator, - boolean useAlternativeReservoirImplementation) { - this.useAlternativeReservoirImplementation = useAlternativeReservoirImplementation; - this.spanExporter = spanExporter; - this.scheduleDelayNanos = scheduleDelayNanos; - this.reservoirSize = reservoirSize; - this.exporterTimeoutNanos = exporterTimeoutNanos; - this.threadSafeRandomGenerator = threadSafeRandomGenerator; - synchronized (reservoirLock) { - this.reservoir = - createReservoir( - reservoirSize, useAlternativeReservoirImplementation, threadSafeRandomGenerator); - } - this.signal = new ArrayBlockingQueue<>(1); - Meter meter = meterProvider.meterBuilder("io.opentelemetry.sdk.trace").build(); - processedSpansCounter = - meter - .counterBuilder("processedSpans") - .setUnit("1") - .setDescription( - "The number of spans processed by the BatchSpanProcessor. " - + "[dropped=true if they were dropped due to high throughput]") - .build(); - droppedAttrs = - Attributes.of( - SPAN_PROCESSOR_TYPE_LABEL, - SPAN_PROCESSOR_TYPE_VALUE, - SPAN_PROCESSOR_DROPPED_LABEL, - true); - exportedAttrs = - Attributes.of( - SPAN_PROCESSOR_TYPE_LABEL, - SPAN_PROCESSOR_TYPE_VALUE, - SPAN_PROCESSOR_DROPPED_LABEL, - false); - } - - private void addSpan(ReadableSpan span) { - ReadableSpanWithPriority readableSpanWithPriority = - ReadableSpanWithPriority.create(span, threadSafeRandomGenerator); - synchronized (reservoirLock) { - reservoir.add(readableSpanWithPriority); - } - processedSpansCounter.add(1, droppedAttrs); - } - - @Override - public void run() { - updateNextExportTime(); - CompletableResultCode completableResultCode = null; - while (continueWork) { - - if (completableResultCode != null || System.nanoTime() >= nextExportTime) { - Reservoir oldReservoir; - Reservoir newReservoir = - createReservoir( - reservoirSize, useAlternativeReservoirImplementation, threadSafeRandomGenerator); - synchronized (reservoirLock) { - oldReservoir = reservoir; - reservoir = newReservoir; - } - exportCurrentBatch(oldReservoir.getResult()); - updateNextExportTime(); - if (completableResultCode != null) { - completableResultCode.succeed(); - } - } - - try { - long pollWaitTime = nextExportTime - System.nanoTime(); - if (pollWaitTime > 0) { - completableResultCode = signal.poll(pollWaitTime, TimeUnit.NANOSECONDS); - } - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - private void updateNextExportTime() { - nextExportTime = System.nanoTime() + scheduleDelayNanos; - } - - private CompletableResultCode shutdown() { - CompletableResultCode result = new CompletableResultCode(); - - CompletableResultCode flushResult = forceFlush(); - flushResult.whenComplete( - () -> { - continueWork = false; - CompletableResultCode shutdownResult = spanExporter.shutdown(); - shutdownResult.whenComplete( - () -> { - if (!flushResult.isSuccess() || !shutdownResult.isSuccess()) { - result.fail(); - } else { - result.succeed(); - } - }); - }); - - return result; - } - - private CompletableResultCode forceFlush() { - CompletableResultCode flushResult = new CompletableResultCode(); - signal.offer(flushResult); - return flushResult; - } - - private void exportCurrentBatch(List batch) { - if (batch.isEmpty()) { - return; - } - - try { - CompletableResultCode result = spanExporter.export(Collections.unmodifiableList(batch)); - result.join(exporterTimeoutNanos, TimeUnit.NANOSECONDS); - if (result.isSuccess()) { - processedSpansCounter.add(batch.size(), exportedAttrs); - } else { - logger.log(Level.FINE, "Exporter failed"); - } - } catch (RuntimeException e) { - logger.log(Level.WARNING, "Exporter threw an Exception", e); - } finally { - batch.clear(); - } - } - - private boolean isReservoirEmpty() { - synchronized (reservoirLock) { - return reservoir.isEmpty(); - } - } - } -} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorBuilder.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorBuilder.java deleted file mode 100644 index f8b27dbcd..000000000 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorBuilder.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.contrib.export; - -import static io.opentelemetry.api.internal.Utils.checkArgument; -import static java.util.Objects.requireNonNull; - -import io.opentelemetry.api.metrics.MeterProvider; -import io.opentelemetry.contrib.util.RandomGenerator; -import io.opentelemetry.sdk.trace.export.SpanExporter; -import java.time.Duration; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; - -/** Builder class for {@link ConsistentReservoirSamplingBatchSpanProcessorBuilder}. */ -public final class ConsistentReservoirSamplingBatchSpanProcessorBuilder { - - // Visible for testing - static final long DEFAULT_SCHEDULE_DELAY_MILLIS = 5000; - // Visible for testing - static final int DEFAULT_RESERVOIR_SIZE = 2048; - // Visible for testing - static final int DEFAULT_EXPORT_TIMEOUT_MILLIS = 30_000; - - private final SpanExporter spanExporter; - private long scheduleDelayNanos = TimeUnit.MILLISECONDS.toNanos(DEFAULT_SCHEDULE_DELAY_MILLIS); - private int reservoirSize = DEFAULT_RESERVOIR_SIZE; - private long exporterTimeoutNanos = TimeUnit.MILLISECONDS.toNanos(DEFAULT_EXPORT_TIMEOUT_MILLIS); - private MeterProvider meterProvider = MeterProvider.noop(); - private RandomGenerator threadSafeRandomGenerator = () -> ThreadLocalRandom.current().nextLong(); - private boolean useAlternativeReservoirImplementation = false; - - ConsistentReservoirSamplingBatchSpanProcessorBuilder(SpanExporter spanExporter) { - this.spanExporter = requireNonNull(spanExporter, "spanExporter"); - } - - // TODO: Consider to add support for constant Attributes and/or Resource. - - /** - * Sets the delay interval between two consecutive exports. If unset, defaults to {@value - * DEFAULT_SCHEDULE_DELAY_MILLIS}ms. - */ - public ConsistentReservoirSamplingBatchSpanProcessorBuilder setScheduleDelay( - long delay, TimeUnit unit) { - requireNonNull(unit, "unit"); - checkArgument(delay >= 0, "delay must be non-negative"); - scheduleDelayNanos = unit.toNanos(delay); - return this; - } - - /** - * Sets the delay interval between two consecutive exports. If unset, defaults to {@value - * DEFAULT_SCHEDULE_DELAY_MILLIS}ms. - */ - public ConsistentReservoirSamplingBatchSpanProcessorBuilder setScheduleDelay(Duration delay) { - requireNonNull(delay, "delay"); - return setScheduleDelay(delay.toNanos(), TimeUnit.NANOSECONDS); - } - - // Visible for testing - long getScheduleDelayNanos() { - return scheduleDelayNanos; - } - - /** - * Sets the maximum time an export will be allowed to run before being cancelled. If unset, - * defaults to {@value DEFAULT_EXPORT_TIMEOUT_MILLIS}ms. - */ - public ConsistentReservoirSamplingBatchSpanProcessorBuilder setExporterTimeout( - long timeout, TimeUnit unit) { - requireNonNull(unit, "unit"); - checkArgument(timeout >= 0, "timeout must be non-negative"); - exporterTimeoutNanos = unit.toNanos(timeout); - return this; - } - - /** - * Sets the maximum time an export will be allowed to run before being cancelled. If unset, - * defaults to {@value DEFAULT_EXPORT_TIMEOUT_MILLIS}ms. - */ - public ConsistentReservoirSamplingBatchSpanProcessorBuilder setExporterTimeout(Duration timeout) { - requireNonNull(timeout, "timeout"); - return setExporterTimeout(timeout.toNanos(), TimeUnit.NANOSECONDS); - } - - // Visible for testing - long getExporterTimeoutNanos() { - return exporterTimeoutNanos; - } - - /** - * Sets the reservoir size, themaximum number of Spans that can be collected. - * - *

See the ConsistentReservoirSamplingBatchSpanProcessor class description for a high-level - * design description of this class. - * - *

Default value is {@code 2048}. - * - * @param reservoirSize the reservoir size, the maximum number of Spans that are kept - * @return this. - * @see ConsistentReservoirSamplingBatchSpanProcessorBuilder#DEFAULT_RESERVOIR_SIZE - */ - public ConsistentReservoirSamplingBatchSpanProcessorBuilder setReservoirSize(int reservoirSize) { - this.reservoirSize = reservoirSize; - return this; - } - - // Visible for testing - int getReservoirSize() { - return reservoirSize; - } - - /** - * Sets the {@link MeterProvider} to use to collect metrics related to batch export. If not set, - * metrics will not be collected. - */ - public ConsistentReservoirSamplingBatchSpanProcessorBuilder setMeterProvider( - MeterProvider meterProvider) { - requireNonNull(meterProvider, "meterProvider"); - this.meterProvider = meterProvider; - return this; - } - - // Visible for testing - ConsistentReservoirSamplingBatchSpanProcessorBuilder setThreadSafeRandomGenerator( - RandomGenerator threadSafeRandomGenerator) { - this.threadSafeRandomGenerator = threadSafeRandomGenerator; - return this; - } - - // Visible for testing - ConsistentReservoirSamplingBatchSpanProcessorBuilder useAlternativeReservoirImplementation( - boolean useAlternativeReservoirImplementation) { - this.useAlternativeReservoirImplementation = useAlternativeReservoirImplementation; - return this; - } - - /** - * Returns a new {@link ConsistentReservoirSamplingBatchSpanProcessorBuilder} that batches, then - * converts spans to proto and forwards them to the given {@code spanExporter}. - * - * @return a new {@link ConsistentReservoirSamplingBatchSpanProcessorBuilder}. - * @throws NullPointerException if the {@code spanExporter} is {@code null}. - */ - public ConsistentReservoirSamplingBatchSpanProcessor build() { - return new ConsistentReservoirSamplingBatchSpanProcessor( - spanExporter, - meterProvider, - scheduleDelayNanos, - reservoirSize, - exporterTimeoutNanos, - threadSafeRandomGenerator, - useAlternativeReservoirImplementation); - } -} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java deleted file mode 100644 index dbcad7c1d..000000000 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/export/ConsistentReservoirSamplingBatchSpanProcessorTest.java +++ /dev/null @@ -1,1114 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.contrib.export; - -import static io.opentelemetry.contrib.util.TestUtil.verifyObservedPvaluesUsingGtest; -import static java.util.stream.Collectors.toList; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatCode; -import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.argThat; -import static org.mockito.Mockito.doThrow; -import static org.mockito.Mockito.reset; -import static org.mockito.Mockito.when; - -import io.opentelemetry.api.internal.GuardedBy; -import io.opentelemetry.api.trace.Span; -import io.opentelemetry.api.trace.Tracer; -import io.opentelemetry.contrib.samplers.ConsistentSampler; -import io.opentelemetry.contrib.state.OtelTraceState; -import io.opentelemetry.contrib.util.RandomGenerator; -import io.opentelemetry.sdk.common.CompletableResultCode; -import io.opentelemetry.sdk.trace.ReadableSpan; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.data.SpanData; -import io.opentelemetry.sdk.trace.export.SpanExporter; -import io.opentelemetry.sdk.trace.samplers.Sampler; -import io.opentelemetry.sdk.trace.samplers.SamplingResult; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.EnumSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.SplittableRandom; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.stream.IntStream; -import java.util.stream.LongStream; -import javax.annotation.Nullable; -import org.assertj.core.api.Assertions; -import org.assertj.core.data.Percentage; -import org.hipparchus.distribution.discrete.BinomialDistribution; -import org.hipparchus.stat.descriptive.StatisticalSummary; -import org.hipparchus.stat.descriptive.StreamingStatistics; -import org.hipparchus.stat.inference.GTest; -import org.hipparchus.stat.inference.TTest; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentMatchers; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.mockito.junit.jupiter.MockitoSettings; -import org.mockito.quality.Strictness; - -@SuppressWarnings("PreferJavaTimeOverload") -@ExtendWith(MockitoExtension.class) -@MockitoSettings(strictness = Strictness.LENIENT) -class ConsistentReservoirSamplingBatchSpanProcessorTest { - - private static final String SPAN_NAME_1 = "MySpanName/1"; - private static final String SPAN_NAME_2 = "MySpanName/2"; - private static final long MAX_SCHEDULE_DELAY_MILLIS = 500; - - @Nullable private SdkTracerProvider sdkTracerProvider; - private final BlockingSpanExporter blockingSpanExporter = new BlockingSpanExporter(); - - @Mock private Sampler mockSampler; - @Mock private SpanExporter mockSpanExporter; - - @BeforeEach - void setUp() { - when(mockSpanExporter.shutdown()).thenReturn(CompletableResultCode.ofSuccess()); - } - - @AfterEach - void cleanup() { - if (sdkTracerProvider != null) { - sdkTracerProvider.shutdown(); - } - } - - @Nullable - private ReadableSpan createEndedSpan(String spanName) { - Tracer tracer = sdkTracerProvider.get(getClass().getName()); - Span span = tracer.spanBuilder(spanName).startSpan(); - span.end(); - if (span instanceof ReadableSpan) { - return (ReadableSpan) span; - } else { - return null; - } - } - - @Test - void configTest_EmptyOptions() { - ConsistentReservoirSamplingBatchSpanProcessorBuilder config = - ConsistentReservoirSamplingBatchSpanProcessor.builder( - new WaitingSpanExporter(0, CompletableResultCode.ofSuccess())); - Assertions.assertThat(config.getScheduleDelayNanos()) - .isEqualTo( - TimeUnit.MILLISECONDS.toNanos( - ConsistentReservoirSamplingBatchSpanProcessorBuilder - .DEFAULT_SCHEDULE_DELAY_MILLIS)); - Assertions.assertThat(config.getReservoirSize()) - .isEqualTo(ConsistentReservoirSamplingBatchSpanProcessorBuilder.DEFAULT_RESERVOIR_SIZE); - Assertions.assertThat(config.getExporterTimeoutNanos()) - .isEqualTo( - TimeUnit.MILLISECONDS.toNanos( - ConsistentReservoirSamplingBatchSpanProcessorBuilder - .DEFAULT_EXPORT_TIMEOUT_MILLIS)); - } - - @Test - void invalidConfig() { - assertThatThrownBy( - () -> - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) - .setScheduleDelay(-1, TimeUnit.MILLISECONDS)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("delay must be non-negative"); - assertThatThrownBy( - () -> - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) - .setScheduleDelay(1, null)) - .isInstanceOf(NullPointerException.class) - .hasMessage("unit"); - assertThatThrownBy( - () -> - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) - .setScheduleDelay(null)) - .isInstanceOf(NullPointerException.class) - .hasMessage("delay"); - assertThatThrownBy( - () -> - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) - .setExporterTimeout(-1, TimeUnit.MILLISECONDS)) - .isInstanceOf(IllegalArgumentException.class) - .hasMessage("timeout must be non-negative"); - assertThatThrownBy( - () -> - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) - .setExporterTimeout(1, null)) - .isInstanceOf(NullPointerException.class) - .hasMessage("unit"); - assertThatThrownBy( - () -> - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) - .setExporterTimeout(null)) - .isInstanceOf(NullPointerException.class) - .hasMessage("timeout"); - } - - @Test - void startEndRequirements() { - ConsistentReservoirSamplingBatchSpanProcessor spansProcessor = - ConsistentReservoirSamplingBatchSpanProcessor.builder( - new WaitingSpanExporter(0, CompletableResultCode.ofSuccess())) - .build(); - Assertions.assertThat(spansProcessor.isStartRequired()).isFalse(); - Assertions.assertThat(spansProcessor.isEndRequired()).isTrue(); - } - - @Test - void exportDifferentSampledSpans() { - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(2, CompletableResultCode.ofSuccess()); - sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor( - ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) - .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) - .build()) - .build(); - - ReadableSpan span1 = createEndedSpan(SPAN_NAME_1); - ReadableSpan span2 = createEndedSpan(SPAN_NAME_2); - List exported = waitingSpanExporter.waitForExport(); - Assertions.assertThat(exported) - .containsExactlyInAnyOrder(span1.toSpanData(), span2.toSpanData()); - } - - @Test - void exportMoreSpansThanTheBufferSize() { - CompletableSpanExporter spanExporter = new CompletableSpanExporter(); - - sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor( - ConsistentReservoirSamplingBatchSpanProcessor.builder(spanExporter) - .setReservoirSize(6) - .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) - .build()) - .build(); - - ReadableSpan span1 = createEndedSpan(SPAN_NAME_1); - ReadableSpan span2 = createEndedSpan(SPAN_NAME_1); - ReadableSpan span3 = createEndedSpan(SPAN_NAME_1); - ReadableSpan span4 = createEndedSpan(SPAN_NAME_1); - ReadableSpan span5 = createEndedSpan(SPAN_NAME_1); - ReadableSpan span6 = createEndedSpan(SPAN_NAME_1); - - spanExporter.succeed(); - - await() - .untilAsserted( - () -> - Assertions.assertThat(spanExporter.getExported()) - .containsExactlyInAnyOrder( - span1.toSpanData(), - span2.toSpanData(), - span3.toSpanData(), - span4.toSpanData(), - span5.toSpanData(), - span6.toSpanData())); - } - - @Test - void forceExport() { - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(100, CompletableResultCode.ofSuccess(), 1); - ConsistentReservoirSamplingBatchSpanProcessor batchSpanProcessor = - ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) - // .setReservoirSize(10_000) - // Force flush should send all spans, make sure the number of spans we check here is - // not divisible by the batch size. - .setReservoirSize(49) - .setScheduleDelay(10, TimeUnit.SECONDS) - .build(); - - sdkTracerProvider = SdkTracerProvider.builder().addSpanProcessor(batchSpanProcessor).build(); - for (int i = 0; i < 100; i++) { - createEndedSpan("notExported"); - } - - batchSpanProcessor.forceFlush().join(10, TimeUnit.SECONDS); - List exported = waitingSpanExporter.getExported(); - Assertions.assertThat(exported).isNotNull(); - Assertions.assertThat(exported.size()).isEqualTo(49); - } - - @Test - void exportSpansToMultipleServices() { - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(2, CompletableResultCode.ofSuccess()); - WaitingSpanExporter waitingSpanExporter2 = - new WaitingSpanExporter(2, CompletableResultCode.ofSuccess()); - sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor( - ConsistentReservoirSamplingBatchSpanProcessor.builder( - SpanExporter.composite( - Arrays.asList(waitingSpanExporter, waitingSpanExporter2))) - .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) - .build()) - .build(); - - ReadableSpan span1 = createEndedSpan(SPAN_NAME_1); - ReadableSpan span2 = createEndedSpan(SPAN_NAME_2); - List exported1 = waitingSpanExporter.waitForExport(); - List exported2 = waitingSpanExporter2.waitForExport(); - Assertions.assertThat(exported1) - .containsExactlyInAnyOrder(span1.toSpanData(), span2.toSpanData()); - Assertions.assertThat(exported2) - .containsExactlyInAnyOrder(span1.toSpanData(), span2.toSpanData()); - } - - @Test - void exportMoreSpansThanTheMaximumLimit() { - int maxQueuedSpans = 8; - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(maxQueuedSpans, CompletableResultCode.ofSuccess()); - sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor( - ConsistentReservoirSamplingBatchSpanProcessor.builder( - SpanExporter.composite( - Arrays.asList(blockingSpanExporter, waitingSpanExporter))) - .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) - .setReservoirSize(maxQueuedSpans) - .build()) - .build(); - - List spansToExport = new ArrayList<>(maxQueuedSpans + 1); - // Wait to block the worker thread in the BatchSampledSpansProcessor. This ensures that no items - // can be removed from the queue. Need to add a span to trigger the export otherwise the - // pipeline is never called. - spansToExport.add(createEndedSpan("blocking_span").toSpanData()); - blockingSpanExporter.waitUntilIsBlocked(); - - for (int i = 0; i < maxQueuedSpans; i++) { - // First export maxQueuedSpans, the worker thread is blocked so all items should be queued. - spansToExport.add(createEndedSpan("span_1_" + i).toSpanData()); - } - - // TODO: assertThat(spanExporter.getReferencedSpans()).isEqualTo(maxQueuedSpans); - - // Now we should start dropping. - for (int i = 0; i < 7; i++) { - createEndedSpan("span_2_" + i); - // TODO: assertThat(getDroppedSpans()).isEqualTo(i + 1); - } - - // TODO: assertThat(getReferencedSpans()).isEqualTo(maxQueuedSpans); - - // Release the blocking exporter - blockingSpanExporter.unblock(); - - // While we wait for maxQueuedSpans we ensure that the queue is also empty after this. - List exported = waitingSpanExporter.waitForExport(); - Assertions.assertThat(exported).isNotNull(); - Assertions.assertThat(exported).hasSize(maxQueuedSpans + 1); - // assertThat(exported).containsExactlyInAnyOrderElementsOf(spansToExport); - exported.clear(); - spansToExport.clear(); - - waitingSpanExporter.reset(); - // We cannot compare with maxReferencedSpans here because the worker thread may get - // unscheduled immediately after exporting, but before updating the pushed spans, if that is - // the case at most bufferSize spans will miss. - // TODO: assertThat(getPushedSpans()).isAtLeast((long) maxQueuedSpans - maxBatchSize); - - for (int i = 0; i < maxQueuedSpans; i++) { - spansToExport.add(createEndedSpan("span_3_" + i).toSpanData()); - // No more dropped spans. - // TODO: assertThat(getDroppedSpans()).isEqualTo(7); - } - - exported = waitingSpanExporter.waitForExport(); - Assertions.assertThat(exported).isNotNull(); - Assertions.assertThat(exported).containsExactlyInAnyOrderElementsOf(spansToExport); - } - - @Test - void ignoresNullSpans() { - ConsistentReservoirSamplingBatchSpanProcessor processor = - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter).build(); - try { - assertThatCode( - () -> { - processor.onStart(null, null); - processor.onEnd(null); - }) - .doesNotThrowAnyException(); - } finally { - processor.shutdown(); - } - } - - @Test - void exporterThrowsException() { - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(1, CompletableResultCode.ofSuccess()); - doThrow(new IllegalArgumentException("No export for you.")) - .when(mockSpanExporter) - .export(ArgumentMatchers.anyList()); - sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor( - ConsistentReservoirSamplingBatchSpanProcessor.builder( - SpanExporter.composite( - Arrays.asList(mockSpanExporter, waitingSpanExporter))) - .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) - .build()) - .build(); - ReadableSpan span1 = createEndedSpan(SPAN_NAME_1); - List exported = waitingSpanExporter.waitForExport(); - Assertions.assertThat(exported).containsExactly(span1.toSpanData()); - waitingSpanExporter.reset(); - // Continue to export after the exception was received. - ReadableSpan span2 = createEndedSpan(SPAN_NAME_2); - exported = waitingSpanExporter.waitForExport(); - Assertions.assertThat(exported).containsExactly(span2.toSpanData()); - } - - @Test - @Timeout(5) - public void continuesIfExporterTimesOut() throws InterruptedException { - int exporterTimeoutMillis = 10; - ConsistentReservoirSamplingBatchSpanProcessor bsp = - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter) - .setExporterTimeout(exporterTimeoutMillis, TimeUnit.MILLISECONDS) - .setScheduleDelay(1, TimeUnit.MILLISECONDS) - .setReservoirSize(1) - .build(); - sdkTracerProvider = SdkTracerProvider.builder().addSpanProcessor(bsp).build(); - - CountDownLatch exported = new CountDownLatch(1); - // We return a result we never complete, meaning it will timeout. - when(mockSpanExporter.export( - argThat( - spans -> { - Assertions.assertThat(spans) - .anySatisfy( - span -> Assertions.assertThat(span.getName()).isEqualTo(SPAN_NAME_1)); - exported.countDown(); - return true; - }))) - .thenReturn(new CompletableResultCode()); - createEndedSpan(SPAN_NAME_1); - exported.await(); - // Timed out so the span was dropped. - await().untilAsserted(() -> Assertions.assertThat(bsp.isReservoirEmpty()).isTrue()); - - // Still processing new spans. - CountDownLatch exportedAgain = new CountDownLatch(1); - reset(mockSpanExporter); - when(mockSpanExporter.shutdown()).thenReturn(CompletableResultCode.ofSuccess()); - when(mockSpanExporter.export( - argThat( - spans -> { - Assertions.assertThat(spans) - .anySatisfy( - span -> Assertions.assertThat(span.getName()).isEqualTo(SPAN_NAME_2)); - exportedAgain.countDown(); - return true; - }))) - .thenReturn(CompletableResultCode.ofSuccess()); - createEndedSpan(SPAN_NAME_2); - exported.await(); - await().untilAsserted(() -> Assertions.assertThat(bsp.isReservoirEmpty()).isTrue()); - } - - @Test - void exportNotSampledSpans() { - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(1, CompletableResultCode.ofSuccess()); - sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor( - ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) - .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) - .build()) - .setSampler(mockSampler) - .build(); - - when(mockSampler.shouldSample(any(), any(), any(), any(), any(), anyList())) - .thenReturn(SamplingResult.drop()); - sdkTracerProvider.get("test").spanBuilder(SPAN_NAME_1).startSpan().end(); - sdkTracerProvider.get("test").spanBuilder(SPAN_NAME_2).startSpan().end(); - when(mockSampler.shouldSample(any(), any(), any(), any(), any(), anyList())) - .thenReturn(SamplingResult.recordAndSample()); - ReadableSpan span = createEndedSpan(SPAN_NAME_2); - // Spans are recorded and exported in the same order as they are ended, we test that a non - // sampled span is not exported by creating and ending a sampled span after a non sampled span - // and checking that the first exported span is the sampled span (the non sampled did not get - // exported). - List exported = waitingSpanExporter.waitForExport(); - // Need to check this because otherwise the variable span1 is unused, other option is to not - // have a span1 variable. - Assertions.assertThat(exported).containsExactly(span.toSpanData()); - } - - @Test - void exportNotSampledSpans_recordOnly() { - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(1, CompletableResultCode.ofSuccess()); - - when(mockSampler.shouldSample(any(), any(), any(), any(), any(), anyList())) - .thenReturn(SamplingResult.recordOnly()); - sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor( - ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) - .setScheduleDelay(MAX_SCHEDULE_DELAY_MILLIS, TimeUnit.MILLISECONDS) - .build()) - .setSampler(mockSampler) - .build(); - - createEndedSpan(SPAN_NAME_1); - when(mockSampler.shouldSample(any(), any(), any(), any(), any(), anyList())) - .thenReturn(SamplingResult.recordAndSample()); - ReadableSpan span = createEndedSpan(SPAN_NAME_2); - - // Spans are recorded and exported in the same order as they are ended, we test that a non - // exported span is not exported by creating and ending a sampled span after a non sampled span - // and checking that the first exported span is the sampled span (the non sampled did not get - // exported). - List exported = waitingSpanExporter.waitForExport(); - // Need to check this because otherwise the variable span1 is unused, other option is to not - // have a span1 variable. - Assertions.assertThat(exported).containsExactly(span.toSpanData()); - } - - @Test - @Timeout(10) - void shutdownFlushes() { - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(1, CompletableResultCode.ofSuccess()); - // Set the export delay to large value, in order to confirm the #flush() below works - - sdkTracerProvider = - SdkTracerProvider.builder() - .addSpanProcessor( - ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) - .setScheduleDelay(10, TimeUnit.SECONDS) - .build()) - .build(); - - ReadableSpan span2 = createEndedSpan(SPAN_NAME_2); - - // Force a shutdown, which forces processing of all remaining spans. - sdkTracerProvider.shutdown().join(10, TimeUnit.SECONDS); - - List exported = waitingSpanExporter.getExported(); - Assertions.assertThat(exported).containsExactly(span2.toSpanData()); - Assertions.assertThat(waitingSpanExporter.shutDownCalled.get()).isTrue(); - } - - @Test - void shutdownPropagatesSuccess() { - ConsistentReservoirSamplingBatchSpanProcessor processor = - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter).build(); - CompletableResultCode result = processor.shutdown(); - result.join(1, TimeUnit.SECONDS); - Assertions.assertThat(result.isSuccess()).isTrue(); - } - - @Test - void shutdownPropagatesFailure() { - when(mockSpanExporter.shutdown()).thenReturn(CompletableResultCode.ofFailure()); - ConsistentReservoirSamplingBatchSpanProcessor processor = - ConsistentReservoirSamplingBatchSpanProcessor.builder(mockSpanExporter).build(); - CompletableResultCode result = processor.shutdown(); - result.join(1, TimeUnit.SECONDS); - Assertions.assertThat(result.isSuccess()).isFalse(); - } - - private static final class BlockingSpanExporter implements SpanExporter { - - final Object monitor = new Object(); - - private enum State { - WAIT_TO_BLOCK, - BLOCKED, - UNBLOCKED - } - - @GuardedBy("monitor") - State state = State.WAIT_TO_BLOCK; - - @Override - public CompletableResultCode export(Collection spanDataList) { - synchronized (monitor) { - while (state != State.UNBLOCKED) { - try { - state = State.BLOCKED; - // Some threads may wait for Blocked State. - monitor.notifyAll(); - monitor.wait(); - } catch (InterruptedException e) { - // Do nothing - } - } - } - return CompletableResultCode.ofSuccess(); - } - - @Override - public CompletableResultCode flush() { - return CompletableResultCode.ofSuccess(); - } - - private void waitUntilIsBlocked() { - synchronized (monitor) { - while (state != State.BLOCKED) { - try { - monitor.wait(); - } catch (InterruptedException e) { - // Do nothing - } - } - } - } - - @Override - public CompletableResultCode shutdown() { - // Do nothing; - return CompletableResultCode.ofSuccess(); - } - - private void unblock() { - synchronized (monitor) { - state = State.UNBLOCKED; - monitor.notifyAll(); - } - } - } - - private static class CompletableSpanExporter implements SpanExporter { - - private final List results = new ArrayList<>(); - - private final List exported = new ArrayList<>(); - - private volatile boolean succeeded; - - List getExported() { - return exported; - } - - void succeed() { - succeeded = true; - results.forEach(CompletableResultCode::succeed); - } - - @Override - public CompletableResultCode export(Collection spans) { - exported.addAll(spans); - if (succeeded) { - return CompletableResultCode.ofSuccess(); - } - CompletableResultCode result = new CompletableResultCode(); - results.add(result); - return result; - } - - @Override - public CompletableResultCode flush() { - if (succeeded) { - return CompletableResultCode.ofSuccess(); - } else { - return CompletableResultCode.ofFailure(); - } - } - - @Override - public CompletableResultCode shutdown() { - return flush(); - } - } - - static class WaitingSpanExporter implements SpanExporter { - - private final List spanDataList = new ArrayList<>(); - private final int numberToWaitFor; - private final CompletableResultCode exportResultCode; - private CountDownLatch countDownLatch; - private int timeout = 10; - private final AtomicBoolean shutDownCalled = new AtomicBoolean(false); - - WaitingSpanExporter(int numberToWaitFor, CompletableResultCode exportResultCode) { - countDownLatch = new CountDownLatch(numberToWaitFor); - this.numberToWaitFor = numberToWaitFor; - this.exportResultCode = exportResultCode; - } - - WaitingSpanExporter(int numberToWaitFor, CompletableResultCode exportResultCode, int timeout) { - this(numberToWaitFor, exportResultCode); - this.timeout = timeout; - } - - List getExported() { - List result = new ArrayList<>(spanDataList); - spanDataList.clear(); - return result; - } - - /** - * Waits until we received numberOfSpans spans to export. Returns the list of exported {@link - * SpanData} objects, otherwise {@code null} if the current thread is interrupted. - * - * @return the list of exported {@link SpanData} objects, otherwise {@code null} if the current - * thread is interrupted. - */ - @Nullable - List waitForExport() { - try { - countDownLatch.await(timeout, TimeUnit.SECONDS); - } catch (InterruptedException e) { - // Preserve the interruption status as per guidance. - Thread.currentThread().interrupt(); - return null; - } - return getExported(); - } - - @Override - public CompletableResultCode export(Collection spans) { - this.spanDataList.addAll(spans); - for (int i = 0; i < spans.size(); i++) { - countDownLatch.countDown(); - } - return exportResultCode; - } - - @Override - public CompletableResultCode flush() { - return CompletableResultCode.ofSuccess(); - } - - @Override - public CompletableResultCode shutdown() { - shutDownCalled.set(true); - return CompletableResultCode.ofSuccess(); - } - - public void reset() { - this.countDownLatch = new CountDownLatch(numberToWaitFor); - } - } - - @Test - void exportDifferentConsistentlySampledSpans() { - int reservoirSize = 10; - int numberOfSpans = 100; - - WaitingSpanExporter waitingSpanExporter = - new WaitingSpanExporter(reservoirSize, CompletableResultCode.ofSuccess()); - - ConsistentReservoirSamplingBatchSpanProcessor spanProcessor = - ConsistentReservoirSamplingBatchSpanProcessor.builder(waitingSpanExporter) - .setReservoirSize(reservoirSize) - .setScheduleDelay(10, TimeUnit.SECONDS) - .setReservoirSize(reservoirSize) - .build(); - - sdkTracerProvider = - SdkTracerProvider.builder() - .setSampler(ConsistentSampler.alwaysOn()) - .addSpanProcessor(spanProcessor) - .build(); - - List spans = - IntStream.range(0, numberOfSpans) - .mapToObj(i -> createEndedSpan("MySpanName/" + i)) - .collect(toList()); - - Assertions.assertThat(spans).hasSize(numberOfSpans); - - spanProcessor.forceFlush().join(10, TimeUnit.SECONDS); - - List exported = waitingSpanExporter.waitForExport(); - Assertions.assertThat(exported).hasSize(reservoirSize); - } - - private enum Tests { - VERIFY_MEAN, - VERIFY_PVALUE_DISTRIBUTION, - VERIFY_ORDER_INDEPENDENCE - } - - private void testConsistentSampling( - boolean useAlternativeReservoirImplementation, - long seed, - int numCycles, - int numberOfSpans, - int reservoirSize, - double samplingProbability, - EnumSet tests) { - - SplittableRandom rng1 = new SplittableRandom(seed); - SplittableRandom rng2 = rng1.split(); - RandomGenerator threadSafeRandomGenerator1 = - () -> { - synchronized (rng1) { - return rng1.nextLong(); - } - }; - RandomGenerator threadSafeRandomGenerator2 = - () -> { - synchronized (rng2) { - return rng1.nextLong(); - } - }; - - WaitingSpanExporter spanExporter = - new WaitingSpanExporter(0, CompletableResultCode.ofSuccess()); - - ConsistentReservoirSamplingBatchSpanProcessor spanProcessor = - ConsistentReservoirSamplingBatchSpanProcessor.builder(spanExporter) - .setReservoirSize(reservoirSize) - .setThreadSafeRandomGenerator(threadSafeRandomGenerator1) - .setScheduleDelay(1000, TimeUnit.SECONDS) - .setReservoirSize(reservoirSize) - .useAlternativeReservoirImplementation(useAlternativeReservoirImplementation) - .build(); - - sdkTracerProvider = - SdkTracerProvider.builder() - .setSampler( - ConsistentSampler.probabilityBased(samplingProbability, threadSafeRandomGenerator2)) - .addSpanProcessor(spanProcessor) - .build(); - - Map observedPvalues = new HashMap<>(); - Map spanNameCounts = new HashMap<>(); - - double[] totalAdjustedCounts = new double[numCycles]; - - for (int k = 0; k < numCycles; ++k) { - String prefixSpanName = "MySpanName/" + k + "/"; - List spans = - LongStream.range(0, numberOfSpans) - .mapToObj(i -> createEndedSpan(prefixSpanName + i)) - .filter(Objects::nonNull) - .collect(toList()); - - if (samplingProbability >= 1.) { - Assertions.assertThat(spans).hasSize(numberOfSpans); - } - - spanProcessor.forceFlush().join(1000, TimeUnit.SECONDS); - - List exported = spanExporter.getExported(); - Assertions.assertThat(exported).hasSize(Math.min(reservoirSize, spans.size())); - - double totalAdjustedCount = 0; - for (SpanData spanData : exported) { - String traceStateString = - spanData.getSpanContext().getTraceState().get(OtelTraceState.TRACE_STATE_KEY); - OtelTraceState traceState = OtelTraceState.parse(traceStateString); - assertTrue(traceState.hasValidR()); - assertTrue(traceState.hasValidP()); - observedPvalues.merge(traceState.getP(), 1L, Long::sum); - totalAdjustedCount += Math.pow(2., traceState.getP()); - spanNameCounts.merge(spanData.getName().split("/")[2], 1L, Long::sum); - } - totalAdjustedCounts[k] = totalAdjustedCount; - } - - long totalNumberOfSpans = numberOfSpans * (long) numCycles; - if (numCycles == 1) { - Assertions.assertThat(observedPvalues).hasSizeLessThanOrEqualTo(2); - } - if (tests.contains(Tests.VERIFY_MEAN)) { - Assertions.assertThat(reservoirSize) - .isGreaterThanOrEqualTo( - 100); // require a lower limit on the reservoir size, to justify the assumption of the - // t-test that values are normally distributed - - Assertions.assertThat( - new TTest().tTest(totalNumberOfSpans / (double) numCycles, totalAdjustedCounts)) - .isGreaterThan(0.01); - } - if (tests.contains(Tests.VERIFY_PVALUE_DISTRIBUTION)) { - Assertions.assertThat(observedPvalues) - .hasSizeLessThanOrEqualTo(2); // test does not work for more than 2 different p-values - - // The expected number of sampled spans is binomially distributed with the given sampling - // probability. However, due to the reservoir sampling buffer the maximum number of sampled - // spans is given by the reservoir size. The effective sampling rate is therefore given by - // sum_{i=0}^n p^i*(1-p)^{n-i}*min(i,k) (n choose i) - // where p denotes the sampling rate, n is the total number of original spans, and k denotes - // the reservoir size - double p1 = - new BinomialDistribution(numberOfSpans - 1, samplingProbability) - .cumulativeProbability(reservoirSize - 1); - double p2 = - new BinomialDistribution(numberOfSpans, samplingProbability) - .cumulativeProbability(reservoirSize); - Assertions.assertThat(p1).isLessThanOrEqualTo(p2); - - double effectiveSamplingProbability = - samplingProbability * p1 + (reservoirSize / (double) numberOfSpans) * (1. - p2); - verifyObservedPvaluesUsingGtest( - totalNumberOfSpans, observedPvalues, effectiveSamplingProbability); - } - if (tests.contains(Tests.VERIFY_ORDER_INDEPENDENCE)) { - Assertions.assertThat(spanNameCounts.size()).isEqualTo(numberOfSpans); - long[] observed = spanNameCounts.values().stream().mapToLong(x -> x).toArray(); - double[] expected = new double[numberOfSpans]; - Arrays.fill(expected, 1.); - Assertions.assertThat(new GTest().gTest(expected, observed)).isGreaterThan(0.01); - } - } - - private void testConsistentSampling(boolean useAlternativeReservoirImplementation) { - testConsistentSampling( - useAlternativeReservoirImplementation, - 0x34e7052af91d5355L, - 10000, - 1000, - 100, - 1., - EnumSet.of(Tests.VERIFY_MEAN, Tests.VERIFY_ORDER_INDEPENDENCE)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0xcd02a41e10ff273dL, - 10000, - 1000, - 100, - 0.8, - EnumSet.of(Tests.VERIFY_MEAN, Tests.VERIFY_ORDER_INDEPENDENCE)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0x2c3d086534e14407L, - 10000, - 1000, - 100, - 0.1, - EnumSet.of( - Tests.VERIFY_MEAN, Tests.VERIFY_PVALUE_DISTRIBUTION, Tests.VERIFY_ORDER_INDEPENDENCE)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0xd3f8a40433cf0522L, - 10000, - 1000, - 200, - 0.9, - EnumSet.of(Tests.VERIFY_MEAN, Tests.VERIFY_ORDER_INDEPENDENCE)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0xf25638ca67eceadcL, - 10000, - 100, - 100, - 1.0, - EnumSet.of(Tests.VERIFY_MEAN)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0x14c5f8f815618ce2L, - 10000, - 200, - 100, - 1.0, - EnumSet.of(Tests.VERIFY_MEAN, Tests.VERIFY_ORDER_INDEPENDENCE)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0xb6c27f1169e128ddL, - 10000, - 1000, - 200, - 0.2, - EnumSet.of( - Tests.VERIFY_MEAN, Tests.VERIFY_PVALUE_DISTRIBUTION, Tests.VERIFY_ORDER_INDEPENDENCE)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0xab558ff7c5c73c18L, - 1000, - 10000, - 200, - 1., - EnumSet.of( - Tests.VERIFY_MEAN, Tests.VERIFY_PVALUE_DISTRIBUTION, Tests.VERIFY_ORDER_INDEPENDENCE)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0xe53010c4b009a6c0L, - 10000, - 1000, - 2000, - 0.2, - EnumSet.of( - Tests.VERIFY_MEAN, Tests.VERIFY_PVALUE_DISTRIBUTION, Tests.VERIFY_ORDER_INDEPENDENCE)); - testConsistentSampling( - useAlternativeReservoirImplementation, - 0xc41d327fd1a6866aL, - 1000000, - 5, - 4, - 1.0, - EnumSet.of(Tests.VERIFY_ORDER_INDEPENDENCE)); - } - - @Test - void testConsistentSampling() { - testConsistentSampling(false); - } - - @Test - void testConsistentSamplingWithAlternativeReservoirImplementation() { - testConsistentSampling(true); - } - - private StatisticalSummary calculateStatisticalSummary( - boolean useAlternativeReservoirImplementation, - long seed, - int numCycles, - int numberOfSpans, - int reservoirSize, - double samplingProbability) { - - SplittableRandom rng1 = new SplittableRandom(seed); - SplittableRandom rng2 = rng1.split(); - RandomGenerator threadSafeRandomGenerator1 = - () -> { - synchronized (rng1) { - return rng1.nextLong(); - } - }; - RandomGenerator threadSafeRandomGenerator2 = - () -> { - synchronized (rng2) { - return rng1.nextLong(); - } - }; - - WaitingSpanExporter spanExporter = - new WaitingSpanExporter(0, CompletableResultCode.ofSuccess()); - - ConsistentReservoirSamplingBatchSpanProcessor spanProcessor = - ConsistentReservoirSamplingBatchSpanProcessor.builder(spanExporter) - .setReservoirSize(reservoirSize) - .setThreadSafeRandomGenerator(threadSafeRandomGenerator1) - .setScheduleDelay(1000, TimeUnit.SECONDS) - .setReservoirSize(reservoirSize) - .useAlternativeReservoirImplementation(useAlternativeReservoirImplementation) - .build(); - - sdkTracerProvider = - SdkTracerProvider.builder() - .setSampler( - ConsistentSampler.probabilityBased(samplingProbability, threadSafeRandomGenerator2)) - .addSpanProcessor(spanProcessor) - .build(); - - StreamingStatistics streamingStatistics = new StreamingStatistics(); - - for (int k = 0; k < numCycles; ++k) { - String prefixSpanName = "MySpanName/" + k + "/"; - List spans = - LongStream.range(0, numberOfSpans) - .mapToObj(i -> createEndedSpan(prefixSpanName + i)) - .filter(Objects::nonNull) - .collect(toList()); - - if (samplingProbability >= 1.) { - Assertions.assertThat(spans).hasSize(numberOfSpans); - } - - spanProcessor.forceFlush().join(1000, TimeUnit.SECONDS); - - List exported = spanExporter.getExported(); - Assertions.assertThat(exported).hasSize(Math.min(reservoirSize, spans.size())); - - double totalAdjustedCount = 0; - for (SpanData spanData : exported) { - String traceStateString = - spanData.getSpanContext().getTraceState().get(OtelTraceState.TRACE_STATE_KEY); - OtelTraceState traceState = OtelTraceState.parse(traceStateString); - assertTrue(traceState.hasValidR()); - assertTrue(traceState.hasValidP()); - totalAdjustedCount += Math.pow(2., traceState.getP()); - } - streamingStatistics.accept(totalAdjustedCount); - } - return streamingStatistics; - } - - @Test - void testVarianceDifferencesBetweenReservoirImplementations1() { - boolean useAlternativeReservoirImplementationFalse = false; - boolean useAlternativeReservoirImplementationTrue = true; - - StatisticalSummary variant1 = - calculateStatisticalSummary( - useAlternativeReservoirImplementationFalse, 0x225de9590067d658L, 10000, 100, 50, 0.8); - - StatisticalSummary variant2 = - calculateStatisticalSummary( - useAlternativeReservoirImplementationTrue, 0x23b418ed68d668L, 10000, 100, 50, 0.8); - - Assertions.assertThat(variant1.getMean()).isCloseTo(100, Percentage.withPercentage(1)); - Assertions.assertThat(variant2.getMean()).isCloseTo(100, Percentage.withPercentage(1)); - - Assertions.assertThat(variant1.getVariance()) - .isCloseTo(111.17153690368997, Percentage.withPercentage(0.01)); - Assertions.assertThat(variant2.getVariance()) - .isCloseTo(95.65024978497851, Percentage.withPercentage(0.01)); - } - - @Test - void testVarianceDifferencesBetweenReservoirImplementations2() { - boolean useAlternativeReservoirImplementationFalse = false; - boolean useAlternativeReservoirImplementationTrue = true; - - StatisticalSummary variant1 = - calculateStatisticalSummary( - useAlternativeReservoirImplementationFalse, 0x7f33baf84d59df65L, 100000, 10, 4, 0.9); - - StatisticalSummary variant2 = - calculateStatisticalSummary( - useAlternativeReservoirImplementationTrue, 0xdb3bf0109e0a4b43L, 100000, 10, 4, 0.9); - - Assertions.assertThat(variant1.getMean()).isCloseTo(10, Percentage.withPercentage(2)); - Assertions.assertThat(variant2.getMean()).isCloseTo(10, Percentage.withPercentage(2)); - Assertions.assertThat(variant1.getVariance()) - .isCloseTo(22.617777121371127, Percentage.withPercentage(0.01)); - Assertions.assertThat(variant2.getVariance()) - .isCloseTo(19.465000847508666, Percentage.withPercentage(0.01)); - } - - @Test - void testVarianceDifferencesBetweenReservoirImplementations3() { - boolean useAlternativeReservoirImplementationFalse = false; - boolean useAlternativeReservoirImplementationTrue = true; - - StatisticalSummary variant1 = - calculateStatisticalSummary( - useAlternativeReservoirImplementationFalse, 0x72d7312adac9c84dL, 10000, 1000, 700, 1); - - StatisticalSummary variant2 = - calculateStatisticalSummary( - useAlternativeReservoirImplementationTrue, 0x7ea0c32d80a319d0L, 10000, 1000, 700, 1.); - - Assertions.assertThat(variant1.getMean()).isCloseTo(1000, Percentage.withPercentage(1)); - Assertions.assertThat(variant2.getMean()).isCloseTo(1000, Percentage.withPercentage(1)); - Assertions.assertThat(variant1.getVariance()) - .isCloseTo(594.9523749875004, Percentage.withPercentage(0.01)); - Assertions.assertThat(variant2.getVariance()) - .isCloseTo(362.2863018701875, Percentage.withPercentage(0.01)); - } -} From 6f731b688c68497e1c288afd8bdfa65bec1b6a83 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Wed, 9 Mar 2022 21:23:31 +0100 Subject: [PATCH 13/47] improved comment --- .../contrib/samplers/ConsistentRateLimitingSampler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 3080971bf..263c194dc 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -20,7 +20,8 @@ * *

This sampler uses exponential smoothing to estimate on irregular data (compare Wright, David * J. "Forecasting data published at irregular time intervals using an extension of Holt's method." - * Management science 32.4 (1986): 499-510.) to estimate the current rate of spans. + * Management science 32.4 (1986): 499-510.) to estimate the average waiting time between spans + * which further allows to estimate the current rate of spans. */ final class ConsistentRateLimitingSampler extends ConsistentSampler { From d40c0f1226bf8780aa1346665dbd4fe861094890 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Thu, 10 Mar 2022 07:59:36 +0100 Subject: [PATCH 14/47] removed unnecessary clipping of sampling probability --- .../contrib/samplers/ConsistentRateLimitingSampler.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 263c194dc..1b80addd0 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -112,10 +112,8 @@ protected int getP(int parentP, boolean isRoot) { State currentState = state.updateAndGet(s -> updateState(s, currentNanoTime)); double samplingProbability = - Math.min( - 1., - (currentState.effectiveWindowNanos * targetSpansPerNanosLimit) - / currentState.effectiveWindowCount); + (currentState.effectiveWindowNanos * targetSpansPerNanosLimit) + / currentState.effectiveWindowCount; if (samplingProbability >= 1.) { return 0; From 775ca5227a97f2b7534aff63078339623ba94e87 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Thu, 10 Mar 2022 08:53:23 +0100 Subject: [PATCH 15/47] added javadoc explaining maths of implementation --- .../ConsistentRateLimitingSampler.java | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 1b80addd0..2e8f0e313 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -21,7 +21,51 @@ *

This sampler uses exponential smoothing to estimate on irregular data (compare Wright, David * J. "Forecasting data published at irregular time intervals using an extension of Holt's method." * Management science 32.4 (1986): 499-510.) to estimate the average waiting time between spans - * which further allows to estimate the current rate of spans. + * which further allows to estimate the current rate of spans. In the paper, Eq. 2 defines the + * weighted average of a sequence of data + * + *

{@code ..., X(n-2), X(n-1), X(n)} + * + *

at irregular times + * + *

{@code ..., t(n-2), t(n-1), t(n)} + * + *

as + * + *

{@code E(X(n)) := A(n) * V(n)}. + * + *

{@code A(n)} and {@code V(n)} are computed recursively using Eq. 5 and Eq. 6 given by + * + *

{@code A(n) = b(n) * A(n-1) + X(n)} and {@code V(n) = V(n-1) / (b(n) + V(n-1))} + * + *

where + * + *

{@code b(n) := (1 - a)^(t(n) - t(n-1)) = exp((t(n) - t(n-1)) * ln(1 - a))}. + * + *

Introducing + * + *

{@code C(n) := 1 / V(n)} + * + *

the recursion can be rewritten as + * + *

{@code A(n) = b(n) * A(n-1) + X(n)} and {@code C(n) = b(n) * C(n-1) + 1}. + * + *

+ * + *

Since we want to estimate the average waiting time, our data is given by + * + *

{@code X(n) := t(n) - t(n-1)}. + * + *

+ * + *

The following correspondence is used for the implementation: + * + *

*/ final class ConsistentRateLimitingSampler extends ConsistentSampler { From a82eee6aa5cab4db71a559470ec6c909845ed5fd Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Sat, 9 Apr 2022 09:11:18 +0200 Subject: [PATCH 16/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java Co-authored-by: Joshua MacDonald --- .../main/java/io/opentelemetry/contrib/state/OtelTraceState.java | 1 - 1 file changed, 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 19522fadf..f4778b2d1 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -231,7 +231,6 @@ public static OtelTraceState parse(@Nullable String ts) { } if (sepPos < len && ts.charAt(sepPos) != ';') { - // error = true; return new OtelTraceState(); } From 6e8a38a3d732905139a52e9fbb5750d231e0ecb3 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Sat, 9 Apr 2022 09:11:24 +0200 Subject: [PATCH 17/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java Co-authored-by: Joshua MacDonald --- .../main/java/io/opentelemetry/contrib/state/OtelTraceState.java | 1 - 1 file changed, 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index f4778b2d1..5dfba3fd7 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -207,7 +207,6 @@ public static OtelTraceState parse(@Nullable String ts) { } } if (eqPos == tsStartPos || eqPos == len || ts.charAt(eqPos) != ':') { - // error = true; return new OtelTraceState(); } From ae9670bedb2957e38818bee4d1e8b07a143a539c Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Sat, 9 Apr 2022 09:11:44 +0200 Subject: [PATCH 18/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java Co-authored-by: Joshua MacDonald --- .../main/java/io/opentelemetry/contrib/state/OtelTraceState.java | 1 - 1 file changed, 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 5dfba3fd7..006f4eab3 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -184,7 +184,6 @@ public static OtelTraceState parse(@Nullable String ts) { List otherKeyValuePairs = null; int p = INVALID_P; int r = INVALID_R; - // boolean error = false; if (ts == null || ts.isEmpty()) { return new OtelTraceState(); From 83c21f46d6c382ed5000eaefe0df5869133d8ce5 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Sat, 9 Apr 2022 09:11:49 +0200 Subject: [PATCH 19/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java Co-authored-by: Joshua MacDonald --- .../main/java/io/opentelemetry/contrib/state/OtelTraceState.java | 1 - 1 file changed, 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 006f4eab3..8c371d6cc 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -190,7 +190,6 @@ public static OtelTraceState parse(@Nullable String ts) { } if (ts.length() > TRACE_STATE_SIZE_LIMIT) { - // error = true; return new OtelTraceState(); } From 35d0373b97f981840bdc4a5b0ac9ddfae4acc809 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:07:01 +0200 Subject: [PATCH 20/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java Co-authored-by: Trask Stalnaker --- .../contrib/samplers/ConsistentComposedAndSampler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java index 003fa187d..1b5e12712 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java @@ -11,7 +11,7 @@ import javax.annotation.concurrent.Immutable; /** - * A consistent sampler composes two consistent samplers. + * A consistent sampler composed of two consistent samplers. * *

This sampler samples if both samplers would sample. */ From 348a240da385cdf723062e7fe1364c44146196d6 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:07:07 +0200 Subject: [PATCH 21/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java Co-authored-by: Trask Stalnaker --- .../contrib/samplers/ConsistentComposedOrSampler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java index a3bd4defb..21150c672 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java @@ -11,7 +11,7 @@ import javax.annotation.concurrent.Immutable; /** - * A consistent sampler composes two consistent samplers. + * A consistent sampler composed of two consistent samplers. * *

This sampler samples if any of the two samplers would sample. */ From 0afaa2f5143a9a6bf46345d694d3b5a663962799 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:07:48 +0200 Subject: [PATCH 22/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java Co-authored-by: Trask Stalnaker --- .../contrib/samplers/ConsistentRateLimitingSampler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 2e8f0e313..d9f32f59e 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -15,7 +15,7 @@ import javax.annotation.concurrent.Immutable; /** - * This consistent {@link Sampler} adjust the sampling probability dynamically to limit the rate of + * This consistent {@link Sampler} adjusts the sampling probability dynamically to limit the rate of * sampled spans. * *

This sampler uses exponential smoothing to estimate on irregular data (compare Wright, David From f63bb0b14bfcd1a3c9337dd525cac770ffe74db3 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:14:28 +0200 Subject: [PATCH 23/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java Co-authored-by: Trask Stalnaker --- .../contrib/samplers/ConsistentParentBasedSampler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java index 8cf18c85b..239c08af6 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java @@ -44,7 +44,7 @@ final class ConsistentParentBasedSampler extends ConsistentSampler { super(threadSafeRandomGenerator); this.rootSampler = requireNonNull(rootSampler); this.description = - "ConsistentComposedSampler{rootSampler=" + rootSampler.getDescription() + '}'; + "ConsistentParentBasedSampler{rootSampler=" + rootSampler.getDescription() + '}'; } @Override From aaffdd7c6766bf2fd88fc86adfb859d4256874f4 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:16:07 +0200 Subject: [PATCH 24/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java Co-authored-by: Trask Stalnaker --- .../io/opentelemetry/contrib/samplers/ConsistentSampler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index 28d30bc2e..08610f71e 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -175,7 +175,7 @@ protected ConsistentSampler(RandomGenerator threadSafeRandomGenerator) { } protected ConsistentSampler() { - this.threadSafeRandomGenerator = DefaultRandomGenerator.get(); + this(DefaultRandomGenerator.get()); } @Override From 56eec3910d7882084e1e6f55d3128f727475f1ae Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:19:18 +0200 Subject: [PATCH 25/47] added component owner for consistent sampling --- .github/component_owners.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/component_owners.yml b/.github/component_owners.yml index 85a68bc74..6c6ef2214 100644 --- a/.github/component_owners.yml +++ b/.github/component_owners.yml @@ -3,6 +3,8 @@ components: aws-xray: - anuraaga - willarmiros + consistent-sampling: + - oertl samplers: - anuraaga - iNikem From 667e1f7dd4e3de30beb7ac590312a5a29c494b1b Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:20:43 +0200 Subject: [PATCH 26/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java Co-authored-by: Trask Stalnaker --- .../java/io/opentelemetry/contrib/state/OtelTraceState.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 8c371d6cc..712757745 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -175,7 +175,7 @@ public static boolean isValidP(int v) { /** * Parses the OtelTraceState from a given string. * - *

If the string cannot be successfully parsed. A new OtelTraceState is returned + *

If the string cannot be successfully parsed, a new empty OtelTraceState is returned. * * @param ts the string * @return the parsed OtelTraceState or a new empty OtelTraceState in case of parsing errors From 3fae630ed251dd7c137dfe95a9b669ed721c998c Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:23:43 +0200 Subject: [PATCH 27/47] removed nonnull annotation --- .../io/opentelemetry/contrib/samplers/ConsistentSampler.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index 08610f71e..55aa67160 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -22,7 +22,6 @@ import io.opentelemetry.sdk.trace.samplers.SamplingResult; import java.util.List; import java.util.function.LongSupplier; -import javax.annotation.Nonnull; /** Abstract base class for consistent samplers. */ public abstract class ConsistentSampler implements Sampler { @@ -73,7 +72,7 @@ public static final ConsistentSampler probabilityBased( * * @param rootSampler the root sampler */ - public static final ConsistentSampler parentBased(@Nonnull ConsistentSampler rootSampler) { + public static final ConsistentSampler parentBased(ConsistentSampler rootSampler) { return new ConsistentParentBasedSampler(rootSampler); } From 5040a743985f5c98b44ae8e445355d007877af54 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:25:00 +0200 Subject: [PATCH 28/47] renamed variable s -> pair --- .../java/io/opentelemetry/contrib/state/OtelTraceState.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 712757745..acbd2db7d 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -101,18 +101,18 @@ public String serialize() { sb.append("r:").append(rval); } if (otherKeyValuePairs != null) { - for (String s : otherKeyValuePairs) { + for (String pair : otherKeyValuePairs) { int ex = sb.length(); if (ex != 0) { ex += 1; } - if (ex + s.length() > TRACE_STATE_SIZE_LIMIT) { + if (ex + pair.length() > TRACE_STATE_SIZE_LIMIT) { break; } if (sb.length() > 0) { sb.append(';'); } - sb.append(s); + sb.append(pair); } } return sb.toString(); From e0041c062c17581447da28daf5a58c0f3678b06f Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:26:47 +0200 Subject: [PATCH 29/47] renamed char parameter r -> c --- .../contrib/state/OtelTraceState.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index acbd2db7d..56ec44f69 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -118,30 +118,30 @@ public String serialize() { return sb.toString(); } - private static boolean isValueByte(char r) { - if (isLowerCaseAlphaNum(r)) { + private static boolean isValueByte(char c) { + if (isLowerCaseAlphaNum(c)) { return true; } - if (isUpperCaseAlpha(r)) { + if (isUpperCaseAlpha(c)) { return true; } - return r == '.' || r == '_' || r == '-'; + return c == '.' || c == '_' || c == '-'; } - private static boolean isLowerCaseAlphaNum(char r) { - return isLowerCaseAlpha(r) || isLowerCaseNum(r); + private static boolean isLowerCaseAlphaNum(char c) { + return isLowerCaseAlpha(c) || isLowerCaseNum(c); } - private static boolean isLowerCaseNum(char r) { - return r >= '0' && r <= '9'; + private static boolean isLowerCaseNum(char c) { + return c >= '0' && c <= '9'; } - private static boolean isLowerCaseAlpha(char r) { - return r >= 'a' && r <= 'z'; + private static boolean isLowerCaseAlpha(char c) { + return c >= 'a' && c <= 'z'; } - private static boolean isUpperCaseAlpha(char r) { - return r >= 'A' && r <= 'Z'; + private static boolean isUpperCaseAlpha(char c) { + return c >= 'A' && c <= 'Z'; } private static int parseOneOrTwoDigitNumber( From 767e52f7af5c2b45f30d74ca8b51330710531666 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:28:29 +0200 Subject: [PATCH 30/47] renamed method isLowerCaseNum -> isDigit --- .../io/opentelemetry/contrib/state/OtelTraceState.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 56ec44f69..e8057da0f 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -129,10 +129,10 @@ private static boolean isValueByte(char c) { } private static boolean isLowerCaseAlphaNum(char c) { - return isLowerCaseAlpha(c) || isLowerCaseNum(c); + return isLowerCaseAlpha(c) || isDigit(c); } - private static boolean isLowerCaseNum(char c) { + private static boolean isDigit(char c) { return c >= '0' && c <= '9'; } @@ -148,13 +148,13 @@ private static int parseOneOrTwoDigitNumber( String ts, int from, int to, int twoDigitMaxValue, int invalidValue) { if (to - from == 1) { char c = ts.charAt(from); - if (isLowerCaseNum(c)) { + if (isDigit(c)) { return c - '0'; } } else if (to - from == 2) { char c1 = ts.charAt(from); char c2 = ts.charAt(from + 1); - if (isLowerCaseNum(c1) && isLowerCaseNum(c2)) { + if (isDigit(c1) && isDigit(c2)) { int v = (c1 - '0') * 10 + (c2 - '0'); if (v <= twoDigitMaxValue) { return v; @@ -200,7 +200,7 @@ public static OtelTraceState parse(@Nullable String ts) { int eqPos = tsStartPos; for (; eqPos < len; eqPos++) { char c = ts.charAt(eqPos); - if (!isLowerCaseAlpha(c) && (!isLowerCaseNum(c) || eqPos == tsStartPos)) { + if (!isLowerCaseAlpha(c) && (!isDigit(c) || eqPos == tsStartPos)) { break; } } From c18dcfe5cd08b3a88c079b8b7c8883ec804ad313 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:34:35 +0200 Subject: [PATCH 31/47] use empty list instead of null for otherKeyValuePairs --- .../contrib/state/OtelTraceState.java | 36 +++++++++---------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index e8057da0f..eea73c89d 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -6,6 +6,7 @@ package io.opentelemetry.contrib.state; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import javax.annotation.Nullable; @@ -24,18 +25,16 @@ public final class OtelTraceState { private int rval; // valid in the interval [0, MAX_R] private int pval; // valid in the interval [0, MAX_P] - @Nullable private final List otherKeyValuePairs; + private final List otherKeyValuePairs; - private OtelTraceState(int rvalue, int pvalue, @Nullable List otherKeyValuePairs) { + private OtelTraceState(int rvalue, int pvalue, List otherKeyValuePairs) { this.rval = rvalue; this.pval = pvalue; this.otherKeyValuePairs = otherKeyValuePairs; } private OtelTraceState() { - this.rval = INVALID_R; - this.pval = INVALID_P; - this.otherKeyValuePairs = null; + this(INVALID_R, INVALID_P, Collections.emptyList()); } public boolean hasValidR() { @@ -100,20 +99,18 @@ public String serialize() { } sb.append("r:").append(rval); } - if (otherKeyValuePairs != null) { - for (String pair : otherKeyValuePairs) { - int ex = sb.length(); - if (ex != 0) { - ex += 1; - } - if (ex + pair.length() > TRACE_STATE_SIZE_LIMIT) { - break; - } - if (sb.length() > 0) { - sb.append(';'); - } - sb.append(pair); + for (String pair : otherKeyValuePairs) { + int ex = sb.length(); + if (ex != 0) { + ex += 1; + } + if (ex + pair.length() > TRACE_STATE_SIZE_LIMIT) { + break; + } + if (sb.length() > 0) { + sb.append(';'); } + sb.append(pair); } return sb.toString(); } @@ -243,7 +240,8 @@ public static OtelTraceState parse(@Nullable String ts) { } } - return new OtelTraceState(r, p, otherKeyValuePairs); + return new OtelTraceState( + r, p, (otherKeyValuePairs != null) ? otherKeyValuePairs : Collections.emptyList()); } public int getR() { From be68ff2d47271a72f3f761aebe756872c6d5fd3d Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:35:43 +0200 Subject: [PATCH 32/47] simplified isValueByte method --- .../io/opentelemetry/contrib/state/OtelTraceState.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index eea73c89d..1b3ce12fb 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -116,13 +116,7 @@ public String serialize() { } private static boolean isValueByte(char c) { - if (isLowerCaseAlphaNum(c)) { - return true; - } - if (isUpperCaseAlpha(c)) { - return true; - } - return c == '.' || c == '_' || c == '-'; + return isLowerCaseAlphaNum(c) || isUpperCaseAlpha(c) || c == '.' || c == '_' || c == '-'; } private static boolean isLowerCaseAlphaNum(char c) { From 30e42e3649e022fcb6eb172089233d527fc6197d Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 08:57:45 +0200 Subject: [PATCH 33/47] Update consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java Co-authored-by: Trask Stalnaker --- .../io/opentelemetry/contrib/state/OtelTraceState.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 712757745..456d168ba 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -209,11 +209,8 @@ public static OtelTraceState parse(@Nullable String ts) { } int sepPos = eqPos + 1; - for (; sepPos < len; sepPos++) { - if (isValueByte(ts.charAt(sepPos))) { - continue; - } - break; + while (sepPos < len && isValueByte(ts.charAt(sepPos))) { + sepPos++; } if (eqPos - tsStartPos == 1 && ts.charAt(tsStartPos) == P_SUBKEY) { From 7f58aa103a50993578546887471e7f597eea6682 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 09:01:10 +0200 Subject: [PATCH 34/47] renamed variable sepPos -> separatorPos --- .../contrib/state/OtelTraceState.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 562993a63..0d1f5fb02 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -199,31 +199,31 @@ public static OtelTraceState parse(@Nullable String ts) { return new OtelTraceState(); } - int sepPos = eqPos + 1; - while (sepPos < len && isValueByte(ts.charAt(sepPos))) { - sepPos++; + int separatorPos = eqPos + 1; + while (separatorPos < len && isValueByte(ts.charAt(separatorPos))) { + separatorPos++; } if (eqPos - tsStartPos == 1 && ts.charAt(tsStartPos) == P_SUBKEY) { - p = parseOneOrTwoDigitNumber(ts, eqPos + 1, sepPos, MAX_P, INVALID_P); + p = parseOneOrTwoDigitNumber(ts, eqPos + 1, separatorPos, MAX_P, INVALID_P); } else if (eqPos - tsStartPos == 1 && ts.charAt(tsStartPos) == R_SUBKEY) { - r = parseOneOrTwoDigitNumber(ts, eqPos + 1, sepPos, MAX_R, INVALID_R); + r = parseOneOrTwoDigitNumber(ts, eqPos + 1, separatorPos, MAX_R, INVALID_R); } else { if (otherKeyValuePairs == null) { otherKeyValuePairs = new ArrayList<>(); } - otherKeyValuePairs.add(ts.substring(tsStartPos, sepPos)); + otherKeyValuePairs.add(ts.substring(tsStartPos, separatorPos)); } - if (sepPos < len && ts.charAt(sepPos) != ';') { + if (separatorPos < len && ts.charAt(separatorPos) != ';') { return new OtelTraceState(); } - if (sepPos == len) { + if (separatorPos == len) { break; } - tsStartPos = sepPos + 1; + tsStartPos = separatorPos + 1; // test for a trailing ; if (tsStartPos == len) { From 938382a6f14ae37ee3a0e59d7bb7a2b9d5e9506c Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 09:04:43 +0200 Subject: [PATCH 35/47] replaced 0. and 1. by 0.0 and 1.0 --- .../opentelemetry/contrib/samplers/ConsistentSampler.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index 55aa67160..b00ad9733 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -300,7 +300,7 @@ public TraceState getUpdatedTraceState(TraceState parentTraceState) { protected static double getSamplingProbability(int p) { if (OtelTraceState.isValidP(p)) { if (p == OtelTraceState.getMaxP()) { - return 0.; + return 0.0; } else { return Double.longBitsToDouble((0x3FFL - p) << 52); } @@ -319,11 +319,11 @@ protected static double getSamplingProbability(int p) { * @return the p-value */ protected static int getLowerBoundP(double samplingProbability) { - if (!(samplingProbability >= 0. && samplingProbability <= 1.)) { + if (!(samplingProbability >= 0.0 && samplingProbability <= 1.0)) { throw new IllegalArgumentException(); } if (samplingProbability <= SMALLEST_POSITIVE_SAMPLING_PROBABILITY) { - return OtelTraceState.getMaxP() - (samplingProbability > 0. ? 1 : 0); + return OtelTraceState.getMaxP() - (samplingProbability > 0.0 ? 1 : 0); } else { long longSamplingProbability = Double.doubleToRawLongBits(samplingProbability); long mantissa = longSamplingProbability & 0x000FFFFFFFFFFFFFL; @@ -340,7 +340,7 @@ protected static int getLowerBoundP(double samplingProbability) { * @return the p-value */ protected static int getUpperBoundP(double samplingProbability) { - if (!(samplingProbability >= 0. && samplingProbability <= 1.)) { + if (!(samplingProbability >= 0.0 && samplingProbability <= 1.0)) { throw new IllegalArgumentException(); } if (samplingProbability <= SMALLEST_POSITIVE_SAMPLING_PROBABILITY) { From 630b157b2aa603addf30c38bf82014a947117ab4 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 09:10:22 +0200 Subject: [PATCH 36/47] improved readability as suggested by @trask --- .../opentelemetry/contrib/samplers/ConsistentSampler.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index b00ad9733..c1b923699 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -322,8 +322,10 @@ protected static int getLowerBoundP(double samplingProbability) { if (!(samplingProbability >= 0.0 && samplingProbability <= 1.0)) { throw new IllegalArgumentException(); } - if (samplingProbability <= SMALLEST_POSITIVE_SAMPLING_PROBABILITY) { - return OtelTraceState.getMaxP() - (samplingProbability > 0.0 ? 1 : 0); + if (samplingProbability == 0.) { + return OtelTraceState.getMaxP(); + } else if (samplingProbability <= SMALLEST_POSITIVE_SAMPLING_PROBABILITY) { + return OtelTraceState.getMaxP() - 1; } else { long longSamplingProbability = Double.doubleToRawLongBits(samplingProbability); long mantissa = longSamplingProbability & 0x000FFFFFFFFFFFFFL; From 1dca6f40c34d7273c167ef62dc93b0a23c9ff15e Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 09:14:09 +0200 Subject: [PATCH 37/47] removed unused methods from RandomUtil --- .../contrib/util/RandomUtil.java | 54 ------------- .../contrib/util/RandomUtilTest.java | 78 ------------------- 2 files changed, 132 deletions(-) delete mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/util/RandomUtilTest.java diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java index c7cd47629..5e6de5bed 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java @@ -5,7 +5,6 @@ package io.opentelemetry.contrib.util; -import java.util.BitSet; import java.util.function.BooleanSupplier; public final class RandomUtil { @@ -40,24 +39,6 @@ public static boolean generateRandomBoolean( } } - /** - * Stochastically rounds the given floating-point value. - * - *

see https://en.wikipedia.org/wiki/Rounding#Stochastic_rounding - * - * @param randomGenerator a random generator - * @param x the value to be rounded - * @return the rounded value - */ - public static long roundStochastically(RandomGenerator randomGenerator, double x) { - long i = (long) Math.floor(x); - if (randomGenerator.nextBoolean(x - i)) { - return i + 1; - } else { - return i; - } - } - /** * Returns the number of leading zeros of a uniform random 64-bit integer. * @@ -71,39 +52,4 @@ public static int numberOfLeadingZerosOfRandomLong(BooleanSupplier randomBoolean } return count; } - - /** - * Generates a random bit set where a given number of 1-bits are randomly set. - * - * @param randomGenerator the random generator - * @param numBits the total number of bits - * @param numOneBits the number of 1-bits - * @return a random bit set - * @throws IllegalArgumentException if {@code 0 <= numOneBits <= numBits} is violated - */ - public static BitSet generateRandomBitSet( - RandomGenerator randomGenerator, int numBits, int numOneBits) { - - if (numOneBits < 0 || numOneBits > numBits) { - throw new IllegalArgumentException(); - } - - BitSet result = new BitSet(numBits); - int numZeroBits = numBits - numOneBits; - - // based on Fisher-Yates shuffling - for (int i = Math.max(numZeroBits, numOneBits); i < numBits; ++i) { - int j = randomGenerator.nextInt(i + 1); - if (result.get(j)) { - result.set(i); - } else { - result.set(j); - } - } - if (numZeroBits < numOneBits) { - result.flip(0, numBits); - } - - return result; - } } diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/RandomUtilTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/RandomUtilTest.java deleted file mode 100644 index 8010c2450..000000000 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/RandomUtilTest.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.contrib.util; - -import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.fail; - -import java.util.BitSet; -import java.util.SplittableRandom; -import java.util.stream.DoubleStream; -import org.hipparchus.stat.inference.GTest; -import org.junit.jupiter.api.Test; - -public class RandomUtilTest { - - private static void testGenerateRandomBitSet(long seed, int numBits, int numOneBits) { - - int numCycles = 100000; - - SplittableRandom splittableRandom = new SplittableRandom(seed); - - long[] observed = new long[numBits]; - double[] expected = DoubleStream.generate(() -> 1.).limit(numBits).toArray(); - - for (int i = 0; i < numCycles; ++i) { - BitSet bitSet = - RandomUtil.generateRandomBitSet(splittableRandom::nextLong, numBits, numOneBits); - bitSet.stream().forEach(k -> observed[k] += 1); - assertThat(bitSet.cardinality()).isEqualTo(numOneBits); - } - if (numBits > 1) { - assertThat(new GTest().gTest(expected, observed)).isGreaterThan(0.01); - } else if (numBits == 1) { - assertThat(observed[0]).isEqualTo(numOneBits * numCycles); - } else { - fail("numBits was non-positive!"); - } - } - - @Test - void testGenerateRandomBitSet() { - testGenerateRandomBitSet(0x4a5580b958d52182L, 1, 0); - testGenerateRandomBitSet(0x529dff14b0ce7414L, 1, 1); - testGenerateRandomBitSet(0x2d3f673a9e1da536L, 2, 0); - testGenerateRandomBitSet(0xb9a6735e64361bacL, 2, 1); - testGenerateRandomBitSet(0xb5aafedc7031506fL, 2, 2); - testGenerateRandomBitSet(0xaecabe7698971ee1L, 3, 0); - testGenerateRandomBitSet(0x119ccf35dc52b34dL, 3, 1); - testGenerateRandomBitSet(0xcaf2b7a98f194ce2L, 3, 2); - testGenerateRandomBitSet(0xe28e8cc3d3de0c2aL, 3, 3); - testGenerateRandomBitSet(0xb69989dce9cc8b34L, 4, 0); - testGenerateRandomBitSet(0x6575d4c848c95dc8L, 4, 1); - testGenerateRandomBitSet(0xed0ad0525ad632e9L, 4, 2); - testGenerateRandomBitSet(0x34db9303b405a706L, 4, 3); - testGenerateRandomBitSet(0x8e97972893044140L, 4, 4); - testGenerateRandomBitSet(0x47f966b8f28dac77L, 5, 0); - testGenerateRandomBitSet(0x7996db4a5f1e4680L, 5, 1); - testGenerateRandomBitSet(0x577fcf18bbc0ba30L, 5, 2); - testGenerateRandomBitSet(0x36b1ed999d2986b0L, 5, 3); - testGenerateRandomBitSet(0xa8e099ed958d03bbL, 5, 4); - testGenerateRandomBitSet(0xc2b50bbf3263b414L, 5, 5); - testGenerateRandomBitSet(0x2994550582b091e9L, 6, 0); - testGenerateRandomBitSet(0xd2797c539136f6faL, 6, 1); - testGenerateRandomBitSet(0xf3ffae1d93983fd9L, 6, 2); - testGenerateRandomBitSet(0x281e0f9873455ea6L, 6, 3); - testGenerateRandomBitSet(0x5344c2887e30d621L, 6, 4); - testGenerateRandomBitSet(0xa8f4ed6e3e1cf385L, 6, 5); - testGenerateRandomBitSet(0x6bd0f9f11520ae57L, 6, 6); - - testGenerateRandomBitSet(0x514f52732c193e62L, 1000, 1); - testGenerateRandomBitSet(0xe214063ae29d9802L, 1000, 10); - testGenerateRandomBitSet(0x602fdb45063e7b0fL, 1000, 990); - testGenerateRandomBitSet(0xe0ef0cb214de3ec0L, 1000, 999); - } -} From 09cc3a0cbcd3232a109475ceaa7116fdd171b57a Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 10:00:23 +0200 Subject: [PATCH 38/47] added javadoc --- .../io/opentelemetry/contrib/util/RandomUtil.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java index 5e6de5bed..34e1cb88d 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java @@ -15,6 +15,18 @@ private RandomUtil() {} * Returns a pseudorandomly chosen {@code boolean} value where the probability of returning {@code * true} is predefined. * + *

{@code true} needs to be returned with a success probability of {@code probability}. If the + * success probability is greater than 50% ({@code probability > 0.5}), the same can be achieved + * by returning {@code true} with a probability of 50%, and returning the result of a Bernoulli + * trial with a probability of {@code 2 * probability - 1}. The resulting success probability will + * be the same as {@code 0.5 + 0.5 * (2 * probability - 1) = probabilty}. Similarly, if the + * success probability is smaller than 50% ({@code probability <= 0.5}), {@code false} is returned + * with a probability of 50%. Otherwise, the result of a Bernoulli trial with success probability + * of {@code 2 * probability} is returned. Again, the resulting success probability is exactly as + * desired because {@code 0.5 * (2 * probability) = probability}. Recursive continuation of this + * approach allows realizing Bernoulli trials with arbitrary success probabilities using just few + * random bits. + * * @param randomBooleanSupplier a random boolean supplier * @param probability the probability of returning {@code true} * @return a random {@code boolean} From 15a8d50e282ff2442f542a66ce1e55d7c7f24239 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 10:03:22 +0200 Subject: [PATCH 39/47] renamed targetSpansPerNanosLimit -> targetSpansPerNanosecondLimit --- .../contrib/samplers/ConsistentRateLimitingSampler.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index d9f32f59e..93401f7ef 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -85,7 +85,7 @@ public State(double effectiveWindowCount, double effectiveWindowNanos, long last private final String description; private final LongSupplier nanoTimeSupplier; private final double inverseAdaptationTimeNanos; - private final double targetSpansPerNanosLimit; + private final double targetSpansPerNanosecondLimit; private final AtomicReference state; /** @@ -132,7 +132,7 @@ public State(double effectiveWindowCount, double effectiveWindowNanos, long last this.nanoTimeSupplier = requireNonNull(nanoTimeSupplier); this.inverseAdaptationTimeNanos = 1e-9 / adaptationTimeSeconds; - this.targetSpansPerNanosLimit = 1e-9 * targetSpansPerSecondLimit; + this.targetSpansPerNanosecondLimit = 1e-9 * targetSpansPerSecondLimit; this.state = new AtomicReference<>(new State(0, 0, nanoTimeSupplier.getAsLong())); } @@ -156,7 +156,7 @@ protected int getP(int parentP, boolean isRoot) { State currentState = state.updateAndGet(s -> updateState(s, currentNanoTime)); double samplingProbability = - (currentState.effectiveWindowNanos * targetSpansPerNanosLimit) + (currentState.effectiveWindowNanos * targetSpansPerNanosecondLimit) / currentState.effectiveWindowCount; if (samplingProbability >= 1.) { From f70454a59e8bad2651ae8b32f755e5ac7fc726c7 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 10:13:44 +0200 Subject: [PATCH 40/47] throw IllegalArgumentException instead of returning NaN + added comments --- .../contrib/samplers/ConsistentSampler.java | 11 +++++++---- .../contrib/samplers/ConsistentSamplerTest.java | 9 ++++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index c1b923699..bfca27ee6 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -295,7 +295,8 @@ public TraceState getUpdatedTraceState(TraceState parentTraceState) { * Returns the sampling probability for a given p-value. * * @param p the p-value - * @return the sampling probability in the range [0,1] or Double.Nan if the p-value is invalid + * @return the sampling probability in the range [0,1] + * @throws IllegalArgumentException if the given p-value is invalid */ protected static double getSamplingProbability(int p) { if (OtelTraceState.isValidP(p)) { @@ -305,7 +306,7 @@ protected static double getSamplingProbability(int p) { return Double.longBitsToDouble((0x3FFL - p) << 52); } } else { - return Double.NaN; + throw new IllegalArgumentException("Invalid p-value!"); } } @@ -329,7 +330,8 @@ protected static int getLowerBoundP(double samplingProbability) { } else { long longSamplingProbability = Double.doubleToRawLongBits(samplingProbability); long mantissa = longSamplingProbability & 0x000FFFFFFFFFFFFFL; - long exponent = longSamplingProbability >>> 52; + long exponent = longSamplingProbability >>> 52; // compare + // https://en.wikipedia.org/wiki/Double-precision_floating-point_format#Exponent_encoding return (int) (0x3FFL - exponent) - (mantissa != 0 ? 1 : 0); } } @@ -349,7 +351,8 @@ protected static int getUpperBoundP(double samplingProbability) { return OtelTraceState.getMaxP(); } else { long longSamplingProbability = Double.doubleToRawLongBits(samplingProbability); - long exponent = longSamplingProbability >>> 52; + long exponent = longSamplingProbability >>> 52; // compare + // https://en.wikipedia.org/wiki/Double-precision_floating-point_format#Exponent_encoding return (int) (0x3FFL - exponent); } } diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java index d0adf2f79..9ca37b867 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java @@ -7,6 +7,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import io.opentelemetry.contrib.state.OtelTraceState; import java.util.SplittableRandom; @@ -16,13 +17,15 @@ class ConsistentSamplerTest { @Test void testGetSamplingRate() { - assertEquals(Double.NaN, ConsistentSampler.getSamplingProbability(-1)); + assertThrows( + IllegalArgumentException.class, () -> ConsistentSampler.getSamplingProbability(-1)); for (int i = 0; i < OtelTraceState.getMaxP() - 1; i += 1) { assertEquals(Math.pow(0.5, i), ConsistentSampler.getSamplingProbability(i)); } assertEquals(0., ConsistentSampler.getSamplingProbability(OtelTraceState.getMaxP())); - assertEquals( - Double.NaN, ConsistentSampler.getSamplingProbability(OtelTraceState.getMaxP() + 1)); + assertThrows( + IllegalArgumentException.class, + () -> ConsistentSampler.getSamplingProbability(OtelTraceState.getMaxP() + 1)); } @Test From 8d3a73197105862f01aa2fc0002e43b470db42b9 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 10:16:30 +0200 Subject: [PATCH 41/47] renamed tsStartPos -> startPos and eqPos -> colonPos --- .../contrib/state/OtelTraceState.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java index 0d1f5fb02..b71ca3e3b 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java @@ -184,35 +184,35 @@ public static OtelTraceState parse(@Nullable String ts) { return new OtelTraceState(); } - int tsStartPos = 0; + int startPos = 0; int len = ts.length(); while (true) { - int eqPos = tsStartPos; - for (; eqPos < len; eqPos++) { - char c = ts.charAt(eqPos); - if (!isLowerCaseAlpha(c) && (!isDigit(c) || eqPos == tsStartPos)) { + int colonPos = startPos; + for (; colonPos < len; colonPos++) { + char c = ts.charAt(colonPos); + if (!isLowerCaseAlpha(c) && (!isDigit(c) || colonPos == startPos)) { break; } } - if (eqPos == tsStartPos || eqPos == len || ts.charAt(eqPos) != ':') { + if (colonPos == startPos || colonPos == len || ts.charAt(colonPos) != ':') { return new OtelTraceState(); } - int separatorPos = eqPos + 1; + int separatorPos = colonPos + 1; while (separatorPos < len && isValueByte(ts.charAt(separatorPos))) { separatorPos++; } - if (eqPos - tsStartPos == 1 && ts.charAt(tsStartPos) == P_SUBKEY) { - p = parseOneOrTwoDigitNumber(ts, eqPos + 1, separatorPos, MAX_P, INVALID_P); - } else if (eqPos - tsStartPos == 1 && ts.charAt(tsStartPos) == R_SUBKEY) { - r = parseOneOrTwoDigitNumber(ts, eqPos + 1, separatorPos, MAX_R, INVALID_R); + if (colonPos - startPos == 1 && ts.charAt(startPos) == P_SUBKEY) { + p = parseOneOrTwoDigitNumber(ts, colonPos + 1, separatorPos, MAX_P, INVALID_P); + } else if (colonPos - startPos == 1 && ts.charAt(startPos) == R_SUBKEY) { + r = parseOneOrTwoDigitNumber(ts, colonPos + 1, separatorPos, MAX_R, INVALID_R); } else { if (otherKeyValuePairs == null) { otherKeyValuePairs = new ArrayList<>(); } - otherKeyValuePairs.add(ts.substring(tsStartPos, separatorPos)); + otherKeyValuePairs.add(ts.substring(startPos, separatorPos)); } if (separatorPos < len && ts.charAt(separatorPos) != ';') { @@ -223,10 +223,10 @@ public static OtelTraceState parse(@Nullable String ts) { break; } - tsStartPos = separatorPos + 1; + startPos = separatorPos + 1; // test for a trailing ; - if (tsStartPos == len) { + if (startPos == len) { return new OtelTraceState(); } } From 3b7f045645548c22f41dcc879e168eb726cfdf47 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 10:35:26 +0200 Subject: [PATCH 42/47] improved readability of invariant check --- .../contrib/samplers/ConsistentSampler.java | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index bfca27ee6..3655f9ee9 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -177,6 +177,22 @@ protected ConsistentSampler() { this(DefaultRandomGenerator.get()); } + private static final boolean isInvariantViolated( + OtelTraceState otelTraceState, boolean isParentSampled) { + if (otelTraceState.hasValidR() && otelTraceState.hasValidP()) { + // if valid p- and r-values are given, they must be consistent with the isParentSampled flag + // see + // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/tracestate-probability-sampling.md#sampled-flag + int p = otelTraceState.getP(); + int r = otelTraceState.getR(); + int maxP = OtelTraceState.getMaxP(); + boolean isInvariantTrue = ((p <= r) == isParentSampled) || (isParentSampled && (p == maxP)); + return !isInvariantTrue; + } else { + return false; + } + } + @Override public final SamplingResult shouldSample( Context parentContext, @@ -195,17 +211,10 @@ public final SamplingResult shouldSample( String otelTraceStateString = parentTraceState.get(OtelTraceState.TRACE_STATE_KEY); OtelTraceState otelTraceState = OtelTraceState.parse(otelTraceStateString); - if (!otelTraceState.hasValidR()) { + if (!otelTraceState.hasValidR() || isInvariantViolated(otelTraceState, isParentSampled)) { + // unset p-value in case of an invalid r-value or in case of any invariant violation otelTraceState.invalidateP(); } - // Invariant checking: unset p-value when p-value, r-value, and isParentSampled are inconsistent - if (otelTraceState.hasValidR() && otelTraceState.hasValidP()) { - if ((((otelTraceState.getP() <= otelTraceState.getR()) == isParentSampled) - || (isParentSampled && (otelTraceState.getP() == OtelTraceState.getMaxP()))) - == false) { - otelTraceState.invalidateP(); - } - } // generate new r-value if not available if (!otelTraceState.hasValidR()) { From 878c236bce2077b2c7f40eb43577a0d74921f957 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 10:52:01 +0200 Subject: [PATCH 43/47] added some more test cases --- .../io/opentelemetry/contrib/state/OtelTraceStateTest.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java index ab371f4e3..118c9d176 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java @@ -70,5 +70,10 @@ public void test() { assertEquals("r:6", OtelTraceState.parse("r:5;r:6").serialize()); assertEquals("p:6;r:10", OtelTraceState.parse("p:5;p:6;r:10").serialize()); assertEquals("", OtelTraceState.parse("p5;p:6;r:10").serialize()); + assertEquals("p:6;r:10;p5:3", OtelTraceState.parse("p5:3;p:6;r:10").serialize()); + assertEquals("", OtelTraceState.parse(":p:6;r:10").serialize()); + assertEquals("", OtelTraceState.parse(";p:6;r:10").serialize()); + assertEquals("", OtelTraceState.parse("_;p:6;r:10").serialize()); + assertEquals("", OtelTraceState.parse("5;p:6;r:10").serialize()); } } From 6fc45cdddc123f61e1210c4e418ffea507d0acdf Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 11:00:13 +0200 Subject: [PATCH 44/47] fixed typo --- .../src/main/java/io/opentelemetry/contrib/util/RandomUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java index 34e1cb88d..57efc7a85 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java @@ -19,7 +19,7 @@ private RandomUtil() {} * success probability is greater than 50% ({@code probability > 0.5}), the same can be achieved * by returning {@code true} with a probability of 50%, and returning the result of a Bernoulli * trial with a probability of {@code 2 * probability - 1}. The resulting success probability will - * be the same as {@code 0.5 + 0.5 * (2 * probability - 1) = probabilty}. Similarly, if the + * be the same as {@code 0.5 + 0.5 * (2 * probability - 1) = probability}. Similarly, if the * success probability is smaller than 50% ({@code probability <= 0.5}), {@code false} is returned * with a probability of 50%. Otherwise, the result of a Bernoulli trial with success probability * of {@code 2 * probability} is returned. Again, the resulting success probability is exactly as From 06cbeeae6b90435965439c2aa8e3be2dbcc2f185 Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 11:07:02 +0200 Subject: [PATCH 45/47] removed unused method --- .../contrib/util/RandomGenerator.java | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java index 114188ca6..dff93dc79 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java @@ -15,37 +15,6 @@ public interface RandomGenerator { */ long nextLong(); - /** - * Returns a pseudorandomly chosen {@code int} value between zero (inclusive) and the specified - * bound (exclusive). - * - *

The implementation is based on Daniel Lemire's algorithm as described in "Fast random - * integer generation in an interval." ACM Transactions on Modeling and Computer Simulation - * (TOMACS) 29.1 (2019): 3. - * - * @param bound the upper bound (exclusive) for the returned value. Must be positive. - * @return a pseudorandomly chosen {@code int} value between zero (inclusive) and the bound - * (exclusive) - * @throws IllegalArgumentException if {@code bound} is not positive - */ - default int nextInt(int bound) { - if (bound <= 0) { - throw new IllegalArgumentException(); - } - long x = nextLong() >>> 33; // use only 31 random bits - long m = x * bound; - int l = (int) m & 0x7FFFFFFF; - if (l < bound) { - int t = (-bound & 0x7FFFFFFF) % bound; - while (l < t) { - x = nextLong() >>> 33; // use only 31 random bits - m = x * bound; - l = (int) m & 0x7FFFFFFF; - } - } - return (int) (m >>> 31); - } - /** * Returns a pseudorandomly chosen {@code boolean} value where the probability of returning {@code * true} is predefined. From edac8a9a517ed79d772d80b758bc56a6e5d7397a Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Tue, 19 Apr 2022 12:02:16 +0200 Subject: [PATCH 46/47] refactored random generator --- .../ConsistentParentBasedSampler.java | 3 +- .../ConsistentProbabilityBasedSampler.java | 12 +- .../ConsistentRateLimitingSampler.java | 11 +- .../contrib/samplers/ConsistentSampler.java | 34 ++--- .../contrib/util/DefaultRandomGenerator.java | 53 ------- .../contrib/util/RandomGenerator.java | 136 ++++++++++++++---- .../contrib/util/RandomUtil.java | 67 --------- ...ConsistentProbabilityBasedSamplerTest.java | 5 +- .../ConsistentRateLimitingSamplerTest.java | 16 ++- 9 files changed, 148 insertions(+), 189 deletions(-) delete mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java delete mode 100644 consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java index 239c08af6..6d7ad4dd9 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java @@ -7,7 +7,6 @@ import static java.util.Objects.requireNonNull; -import io.opentelemetry.contrib.util.DefaultRandomGenerator; import io.opentelemetry.contrib.util.RandomGenerator; import javax.annotation.concurrent.Immutable; @@ -29,7 +28,7 @@ final class ConsistentParentBasedSampler extends ConsistentSampler { * @param rootSampler the root sampler */ ConsistentParentBasedSampler(ConsistentSampler rootSampler) { - this(rootSampler, DefaultRandomGenerator.get()); + this(rootSampler, RandomGenerator.getDefault()); } /** diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java index aff3d5cd4..3b05549af 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java @@ -5,7 +5,6 @@ package io.opentelemetry.contrib.samplers; -import io.opentelemetry.contrib.util.DefaultRandomGenerator; import io.opentelemetry.contrib.util.RandomGenerator; import javax.annotation.concurrent.Immutable; @@ -24,18 +23,17 @@ final class ConsistentProbabilityBasedSampler extends ConsistentSampler { * @param samplingProbability the sampling probability */ ConsistentProbabilityBasedSampler(double samplingProbability) { - this(samplingProbability, DefaultRandomGenerator.get()); + this(samplingProbability, RandomGenerator.getDefault()); } /** * Constructor. * * @param samplingProbability the sampling probability - * @param threadSafeRandomGenerator a thread-safe random generator + * @param randomGenerator a random generator */ - ConsistentProbabilityBasedSampler( - double samplingProbability, RandomGenerator threadSafeRandomGenerator) { - super(threadSafeRandomGenerator); + ConsistentProbabilityBasedSampler(double samplingProbability, RandomGenerator randomGenerator) { + super(randomGenerator); if (samplingProbability < 0.0 || samplingProbability > 1.0) { throw new IllegalArgumentException("Sampling probability must be in range [0.0, 1.0]!"); } @@ -58,7 +56,7 @@ final class ConsistentProbabilityBasedSampler extends ConsistentSampler { @Override protected int getP(int parentP, boolean isRoot) { - if (threadSafeRandomGenerator.nextBoolean(probabilityToUseLowerPValue)) { + if (randomGenerator.nextBoolean(probabilityToUseLowerPValue)) { return lowerPValue; } else { return upperPValue; diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 93401f7ef..117501718 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -7,7 +7,6 @@ import static java.util.Objects.requireNonNull; -import io.opentelemetry.contrib.util.DefaultRandomGenerator; import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.samplers.Sampler; import java.util.concurrent.atomic.AtomicReference; @@ -99,7 +98,7 @@ public State(double effectiveWindowCount, double effectiveWindowNanos, long last this( targetSpansPerSecondLimit, adaptationTimeSeconds, - DefaultRandomGenerator.get(), + RandomGenerator.getDefault(), System::nanoTime); } @@ -109,15 +108,15 @@ public State(double effectiveWindowCount, double effectiveWindowNanos, long last * @param targetSpansPerSecondLimit the desired spans per second limit * @param adaptationTimeSeconds the typical time to adapt to a new load (time constant used for * exponential smoothing) - * @param threadSafeRandomGenerator a thread-safe random generator + * @param randomGenerator a random generator * @param nanoTimeSupplier a supplier for the current nano time */ ConsistentRateLimitingSampler( double targetSpansPerSecondLimit, double adaptationTimeSeconds, - RandomGenerator threadSafeRandomGenerator, + RandomGenerator randomGenerator, LongSupplier nanoTimeSupplier) { - super(threadSafeRandomGenerator); + super(randomGenerator); if (targetSpansPerSecondLimit < 0.0) { throw new IllegalArgumentException("Limit for sampled spans per second must be nonnegative!"); @@ -175,7 +174,7 @@ protected int getP(int parentP, boolean isRoot) { double probabilityToUseLowerPValue = (samplingProbability - lowerSamplingRate) / (upperSamplingRate - lowerSamplingRate); - if (threadSafeRandomGenerator.nextBoolean(probabilityToUseLowerPValue)) { + if (randomGenerator.nextBoolean(probabilityToUseLowerPValue)) { return lowerPValue; } else { return upperPValue; diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index 3655f9ee9..31f7428dd 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -14,7 +14,6 @@ import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.context.Context; import io.opentelemetry.contrib.state.OtelTraceState; -import io.opentelemetry.contrib.util.DefaultRandomGenerator; import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.samplers.Sampler; @@ -58,12 +57,12 @@ public static final ConsistentSampler probabilityBased(double samplingProbabilit * Returns a {@link ConsistentSampler} that samples each span with a fixed probability. * * @param samplingProbability the sampling probability - * @param threadSafeRandomGenerator a thread-safe random generator + * @param randomGenerator a random generator * @return a sampler */ public static final ConsistentSampler probabilityBased( - double samplingProbability, RandomGenerator threadSafeRandomGenerator) { - return new ConsistentProbabilityBasedSampler(samplingProbability, threadSafeRandomGenerator); + double samplingProbability, RandomGenerator randomGenerator) { + return new ConsistentProbabilityBasedSampler(samplingProbability, randomGenerator); } /** @@ -81,11 +80,11 @@ public static final ConsistentSampler parentBased(ConsistentSampler rootSampler) * or falls-back to the given sampler if it is a root span. * * @param rootSampler the root sampler - * @param threadSafeRandomGenerator a thread-safe random generator + * @param randomGenerator a random generator */ public static final ConsistentSampler parentBased( - ConsistentSampler rootSampler, RandomGenerator threadSafeRandomGenerator) { - return new ConsistentParentBasedSampler(rootSampler, threadSafeRandomGenerator); + ConsistentSampler rootSampler, RandomGenerator randomGenerator) { + return new ConsistentParentBasedSampler(rootSampler, randomGenerator); } /** @@ -108,19 +107,16 @@ public static final ConsistentSampler rateLimited( * @param targetSpansPerSecondLimit the desired spans per second limit * @param adaptationTimeSeconds the typical time to adapt to a new load (time constant used for * exponential smoothing) - * @param threadSafeRandomGenerator a thread-safe random generator + * @param randomGenerator a random generator * @param nanoTimeSupplier a supplier for the current nano time */ public static final ConsistentSampler rateLimited( double targetSpansPerSecondLimit, double adaptationTimeSeconds, - RandomGenerator threadSafeRandomGenerator, + RandomGenerator randomGenerator, LongSupplier nanoTimeSupplier) { return new ConsistentRateLimitingSampler( - targetSpansPerSecondLimit, - adaptationTimeSeconds, - threadSafeRandomGenerator, - nanoTimeSupplier); + targetSpansPerSecondLimit, adaptationTimeSeconds, randomGenerator, nanoTimeSupplier); } /** @@ -167,14 +163,14 @@ public ConsistentSampler or(ConsistentSampler otherConsistentSampler) { return new ConsistentComposedOrSampler(this, otherConsistentSampler); } - protected final RandomGenerator threadSafeRandomGenerator; + protected final RandomGenerator randomGenerator; - protected ConsistentSampler(RandomGenerator threadSafeRandomGenerator) { - this.threadSafeRandomGenerator = requireNonNull(threadSafeRandomGenerator); + protected ConsistentSampler(RandomGenerator randomGenerator) { + this.randomGenerator = requireNonNull(randomGenerator); } protected ConsistentSampler() { - this(DefaultRandomGenerator.get()); + this(RandomGenerator.getDefault()); } private static final boolean isInvariantViolated( @@ -219,9 +215,7 @@ public final SamplingResult shouldSample( // generate new r-value if not available if (!otelTraceState.hasValidR()) { otelTraceState.setR( - Math.min( - threadSafeRandomGenerator.numberOfLeadingZerosOfRandomLong(), - OtelTraceState.getMaxR())); + Math.min(randomGenerator.numberOfLeadingZerosOfRandomLong(), OtelTraceState.getMaxR())); } // determine and set new p-value that is used for the sampling decision diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java deleted file mode 100644 index 4985473a5..000000000 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/DefaultRandomGenerator.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.contrib.util; - -import java.util.concurrent.ThreadLocalRandom; - -public class DefaultRandomGenerator implements RandomGenerator { - - private static final class ThreadLocalData { - private long randomBits = 0; - private int bitCount = 0; - - boolean nextRandomBit() { - if ((bitCount & 0x3F) == 0) { - randomBits = ThreadLocalRandom.current().nextLong(); - } - boolean randomBit = ((randomBits >>> bitCount) & 1L) != 0L; - bitCount += 1; - return randomBit; - } - } - - private static final ThreadLocal THREAD_LOCAL_DATA = - ThreadLocal.withInitial(ThreadLocalData::new); - - private static final DefaultRandomGenerator INSTANCE = new DefaultRandomGenerator(); - - private DefaultRandomGenerator() {} - - public static RandomGenerator get() { - return INSTANCE; - } - - @Override - public long nextLong() { - return ThreadLocalRandom.current().nextLong(); - } - - @Override - public boolean nextBoolean(double probability) { - return RandomUtil.generateRandomBoolean( - () -> THREAD_LOCAL_DATA.get().nextRandomBit(), probability); - } - - @Override - public int numberOfLeadingZerosOfRandomLong() { - return RandomUtil.numberOfLeadingZerosOfRandomLong( - () -> THREAD_LOCAL_DATA.get().nextRandomBit()); - } -} diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java index dff93dc79..b2277cb83 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java @@ -5,15 +5,112 @@ package io.opentelemetry.contrib.util; -/** A random generator. */ -public interface RandomGenerator { +import static java.util.Objects.requireNonNull; + +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.LongSupplier; + +public final class RandomGenerator { + + private final LongSupplier threadSafeRandomLongSupplier; + + private static final class ThreadLocalData { + private long randomBits = 0; + private int bitCount = 0; + + private boolean nextRandomBit(LongSupplier threadSafeRandomLongSupplier) { + if ((bitCount & 0x3F) == 0) { + randomBits = threadSafeRandomLongSupplier.getAsLong(); + } + boolean randomBit = ((randomBits >>> bitCount) & 1L) != 0L; + bitCount += 1; + return randomBit; + } + + /** + * Returns a pseudorandomly chosen {@code boolean} value where the probability of returning + * {@code true} is predefined. + * + *

{@code true} needs to be returned with a success probability of {@code probability}. If + * the success probability is greater than 50% ({@code probability > 0.5}), the same can be + * achieved by returning {@code true} with a probability of 50%, and returning the result of a + * Bernoulli trial with a probability of {@code 2 * probability - 1}. The resulting success + * probability will be the same as {@code 0.5 + 0.5 * (2 * probability - 1) = probability}. + * Similarly, if the success probability is smaller than 50% ({@code probability <= 0.5}), + * {@code false} is returned with a probability of 50%. Otherwise, the result of a Bernoulli + * trial with success probability of {@code 2 * probability} is returned. Again, the resulting + * success probability is exactly as desired because {@code 0.5 * (2 * probability) = + * probability}. Recursive continuation of this approach allows realizing Bernoulli trials with + * arbitrary success probabilities using just few random bits. + * + * @param threadSafeRandomLongSupplier a thread-safe random long supplier + * @param probability the probability of returning {@code true} + * @return a random {@code boolean} + */ + private boolean generateRandomBoolean( + LongSupplier threadSafeRandomLongSupplier, double probability) { + while (true) { + if (probability <= 0) { + return false; + } + if (probability >= 1) { + return true; + } + boolean b = probability > 0.5; + if (nextRandomBit(threadSafeRandomLongSupplier)) { + return b; + } + probability += probability; + if (b) { + probability -= 1; + } + } + } + + /** + * Returns the number of leading zeros of a uniform random 64-bit integer. + * + * @param threadSafeRandomLongSupplier a thread-safe random long supplier + * @return the number of leading zeros + */ + private int numberOfLeadingZerosOfRandomLong(LongSupplier threadSafeRandomLongSupplier) { + int count = 0; + while (count < Long.SIZE && nextRandomBit(threadSafeRandomLongSupplier)) { + count += 1; + } + return count; + } + } + + private static final ThreadLocal THREAD_LOCAL_DATA = + ThreadLocal.withInitial(ThreadLocalData::new); + + private static final RandomGenerator INSTANCE = + new RandomGenerator(() -> ThreadLocalRandom.current().nextLong()); + + private RandomGenerator(LongSupplier threadSafeRandomLongSupplier) { + this.threadSafeRandomLongSupplier = requireNonNull(threadSafeRandomLongSupplier); + } /** - * Returns a pseudorandomly chosen {@code long} value. + * Creates a new random generator using the given thread-safe random long supplier as random + * source. * - * @return a pseudorandomly chosen {@code long} value + * @param threadSafeRandomLongSupplier a thread-safe random long supplier + * @return a random generator */ - long nextLong(); + public static RandomGenerator create(LongSupplier threadSafeRandomLongSupplier) { + return new RandomGenerator(threadSafeRandomLongSupplier); + } + + /** + * Returns a default random generator. + * + * @return a random generator + */ + public static RandomGenerator getDefault() { + return INSTANCE; + } /** * Returns a pseudorandomly chosen {@code boolean} value where the probability of returning {@code @@ -22,29 +119,8 @@ public interface RandomGenerator { * @param probability the probability of returning {@code true} * @return a random {@code boolean} */ - default boolean nextBoolean(double probability) { - long randomBits = 0; - int bitCounter = 0; - while (true) { - if (probability <= 0) { - return false; - } - if (probability >= 1) { - return true; - } - boolean b = probability > 0.5; - if ((bitCounter & 0x3f) == 0) { - randomBits = nextLong(); - } - if (((randomBits >>> bitCounter) & 1L) == 1L) { - return b; - } - bitCounter += 1; - probability += probability; - if (b) { - probability -= 1; - } - } + public boolean nextBoolean(double probability) { + return THREAD_LOCAL_DATA.get().generateRandomBoolean(threadSafeRandomLongSupplier, probability); } /** @@ -52,7 +128,7 @@ default boolean nextBoolean(double probability) { * * @return the number of leading zeros */ - default int numberOfLeadingZerosOfRandomLong() { - return Long.numberOfLeadingZeros(nextLong()); + public int numberOfLeadingZerosOfRandomLong() { + return THREAD_LOCAL_DATA.get().numberOfLeadingZerosOfRandomLong(threadSafeRandomLongSupplier); } } diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java deleted file mode 100644 index 57efc7a85..000000000 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomUtil.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.contrib.util; - -import java.util.function.BooleanSupplier; - -public final class RandomUtil { - - private RandomUtil() {} - - /** - * Returns a pseudorandomly chosen {@code boolean} value where the probability of returning {@code - * true} is predefined. - * - *

{@code true} needs to be returned with a success probability of {@code probability}. If the - * success probability is greater than 50% ({@code probability > 0.5}), the same can be achieved - * by returning {@code true} with a probability of 50%, and returning the result of a Bernoulli - * trial with a probability of {@code 2 * probability - 1}. The resulting success probability will - * be the same as {@code 0.5 + 0.5 * (2 * probability - 1) = probability}. Similarly, if the - * success probability is smaller than 50% ({@code probability <= 0.5}), {@code false} is returned - * with a probability of 50%. Otherwise, the result of a Bernoulli trial with success probability - * of {@code 2 * probability} is returned. Again, the resulting success probability is exactly as - * desired because {@code 0.5 * (2 * probability) = probability}. Recursive continuation of this - * approach allows realizing Bernoulli trials with arbitrary success probabilities using just few - * random bits. - * - * @param randomBooleanSupplier a random boolean supplier - * @param probability the probability of returning {@code true} - * @return a random {@code boolean} - */ - public static boolean generateRandomBoolean( - BooleanSupplier randomBooleanSupplier, double probability) { - while (true) { - if (probability <= 0) { - return false; - } - if (probability >= 1) { - return true; - } - boolean b = probability > 0.5; - if (randomBooleanSupplier.getAsBoolean()) { - return b; - } - probability += probability; - if (b) { - probability -= 1; - } - } - } - - /** - * Returns the number of leading zeros of a uniform random 64-bit integer. - * - * @param randomBooleanSupplier a random boolean supplier - * @return the truncated geometrically distributed random value - */ - public static int numberOfLeadingZerosOfRandomLong(BooleanSupplier randomBooleanSupplier) { - int count = 0; - while (count < Long.SIZE && randomBooleanSupplier.getAsBoolean()) { - count += 1; - } - return count; - } -} diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java index 03333659d..3a1b67181 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java @@ -13,6 +13,7 @@ import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.context.Context; import io.opentelemetry.contrib.state.OtelTraceState; +import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; @@ -48,7 +49,9 @@ public void init() { private void test(SplittableRandom rng, double samplingProbability) { int numSpans = 1000000; - Sampler sampler = ConsistentSampler.probabilityBased(samplingProbability, rng::nextLong); + Sampler sampler = + ConsistentSampler.probabilityBased( + samplingProbability, RandomGenerator.create(rng::nextLong)); Map observedPvalues = new HashMap<>(); for (long i = 0; i < numSpans; ++i) { diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java index 6446296b0..42117621b 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java @@ -10,6 +10,7 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; +import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; @@ -63,7 +64,10 @@ void testConstantRate() { ConsistentSampler sampler = ConsistentSampler.rateLimited( - targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); + targetSpansPerSecondLimit, + adaptationTimeSeconds, + RandomGenerator.create(random::nextLong), + nanoTimeSupplier); long nanosBetweenSpans = TimeUnit.MICROSECONDS.toNanos(100); int numSpans = 1000000; @@ -97,7 +101,10 @@ void testRateIncrease() { ConsistentSampler sampler = ConsistentSampler.rateLimited( - targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); + targetSpansPerSecondLimit, + adaptationTimeSeconds, + RandomGenerator.create(random::nextLong), + nanoTimeSupplier); long nanosBetweenSpans1 = TimeUnit.MICROSECONDS.toNanos(100); long nanosBetweenSpans2 = TimeUnit.MICROSECONDS.toNanos(10); @@ -153,7 +160,10 @@ void testRateDecrease() { ConsistentSampler sampler = ConsistentSampler.rateLimited( - targetSpansPerSecondLimit, adaptationTimeSeconds, random::nextLong, nanoTimeSupplier); + targetSpansPerSecondLimit, + adaptationTimeSeconds, + RandomGenerator.create(random::nextLong), + nanoTimeSupplier); long nanosBetweenSpans1 = TimeUnit.MICROSECONDS.toNanos(10); long nanosBetweenSpans2 = TimeUnit.MICROSECONDS.toNanos(100); From da4201893e6ea2674fb4b4f5f7a5337de8d1d23f Mon Sep 17 00:00:00 2001 From: Otmar Ertl Date: Wed, 20 Apr 2022 06:55:27 +0200 Subject: [PATCH 47/47] made OtelTraceState and RandomGenerator package private and moved them to samplers package --- .../samplers/ConsistentAlwaysOffSampler.java | 1 - .../ConsistentComposedAndSampler.java | 1 - .../samplers/ConsistentComposedOrSampler.java | 1 - .../ConsistentParentBasedSampler.java | 1 - .../ConsistentProbabilityBasedSampler.java | 1 - .../ConsistentRateLimitingSampler.java | 1 - .../contrib/samplers/ConsistentSampler.java | 8 +- .../{state => samplers}/OtelTraceState.java | 4 +- .../{util => samplers}/RandomGenerator.java | 4 +- ...ConsistentProbabilityBasedSamplerTest.java | 65 +++++++++++++++- .../ConsistentRateLimitingSamplerTest.java | 1 - .../samplers/ConsistentSamplerTest.java | 1 - .../OtelTraceStateTest.java | 2 +- .../opentelemetry/contrib/util/TestUtil.java | 77 ------------------- 14 files changed, 70 insertions(+), 98 deletions(-) rename consistent-sampling/src/main/java/io/opentelemetry/contrib/{state => samplers}/OtelTraceState.java (98%) rename consistent-sampling/src/main/java/io/opentelemetry/contrib/{util => samplers}/RandomGenerator.java (98%) rename consistent-sampling/src/test/java/io/opentelemetry/contrib/{state => samplers}/OtelTraceStateTest.java (98%) delete mode 100644 consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java index ae0af4cbc..37759d670 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentAlwaysOffSampler.java @@ -5,7 +5,6 @@ package io.opentelemetry.contrib.samplers; -import io.opentelemetry.contrib.state.OtelTraceState; import javax.annotation.concurrent.Immutable; @Immutable diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java index 1b5e12712..17f758b46 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedAndSampler.java @@ -7,7 +7,6 @@ import static java.util.Objects.requireNonNull; -import io.opentelemetry.contrib.state.OtelTraceState; import javax.annotation.concurrent.Immutable; /** diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java index 21150c672..b090dd399 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentComposedOrSampler.java @@ -7,7 +7,6 @@ import static java.util.Objects.requireNonNull; -import io.opentelemetry.contrib.state.OtelTraceState; import javax.annotation.concurrent.Immutable; /** diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java index 6d7ad4dd9..4db1ad196 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentParentBasedSampler.java @@ -7,7 +7,6 @@ import static java.util.Objects.requireNonNull; -import io.opentelemetry.contrib.util.RandomGenerator; import javax.annotation.concurrent.Immutable; /** diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java index 3b05549af..2e55d7409 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSampler.java @@ -5,7 +5,6 @@ package io.opentelemetry.contrib.samplers; -import io.opentelemetry.contrib.util.RandomGenerator; import javax.annotation.concurrent.Immutable; /** A consistent sampler that samples with a fixed probability. */ diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java index 117501718..9f1edbbd1 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSampler.java @@ -7,7 +7,6 @@ import static java.util.Objects.requireNonNull; -import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.samplers.Sampler; import java.util.concurrent.atomic.AtomicReference; import java.util.function.LongSupplier; diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java index 31f7428dd..6f9cd76e2 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/ConsistentSampler.java @@ -13,8 +13,6 @@ import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.context.Context; -import io.opentelemetry.contrib.state.OtelTraceState; -import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; @@ -60,7 +58,7 @@ public static final ConsistentSampler probabilityBased(double samplingProbabilit * @param randomGenerator a random generator * @return a sampler */ - public static final ConsistentSampler probabilityBased( + static final ConsistentSampler probabilityBased( double samplingProbability, RandomGenerator randomGenerator) { return new ConsistentProbabilityBasedSampler(samplingProbability, randomGenerator); } @@ -82,7 +80,7 @@ public static final ConsistentSampler parentBased(ConsistentSampler rootSampler) * @param rootSampler the root sampler * @param randomGenerator a random generator */ - public static final ConsistentSampler parentBased( + static final ConsistentSampler parentBased( ConsistentSampler rootSampler, RandomGenerator randomGenerator) { return new ConsistentParentBasedSampler(rootSampler, randomGenerator); } @@ -110,7 +108,7 @@ public static final ConsistentSampler rateLimited( * @param randomGenerator a random generator * @param nanoTimeSupplier a supplier for the current nano time */ - public static final ConsistentSampler rateLimited( + static final ConsistentSampler rateLimited( double targetSpansPerSecondLimit, double adaptationTimeSeconds, RandomGenerator randomGenerator, diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/OtelTraceState.java similarity index 98% rename from consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java rename to consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/OtelTraceState.java index b71ca3e3b..f29d40b6d 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/state/OtelTraceState.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/OtelTraceState.java @@ -3,14 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -package io.opentelemetry.contrib.state; +package io.opentelemetry.contrib.samplers; import java.util.ArrayList; import java.util.Collections; import java.util.List; import javax.annotation.Nullable; -public final class OtelTraceState { +final class OtelTraceState { public static final String TRACE_STATE_KEY = "ot"; diff --git a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/RandomGenerator.java similarity index 98% rename from consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java rename to consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/RandomGenerator.java index b2277cb83..3de2260a2 100644 --- a/consistent-sampling/src/main/java/io/opentelemetry/contrib/util/RandomGenerator.java +++ b/consistent-sampling/src/main/java/io/opentelemetry/contrib/samplers/RandomGenerator.java @@ -3,14 +3,14 @@ * SPDX-License-Identifier: Apache-2.0 */ -package io.opentelemetry.contrib.util; +package io.opentelemetry.contrib.samplers; import static java.util.Objects.requireNonNull; import java.util.concurrent.ThreadLocalRandom; import java.util.function.LongSupplier; -public final class RandomGenerator { +final class RandomGenerator { private final LongSupplier threadSafeRandomLongSupplier; diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java index 3a1b67181..e37861d24 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentProbabilityBasedSamplerTest.java @@ -5,15 +5,13 @@ package io.opentelemetry.contrib.samplers; -import static io.opentelemetry.contrib.util.TestUtil.verifyObservedPvaluesUsingGtest; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertTrue; import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.api.trace.TraceState; import io.opentelemetry.context.Context; -import io.opentelemetry.contrib.state.OtelTraceState; -import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.samplers.Sampler; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; @@ -23,6 +21,7 @@ import java.util.List; import java.util.Map; import java.util.SplittableRandom; +import org.hipparchus.stat.inference.GTest; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -87,4 +86,64 @@ public void test() { test(random, 0.13); test(random, 0.05); } + + private static void verifyObservedPvaluesUsingGtest( + long originalNumberOfSpans, Map observedPvalues, double samplingProbability) { + + Object notSampled = + new Object() { + @Override + public String toString() { + return "NOT SAMPLED"; + } + }; + + Map expectedProbabilities = new HashMap<>(); + if (samplingProbability >= 1.) { + expectedProbabilities.put(0, 1.); + } else if (samplingProbability <= 0.) { + expectedProbabilities.put(notSampled, 1.); + } else { + int exponent = 0; + while (true) { + if (Math.pow(0.5, exponent + 1) < samplingProbability + && Math.pow(0.5, exponent) >= samplingProbability) { + break; + } + exponent += 1; + } + if (samplingProbability == Math.pow(0.5, exponent)) { + expectedProbabilities.put(notSampled, 1 - samplingProbability); + expectedProbabilities.put(exponent, samplingProbability); + } else { + expectedProbabilities.put(notSampled, 1 - samplingProbability); + expectedProbabilities.put(exponent, 2 * samplingProbability - Math.pow(0.5, exponent)); + expectedProbabilities.put(exponent + 1, Math.pow(0.5, exponent) - samplingProbability); + } + } + + Map extendedObservedAdjustedCounts = new HashMap<>(observedPvalues); + long numberOfSpansNotSampled = + originalNumberOfSpans - observedPvalues.values().stream().mapToLong(i -> i).sum(); + if (numberOfSpansNotSampled > 0) { + extendedObservedAdjustedCounts.put(notSampled, numberOfSpansNotSampled); + } + + double[] expectedValues = new double[expectedProbabilities.size()]; + long[] observedValues = new long[expectedProbabilities.size()]; + + int counter = 0; + for (Object key : expectedProbabilities.keySet()) { + observedValues[counter] = extendedObservedAdjustedCounts.getOrDefault(key, 0L); + double p = expectedProbabilities.get(key); + expectedValues[counter] = p * originalNumberOfSpans; + counter += 1; + } + + if (expectedProbabilities.size() > 1) { + assertThat(new GTest().gTest(expectedValues, observedValues)).isGreaterThan(0.01); + } else { + assertThat((double) observedValues[0]).isEqualTo(expectedValues[0]); + } + } } diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java index 42117621b..c54d22d2f 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentRateLimitingSamplerTest.java @@ -10,7 +10,6 @@ import io.opentelemetry.api.common.Attributes; import io.opentelemetry.api.trace.SpanKind; import io.opentelemetry.context.Context; -import io.opentelemetry.contrib.util.RandomGenerator; import io.opentelemetry.sdk.trace.data.LinkData; import io.opentelemetry.sdk.trace.samplers.SamplingDecision; import io.opentelemetry.sdk.trace.samplers.SamplingResult; diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java index 9ca37b867..d2c5128be 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/ConsistentSamplerTest.java @@ -9,7 +9,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; -import io.opentelemetry.contrib.state.OtelTraceState; import java.util.SplittableRandom; import org.junit.jupiter.api.Test; diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/OtelTraceStateTest.java similarity index 98% rename from consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java rename to consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/OtelTraceStateTest.java index 118c9d176..716321495 100644 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/state/OtelTraceStateTest.java +++ b/consistent-sampling/src/test/java/io/opentelemetry/contrib/samplers/OtelTraceStateTest.java @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -package io.opentelemetry.contrib.state; +package io.opentelemetry.contrib.samplers; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java b/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java deleted file mode 100644 index af196c1e9..000000000 --- a/consistent-sampling/src/test/java/io/opentelemetry/contrib/util/TestUtil.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.contrib.util; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.HashMap; -import java.util.Map; -import org.hipparchus.stat.inference.GTest; - -public final class TestUtil { - - private TestUtil() {} - - public static void verifyObservedPvaluesUsingGtest( - long originalNumberOfSpans, Map observedPvalues, double samplingProbability) { - - Object notSampled = - new Object() { - @Override - public String toString() { - return "NOT SAMPLED"; - } - }; - - Map expectedProbabilities = new HashMap<>(); - if (samplingProbability >= 1.) { - expectedProbabilities.put(0, 1.); - } else if (samplingProbability <= 0.) { - expectedProbabilities.put(notSampled, 1.); - } else { - int exponent = 0; - while (true) { - if (Math.pow(0.5, exponent + 1) < samplingProbability - && Math.pow(0.5, exponent) >= samplingProbability) { - break; - } - exponent += 1; - } - if (samplingProbability == Math.pow(0.5, exponent)) { - expectedProbabilities.put(notSampled, 1 - samplingProbability); - expectedProbabilities.put(exponent, samplingProbability); - } else { - expectedProbabilities.put(notSampled, 1 - samplingProbability); - expectedProbabilities.put(exponent, 2 * samplingProbability - Math.pow(0.5, exponent)); - expectedProbabilities.put(exponent + 1, Math.pow(0.5, exponent) - samplingProbability); - } - } - - Map extendedObservedAdjustedCounts = new HashMap<>(observedPvalues); - long numberOfSpansNotSampled = - originalNumberOfSpans - observedPvalues.values().stream().mapToLong(i -> i).sum(); - if (numberOfSpansNotSampled > 0) { - extendedObservedAdjustedCounts.put(notSampled, numberOfSpansNotSampled); - } - - double[] expectedValues = new double[expectedProbabilities.size()]; - long[] observedValues = new long[expectedProbabilities.size()]; - - int counter = 0; - for (Object key : expectedProbabilities.keySet()) { - observedValues[counter] = extendedObservedAdjustedCounts.getOrDefault(key, 0L); - double p = expectedProbabilities.get(key); - expectedValues[counter] = p * originalNumberOfSpans; - counter += 1; - } - - if (expectedProbabilities.size() > 1) { - assertThat(new GTest().gTest(expectedValues, observedValues)).isGreaterThan(0.01); - } else { - assertThat((double) observedValues[0]).isEqualTo(expectedValues[0]); - } - } -}