Add request tracing framework (opensearch-project#7648)

* Add request tracing framework

Signed-off-by: suranjay <surajkumar.tu@gmail.com>

* Introduce ThreadContextStatePropagator to propagate request and transient headers within ThreadContext

Signed-off-by: Andriy Redko <andriy.redko@aiven.io>

---------

Signed-off-by: suranjay <surajkumar.tu@gmail.com>
Signed-off-by: Andriy Redko <andriy.redko@aiven.io>
Co-authored-by: Andriy Redko <andriy.redko@aiven.io>
Signed-off-by: Shivansh Arora <hishiv@amazon.com>

CHANGELOG.md

@@ -35,6 +35,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
 - OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344)
 - Bump `com.google.http-client:google-http-client:1.43.2` from 1.42.0 to 1.43.2 ([7928](https://github.com/opensearch-project/OpenSearch/pull/7928)))
+- Add Opentelemetry dependencies ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -83,6 +84,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update components of segrep backpressure to support remote store. ([#8020](https://github.com/opensearch-project/OpenSearch/pull/8020))
 - Make remote cluster connection setup in async ([#8038](https://github.com/opensearch-project/OpenSearch/pull/8038))
 - Add API to initialize extensions ([#8029]()https://github.com/opensearch-project/OpenSearch/pull/8029)
+- Add distributed tracing framework ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543))
 
 ### Dependencies
 - Bump `com.azure:azure-storage-common` from 12.21.0 to 12.21.1 (#7566, #7814)

buildSrc/version.properties

@@ -66,3 +66,7 @@ zstd              = 1.5.5-3
 jzlib             = 1.1.3
 
 resteasy          = 6.2.4.Final
+
+# opentelemetry dependencies
+opentelemetry    = 1.26.0
+

libs/telemetry/build.gradle

@@ -0,0 +1,23 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+dependencies {
+  testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.opensearch', module: 'opensearch-telemetry'
+  }
+}
+
+tasks.named('forbiddenApisMain').configure {
+  replaceSignatureFiles 'jdk-signatures'
+}

libs/telemetry/src/main/java/org/opensearch/telemetry/Telemetry.java

@@ -0,0 +1,31 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry;
+
+import org.opensearch.telemetry.metrics.MetricsTelemetry;
+import org.opensearch.telemetry.tracing.TracingTelemetry;
+
+/**
+ * Interface defining telemetry
+ */
+public interface Telemetry {
+
+    /**
+     * Provides tracing telemetry
+     * @return tracing telemetry instance
+     */
+    TracingTelemetry getTracingTelemetry();
+
+    /**
+     * Provides metrics telemetry
+     * @return metrics telemetry instance
+     */
+    MetricsTelemetry getMetricsTelemetry();
+
+}

libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/MetricsTelemetry.java

@@ -0,0 +1,16 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.metrics;
+
+/**
+ * Interface for metrics telemetry providers
+ */
+public interface MetricsTelemetry {
+
+}

libs/telemetry/src/main/java/org/opensearch/telemetry/metrics/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Contains metrics related classes
+ */
+package org.opensearch.telemetry.metrics;

libs/telemetry/src/main/java/org/opensearch/telemetry/package-info.java

@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * Contains telemetry related classes
+ */
+package org.opensearch.telemetry;

libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/AbstractSpan.java

@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.tracing;
+
+/**
+ * Base span
+ */
+public abstract class AbstractSpan implements Span {
+
+    /**
+     * name of the span
+     */
+    private final String spanName;
+    /**
+     * span's parent span
+     */
+    private final Span parentSpan;
+
+    /**
+     * Base constructor
+     * @param spanName name of the span
+     * @param parentSpan span's parent span
+     */
+    protected AbstractSpan(String spanName, Span parentSpan) {
+        this.spanName = spanName;
+        this.parentSpan = parentSpan;
+    }
+
+    @Override
+    public Span getParentSpan() {
+        return parentSpan;
+    }
+
+    @Override
+    public String getSpanName() {
+        return spanName;
+    }
+
+}

libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/DefaultTracer.java

@@ -0,0 +1,109 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.tracing;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+/**
+ *
+ * The default tracer implementation. This class implements the basic logic for span lifecycle and its state management.
+ * It also handles tracing context propagation between spans.
+ *
+ *
+ */
+public class DefaultTracer implements Tracer {
+    static final String THREAD_NAME = "th_name";
+
+    private final TracingTelemetry tracingTelemetry;
+    private final TracerContextStorage<String, Span> tracerContextStorage;
+
+    /**
+     * Creates DefaultTracer instance
+     *
+     * @param tracingTelemetry tracing telemetry instance
+     * @param tracerContextStorage storage used for storing current span context
+     */
+    public DefaultTracer(TracingTelemetry tracingTelemetry, TracerContextStorage<String, Span> tracerContextStorage) {
+        this.tracingTelemetry = tracingTelemetry;
+        this.tracerContextStorage = tracerContextStorage;
+    }
+
+    @Override
+    public Scope startSpan(String spanName) {
+        Span span = createSpan(spanName, getCurrentSpan());
+        setCurrentSpanInContext(span);
+        addDefaultAttributes(span);
+        return new ScopeImpl(() -> endSpan(span));
+    }
+
+    @Override
+    public void addSpanAttribute(String key, String value) {
+        Span currentSpan = getCurrentSpan();
+        currentSpan.addAttribute(key, value);
+    }
+
+    @Override
+    public void addSpanAttribute(String key, long value) {
+        Span currentSpan = getCurrentSpan();
+        currentSpan.addAttribute(key, value);
+    }
+
+    @Override
+    public void addSpanAttribute(String key, double value) {
+        Span currentSpan = getCurrentSpan();
+        currentSpan.addAttribute(key, value);
+    }
+
+    @Override
+    public void addSpanAttribute(String key, boolean value) {
+        Span currentSpan = getCurrentSpan();
+        currentSpan.addAttribute(key, value);
+    }
+
+    @Override
+    public void addSpanEvent(String event) {
+        Span currentSpan = getCurrentSpan();
+        currentSpan.addEvent(event);
+    }
+
+    @Override
+    public void close() throws IOException {
+        ((Closeable) tracingTelemetry).close();
+    }
+
+    // Visible for testing
+    Span getCurrentSpan() {
+        return tracerContextStorage.get(TracerContextStorage.CURRENT_SPAN);
+    }
+
+    private void endSpan(Span span) {
+        if (span != null) {
+            span.endSpan();
+            setCurrentSpanInContext(span.getParentSpan());
+        }
+    }
+
+    private Span createSpan(String spanName, Span parentSpan) {
+        return tracingTelemetry.createSpan(spanName, parentSpan);
+    }
+
+    private void setCurrentSpanInContext(Span span) {
+        tracerContextStorage.put(TracerContextStorage.CURRENT_SPAN, span);
+    }
+
+    /**
+     * Adds default attributes in the span
+     * @param span the current active span
+     */
+    protected void addDefaultAttributes(Span span) {
+        span.addAttribute(THREAD_NAME, Thread.currentThread().getName());
+    }
+
+}

libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/Scope.java

@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.tracing;
+
+/**
+ * An auto-closeable that represents scope of the span.
+ * It is recommended that you use this class with a try-with-resources block:
+ */
+public interface Scope extends AutoCloseable {
+    /**
+     * No-op Scope implementation
+     */
+    Scope NO_OP = () -> {};
+
+    /**
+     * closes the scope
+     */
+    @Override
+    void close();
+}

libs/telemetry/src/main/java/org/opensearch/telemetry/tracing/ScopeImpl.java

@@ -0,0 +1,33 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.telemetry.tracing;
+
+/**
+ * Executes the runnable on close
+ */
+public class ScopeImpl implements Scope {
+
+    private Runnable runnableOnClose;
+
+    /**
+     * Creates Scope instance
+     * @param runnableOnClose runnable to execute on scope close
+     */
+    public ScopeImpl(Runnable runnableOnClose) {
+        this.runnableOnClose = runnableOnClose;
+    }
+
+    /**
+     * Executes the runnable to end the scope
+     */
+    @Override
+    public void close() {
+        runnableOnClose.run();
+    }
+}