opentelemetrytracer: Dynatrace sampler: Enable adaptative sampling (e…

…nvoyproxy#32848) * opentelemetrytracer: Dynatrace sampler to fetch configuration from an API (#21) * Dynatrace sampler to fetch configuration from an API Signed-off-by: Thomas Ebner <96168670+samohte@users.noreply.github.com> Co-authored-by: Joao Grassi <5938087+joaopgrassi@users.noreply.github.com> * review feedback: add log msg, remove timeout adjustment Signed-off-by: thomas.ebner <thomas.ebner@dynatrace.com> --------- Signed-off-by: Thomas Ebner <96168670+samohte@users.noreply.github.com> Signed-off-by: thomas.ebner <thomas.ebner@dynatrace.com> Co-authored-by: Joao Grassi <5938087+joaopgrassi@users.noreply.github.com>
ravenblackx · Mar 21, 2024 · c4fe01c · c4fe01c
1 parent 05424b2
commit c4fe01c
Show file tree

Hide file tree

Showing 8 changed files with 304 additions and 22 deletions.
diff --git a/api/envoy/extensions/tracers/opentelemetry/samplers/v3/dynatrace_sampler.proto b/api/envoy/extensions/tracers/opentelemetry/samplers/v3/dynatrace_sampler.proto
@@ -33,7 +33,7 @@ message DynatraceSamplerConfig {
   // .. code-block:: yaml
   //
   //    http_uri:
-  //      uri: <tenant>.dev.dynatracelabs.com/api/v2/otlp/v1/traces
+  //      uri: <tenant>.dev.dynatracelabs.com/api/v2/samplingConfiguration
   //      cluster: dynatrace
   //      timeout: 10s
   //

diff --git a/changelogs/current.yaml b/changelogs/current.yaml
@@ -365,6 +365,9 @@ new_features:
   change: |
     Added :ref:`rules_stat_prefix <envoy_v3_api_field_extensions.filters.http.rbac.v3.RBAC.rules_stat_prefix>`
     to allow adding custom prefix to the stats emitted by rules.
+- area: tracing
+  change: |
+    Dynatrace sampler fetches configuration from Dynatrace API.
 
 deprecated:
 - area: listener

diff --git a/source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config.cc b/source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config.cc
@@ -8,18 +8,19 @@ namespace Extensions {
 namespace Tracers {
 namespace OpenTelemetry {
 
-void SamplerConfig::parse(const std::string& json) {
+bool SamplerConfig::parse(const std::string& json) {
   const auto result = Envoy::Json::Factory::loadFromStringNoThrow(json);
   if (result.ok()) {
     const auto& obj = result.value();
     if (obj->hasObject("rootSpansPerMinute")) {
       const auto value = obj->getInteger("rootSpansPerMinute", default_root_spans_per_minute_);
       root_spans_per_minute_.store(value);
-      return;
+      return true;
     }
   }
   // Didn't get a value, reset to default
   root_spans_per_minute_.store(default_root_spans_per_minute_);
+  return false;
 }
 
 } // namespace OpenTelemetry

diff --git a/source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config.h b/source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config.h
@@ -22,12 +22,18 @@ class SamplerConfig {
                                            ? default_root_spans_per_minute
                                            : ROOT_SPANS_PER_MINUTE_DEFAULT),
         root_spans_per_minute_(default_root_spans_per_minute_) {}
+
+  SamplerConfig(const SamplerConfig&) = delete;
+  SamplerConfig& operator=(const SamplerConfig&) = delete;
+
   /**
    * @brief Parses a json string containing the expected root spans per minute.
    *
    * @param json A string containing the configuration.
+   *
+   * @return true if parsing was successful, false otherwise
    */
-  void parse(const std::string& json);
+  bool parse(const std::string& json);
 
   /**
    * @brief Returns wanted root spans per minute

diff --git a/source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config_provider.cc b/source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config_provider.cc
@@ -1,5 +1,7 @@
 #include "source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config_provider.h"
 
+#include <chrono>
+
 #include "source/common/common/enum_to_int.h"
 #include "source/common/http/utility.h"
 
@@ -8,10 +10,92 @@ namespace Extensions {
 namespace Tracers {
 namespace OpenTelemetry {
 
+static constexpr std::chrono::seconds INITIAL_TIMER_DURATION{10};
+static constexpr std::chrono::minutes TIMER_INTERVAL{5};
+
+namespace {
+
+bool reEnableTimer(Http::Code response_code) {
+  switch (response_code) {
+  case Http::Code::OK:
+  case Http::Code::TooManyRequests:
+  case Http::Code::InternalServerError:
+  case Http::Code::BadGateway:
+  case Http::Code::ServiceUnavailable:
+  case Http::Code::GatewayTimeout:
+    return true;
+  default:
+    return false;
+  }
+}
+
+} // namespace
+
 SamplerConfigProviderImpl::SamplerConfigProviderImpl(
-    Server::Configuration::TracerFactoryContext& /*context*/,
+    Server::Configuration::TracerFactoryContext& context,
     const envoy::extensions::tracers::opentelemetry::samplers::v3::DynatraceSamplerConfig& config)
-    : sampler_config_(config.root_spans_per_minute()) {}
+    : cluster_manager_(context.serverFactoryContext().clusterManager()),
+      http_uri_(config.http_uri()),
+      authorization_header_value_(absl::StrCat("Api-Token ", config.token())),
+      sampler_config_(config.root_spans_per_minute()) {
+
+  timer_ = context.serverFactoryContext().mainThreadDispatcher().createTimer([this]() -> void {
+    const auto thread_local_cluster = cluster_manager_.getThreadLocalCluster(http_uri_.cluster());
+    if (thread_local_cluster == nullptr) {
+      ENVOY_LOG(error, "SamplerConfigProviderImpl failed: [cluster = {}] is not configured",
+                http_uri_.cluster());
+    } else {
+      Http::RequestMessagePtr message = Http::Utility::prepareHeaders(http_uri_);
+      message->headers().setReferenceMethod(Http::Headers::get().MethodValues.Get);
+      message->headers().setReference(Http::CustomHeaders::get().Authorization,
+                                      authorization_header_value_);
+      active_request_ = thread_local_cluster->httpAsyncClient().send(
+          std::move(message), *this,
+          Http::AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds(
+              DurationUtil::durationToMilliseconds(http_uri_.timeout()))));
+    }
+  });
+
+  timer_->enableTimer(std::chrono::seconds(INITIAL_TIMER_DURATION));
+}
+
+SamplerConfigProviderImpl::~SamplerConfigProviderImpl() {
+  if (active_request_) {
+    active_request_->cancel();
+  }
+}
+
+void SamplerConfigProviderImpl::onSuccess(const Http::AsyncClient::Request& /*request*/,
+                                          Http::ResponseMessagePtr&& http_response) {
+  active_request_ = nullptr;
+  const auto response_code = Http::Utility::getResponseStatus(http_response->headers());
+  bool json_valid = false;
+  if (response_code == enumToInt(Http::Code::OK)) {
+    ENVOY_LOG(debug, "Received sampling configuration from Dynatrace: {}",
+              http_response->bodyAsString());
+    json_valid = sampler_config_.parse(http_response->bodyAsString());
+    if (!json_valid) {
+      ENVOY_LOG(warn, "Failed to parse sampling configuration received from Dynatrace: {}",
+                http_response->bodyAsString());
+    }
+  } else {
+    ENVOY_LOG(warn, "Failed to get sampling configuration from Dynatrace: {}", response_code);
+  }
+
+  if (json_valid || reEnableTimer(static_cast<Http::Code>(response_code))) {
+    timer_->enableTimer(std::chrono::seconds(TIMER_INTERVAL));
+  } else {
+    ENVOY_LOG(error, "Stopped to query sampling configuration from Dynatrace.");
+  }
+}
+
+void SamplerConfigProviderImpl::onFailure(const Http::AsyncClient::Request& /*request*/,
+                                          Http::AsyncClient::FailureReason reason) {
+  active_request_ = nullptr;
+  timer_->enableTimer(std::chrono::seconds(TIMER_INTERVAL));
+  ENVOY_LOG(warn, "Failed to get sampling configuration from Dynatrace. Reason {}",
+            enumToInt(reason));
+}
 
 const SamplerConfig& SamplerConfigProviderImpl::getSamplerConfig() const { return sampler_config_; }
 

diff --git a/source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config_provider.h b/source/extensions/tracers/opentelemetry/samplers/dynatrace/sampler_config_provider.h
@@ -40,18 +40,33 @@ class SamplerConfigProvider {
 };
 
 class SamplerConfigProviderImpl : public SamplerConfigProvider,
-                                  public Logger::Loggable<Logger::Id::tracing> {
+                                  public Logger::Loggable<Logger::Id::tracing>,
+                                  public Http::AsyncClient::Callbacks {
 public:
   SamplerConfigProviderImpl(
       Server::Configuration::TracerFactoryContext& context,
       const envoy::extensions::tracers::opentelemetry::samplers::v3::DynatraceSamplerConfig&
           config);
 
+  void onSuccess(const Http::AsyncClient::Request& request,
+                 Http::ResponseMessagePtr&& response) override;
+
+  void onFailure(const Http::AsyncClient::Request& request,
+                 Http::AsyncClient::FailureReason reason) override;
+
+  void onBeforeFinalizeUpstreamSpan(Envoy::Tracing::Span& /*span*/,
+                                    const Http::ResponseHeaderMap* /*response_headers*/) override{};
+
   const SamplerConfig& getSamplerConfig() const override;
 
-  ~SamplerConfigProviderImpl() override = default;
+  ~SamplerConfigProviderImpl() override;
 
 private:
+  Event::TimerPtr timer_;
+  Upstream::ClusterManager& cluster_manager_;
+  envoy::config::core::v3::HttpUri http_uri_;
+  const std::string authorization_header_value_;
+  Http::AsyncClient::Request* active_request_{};
   SamplerConfig sampler_config_;
 };