refactor: update opentelemetry in all our crates (#652)

* refactor: update gateway, deployer, and auth

* feat: tracing on provisioner

* refactor: propagation to auth

* feat: otel container

* refactor: propagate to gateway better

* refactor: clear build queue faster

* refactor: lower build queue timeout

* refactor: bigger buffer and more logs

* refactor: batch DataDog data better

* refactor: sync expirations

Cargo.toml

@@ -46,7 +46,6 @@ hyper = "0.14.23"
 jsonwebtoken = { version = "8.2.0" }
 once_cell = "1.16.0"
 opentelemetry = { version = "0.18.0", features = ["rt-tokio"] }
-opentelemetry-datadog = { version = "0.6.0", features = ["reqwest-client"] }
 opentelemetry-http = "0.7.0"
 pin-project = "1.0.12"
 rand = "0.8.5"

Makefile

@@ -50,6 +50,7 @@ DD_ENV=production
 # make sure we only ever go to production with `--tls=enable`
 USE_TLS=enable
 CARGO_PROFILE=release
+RUST_LOG=debug
 else
 DOCKER_COMPOSE_FILES=-f docker-compose.yml -f docker-compose.dev.yml
 STACK?=shuttle-dev
@@ -59,6 +60,7 @@ CONTAINER_REGISTRY=public.ecr.aws/shuttle-dev
 DD_ENV=unstable
 USE_TLS?=disable
 CARGO_PROFILE=debug
+RUST_LOG?=shuttle=trace,debug
 endif
 
 POSTGRES_EXTRA_PATH?=./extras/postgres
@@ -67,17 +69,18 @@ POSTGRES_TAG?=14
 PANAMAX_EXTRA_PATH?=./extras/panamax
 PANAMAX_TAG?=1.0.6
 
-RUST_LOG?=debug
+OTEL_EXTRA_PATH?=./extras/otel
+OTEL_TAG?=0.72.0
 
-DOCKER_COMPOSE_ENV=STACK=$(STACK) BACKEND_TAG=$(BACKEND_TAG) DEPLOYER_TAG=$(DEPLOYER_TAG) PROVISIONER_TAG=$(PROVISIONER_TAG) POSTGRES_TAG=${POSTGRES_TAG} PANAMAX_TAG=${PANAMAX_TAG} APPS_FQDN=$(APPS_FQDN) DB_FQDN=$(DB_FQDN) POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) RUST_LOG=$(RUST_LOG) CONTAINER_REGISTRY=$(CONTAINER_REGISTRY) MONGO_INITDB_ROOT_USERNAME=$(MONGO_INITDB_ROOT_USERNAME) MONGO_INITDB_ROOT_PASSWORD=$(MONGO_INITDB_ROOT_PASSWORD) DD_ENV=$(DD_ENV) USE_TLS=$(USE_TLS)
+DOCKER_COMPOSE_ENV=STACK=$(STACK) BACKEND_TAG=$(BACKEND_TAG) DEPLOYER_TAG=$(DEPLOYER_TAG) PROVISIONER_TAG=$(PROVISIONER_TAG) POSTGRES_TAG=${POSTGRES_TAG} PANAMAX_TAG=${PANAMAX_TAG} OTEL_TAG=${OTEL_TAG} APPS_FQDN=$(APPS_FQDN) DB_FQDN=$(DB_FQDN) POSTGRES_PASSWORD=$(POSTGRES_PASSWORD) RUST_LOG=$(RUST_LOG) CONTAINER_REGISTRY=$(CONTAINER_REGISTRY) MONGO_INITDB_ROOT_USERNAME=$(MONGO_INITDB_ROOT_USERNAME) MONGO_INITDB_ROOT_PASSWORD=$(MONGO_INITDB_ROOT_PASSWORD) DD_ENV=$(DD_ENV) USE_TLS=$(USE_TLS)
 
 .PHONY: images clean src up down deploy shuttle-% postgres docker-compose.rendered.yml test bump-% deploy-examples publish publish-% --validate-version
 
 clean:
 	rm .shuttle-*
 	rm docker-compose.rendered.yml
 
-images: shuttle-provisioner shuttle-deployer shuttle-gateway shuttle-auth postgres panamax
+images: shuttle-provisioner shuttle-deployer shuttle-gateway shuttle-auth postgres panamax otel
 
 postgres:
 	docker buildx build \
@@ -95,6 +98,14 @@ panamax:
 	       -f $(PANAMAX_EXTRA_PATH)/Containerfile \
 	       $(PANAMAX_EXTRA_PATH)
 
+otel:
+	docker buildx build \
+	       --build-arg OTEL_TAG=$(OTEL_TAG) \
+	       --tag $(CONTAINER_REGISTRY)/otel:$(OTEL_TAG) \
+	       $(BUILDX_FLAGS) \
+	       -f $(OTEL_EXTRA_PATH)/Containerfile \
+	       $(OTEL_EXTRA_PATH)
+
 docker-compose.rendered.yml: docker-compose.yml docker-compose.dev.yml
 	$(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) $(DOCKER_COMPOSE_FILES) $(DOCKER_COMPOSE_CONFIG_FLAGS) -p $(STACK) config > $@
 

auth/Cargo.toml

@@ -15,7 +15,6 @@ clap = { workspace = true }
 http = { workspace = true }
 jsonwebtoken = { workspace = true }
 opentelemetry = { workspace = true }
-opentelemetry-datadog = { workspace = true }
 rand = { workspace = true }
 ring = { workspace = true }
 serde = { workspace = true, features = ["derive"] }
@@ -25,7 +24,7 @@ thiserror = { workspace = true }
 tokio = { version = "1.22.0", features = ["full"] }
 tracing = { workspace = true }
 tracing-opentelemetry = { workspace = true }
-tracing-subscriber = { workspace = true, features = ["env-filter"] }
+tracing-subscriber = { workspace = true }
 
 [dependencies.shuttle-common]
 workspace = true

auth/src/main.rs

@@ -1,10 +1,9 @@
 use std::io;
 
 use clap::Parser;
-use opentelemetry::global;
+use shuttle_common::backends::tracing::setup_tracing;
 use sqlx::migrate::Migrator;
 use tracing::{info, trace};
-use tracing_subscriber::{fmt, prelude::*, EnvFilter};
 
 use shuttle_auth::{init, sqlite_init, start, Args, Commands};
 
@@ -16,24 +15,7 @@ async fn main() -> io::Result<()> {
 
     trace!(args = ?args, "parsed args");
 
-    global::set_text_map_propagator(opentelemetry_datadog::DatadogPropagator::new());
-
-    let fmt_layer = fmt::layer();
-    let filter_layer = EnvFilter::try_from_default_env()
-        .or_else(|_| EnvFilter::try_new("info"))
-        .unwrap();
-
-    let tracer = opentelemetry_datadog::new_pipeline()
-        .with_service_name("gateway")
-        .with_agent_endpoint("http://datadog-agent:8126")
-        .install_batch(opentelemetry::runtime::Tokio)
-        .unwrap();
-    let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer);
-    tracing_subscriber::registry()
-        .with(filter_layer)
-        .with(fmt_layer)
-        .with(opentelemetry)
-        .init();
+    setup_tracing(tracing_subscriber::registry(), "auth");
 
     let db_path = args.state.join("authentication.sqlite");
 

common/Cargo.toml

@@ -23,7 +23,8 @@ jsonwebtoken = { workspace = true, optional = true }
 once_cell = { workspace = true }
 opentelemetry = { workspace = true, optional = true }
 opentelemetry-http = { workspace = true, optional = true }
-pin-project = { workspace = true } 
+opentelemetry-otlp = { version = "0.11.0", optional = true }
+pin-project = { workspace = true }
 reqwest = { version = "0.11.13", optional = true }
 rustrict = "0.5.5"
 serde = { workspace = true, features = ["derive"] }
@@ -35,11 +36,12 @@ tower = { workspace = true, optional = true }
 tower-http = { workspace = true, optional = true }
 tracing = { workspace = true }
 tracing-opentelemetry = { workspace = true, optional = true }
+tracing-subscriber = { workspace = true, optional = true }
 ttl_cache = { workspace = true, optional = true }
 uuid = { workspace = true, features = ["v4", "serde"] }
 
 [features]
-backend = ["async-trait", "axum", "bytes", "http", "http-body", "hyper/client", "jsonwebtoken", "opentelemetry", "opentelemetry-http", "thiserror", "tower", "tower-http", "tracing-opentelemetry", "ttl_cache"]
+backend = ["async-trait", "axum", "bytes", "http", "http-body", "hyper/client", "jsonwebtoken", "opentelemetry", "opentelemetry-http", "opentelemetry-otlp", "thiserror", "tower", "tower-http", "tracing-opentelemetry", "tracing-subscriber/env-filter", "ttl_cache"]
 display = ["comfy-table", "crossterm"]
 models = ["anyhow", "async-trait", "display", "http", "reqwest", "serde_json"]
 

common/src/backends/auth.rs

@@ -15,18 +15,21 @@ use http::{Request, Response, StatusCode, Uri};
 use http_body::combinators::UnsyncBoxBody;
 use hyper::{body, Body, Client};
 use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header as JwtHeader, Validation};
+use opentelemetry::global;
+use opentelemetry_http::HeaderInjector;
 use pin_project::pin_project;
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 use tower::{Layer, Service};
-use tracing::{error, trace};
+use tracing::{error, trace, Span};
+use tracing_opentelemetry::OpenTelemetrySpanExt;
 
 use super::{
     cache::{CacheManagement, CacheManager},
     headers::XShuttleAdminSecret,
 };
 
-const EXP_MINUTES: i64 = 5;
+pub const EXP_MINUTES: i64 = 5;
 const ISS: &str = "shuttle";
 const PUBLIC_KEY_CACHE_KEY: &str = "shuttle.public-key";
 
@@ -307,8 +310,18 @@ impl PublicKeyFn for AuthPublicKey {
             Ok(public_key)
         } else {
             let client = Client::new();
-            let uri = format!("{}public-key", self.auth_uri).parse()?;
-            let res = client.get(uri).await?;
+            let uri: Uri = format!("{}public-key", self.auth_uri).parse()?;
+            let mut request = Request::builder().uri(uri);
+
+            // Safe to unwrap since we just build it
+            let headers = request.headers_mut().unwrap();
+
+            let cx = Span::current().context();
+            global::get_text_map_propagator(|propagator| {
+                propagator.inject_context(&cx, &mut HeaderInjector(headers))
+            });
+
+            let res = client.request(request.body(Body::empty())?).await?;
             let buf = body::to_bytes(res).await?;
 
             trace!("inserting public key from auth service into cache");
@@ -330,6 +343,9 @@ pub enum PublicKeyFnError {
 
     #[error("hyper error: {0}")]
     Hyper(#[from] hyper::Error),
+
+    #[error("http error: {0}")]
+    Http(#[from] http::Error),
 }
 
 /// Layer to validate JWT tokens with a public key. Valid claims are added to the request extension

common/src/backends/mod.rs

@@ -2,3 +2,4 @@ pub mod auth;
 pub mod cache;
 pub mod headers;
 pub mod metrics;
+pub mod tracing;